frontend,backend,rpmbuild,cli,python: uploadrpm - multiple RPMs, optional srpm/logs

backend/copr_backend/background_worker_build.py

@@ -8,6 +8,7 @@
 import os
 import shutil
 import statistics
+import tarfile
 import time
 import json
 import shlex
@@ -581,6 +582,22 @@ def _transfer_log_file(self):
         except SSHConnectionError as exc:
             return "Stopped following builder for broken SSH: {}".format(exc)
 
+    def _archive_uploaded_logs(self):
+        uploaded_logs_dir = os.path.join(self.job.results_dir, "uploaded-logs")
+        if not os.path.isdir(uploaded_logs_dir):
+            return
+
+        tarball_path = os.path.join(self.job.results_dir, "uploaded-logs.tar.gz")
+        try:
+            with tarfile.open(tarball_path, "w:gz") as tar:
+                for entry in sorted(os.listdir(uploaded_logs_dir)):
+                    tar.add(os.path.join(uploaded_logs_dir, entry), arcname=entry)
+        except OSError as ex:
+            self.log.error("Unable to archive uploaded logs: %s", ex)
+            return
+
+        shutil.rmtree(uploaded_logs_dir, ignore_errors=True)
+
     def _compress_logs(self):
         """
         Compress builder-live.log, backend.log, and fedora-review.log using gzip.
@@ -1106,6 +1123,7 @@ def handle_task(self):
             self._drop_host()
             if self.job:
                 self._mark_finished()
+                self._archive_uploaded_logs()
                 self._compress_logs()
             else:
                 self.log.error("No job object from Frontend")

backend/tests/test_background_worker_build.py

@@ -11,6 +11,7 @@
 import os
 import shutil
 import subprocess
+import tarfile
 import time
 import tempfile
 from unittest import mock
@@ -895,3 +896,55 @@ def test_buildjob_chroot_dir(f_build_rpm_case):
     job_dict = copy.deepcopy(testlib.VALID_SUBPROJECT_PRM_JOB)
     job = BuildJob(job_dict, worker.opts)
     assert job.chroot_dir.endswith("copr-pull-requests:pr:3568/fedora-40-x86_64")
+
+def test_archive_uploaded_logs_creates_tarball(f_build_rpm_case):
+    config = f_build_rpm_case
+    worker = config.bw
+    worker.job = _get_rpm_job_object(worker.opts)
+
+    uploaded_logs_dir = os.path.join(worker.job.results_dir, "uploaded-logs")
+    os.makedirs(uploaded_logs_dir)
+    with open(os.path.join(uploaded_logs_dir, "builder-live.log"), "w",
+             encoding="utf-8") as handle:
+        handle.write("live log")
+    with open(os.path.join(uploaded_logs_dir, "notes.txt.gz"), "wb") as handle:
+        handle.write(b"gz data")
+
+    worker._archive_uploaded_logs()
+
+    tarball_path = os.path.join(worker.job.results_dir, "uploaded-logs.tar.gz")
+    assert os.path.exists(tarball_path)
+    assert not os.path.exists(uploaded_logs_dir)
+    with tarfile.open(tarball_path, "r:gz") as tar:
+        names = sorted(tar.getnames())
+    assert names == ["builder-live.log", "notes.txt.gz"]
+
+def test_archive_uploaded_logs_noop_without_dir(f_build_rpm_case):
+    config = f_build_rpm_case
+    worker = config.bw
+    worker.job = _get_rpm_job_object(worker.opts)
+    os.makedirs(worker.job.results_dir, exist_ok=True)
+
+    worker._archive_uploaded_logs()
+
+    tarball_path = os.path.join(worker.job.results_dir, "uploaded-logs.tar.gz")
+    assert not os.path.exists(tarball_path)
+
+def test_archive_uploaded_logs_tar_error(f_build_rpm_case, caplog):
+    config = f_build_rpm_case
+    worker = config.bw
+    worker.job = _get_rpm_job_object(worker.opts)
+
+    uploaded_logs_dir = os.path.join(worker.job.results_dir, "uploaded-logs")
+    os.makedirs(uploaded_logs_dir)
+    with open(os.path.join(uploaded_logs_dir, "builder-live.log"), "w",
+             encoding="utf-8") as handle:
+        handle.write("live log")
+
+    with mock.patch("copr_backend.background_worker_build.tarfile.open",
+                    side_effect=OSError("disk full")):
+        worker._archive_uploaded_logs()
+
+    assert_logs_exist(["Unable to archive uploaded logs"], caplog)
+    # the raw directory is left in place, nothing crashed
+    assert os.path.isdir(uploaded_logs_dir)

beaker-tests/Sanity/copr-cli-basic-operations/runtest-rpm-upload.sh

@@ -23,6 +23,7 @@ source "$HERE/config"
 source "$HERE/helpers"
 
 PACKAGE=copr-rpm-upload-sanity-test
+PACKAGE_MULTI=copr-rpm-upload-multi-sanity-test
 
 # Build a throwaway binary RPM locally
 build_local_rpm()
@@ -49,6 +50,41 @@ EOF
     find "$workdir" -name '*.rpm'
 }
 
+# Build a throwaway package with a sub-package (-> multiple binary RPMs) plus
+# its srpm, to exercise the multi-file "uploadrpm" scenario (RPM array +
+# optional accompanying srpm)
+build_local_rpms_with_subpackage_and_srpm()
+{
+    local workdir
+    workdir=$(mktemp -d)
+    cat > "$workdir/$PACKAGE_MULTI.spec" <<EOF
+Name: $PACKAGE_MULTI
+Version: 1
+Release: 1
+Summary: Throwaway package for the direct RPM upload sanity test
+License: MIT
+BuildArch: $(rpm --eval '%_arch')
+
+%description
+Throwaway package for the direct RPM upload sanity test.
+
+%package subpkg
+Summary: Throwaway sub-package for the direct RPM upload sanity test
+%description subpkg
+Throwaway sub-package for the direct RPM upload sanity test.
+
+%files
+
+%files subpkg
+EOF
+    rpmbuild -ba "$workdir/$PACKAGE_MULTI.spec" \
+        --define "_topdir $workdir" \
+        --define "_rpmdir $workdir" \
+        --define "_srcrpmdir $workdir" \
+        --define "_build_id_links none" >&2
+    find "$workdir" -name '*.rpm'
+}
+
 rlJournalStart
     rlPhaseStartSetup
         setup_checks
@@ -93,8 +129,73 @@ rlJournalStart
             $PROJECT $RPM_PATH" 1 "Upload with wrong SHA256 should fail"
     rlPhaseEnd
 
+    rlPhaseStartTest "multi-file uploadrpm: RPM array + --name + --srpm + --logs"
+        if [[ $FRONTEND_URL == "https://copr.stg.fedoraproject.org" ]]; then
+            rlLog "Skipping, RPM uploads are not enabled for the Fedora Copr instance"
+            exit 0
+        fi
+
+        rlRun "RPM_PATHS=(\$(build_local_rpms_with_subpackage_and_srpm))" \
+            0 "Building local test RPMs (main + sub-package + srpm)"
+
+        SRPM_PATH=
+        BINARY_RPMS=()
+        for _path in "${RPM_PATHS[@]}"; do
+            case "$_path" in
+                *.src.rpm) SRPM_PATH=$_path ;;
+                *) BINARY_RPMS+=("$_path") ;;
+            esac
+        done
+        rlAssertExists "$SRPM_PATH"
+        rlRun "test ${#BINARY_RPMS[@]} -eq 2" 0 \
+            "Expecting 2 binary RPMs (main package + sub-package)"
+
+        # a handful of throwaway log/text files to upload alongside the
+        # RPMs, covering all four supported extensions
+        LOG1=$(mktemp --suffix=.log)
+        echo "fake builder-live log" > "$LOG1"
+        LOG2=$(mktemp --suffix=.log.gz)
+        echo "fake gzipped log" | gzip > "$LOG2"
+        LOG3=$(mktemp --suffix=.txt)
+        echo "fake notes" > "$LOG3"
+        LOG4=$(mktemp --suffix=.txt.gz)
+        echo "fake gzipped notes" | gzip > "$LOG4"
+
+        # one SHA256 checksum per binary RPM, in the same order as
+        # BINARY_RPMS, to exercise the multi-RPM checksum verification
+        CHECKSUMS=()
+        for _rpm in "${BINARY_RPMS[@]}"; do
+            CHECKSUMS+=("$(sha256sum "$_rpm" | cut -d' ' -f1)")
+        done
+
+        # --name is required here since more than one RPM is uploaded and
+        # the package name can't be reliably guessed from the filenames
+        rlRun -s "copr-cli uploadrpm --nowait --chroot $CHROOT \
+            --name $PACKAGE_MULTI \
+            --srpm $SRPM_PATH \
+            --logs $LOG1 $LOG2 $LOG3 $LOG4 \
+            --sha256 ${CHECKSUMS[*]} \
+            $PROJECT ${BINARY_RPMS[*]}"
+        rlRun "parse_build_id"
+        rlRun "copr watch-build $BUILD_ID"
+
+        # verify both the main package and its sub-package are installable
+        rlRun "dnf install -y --disablerepo='*' \
+            --enablerepo=\"copr:${FRONTEND_PUBLIC_HOST}:$(repo_owner):${PROJECTNAME}\" \
+            $PACKAGE_MULTI $PACKAGE_MULTI-subpkg"
+        rlAssertRpm "$PACKAGE_MULTI"
+        rlAssertRpm "$PACKAGE_MULTI-subpkg"
+
+        # the uploaded logs must be auto-compressed into a single tarball
+        # and downloadable the same way as regular build logs (never via
+        # Pulp -- they only ever live on the backend filesystem)
+        DOWNLOAD_DEST=$(mktemp -d)
+        rlRun "copr-cli download-build --dest $DOWNLOAD_DEST --logs $BUILD_ID"
+        rlRun "find $DOWNLOAD_DEST -name 'uploaded-logs.tar.gz'"
+    rlPhaseEnd
+
     rlPhaseStartCleanup
-        rlRun "dnf -y remove $PACKAGE"
+        rlRun "dnf -y remove $PACKAGE $PACKAGE_MULTI $PACKAGE_MULTI-subpkg"
         rlRun "dnf -y copr remove $DNF_COPR_ID/$PROJECT"
         cleanProject
     rlPhaseEnd

cli/copr_cli/main.py

@@ -464,17 +464,31 @@ def action_upload_rpm(self, args):
         username, projectname, project_dirname = self.parse_dirname(args.copr_repo)
         buildopts = buildopts_from_args(args)
 
-        if not os.path.exists(args.rpm):
-            raise CoprException("File {0} not found".format(args.rpm))
+        if len(args.rpms) > 1 and not args.pkgname:
+            raise CoprException(
+                "--name is required when uploading more than one RPM "
+                "(it can't be reliably guessed).")
+
+        all_paths = list(args.rpms)
+        if args.srpm:
+            all_paths.append(args.srpm)
+        if args.logs:
+            all_paths.extend(args.logs)
+
+        for path in all_paths:
+            if not os.path.exists(path):
+                raise CoprException("File {0} not found".format(path))
 
-        progress_callback = get_progress_callback(os.path.getsize(args.rpm))
+        total_size = sum(os.path.getsize(path) for path in all_paths)
+        progress_callback = get_progress_callback(total_size)
         buildopts["progress_callback"] = progress_callback
-        print('Uploading package {0}'.format(args.rpm))
+        print('Uploading package(s) {0}'.format(', '.join(all_paths)))
         try:
             build = self.client.build_proxy.create_from_rpm_upload(
                 ownername=username, projectname=projectname,
                 project_dirname=project_dirname, buildopts=buildopts,
-                path=args.rpm,
+                paths=args.rpms, name=args.pkgname,
+                srpm_path=args.srpm, log_paths=args.logs,
                 sha256=getattr(args, "sha256", None))
         finally:
             if progress_callback:
@@ -884,6 +898,8 @@ def action_download_build(self, args):
 
             if args.logs:
                 cmd.extend(["-A", "*.log.gz"])
+                # tarball of client-uploaded logs for "uploadrpm" builds
+                cmd.extend(["-A", "uploaded-logs.tar.gz"])
 
             if args.review:
                 cmd.extend([
@@ -1739,13 +1755,30 @@ def setup_parser():
     # create the parser for the "uploadrpm" command
     parser_upload_rpm = subparsers.add_parser(
         "uploadrpm", parents=[parser_build_parent],
-        help="Publish an already-built local RPM directly to a specified copr, "
-             "skipping the SRPM build phase entirely")
+        help="Publish one or more already-built local RPMs directly to a "
+             "specified copr, skipping the SRPM build phase entirely")
+    parser_upload_rpm.add_argument(
+        "rpms", nargs="+",
+        help="Local path(s) to the already-built .rpm file(s) to publish")
+    parser_upload_rpm.add_argument(
+        "--name", dest="pkgname", required=False,
+        help=("Package name. Optional when uploading a single RPM "
+              "(guessed from its filename), required when uploading "
+              "more than one RPM."))
+    parser_upload_rpm.add_argument(
+        "--srpm", dest="srpm", metavar="SRPM", required=False,
+        help=("Optional local path to an accompanying .src.rpm/.nosrc.rpm "
+              "file, published alongside the uploaded RPM(s)"))
     parser_upload_rpm.add_argument(
-        "rpm", help="Local path to the already-built .rpm file to publish")
+        "--logs", dest="logs", nargs="+", metavar="LOG", required=False,
+        help=("Optional local path(s) to .log/.log.gz/.txt/.txt.gz "
+              "file(s), auto-compressed into a single tarball and "
+              "stored on the backend filesystem"))
     parser_upload_rpm.add_argument(
-        "--sha256", help="Expected SHA256 hex digest of the uploaded file; "
-        "the server rejects the build on mismatch")
+        "--sha256", nargs="+", metavar="SHA256", required=False,
+        help=("Optional expected SHA256 hex digest(s) of the uploaded "
+              "RPM(s), one per RPM in the same order as the RPM "
+              "arguments; the server rejects the build on mismatch"))
     parser_upload_rpm.set_defaults(func="action_upload_rpm")
 
     # create the parser for the "buildpypi" command

cli/man/copr-cli.1.asciidoc

@@ -268,16 +268,29 @@ This limitation comes from the COPR API.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 usage: copr-cli uploadrpm [-h] [-r, --chroot CHROOTS] [--memory MEMORY] [--timeout TIMEOUT] [--nowait]
-                          [--background]
-                          copr_repo RPM
+                          [--background] [--name NAME] [--srpm SRPM] [--logs LOG [LOG ...]]
+                          copr_repo RPMS [RPMS ...]
 
-Publish an already-built local RPM directly to a project, skipping the SRPM
-build phase entirely.
+Publish one or more already-built local RPMs directly to a project, skipping
+the SRPM build phase entirely.
 
 For the arguments, see `copr-cli build` command above.
 
-RPM::
-Local path to the already-built .rpm file to publish.
+RPMS::
+Local path(s) to the already-built .rpm file(s) to publish.
+
+--name NAME::
+Package name. Optional when uploading a single RPM (guessed from its
+filename), required when uploading more than one RPM.
+
+--srpm SRPM::
+Optional local path to an accompanying .src.rpm/.nosrc.rpm file, published
+alongside the uploaded RPM(s).
+
+--logs LOG [LOG ...]::
+Optional local path(s) to .log/.log.gz/.txt/.txt.gz file(s). They are
+auto-compressed into a single tarball and stored to Copr, downloadable the same way
+as regular build logs e.g. via `copr-cli download-build --logs`.
 
 
 `copr-cli buildpypi [options]`

cli/man/copr-cli.cheat

@@ -34,5 +34,13 @@ copr-cli buildpypi test-project --packagename pydantic-ai --with-deps --chroot f
 # to publish an already-built local RPM directly, skipping the SRPM build
 copr-cli uploadrpm test-project ~/packages/package-1.6-1.fc28.x86_64.rpm
 
+# to publish multiple built RPMs (plus their srpm and build logs) directly,
+# skipping the SRPM build; --name is required with more than one RPM
+copr-cli uploadrpm test-project --name package \
+    --srpm ~/packages/package-1.6-1.fc28.src.rpm \  # optional
+    --logs ~/packages/build.log ~/packages/build.txt \
+    ~/packages/package-1.6-1.fc28.x86_64.rpm \
+    ~/packages/package-devel-1.6-1.fc28.x86_64.rpm
+
 # to regenerate repository metadata for a project
 copr-cli regenerate-repos test-project