diff --git a/.github/pkg-workflows/debian/pkg-pr-hook.yml b/.github/pkg-workflows/debian/pkg-pr-hook.yml index 0bb77583..c52b5876 100644 --- a/.github/pkg-workflows/debian/pkg-pr-hook.yml +++ b/.github/pkg-workflows/debian/pkg-pr-hook.yml @@ -1,4 +1,4 @@ -name: PR Pre and Post Merge Build +name: PR Build on: pull_request: @@ -10,33 +10,8 @@ permissions: packages: read jobs: - resolve-suite: - name: Resolve suite from branch - if: github.repository_owner == 'qualcomm-linux' && github.repository != 'qualcomm-linux/pkg-template' - runs-on: ubuntu-latest - outputs: - suite: ${{ steps.resolve.outputs.suite }} - steps: - - name: Derive suite from target branch - id: resolve - env: - BASE_REF: ${{ github.base_ref }} - run: | - set -euo pipefail - case "$BASE_REF" in - qcom/ubuntu/*|qcom/debian/*) - suite="${BASE_REF##*/}" - ;; - *) - suite=sid - ;; - esac - echo "suite=$suite" >> "$GITHUB_OUTPUT" - echo "Resolved suite: $suite (from branch: $BASE_REF)" - build: - name: Build Debian Package - needs: resolve-suite + name: PR Build if: ${{ github.event.action != 'closed' || github.event.pull_request.merged == true }} uses: qualcomm-linux/qcom-build-utils/.github/workflows/pkg-build-reusable-workflow.yml@main with: @@ -44,7 +19,6 @@ jobs: # PRE-MERGE: use the PR head branch (github.head_ref) # POST-MERGE: use the base branch name from the PR debian-ref: ${{ (github.event.action == 'closed' && github.event.pull_request.merged) && github.event.pull_request.base.ref || github.head_ref }} - suite: ${{ needs.resolve-suite.outputs.suite }} debusine-parent-workspace: ${{ vars.DEBUSINE_PARENT_WORKSPACE }} secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} diff --git a/.github/pkg-workflows/main/pkg-build.yml b/.github/pkg-workflows/qli-ci/pkg-build.yml similarity index 70% rename from .github/pkg-workflows/main/pkg-build.yml rename to .github/pkg-workflows/qli-ci/pkg-build.yml index 958b0872..008cc160 100644 --- a/.github/pkg-workflows/main/pkg-build.yml +++ b/.github/pkg-workflows/qli-ci/pkg-build.yml @@ -1,4 +1,4 @@ -name: Build Debian Package +name: Build description: | Builds the debian package represented by this repo at the ref pointed by the debain-ref argument. @@ -12,20 +12,6 @@ on: required: true default: qcom/debian/latest - suite: - description: The distribution codename or Debian suite to build for. Ex noble, questing, resolute, trixie, sid, unstable - type: choice - default: unstable - options: - - noble - - questing - - resolute - - unstable - - forky - - trixie - - bookworm - - sid - force-docker-build: description: Force local pkg-builder instead of Debusine for Debian-family suites type: boolean @@ -37,11 +23,11 @@ permissions: jobs: build: + name: Build uses: qualcomm-linux/qcom-build-utils/.github/workflows/pkg-build-reusable-workflow.yml@main with: qcom-build-utils-ref: main debian-ref: ${{ inputs.debian-ref }} - suite: ${{ inputs.suite }} force-docker-build: ${{ inputs.force-docker-build }} debusine-parent-workspace: ${{ vars.DEBUSINE_PARENT_WORKSPACE }} secrets: diff --git a/.github/pkg-workflows/main/pkg-promote-prebuilt.yml b/.github/pkg-workflows/qli-ci/pkg-promote-prebuilt.yml similarity index 97% rename from .github/pkg-workflows/main/pkg-promote-prebuilt.yml rename to .github/pkg-workflows/qli-ci/pkg-promote-prebuilt.yml index e1b2bb9e..415a2afe 100644 --- a/.github/pkg-workflows/main/pkg-promote-prebuilt.yml +++ b/.github/pkg-workflows/qli-ci/pkg-promote-prebuilt.yml @@ -1,4 +1,4 @@ -name: Promote New Prebuilt Binary Version +name: Promote Prebuilt on: workflow_dispatch: diff --git a/.github/pkg-workflows/main/pkg-promote.yml b/.github/pkg-workflows/qli-ci/pkg-promote.yml similarity index 90% rename from .github/pkg-workflows/main/pkg-promote.yml rename to .github/pkg-workflows/qli-ci/pkg-promote.yml index 564114fb..3b195b9b 100644 --- a/.github/pkg-workflows/main/pkg-promote.yml +++ b/.github/pkg-workflows/qli-ci/pkg-promote.yml @@ -1,4 +1,4 @@ -name: Promote New Upstream Version +name: Promote on: workflow_dispatch: @@ -7,8 +7,7 @@ on: debian-branch: description: The debian branch to apply the promotion to. For example branch "qcom/debian/latest" type: string - required: false - default: qcom/debian/latest + required: true upstream-tag: description: The upstream tag to promote this package repo to. Eg, v1.1.0 or 1.2.0, depending on the versioning style @@ -22,6 +21,7 @@ permissions: jobs: promote: + name: Promote uses: qualcomm-linux/qcom-build-utils/.github/workflows/pkg-promote-reusable-workflow.yml@main with: diff --git a/.github/pkg-workflows/main/pkg-release.yml b/.github/pkg-workflows/qli-ci/pkg-release.yml similarity index 52% rename from .github/pkg-workflows/main/pkg-release.yml rename to .github/pkg-workflows/qli-ci/pkg-release.yml index 542be328..f5922d93 100644 --- a/.github/pkg-workflows/main/pkg-release.yml +++ b/.github/pkg-workflows/qli-ci/pkg-release.yml @@ -1,4 +1,4 @@ -name: Release Version +name: Release description: | Release a new package version. This workflow is manually triggered via workflow_dispatch. @@ -7,30 +7,15 @@ on: workflow_dispatch: inputs: - suite: - description: The distribution codename or Debian suite to release for. Ex noble, questing, resolute, trixie, bookworm, sid, unstable - type: choice - default: noble - options: - - noble - - questing - - resolute - - unstable - - forky - - trixie - - bookworm - - sid - debian-branch: - description: The debian branch to use to execute the release from. For example branch "qcom/debian/latest" + description: The debian branch to release. For example branch "qcom/debian/trixie" type: string - required: false - default: qcom/debian/latest + required: true test-run: description: | - Debian suites: if true, stop after the Debusine build and installability test. - Ubuntu codenames: preserve the previous release flow, including upload to the test S3 location. + Debian: if true, stop after build/test. + Ubuntu: still runs the full release flow. type: boolean default: true @@ -39,12 +24,10 @@ permissions: packages: read jobs: - release: - + Release: uses: qualcomm-linux/qcom-build-utils/.github/workflows/pkg-release-reusable-workflow.yml@main with: qcom-build-utils-ref: main - suite: ${{ github.event.inputs.suite }} debian-branch: ${{ github.event.inputs.debian-branch }} test-run: ${{ github.event.inputs.test-run == 'true' && true || false }} debusine-parent-workspace: ${{ vars.DEBUSINE_PARENT_WORKSPACE }} @@ -53,3 +36,4 @@ jobs: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + QSC_API_KEY: ${{ secrets.DEB_PKG_BOT_CI_QSC_TOKEN }} diff --git a/.github/workflows/pkg-build-reusable-workflow.yml b/.github/workflows/pkg-build-reusable-workflow.yml index 35f48a43..504f5242 100644 --- a/.github/workflows/pkg-build-reusable-workflow.yml +++ b/.github/workflows/pkg-build-reusable-workflow.yml @@ -1,4 +1,4 @@ -name: Qualcomm Build Debian Package Reusable Workflow +name: Build Reusable Workflow description: | This reusable workflow is called by debian-packaging repos to offer a consistent build-and-test process. @@ -18,11 +18,6 @@ on: type: string required: true - suite: - description: The distribution codename or Debian suite to build for. Ex noble, questing, resolute, trixie, sid, unstable - type: string - default: unstable - run-lintian: description: Run lintian or not during the Docker pkg-builder build path type: boolean @@ -42,11 +37,6 @@ on: type: string default: "" - job-index: - description: Uniquely identifying index for matrix-expanded callers so Debusine child workspaces stay distinct - type: string - default: "0" - release: description: Prepare a release bundle (finalizes changelog, creates git bundle for push-release) for the Debian Debusine path type: boolean @@ -58,21 +48,11 @@ on: force-docker-build: description: | - When the suite is Debian-family, force the local sbuild pkg-builder path instead of Debusine. + When the resolved family is Debian, force the local sbuild pkg-builder path instead of Debusine. Automatically set to true when DEBUSINE_TOKEN is absent. type: boolean default: false - srcpkg-artifact: - description: | - Name of a GitHub Actions artifact containing a pre-prepared source tree - to use as srcpkg/. When set, the debian-ref checkout is skipped and the - artifact is downloaded instead. Used for packages like pkg-linux-qcom - where the source tree is assembled at CI time rather than managed in a - single git repository. - type: string - default: "" - assemble-orig: description: | When true, generate-source-package creates the .orig.tar.gz directly @@ -89,21 +69,24 @@ on: required: false outputs: - target_suite: + family: + description: The resolved distro family (`debian` or `ubuntu`) + value: ${{ jobs.resolve.outputs.family }} + suite: description: The resolved suite/codename actually used by the workflow - value: ${{ jobs.finalize.outputs.target_suite }} + value: ${{ jobs.resolve.outputs.suite }} workspace: description: Debusine workspace ID for Debusine builds; empty for Docker builds - value: ${{ jobs.finalize.outputs.workspace }} + value: ${{ jobs.test.outputs.workspace }} workspace_url: description: Debusine workspace URL for Debusine builds; empty for Docker builds - value: ${{ jobs.finalize.outputs.workspace_url }} + value: ${{ jobs.test.outputs.workspace_url }} srcpkg_name: description: Source package name - value: ${{ jobs.finalize.outputs.srcpkg_name }} + value: ${{ jobs.test.outputs.srcpkg_name }} srcpkg_version: description: Source package version - value: ${{ jobs.finalize.outputs.srcpkg_version }} + value: ${{ jobs.test.outputs.srcpkg_version }} concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.release && 'release' || github.run_id }} @@ -117,56 +100,89 @@ env: jobs: resolve: - name: Resolve suite family + name: Resolve branch target runs-on: ubuntu-latest outputs: family: ${{ steps.resolve.outputs.family }} force_docker_build: ${{ steps.resolve.outputs.force_docker_build }} - target_suite: ${{ steps.resolve.outputs.target_suite }} + suite: ${{ steps.resolve.outputs.suite }} steps: - - name: Classify suite + - name: Checkout qcom-build-utils scripts + uses: actions/checkout@v5 + with: + repository: qualcomm-linux/qcom-build-utils + ref: ${{ inputs.qcom-build-utils-ref }} + path: qcom-build-utils + + - name: Resolve family and suite from branch id: resolve shell: bash env: - SUITE_INPUT: ${{ inputs.suite }} + DEBIAN_REF_INPUT: ${{ inputs.debian-ref }} + BASE_REF_INPUT: ${{ github.base_ref }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} FORCE_DOCKER_BUILD: ${{ inputs.force-docker-build }} run: | set -euo pipefail - target_suite="$SUITE_INPUT" - - # 'latest' is not a valid suite name — map to sid (Debian unstable) - if [[ "$target_suite" == "latest" ]]; then - target_suite=sid + RESOLVER_SCRIPT="${GITHUB_WORKSPACE}/qcom-build-utils/scripts/resolve_branch_family_suite.sh" + + parse_resolved_values() { + local resolved_kv="$1" + local key value + + normalized_ref="" + family="" + suite="" + while IFS='=' read -r key value; do + case "$key" in + normalized_ref) normalized_ref="$value" ;; + family) family="$value" ;; + suite) suite="$value" ;; + esac + done <<< "$resolved_kv" + } + + source_ref="" + if resolved_kv="$("$RESOLVER_SCRIPT" "$DEBIAN_REF_INPUT")"; then + parse_resolved_values "$resolved_kv" + source_ref="$normalized_ref" + else + if [[ -n "$BASE_REF_INPUT" ]]; then + if resolved_kv="$("$RESOLVER_SCRIPT" "$BASE_REF_INPUT")"; then + parse_resolved_values "$resolved_kv" + source_ref="$normalized_ref" + echo "::warning::debian-ref '$DEBIAN_REF_INPUT' does not end with '//'; falling back to base-ref '$BASE_REF_INPUT' for suite routing." + else + echo "::error::Unable to resolve build target from debian-ref '$DEBIAN_REF_INPUT' or base-ref '$BASE_REF_INPUT'. Expected a branch ending in '//' (for example 'ubuntu/resolute' or 'qcom/ubuntu/resolute')." + exit 1 + fi + else + echo "::error::Unable to resolve build target from debian-ref '$DEBIAN_REF_INPUT'. Expected a branch ending in '//' (for example 'ubuntu/resolute' or 'qcom/ubuntu/resolute')." + exit 1 + fi fi - case "$target_suite" in - trixie|sid|unstable|bookworm|forky) - family=debian - ;; - *) - family=ubuntu - ;; - esac - - if [[ "$target_suite" == "unstable" ]]; then - target_suite=sid + # Preserve Debian aliases used in branch names. + if [[ "$family" == "debian" ]] && [[ "$suite" == "latest" || "$suite" == "unstable" ]]; then + suite=sid fi force_docker_build=false if [[ "$family" == "debian" ]] && [[ -z "$DEBUSINE_TOKEN" || "$FORCE_DOCKER_BUILD" == "true" ]]; then force_docker_build=true if [[ -z "$DEBUSINE_TOKEN" ]]; then - echo "::warning::DEBUSINE_TOKEN is not set or not accessible (fork PR secrets are unavailable to workflows triggered by external contributors) — falling back to local pkg-builder for suite '$target_suite'" + echo "::warning::DEBUSINE_TOKEN is not set or not accessible (fork PR secrets are unavailable to workflows triggered by external contributors) — falling back to local pkg-builder for suite '$suite'" else - echo "::notice::force-docker-build is set — using local pkg-builder for suite '$target_suite'" + echo "::notice::force-docker-build is set — using local pkg-builder for suite '$suite'" fi fi + echo "::notice title=Resolved build target::debian-ref='${DEBIAN_REF_INPUT}', source-ref='${source_ref}', family='${family}', suite='${suite}', force-docker-build='${force_docker_build}'" + echo "family=$family" >> "$GITHUB_OUTPUT" echo "force_docker_build=$force_docker_build" >> "$GITHUB_OUTPUT" - echo "target_suite=$target_suite" >> "$GITHUB_OUTPUT" + echo "suite=$suite" >> "$GITHUB_OUTPUT" docker-build: name: Build (Docker) @@ -180,7 +196,7 @@ jobs: run: shell: bash container: - image: ghcr.io/qualcomm-linux/pkg-builder:${{ needs.resolve.outputs.target_suite }} + image: ghcr.io/qualcomm-linux/pkg-builder:${{ needs.resolve.outputs.suite }} options: --privileged credentials: username: ${{ github.actor }} @@ -215,7 +231,7 @@ jobs: - name: Build Debian Packages uses: ./qcom-build-utils/.github/actions/build_package with: - suite: ${{ needs.resolve.outputs.target_suite }} + suite: ${{ needs.resolve.outputs.suite }} pkg-dir: package-repo build-dir: build-area run-lintian: ${{ inputs.run-lintian }} @@ -225,7 +241,7 @@ jobs: if: ${{ inputs.run-abi-checker }} uses: ./qcom-build-utils/.github/actions/abi_checker with: - apt-repository: "deb [arch=arm64 trusted=yes] https://qartifactory-edge.qualcomm.com/artifactory/qsc-deb-releases ${{ needs.resolve.outputs.target_suite }} main" + apt-repository: "deb [arch=arm64 trusted=yes] https://qartifactory-edge.qualcomm.com/artifactory/qsc-deb-releases ${{ needs.resolve.outputs.suite }} main" - name: Upload Docker build artifacts run: | @@ -238,6 +254,7 @@ jobs: name: docker-build-area path: docker-build-area.tgz if-no-files-found: error + retention-days: 1 debian-build: name: Build (Debusine) @@ -245,7 +262,7 @@ jobs: needs: resolve runs-on: ubuntu-latest container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:${{ needs.resolve.outputs.target_suite }} + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:${{ needs.resolve.outputs.suite }} options: --user 0:0 credentials: username: ${{ github.actor }} @@ -270,7 +287,6 @@ jobs: lib - name: Checkout Repository - if: ${{ inputs.srcpkg-artifact == '' }} uses: actions/checkout@v5 with: ref: ${{ inputs.debian-ref }} @@ -278,31 +294,10 @@ jobs: fetch-depth: 0 fetch-tags: true - - name: Download pre-prepared source tree - if: ${{ inputs.srcpkg-artifact != '' }} - uses: actions/download-artifact@v8 - with: - name: ${{ inputs.srcpkg-artifact }} - path: /tmp/srcpkg-artifact - - - name: Extract pre-prepared source tree - if: ${{ inputs.srcpkg-artifact != '' }} - run: | - # The artifact is a tar.gz archive created by the caller to preserve - # Unix execute permissions on kernel build scripts. zip format (used - # by actions/upload-artifact) strips execute bits; tar does not. - # --strip-components=1 extracts the top-level directory directly into - # srcpkg/ without depending on its name inside the archive. - # The artifact contains exactly one *.tar.gz, so the glob below stays - # unambiguous while decoupling this step from the producer's filename. - mkdir srcpkg - tar xzf /tmp/srcpkg-artifact/*.tar.gz \ - -C srcpkg --strip-components=1 - - name: Prepare release if: ${{ inputs.release }} env: - SUITE_INPUT: ${{ needs.resolve.outputs.target_suite }} + SUITE_INPUT: ${{ needs.resolve.outputs.suite }} run: | set -ex echo "::group::Prepare release" @@ -312,7 +307,7 @@ jobs: - name: Generate source package id: generate-source-package env: - SUITE_INPUT: ${{ needs.resolve.outputs.target_suite }} + SUITE_INPUT: ${{ needs.resolve.outputs.suite }} DEBUSINE_ASSEMBLE_ORIG: ${{ inputs.assemble-orig }} run: | set -ex @@ -324,12 +319,11 @@ jobs: GITHUB_REPOSITORY_ID: ${{ github.repository_id }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - JOB_INDEX: ${{ inputs.job-index }} DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} - SUITE: ${{ needs.resolve.outputs.target_suite }} + SUITE: ${{ needs.resolve.outputs.suite }} run: | set -euxo pipefail debusine-action/lib/build @@ -348,10 +342,26 @@ jobs: name: release-bundle path: release.bundle if-no-files-found: error + retention-days: 1 test: name: Test - if: ${{ always() && ((needs.resolve.outputs.family == 'debian' && needs.resolve.outputs.force_docker_build != 'true' && needs.debian-build.result == 'success') || ((needs.resolve.outputs.family == 'ubuntu' || needs.resolve.outputs.force_docker_build == 'true') && needs.docker-build.result == 'success')) }} + if: >- + ${{ + always() && ( + ( + needs.resolve.outputs.family == 'debian' && + needs.resolve.outputs.force_docker_build != 'true' && + needs.debian-build.result == 'success' + ) || ( + ( + needs.resolve.outputs.family == 'ubuntu' || + needs.resolve.outputs.force_docker_build == 'true' + ) && + needs.docker-build.result == 'success' + ) + ) + }} needs: - resolve - debian-build @@ -363,7 +373,7 @@ jobs: srcpkg_version: ${{ steps.select.outputs.srcpkg_version }} runs-on: ubuntu-24.04-arm container: - image: ${{ format('ghcr.io/qualcomm-linux/pkg-builder:{0}', needs.resolve.outputs.target_suite) }} + image: ${{ format('ghcr.io/qualcomm-linux/pkg-builder:{0}', needs.resolve.outputs.suite) }} options: --user 0:0 credentials: username: ${{ github.actor }} @@ -395,29 +405,13 @@ jobs: lib - name: Checkout Repository - if: ${{ needs.resolve.outputs.family == 'debian' && needs.resolve.outputs.force_docker_build != 'true' && inputs.srcpkg-artifact == '' }} + if: ${{ needs.resolve.outputs.family == 'debian' && needs.resolve.outputs.force_docker_build != 'true' }} uses: actions/checkout@v5 with: ref: ${{ inputs.debian-ref }} path: srcpkg fetch-depth: 1 - - name: Download pre-prepared source tree - if: ${{ needs.resolve.outputs.family == 'debian' && inputs.srcpkg-artifact != '' }} - uses: actions/download-artifact@v8 - with: - name: ${{ inputs.srcpkg-artifact }} - path: /tmp/srcpkg-artifact - - - name: Extract pre-prepared source tree - if: ${{ needs.resolve.outputs.family == 'debian' && inputs.srcpkg-artifact != '' }} - run: | - # Single *.tar.gz in the artifact; glob decouples from the - # producer's filename (see build-job extract step for rationale). - mkdir srcpkg - tar xzf /tmp/srcpkg-artifact/*.tar.gz \ - -C srcpkg --strip-components=1 - - name: Validate installability from Debusine CI workspace # Temporarily disabled while dependency repository injection is unresolved. if: ${{ false }} @@ -427,7 +421,7 @@ jobs: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} DEBUSINE_WORKSPACE: ${{ needs.debian-build.outputs.workspace }} - SUITE: ${{ needs.resolve.outputs.target_suite }} + SUITE: ${{ needs.resolve.outputs.suite }} run: | set -euxo pipefail @@ -501,7 +495,7 @@ jobs: set -euxo pipefail # Add qsc-deb-releases so Qualcomm package dependencies are resolvable - suite="${{ needs.resolve.outputs.target_suite }}" + suite="${{ needs.resolve.outputs.suite }}" install -d /etc/apt/keyrings /etc/apt/sources.list.d cat > /etc/apt/keyrings/qsc-deb-releases.asc <<'KEYEOF' -----BEGIN PGP PUBLIC KEY BLOCK----- @@ -553,30 +547,6 @@ jobs: echo "srcpkg_version=${{ needs.docker-build.outputs.srcpkg_version }}" >> "$GITHUB_OUTPUT" fi - finalize: - name: Finalize outputs - if: ${{ always() && needs.resolve.result == 'success' && needs.test.result == 'success' }} - needs: - - resolve - - test - runs-on: ubuntu-latest - outputs: - target_suite: ${{ steps.select.outputs.target_suite }} - workspace: ${{ steps.select.outputs.workspace }} - workspace_url: ${{ steps.select.outputs.workspace_url }} - srcpkg_name: ${{ steps.select.outputs.srcpkg_name }} - srcpkg_version: ${{ steps.select.outputs.srcpkg_version }} - steps: - - name: Select workflow outputs - id: select - shell: bash - run: | - echo "target_suite=${{ needs.resolve.outputs.target_suite }}" >> "$GITHUB_OUTPUT" - echo "workspace=${{ needs.test.outputs.workspace }}" >> "$GITHUB_OUTPUT" - echo "workspace_url=${{ needs.test.outputs.workspace_url }}" >> "$GITHUB_OUTPUT" - echo "srcpkg_name=${{ needs.test.outputs.srcpkg_name }}" >> "$GITHUB_OUTPUT" - echo "srcpkg_version=${{ needs.test.outputs.srcpkg_version }}" >> "$GITHUB_OUTPUT" - - name: Note temporary installability policy run: | { diff --git a/.github/workflows/pkg-promote-reusable-workflow.yml b/.github/workflows/pkg-promote-reusable-workflow.yml index 0cdc2020..b87f0dbb 100644 --- a/.github/workflows/pkg-promote-reusable-workflow.yml +++ b/.github/workflows/pkg-promote-reusable-workflow.yml @@ -18,7 +18,7 @@ on: debian-branch: description: The debian branch to apply the promotion to. For example branch "qcom/debian/latest" type: string - required: false + required: true upstream-tag: description: The tag in the upstream repo to promote to. @@ -45,6 +45,7 @@ env: jobs: promote-upstream-version: + name: Promote runs-on: ubuntu-24.04-arm diff --git a/.github/workflows/pkg-release-reusable-workflow.yml b/.github/workflows/pkg-release-reusable-workflow.yml index a753286d..dcb6c760 100644 --- a/.github/workflows/pkg-release-reusable-workflow.yml +++ b/.github/workflows/pkg-release-reusable-workflow.yml @@ -2,7 +2,7 @@ name: Qualcomm Release Debian Package Reusable Workflow description: | This reusable workflow performs a package release. Debian suites use the Debusine-backed release path, while Ubuntu codenames keep - using the local pkg-builder and S3 release flow. + using the local pkg-builder release flow with direct apt-artifactory upload. on: workflow_call: @@ -13,19 +13,14 @@ on: required: true debian-branch: - description: The debian branch to use to execute the release from. For example branch "qcom/debian/latest" + description: The debian ref to use to execute the release from. For example branch "qcom/debian/latest" or tag "debian/1.0.0-1" type: string - required: false - - suite: - description: The distribution codename or Debian suite to build and release for. Ex noble, questing, resolute, trixie, sid - type: string - default: noble + required: true test-run: description: | Debian path: if true, stop after the Debusine build and installability test. - Ubuntu path: preserve the previous release flow, including upload to the test S3 location. + Ubuntu path remains stateful and still runs the full release flow. type: boolean default: true @@ -46,23 +41,9 @@ on: DEBUSINE_RELEASE_TOKEN: description: Debusine production token used only for the Debian publish-to-prod step. required: false - - outputs: - workspace: - description: Debusine child workspace name for Debian releases; empty for Ubuntu releases - value: ${{ jobs.finalize.outputs.workspace }} - workspace_url: - description: Debusine web URL for the child workspace for Debian releases; empty for Ubuntu releases - value: ${{ jobs.finalize.outputs.workspace_url }} - srcpkg_name: - description: Source package name prepared for release - value: ${{ jobs.finalize.outputs.srcpkg_name }} - srcpkg_version: - description: Source package version prepared for release - value: ${{ jobs.finalize.outputs.srcpkg_version }} - complete_version: - description: Alias for the released source package version - value: ${{ jobs.finalize.outputs.complete_version }} + QSC_API_KEY: + description: QArtifactory API key used for Ubuntu apt upload. + required: false permissions: contents: read @@ -72,72 +53,174 @@ env: DEBUSINE_ACTION_REF: main jobs: - validate-inputs: - name: Validate suite/branch combination - runs-on: ubuntu-latest - steps: - - name: Check resolute uses qcom/ubuntu/resolute + prepare-release-source: + name: Prepare Release Source + runs-on: ubuntu-24.04-arm + outputs: + family: ${{ steps.resolve.outputs.family }} + suite: ${{ steps.resolve.outputs.suite }} + branch_ref: ${{ steps.resolve.outputs.normalized_ref }} + srcpkg_artifact: ${{ steps.select.outputs.srcpkg_artifact }} + srcpkg_name: ${{ steps.select.outputs.srcpkg_name }} + release_version: ${{ steps.select.outputs.release_version }} + defaults: + run: shell: bash + steps: + - name: Checkout qcom-build-utils scripts + uses: actions/checkout@v5 + with: + repository: qualcomm-linux/qcom-build-utils + ref: ${{ inputs.qcom-build-utils-ref }} + path: qcom-build-utils + + - name: Resolve family and suite from branch + id: resolve env: - SUITE: ${{ inputs.suite }} - DEBIAN_BRANCH: ${{ inputs.debian-branch }} + DEBIAN_REF_INPUT: ${{ inputs.debian-branch }} run: | set -euo pipefail - if [[ "$SUITE" == "resolute" && "$DEBIAN_BRANCH" != "qcom/ubuntu/resolute" ]]; then - echo "::error::Suite 'resolute' must be released from branch 'qcom/ubuntu/resolute' but '$DEBIAN_BRANCH' was provided." + + RESOLVER_SCRIPT="${GITHUB_WORKSPACE}/qcom-build-utils/scripts/resolve_branch_family_suite.sh" + if ! resolved_kv="$("$RESOLVER_SCRIPT" "$DEBIAN_REF_INPUT")"; then + echo "::error::Unable to resolve release target from debian-branch '$DEBIAN_REF_INPUT'. Expected a branch ending in '//' (for example 'ubuntu/resolute' or 'qcom/ubuntu/resolute')." exit 1 fi - echo "✅ Input validation passed" - resolve: - name: Resolve suite family - needs: validate-inputs - runs-on: ubuntu-latest - outputs: - family: ${{ steps.resolve.outputs.family }} - debian_builder_suite: ${{ steps.resolve.outputs.debian_builder_suite }} - steps: - - name: Classify suite - id: resolve - shell: bash + normalized_ref="" + family="" + suite="" + while IFS='=' read -r key value; do + case "$key" in + normalized_ref) normalized_ref="$value" ;; + family) family="$value" ;; + suite) suite="$value" ;; + esac + done <<< "$resolved_kv" + + if [[ "$family" == "debian" ]] && [[ "$suite" == "latest" || "$suite" == "unstable" ]]; then + suite=sid + fi + + echo "::notice title=Resolved release target::debian-branch='${DEBIAN_REF_INPUT}', family='${family}', suite='${suite}'" + + echo "family=$family" >> "$GITHUB_OUTPUT" + echo "suite=$suite" >> "$GITHUB_OUTPUT" + echo "normalized_ref=$normalized_ref" >> "$GITHUB_OUTPUT" + + - name: Install packaging tools + run: | + set -euxo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends devscripts dpkg-dev + + - name: Checkout Packaging Repo + uses: actions/checkout@v5 + with: + token: ${{ secrets.PAT }} + persist-credentials: false + path: package-repo + ref: ${{ steps.resolve.outputs.normalized_ref }} + fetch-depth: 0 + fetch-tags: true + + - name: Prepare release source state + id: prepare env: - TARGET_SUITE: ${{ inputs.suite }} + FAMILY: ${{ steps.resolve.outputs.family }} + DISTRO_CODENAME: ${{ steps.resolve.outputs.suite }} + BOT_NAME: ${{ vars.DEB_PKG_BOT_CI_NAME }} + BOT_EMAIL: ${{ vars.DEB_PKG_BOT_CI_EMAIL }} run: | - set -euo pipefail + set -euxo pipefail + + cd package-repo - # 'latest' is not a valid suite name — map to sid (Debian unstable) - if [[ "$TARGET_SUITE" == "latest" ]]; then - TARGET_SUITE=sid + git config user.name "${BOT_NAME}" + git config user.email "${BOT_EMAIL}" + + export DEBFULLNAME="${BOT_NAME}" + export DEBEMAIL="${BOT_EMAIL}" + + echo "Initial changelog content:" + cat debian/changelog | sed 's/^/\x1b[34m/' | sed 's/$/\x1b[0m/' + + if ! head -1 debian/changelog | grep -q 'UNRELEASED'; then + echo "Error: The suite in the latest changelog entry is not UNRELEASED." \ + "The release process expects the latest entry to be UNRELEASED at any point except in the commit (this one) that represents the release" + exit 1 fi - case "$TARGET_SUITE" in - trixie|sid|unstable|bookworm|forky) - family=debian - ;; - *) - family=ubuntu - ;; - esac - - debian_builder_suite="$TARGET_SUITE" - if [[ "$TARGET_SUITE" == "unstable" ]]; then - debian_builder_suite=sid + version=$(dpkg-parsechangelog --show-field Version) + srcpkg_name=$(dpkg-parsechangelog --show-field Source) + + if [[ "${FAMILY}" == "ubuntu" ]]; then + # Guard: refuse to release a WIP placeholder + if [[ "${version}" == *~ ]]; then + echo "Error: version '${version}' ends with '~' — update debian/changelog and remove the '~' suffix before releasing" >&2 + exit 1 + fi + if grep -q 'WIP' debian/changelog; then + echo "Error: debian/changelog contains 'WIP' — update changelog entry before releasing" >&2 + exit 1 + fi + + # dch expects the suite name; sid is the codename for unstable + dch_suite="${DISTRO_CODENAME}" + if [[ "${dch_suite}" == "sid" ]]; then + dch_suite=unstable + fi + + echo "Releasing version: ${version} for suite '${dch_suite}'" | sed 's/^/\x1b[32m/' | sed 's/$/\x1b[0m/' + + dch --release --distribution="${dch_suite}" "Release" + + echo "Updated changelog content:" + cat debian/changelog | sed 's/^/\x1b[34m/' | sed 's/$/\x1b[0m/' + + git commit -a -m "debian/changelog: Release version ${version} for suite '${dch_suite}'" + + tag_name="${DISTRO_CODENAME}/${version}" + if git rev-parse -q --verify "refs/tags/${tag_name}" >/dev/null; then + echo "Error: local tag '${tag_name}' already exists" >&2 + exit 1 + fi + git tag "${tag_name}" fi - echo "family=$family" >> "$GITHUB_OUTPUT" - echo "debian_builder_suite=$debian_builder_suite" >> "$GITHUB_OUTPUT" + git log --graph --oneline -n 10 --color=always - debian-build: - name: Build and Test (Debian) - if: ${{ needs.resolve.outputs.family == 'debian' }} + echo "release_version=${version}" >> "$GITHUB_OUTPUT" + echo "srcpkg_name=${srcpkg_name}" >> "$GITHUB_OUTPUT" + + - name: Archive prepared source repository + run: | + set -euxo pipefail + tar -C package-repo -czf prepared-release-source.tar.gz . + + - name: Upload prepared source repository + uses: actions/upload-artifact@v6 + with: + name: prepared-release-source + path: prepared-release-source.tar.gz + if-no-files-found: error + retention-days: 1 + + - name: Select source preparation outputs + id: select + run: | + echo "srcpkg_artifact=prepared-release-source" >> "$GITHUB_OUTPUT" + echo "srcpkg_name=${{ steps.prepare.outputs.srcpkg_name }}" >> "$GITHUB_OUTPUT" + echo "release_version=${{ steps.prepare.outputs.release_version }}" >> "$GITHUB_OUTPUT" + + build-and-test: + name: Build and Test needs: - - validate-inputs - - resolve + - prepare-release-source uses: ./.github/workflows/pkg-build-reusable-workflow.yml with: qcom-build-utils-ref: ${{ inputs.qcom-build-utils-ref }} debian-ref: ${{ inputs.debian-branch }} - suite: ${{ inputs.suite }} release: true debusine-parent-workspace: ${{ inputs.debusine-parent-workspace }} secrets: @@ -146,11 +229,11 @@ jobs: debian-release: name: Release (Debian) - if: ${{ needs.resolve.outputs.family == 'debian' && !inputs.test-run }} + if: ${{ needs.build-and-test.outputs.family == 'debian' && !inputs.test-run }} needs: - - resolve - - debian-build + - build-and-test runs-on: ubuntu-latest + environment: pkg-release-approval permissions: contents: write packages: read @@ -158,7 +241,8 @@ jobs: run: shell: bash container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:${{ needs.resolve.outputs.debian_builder_suite }} + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:${{ needs.build-and-test.outputs.suite }} + options: --user 0:0 credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -201,11 +285,11 @@ jobs: DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - DEBUSINE_CI_WORKSPACE: ${{ needs.debian-build.outputs.workspace }} + DEBUSINE_CI_WORKSPACE: ${{ needs.build-and-test.outputs.workspace }} DEBUSINE_TARGET_WORKSPACE: prod - SRCPKG_NAME: ${{ needs.debian-build.outputs.srcpkg_name }} - SRCPKG_VERSION: ${{ needs.debian-build.outputs.srcpkg_version }} - SUITE: ${{ inputs.suite }} + SRCPKG_NAME: ${{ needs.build-and-test.outputs.srcpkg_name }} + SRCPKG_VERSION: ${{ needs.build-and-test.outputs.srcpkg_version }} + SUITE: ${{ needs.build-and-test.outputs.suite }} run: | set -euxo pipefail @@ -217,135 +301,253 @@ jobs: debusine-action/lib/push-release - ubuntu-pkg-release: + ubuntu-release: name: Release (Ubuntu) - if: ${{ needs.resolve.outputs.family == 'ubuntu' }} + if: ${{ needs.build-and-test.outputs.family == 'ubuntu' }} needs: - - validate-inputs - - resolve - runs-on: ubuntu-24.04-arm - outputs: - complete_version: ${{ steps.changelog.outputs.version }} - srcpkg_name: ${{ steps.metadata.outputs.srcpkg_name }} - srcpkg_version: ${{ steps.changelog.outputs.version }} + - prepare-release-source + - build-and-test + runs-on: lecore-prd-u2404-arm64-xlrg-od-ephem + environment: pkg-release-approval defaults: run: shell: bash - container: - image: ghcr.io/qualcomm-linux/pkg-builder:${{ inputs.suite }} - options: --privileged - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Checkout qcom-build-utils - uses: actions/checkout@v5 - with: - repository: qualcomm-linux/qcom-build-utils - ref: ${{ inputs.qcom-build-utils-ref }} - path: qcom-build-utils - fetch-depth: 1 - sparse-checkout: | - .github - scripts + - name: Install release tools + run: | + set -euxo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends devscripts jq - - name: Checkout Packaging Repo - uses: actions/checkout@v5 + - name: Download prepared source repository + uses: actions/download-artifact@v8 with: - token: ${{ secrets.PAT }} - path: package-repo - ref: ${{ inputs.debian-branch }} - fetch-depth: 0 - fetch-tags: true + name: ${{ needs.prepare-release-source.outputs.srcpkg_artifact }} + path: . - - name: Collect source package metadata - id: metadata + - name: Extract prepared source repository run: | - cd package-repo - echo "srcpkg_name=$(dpkg-parsechangelog --show-field Source)" >> "$GITHUB_OUTPUT" + set -euxo pipefail + mkdir package-repo + tar -C package-repo -xzf prepared-release-source.tar.gz --strip-components=1 --no-same-owner - - name: Changelog suite name change - id: changelog + - name: Configure git identity + working-directory: ./package-repo env: - DEBIAN_BRANCH: ${{ inputs.debian-branch }} - DISTRO_CODENAME: ${{ inputs.suite }} BOT_NAME: ${{ vars.DEB_PKG_BOT_CI_NAME }} BOT_EMAIL: ${{ vars.DEB_PKG_BOT_CI_EMAIL }} run: | - cd ./package-repo - + set -euxo pipefail git config user.name "${BOT_NAME}" git config user.email "${BOT_EMAIL}" + export DEBFULLNAME="${BOT_NAME}" + export DEBEMAIL="${BOT_EMAIL}" + + - name: Commit changelog suite change + working-directory: ./package-repo + env: + RELEASE_VERSION: ${{ needs.prepare-release-source.outputs.release_version }} + DISTRO_CODENAME: ${{ needs.build-and-test.outputs.suite }} + BOT_NAME: ${{ vars.DEB_PKG_BOT_CI_NAME }} + BOT_EMAIL: ${{ vars.DEB_PKG_BOT_CI_EMAIL }} + run: | + set -euxo pipefail export DEBFULLNAME="${BOT_NAME}" export DEBEMAIL="${BOT_EMAIL}" - git checkout "${DEBIAN_BRANCH}" + version=$(dpkg-parsechangelog --show-field Version) + if [[ "${version}" != "${RELEASE_VERSION}" ]]; then + echo "Error: expected release changelog version '${RELEASE_VERSION}', found '${version}'" >&2 + exit 1 + fi + + if [[ "$version" != *-* ]]; then + echo "Error: version '$version' has no debian revision part" >&2 + exit 1 + fi - echo "Initial changelog content:" - cat debian/changelog | sed 's/^/\x1b[34m/' | sed 's/$/\x1b[0m/' + upstream="${version%-*}" + rev="${version##*-}" - if ! head -1 debian/changelog | grep -q 'UNRELEASED'; then - echo "Error: The suite in the latest changelog entry is not UNRELEASED." \ - "The release process expects the latest entry to be UNRELEASED at any point except in the commit (this one) that represents the release" + if ! [[ "$rev" =~ ^[0-9]+$ ]]; then + echo "Error: debian revision '$rev' is not numeric" >&2 exit 1 fi - version=$(dpkg-parsechangelog --show-field Version) - echo "version=${version}" >> "$GITHUB_OUTPUT" + rev=$((rev + 1)) + newversion="${upstream}-${rev}~" + echo "Bumped version -> ${newversion}" + dch --newversion="${newversion}" --distribution=UNRELEASED --controlmaint "WIP — update changelog before next release" + + git commit -a -m "debian/changelog: bump to ${newversion}" - # Guard: refuse to release a WIP placeholder - if [[ "${version}" == *~ ]]; then - echo "Error: version '${version}' ends with '~' — update debian/changelog and remove the '~' suffix before releasing" >&2 + git log --graph --oneline -n 10 --color=always + + - name: Push release git state + working-directory: ./package-repo + env: + GH_PAT: ${{ secrets.PAT }} + DEBIAN_BRANCH: ${{ needs.prepare-release-source.outputs.branch_ref }} + DISTRO_CODENAME: ${{ needs.build-and-test.outputs.suite }} + RELEASE_VERSION: ${{ needs.prepare-release-source.outputs.release_version }} + run: | + set -euxo pipefail + + tag_name="${DISTRO_CODENAME}/${RELEASE_VERSION}" + git remote set-url origin "https://x-access-token:${GH_PAT}@github.com/${{ github.repository }}.git" + + if git ls-remote --exit-code --tags origin "refs/tags/${tag_name}" >/dev/null; then + echo "Error: remote tag '${tag_name}' already exists" >&2 exit 1 fi - if grep -q 'WIP' debian/changelog; then - echo "Error: debian/changelog contains 'WIP' — update changelog entry before releasing" >&2 + + git push --atomic origin "HEAD:${DEBIAN_BRANCH}" "${tag_name}" + + - name: Download Docker build artifacts + uses: actions/download-artifact@v8 + with: + name: docker-build-area + path: . + + - name: Extract Docker build artifacts + run: | + set -euxo pipefail + mkdir -p build + tar -C build -xzf docker-build-area.tgz + + - name: Prepare build logs for upload + working-directory: ./build/ + run: | + set -euxo pipefail + + shopt -s nullglob + for build_file in *.build; do + if [ -L "$build_file" ]; then + BUILD_TARGET=$(readlink "$build_file") + rm "$build_file" + mv "$BUILD_TARGET" "$build_file" + fi + done + + for build_file in *.build; do + renamed="${build_file//:/_}" + if [ "$renamed" != "$build_file" ]; then + mv "$build_file" "$renamed" + fi + done + + - name: Require QArtifactory API key + env: + QSC_API_KEY: ${{ secrets.QSC_API_KEY }} + run: | + if [ -z "$QSC_API_KEY" ]; then + echo "QSC_API_KEY is required when publishing Ubuntu releases" >&2 exit 1 fi - # dch expects the suite name; sid is the codename for unstable - dch_suite="${DISTRO_CODENAME}" - if [[ "${dch_suite}" == "sid" ]]; then - dch_suite=unstable + - name: Prepare APT artifactory upload + id: pre-upload + working-directory: ./build + run: | + set -euo pipefail + + CHANGES_FILE=$(find . -maxdepth 1 -type f -name "*.changes" | head -n 1) + if [ -z "$CHANGES_FILE" ]; then + echo "ERROR: No .changes file found in build artifacts." >&2 + exit 1 fi + CHANGES_FILE_ABS=$(realpath "$CHANGES_FILE") + echo "changes_path=$CHANGES_FILE_ABS" >> "$GITHUB_OUTPUT" - echo "Releasing version: ${version} for suite '${dch_suite}'" | sed 's/^/\x1b[32m/' | sed 's/$/\x1b[0m/' + COMMIT=$(git -C ../package-repo rev-parse HEAD) + echo "commit=$COMMIT" >> "$GITHUB_OUTPUT" - dch --release --distribution="${dch_suite}" "Release" + WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "workflow_url=$WORKFLOW_URL" >> "$GITHUB_OUTPUT" - echo "Updated changelog content:" - cat debian/changelog | sed 's/^/\x1b[34m/' | sed 's/$/\x1b[0m/' + - name: Upload to apt artifactory + uses: qualcomm-linux/apt-artifactory-action/upload@main + with: + qsc_api_key: ${{ secrets.QSC_API_KEY }} + server_url: https://qartifactory.pe.jfrog.io + target_base_repo: qsc-debian-local + changes_path: ${{ steps.pre-upload.outputs.changes_path }} + provenance_info: | + { + "commit": "${{ steps.pre-upload.outputs.commit }}", + "workflow": "${{ steps.pre-upload.outputs.workflow_url }}" + } + debug: ${{ inputs.test-run }} - git commit -a -m "debian/changelog: Release version ${version} for suite '${dch_suite}'" + persistance: + name: Release Metadata Persistence + needs: + - prepare-release-source + - build-and-test + - debian-release + - ubuntu-release + runs-on: ubuntu-latest + if: ${{ always() && ((needs.build-and-test.outputs.family == 'ubuntu' && needs.ubuntu-release.result == 'success') || (needs.build-and-test.outputs.family == 'debian' && !inputs.test-run && needs.debian-release.result == 'success')) }} + permissions: + contents: read + steps: + - name: Checkout Packaging Repo + if: ${{ needs.build-and-test.outputs.family != 'ubuntu' }} + uses: actions/checkout@v5 + with: + token: ${{ secrets.PAT }} + path: package-repo + ref: ${{ inputs.debian-branch }} + fetch-depth: 0 + fetch-tags: true - git tag "${DISTRO_CODENAME}/${version}" + - name: Download prepared source repository + if: ${{ needs.build-and-test.outputs.family == 'ubuntu' }} + uses: actions/download-artifact@v8 + with: + name: ${{ needs.prepare-release-source.outputs.srcpkg_artifact }} + path: . - git log --graph --oneline -n 10 --color=always + - name: Extract prepared source repository + if: ${{ needs.build-and-test.outputs.family == 'ubuntu' }} + run: | + set -euxo pipefail + mkdir package-repo + tar -C package-repo -xzf prepared-release-source.tar.gz --strip-components=1 --no-same-owner - name: Create provenance file env: - DISTRO_CODENAME: ${{ inputs.suite }} - DEBIAN_BRANCH: ${{ inputs.debian-branch }} + DISTRO_CODENAME: ${{ needs.build-and-test.outputs.suite }} UPSTREAM_REPO: ${{ vars.UPSTREAM_REPO_GITHUB_NAME }} - PKG_VERSION: ${{ steps.changelog.outputs.version }} + PKG_VERSION: ${{ needs.build-and-test.outputs.srcpkg_version }} PKG_REPO: ${{ github.repository }} + SRCPKG_NAME: ${{ needs.build-and-test.outputs.srcpkg_name }} run: | - mkdir build + set -euxo pipefail + mkdir build cd package-repo - SOURCE=$(grep-dctrl -n -s Source -r '' debian/control | head -n1) - ALL_PKGS=$(grep-dctrl -n -s Package -r '' debian/control | sort -u) + SOURCE="${SRCPKG_NAME}" + ALL_PKGS=$(awk '/^Package:/{print $2}' debian/control | sort -u) ALL_PKGS_JSON=$(printf '%s\n' "$ALL_PKGS" | jq -c -R -s 'split("\n") | map(select(length>0))') - PACKAGE_REPO_TAG=$(git describe --tags --match "${DISTRO_CODENAME}/*" --abbrev=0 "${DEBIAN_BRANCH}") + if PACKAGE_REPO_TAG=$(git describe --tags --match "${DISTRO_CODENAME}/*" --abbrev=0 2>/dev/null); then + echo "Using package repo tag from git history: ${PACKAGE_REPO_TAG}" + else + PACKAGE_REPO_TAG="${DISTRO_CODENAME}/${PKG_VERSION}" + echo "No matching local tag found; using derived tag: ${PACKAGE_REPO_TAG}" + fi + + PACKAGE_REPO_COMMIT="$(git rev-parse HEAD)" if [[ -f "upstream.conf" ]]; then echo "ℹ️ upstream.conf found — generating provenance for prebuilt binary package" + # shellcheck source=/dev/null source upstream.conf - cat > ../build/provenance.json << EOF + cat > ../build/provenance.json << EOF2 { "$SOURCE" : { "source_pkg_version": "${PKG_VERSION}", @@ -358,21 +560,26 @@ jobs: "pkg_repo": "${PKG_REPO}", "pkg_repo_tag": "$PACKAGE_REPO_TAG", - "pkg_repo_commit": "$(git rev-parse HEAD)", + "pkg_repo_commit": "$PACKAGE_REPO_COMMIT", "binary_pkgs": $ALL_PKGS_JSON } } - EOF + EOF2 else echo "ℹ️ No upstream.conf — generating provenance for source package" NEAREST_UPSTREAM_BRANCH_TAG=$(git describe --tags --match 'upstream/*' --abbrev=0) NEAREST_UPSTREAM_COMMIT=$(git rev-list -n 1 "$NEAREST_UPSTREAM_BRANCH_TAG") - NEAREST_UPSTREAM_TAG=$(git ls-remote --tags "https://github.com/${UPSTREAM_REPO}.git" | \ - awk -v commit="$NEAREST_UPSTREAM_COMMIT" '$1 == commit && $2 ~ /refs\/tags\// { sub("refs/tags/", "", $2); print $2 }' | head -n1) + NEAREST_UPSTREAM_TAG="${NEAREST_UPSTREAM_BRANCH_TAG#upstream/}" - cat > ../build/provenance.json << EOF + if [[ "$NEAREST_UPSTREAM_TAG" == "$NEAREST_UPSTREAM_BRANCH_TAG" ]]; then + echo "::warning::Unexpected upstream tag format '${NEAREST_UPSTREAM_BRANCH_TAG}'; using it as-is" + else + echo "::notice::Using upstream tag '${NEAREST_UPSTREAM_TAG}' derived from '${NEAREST_UPSTREAM_BRANCH_TAG}'" + fi + + cat > ../build/provenance.json << EOF2 { "$SOURCE" : { "source_pkg_version": "${PKG_VERSION}", @@ -384,66 +591,36 @@ jobs: "pkg_repo": "${PKG_REPO}", "pkg_repo_tag": "$PACKAGE_REPO_TAG", - "pkg_repo_commit": "$(git rev-parse HEAD)", + "pkg_repo_commit": "$PACKAGE_REPO_COMMIT", "pkg_repo_upstream_tag": "$NEAREST_UPSTREAM_BRANCH_TAG", "binary_pkgs": $ALL_PKGS_JSON } } - EOF + EOF2 fi echo "Content of the provenance file:" cat ../build/provenance.json | sed 's/^/\x1b[34m/' | sed 's/$/\x1b[0m/' - - name: Build Debian Packages - uses: ./qcom-build-utils/.github/actions/build_package + - name: Upload provenance artifact + uses: actions/upload-artifact@v6 with: - suite: ${{ inputs.suite }} - pkg-dir: package-repo - build-dir: build + name: release-provenance + path: build/provenance.json + if-no-files-found: error + retention-days: 1 - - name: Commit changelog suite change - working-directory: ./package-repo - env: - DEBIAN_BRANCH: ${{ inputs.debian-branch }} - DISTRO_CODENAME: ${{ inputs.suite }} - run: | - version=$(dpkg-parsechangelog --show-field Version) - - if [[ "$version" != *-* ]]; then - echo "Error: version '$version' has no debian revision part" >&2 - exit 1 - fi - - upstream="${version%-*}" - rev="${version##*-}" - - if ! [[ "$rev" =~ ^[0-9]+$ ]]; then - echo "Error: debian revision '$rev' is not numeric" >&2 - exit 1 - fi - - rev=$((rev + 1)) - newversion="${upstream}-${rev}~" - echo "Bumped version -> ${newversion}" - dch --newversion="${newversion}" --distribution=UNRELEASED --controlmaint "WIP — update changelog before next release" - - git commit -a -m "debian/changelog: bump to ${newversion}" - - git push origin "${DEBIAN_BRANCH}" - git push origin "${DISTRO_CODENAME}/${version}" - - git log --graph --oneline -n 10 --color=always - - - name: Push provenance to qcom-distro-artifacts - if: ${{ !inputs.test-run }} + - name: Persist metadata to qcom-distro-artifacts env: GH_PAT: ${{ secrets.PAT }} - SUITE: ${{ inputs.suite }} + SUITE: ${{ needs.build-and-test.outputs.suite }} BOT_NAME: ${{ vars.DEB_PKG_BOT_CI_NAME }} BOT_EMAIL: ${{ vars.DEB_PKG_BOT_CI_EMAIL }} + NEW_PROVENANCE: ${{ github.workspace }}/build/provenance.json run: | + set -euxo pipefail + git clone "https://x-access-token:${GH_PAT}@github.com/qualcomm-linux/qcom-distro-artifacts.git" ./qcom-distro-artifacts cd qcom-distro-artifacts @@ -454,7 +631,6 @@ jobs: mkdir -p "${SUITE}" SUITE_PROVENANCE="${SUITE}/provenance.json" - NEW_PROVENANCE="../build/provenance.json" if [[ -f "${SUITE_PROVENANCE}" ]]; then jq -s --indent 2 '.[0] * .[1]' "${SUITE_PROVENANCE}" "${NEW_PROVENANCE}" > /tmp/merged_provenance.json @@ -478,110 +654,3 @@ jobs: git pull --rebase origin main done fi - - - name: Prepare build logs for upload - working-directory: ./build/ - run: | - BUILD_LINK=$(find . -maxdepth 1 -name "*.build" -type l) - if [ -n "$BUILD_LINK" ]; then - BUILD_TARGET=$(readlink "$BUILD_LINK") - rm "$BUILD_LINK" - mv "$BUILD_TARGET" "$BUILD_LINK" - fi - - sed -i 's/:/_/g' *.build - - - name: Upload build artifacts - uses: actions/upload-artifact@v6 - with: - name: build-artifacts - path: build/ - - ubuntu-upload-to-s3: - name: Upload to S3 (Ubuntu) - if: ${{ needs.resolve.outputs.family == 'ubuntu' }} - needs: - - resolve - - ubuntu-pkg-release - runs-on: lecore-prd-u2404-arm64-xlrg-od-ephem - steps: - - name: Download build artifacts - uses: actions/download-artifact@v8 - with: - name: build-artifacts - path: build/ - - - name: Build S3 destination environment variables - run: | - OWNER_NAME="${{ github.repository_owner }}" - REPO_NAME="${{ github.event.repository.name }}" - CATEGORY_FOLDER="${{ inputs.test-run == true && 'test' || 'proposed' }}" - SUITE_NAME="${{ inputs.suite }}" - VERSION_NAME="${{ needs.ubuntu-pkg-release.outputs.complete_version }}" - WORKFLOW_BUILD_ID="${{ github.run_id }}" - WORKFLOW_RUN_ID="${{ github.run_attempt }}" - - S3_BUCKET_PATH="${OWNER_NAME}/pkg/${CATEGORY_FOLDER}/${WORKFLOW_BUILD_ID}-${WORKFLOW_RUN_ID}/${REPO_NAME}/${SUITE_NAME}/${VERSION_NAME}/" - - echo "ORG_NAME=${OWNER_NAME}" - echo "REPO_NAME=${REPO_NAME}" - echo "CATEGORY_FOLDER=${CATEGORY_FOLDER}" - echo "SUITE_NAME=${SUITE_NAME}" - echo "VERSION_NAME=${VERSION_NAME}" - - echo "S3 Bucket Path: ${S3_BUCKET_PATH}" - - echo "S3_BUCKET_PATH=${S3_BUCKET_PATH}" >> "$GITHUB_ENV" - - - name: Upload released debian package to S3 - uses: qualcomm-linux/upload-private-artifact-action@aws - with: - s3_bucket: qli-prd-lecore-gh-artifacts - path: build - destination: ${{ env.S3_BUCKET_PATH }} - - - name: Notify qcom-distro-images of new release via repository dispatch - uses: peter-evans/repository-dispatch@v3 - with: - token: ${{ secrets.PAT }} - repository: qualcomm-linux/qcom-distro-images - event-type: pkg-repo-release - client-payload: >- - { - "artifact_path":"${{ env.S3_BUCKET_PATH }}" - } - - finalize: - name: Finalize outputs - needs: - - validate-inputs - - resolve - - debian-build - - debian-release - - ubuntu-pkg-release - - ubuntu-upload-to-s3 - runs-on: ubuntu-latest - outputs: - workspace: ${{ steps.select.outputs.workspace }} - workspace_url: ${{ steps.select.outputs.workspace_url }} - srcpkg_name: ${{ steps.select.outputs.srcpkg_name }} - srcpkg_version: ${{ steps.select.outputs.srcpkg_version }} - complete_version: ${{ steps.select.outputs.complete_version }} - steps: - - name: Select workflow outputs - id: select - shell: bash - run: | - if [[ "${{ needs.resolve.outputs.family }}" == "debian" ]]; then - echo "workspace=${{ needs.debian-build.outputs.workspace }}" >> "$GITHUB_OUTPUT" - echo "workspace_url=${{ needs.debian-build.outputs.workspace_url }}" >> "$GITHUB_OUTPUT" - echo "srcpkg_name=${{ needs.debian-build.outputs.srcpkg_name }}" >> "$GITHUB_OUTPUT" - echo "srcpkg_version=${{ needs.debian-build.outputs.srcpkg_version }}" >> "$GITHUB_OUTPUT" - echo "complete_version=${{ needs.debian-build.outputs.srcpkg_version }}" >> "$GITHUB_OUTPUT" - else - echo "workspace=" >> "$GITHUB_OUTPUT" - echo "workspace_url=" >> "$GITHUB_OUTPUT" - echo "srcpkg_name=${{ needs.ubuntu-pkg-release.outputs.srcpkg_name }}" >> "$GITHUB_OUTPUT" - echo "srcpkg_version=${{ needs.ubuntu-pkg-release.outputs.srcpkg_version }}" >> "$GITHUB_OUTPUT" - echo "complete_version=${{ needs.ubuntu-pkg-release.outputs.complete_version }}" >> "$GITHUB_OUTPUT" - fi diff --git a/.github/workflows/workflows_sync.yml b/.github/workflows/workflows_sync.yml index 79ddc58d..b0c6802c 100644 --- a/.github/workflows/workflows_sync.yml +++ b/.github/workflows/workflows_sync.yml @@ -64,6 +64,7 @@ jobs: DEBIAN_LATEST_WORKFLOW_FILE="$(pwd)/qcom-build-utils/.github/pkg-workflows/debian/pkg-pr-hook.yml" DEBIAN_TARGET_WORKFLOW_FILE=".github/workflows/pkg-pr-hook.yml" DEBIAN_LEGACY_WORKFLOW_FILE=".github/workflows/pr-pre-post-merge.yml" + LEGACY_UPSTREAM_HELPER_DIR=".github/TO_PASTE_IN_UPSTREAM_REPO" echo "📄 Using pkg-pr-hook.yml from pkg-workflows/debian/" for repo in $repos; do @@ -123,8 +124,31 @@ jobs: isDirty=true fi - # Remove legacy main-branch caller workflow names. New names are - # synced from .github/pkg-workflows/main/*.yml below. + # Remove obsolete helper folder that used to be copied manually into + # upstream repos. + if [[ -d "$LEGACY_UPSTREAM_HELPER_DIR" ]]; then + echo " 🗑️ Removing obsolete $LEGACY_UPSTREAM_HELPER_DIR" + rm -rf "$LEGACY_UPSTREAM_HELPER_DIR" + isDirty=true + fi + + # Remove any workflow files that still use pull_request_target. + # Those are legacy/stale and should not remain in pkg-repos. + if [[ -d ".github/workflows" ]]; then + pr_target_workflows=$(grep -R -l --include='*.yml' --include='*.yaml' 'pull_request_target' .github/workflows 2>/dev/null || true) + if [[ -n "$pr_target_workflows" ]]; then + echo " 🗑️ Removing workflows containing pull_request_target" + while IFS= read -r workflow_file; do + [[ -z "$workflow_file" ]] && continue + echo " - $workflow_file" + rm -f "$workflow_file" + done <<< "$pr_target_workflows" + isDirty=true + fi + fi + + # Remove legacy default-branch caller workflow names. New names are + # synced from .github/pkg-workflows/qli-ci/*.yml below. for legacy_main_workflow in build-debian-package.yml promote-prebuilt.yml promote-upstream.yml release.yml; do legacy_main_path=".github/workflows/$legacy_main_workflow" if [[ -f "$legacy_main_path" ]]; then @@ -135,7 +159,7 @@ jobs: done # Compare and sync workflow files - for workflow in ../qcom-build-utils/.github/pkg-workflows/main/*.yml; do + for workflow in ../qcom-build-utils/.github/pkg-workflows/qli-ci/*.yml; do workflow_name=$(basename "$workflow") target_workflow=".github/workflows/$workflow_name" @@ -156,11 +180,11 @@ jobs: fi done - # Create branch, commit, and push main branch changes if needed + # Create branch, commit, and push default branch changes if needed if [[ "$isDirty" == "true" ]]; then - echo " 📝 Creating branch and committing main branch changes" + echo " 📝 Creating branch and committing default branch changes" git checkout -b sync/qcom-build-utils-workflows 2>&1 | sed 's/^/ /' - git add -A .github/workflows/ + git add -A .github/ git commit -s -m "chore: sync workflows from qcom-build-utils" 2>&1 | sed 's/^/ /' if [[ "$CONFIRMATION_INPUT" == "true" ]]; then @@ -182,9 +206,74 @@ jobs: # Return to default branch before packaging-branch sync git checkout "$default_branch" 2>&1 | sed 's/^/ /' else - echo " ✨ No main branch changes needed for $repo" + echo " ✨ No default branch changes needed for $repo" fi + # Cleanup-only pass for non-packaging branches: + # delete stale workflows that still contain pull_request_target. + other_branches=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/* \ + | sed 's|^origin/||' \ + | grep -Ev '^HEAD$' \ + | grep -Ev '^origin$' \ + | grep -Ev '(^|/)pr/' \ + | grep -Ev '^sync/' \ + | grep -v "^${default_branch}$" \ + | grep -Ev '^(debian/|ubuntu/|qcom/debian/|qcom/ubuntu/)' \ + | sort -u) + + for other_branch in $other_branches; do + echo " 🌿 Checking stale pull_request_target workflows on branch: $other_branch" + safe_branch_name="${other_branch//\//-}" + cleanup_sync_branch="sync/qcom-build-utils-pr-target-cleanup-${safe_branch_name}" + + git checkout "$other_branch" 2>&1 | sed 's/^/ /' + isCleanupDirty=false + + if [[ -d ".github/workflows" ]]; then + pr_target_workflows=$(grep -R -l --include='*.yml' --include='*.yaml' 'pull_request_target' .github/workflows 2>/dev/null || true) + if [[ -n "$pr_target_workflows" ]]; then + echo " 🗑️ Removing workflows containing pull_request_target on $other_branch" + while IFS= read -r workflow_file; do + [[ -z "$workflow_file" ]] && continue + echo " - $workflow_file" + rm -f "$workflow_file" + done <<< "$pr_target_workflows" + isCleanupDirty=true + fi + fi + + if [[ "$isCleanupDirty" == "true" ]]; then + if git ls-remote --exit-code --heads origin "$cleanup_sync_branch" > /dev/null 2>&1; then + echo " 🧹 Deleting existing $cleanup_sync_branch" + git push origin --delete "$cleanup_sync_branch" 2>&1 | sed 's/^/ /' + fi + + git checkout -b "$cleanup_sync_branch" 2>&1 | sed 's/^/ /' + git add -A .github/ + git commit -s -m "chore: remove pull_request_target workflows" 2>&1 | sed 's/^/ /' + + if [[ "$CONFIRMATION_INPUT" == "true" ]]; then + echo " 🚀 Pushing changes and creating PR for $other_branch" + git push origin "$cleanup_sync_branch" 2>&1 | sed 's/^/ /' + + pr_url=$(gh pr create --repo "$ORG/$repo" \ + --title "chore: remove pull_request_target workflows" \ + --body "This PR removes stale workflow files that still contain \`pull_request_target\`." \ + --head "$cleanup_sync_branch" \ + --base "$other_branch") + + PR_URLS+=("$pr_url") + echo " ✅ PR created for $other_branch: $pr_url" + else + echo " 👀 Dry-run mode - No PR will be created for $other_branch" + fi + else + echo " ✨ No pull_request_target workflow cleanup needed on $other_branch" + fi + + git checkout "$default_branch" 2>&1 | sed 's/^/ /' + done + # Sync pkg-pr-hook.yml to every managed packaging branch family. # Supported families: # - debian/* @@ -192,8 +281,9 @@ jobs: # - qcom/debian/* # - qcom/ubuntu/* # Exclude transient promotion branches (*/pr/*). - packaging_branches=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin \ + packaging_branches=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/* \ | sed 's|^origin/||' \ + | grep -Ev '^origin$' \ | grep -E '^(debian/|ubuntu/|qcom/debian/|qcom/ubuntu/)' \ | grep -Ev '(^|/)pr/' \ | sort -u) @@ -219,6 +309,25 @@ jobs: isDebianDirty=true fi + if [[ -d "$LEGACY_UPSTREAM_HELPER_DIR" ]]; then + echo " 🗑️ Removing obsolete $LEGACY_UPSTREAM_HELPER_DIR on $packaging_branch" + rm -rf "$LEGACY_UPSTREAM_HELPER_DIR" + isDebianDirty=true + fi + + if [[ -d ".github/workflows" ]]; then + pr_target_workflows=$(grep -R -l --include='*.yml' --include='*.yaml' 'pull_request_target' .github/workflows 2>/dev/null || true) + if [[ -n "$pr_target_workflows" ]]; then + echo " 🗑️ Removing workflows containing pull_request_target on $packaging_branch" + while IFS= read -r workflow_file; do + [[ -z "$workflow_file" ]] && continue + echo " - $workflow_file" + rm -f "$workflow_file" + done <<< "$pr_target_workflows" + isDebianDirty=true + fi + fi + if [[ -f "$DEBIAN_TARGET_WORKFLOW_FILE" ]]; then if diff -q "$DEBIAN_LATEST_WORKFLOW_FILE" "$DEBIAN_TARGET_WORKFLOW_FILE" > /dev/null; then echo " ✅ pkg-pr-hook.yml is up to date on $packaging_branch" @@ -236,7 +345,7 @@ jobs: if [[ "$isDebianDirty" == "true" ]]; then git checkout -b "$debian_sync_branch" 2>&1 | sed 's/^/ /' - git add -A .github/workflows/ + git add -A .github/ git commit -s -m "chore: sync pkg-pr-hook from qcom-build-utils" 2>&1 | sed 's/^/ /' if [[ "$CONFIRMATION_INPUT" == "true" ]]; then diff --git a/AGENTS.md b/AGENTS.md index 451f4bfa..a8622974 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,10 @@ orchestration around build, test, promotion, and release flows. `qualcomm-linux/debusine-action` and Debusine builder images - Ubuntu codenames (`noble`, `questing`, `resolute`, and similar Ubuntu targets) use the older local `pkg-builder` path with qcom-build-utils composite actions +- Ubuntu release path now prepares release changelog/tag state as an artifact, + builds once via `pkg-build-reusable-workflow.yml`, then gates on environment + `pkg-release-approval` before pushing git state, publishing provenance, and + uploading the already-built artifacts directly to apt artifactory. - Debian-path helper entrypoints come from checked-out `debusine-action/lib/`: - `prepare-release` - `generate-source-package` @@ -33,12 +37,44 @@ orchestration around build, test, promotion, and release flows. installed broadly (for example `qcom-preflight-checks.yml`). - Keep this split so package-specific automation remains easy to identify in `pkg-*` repositories. +## Build Branch Convention (Caller Contract) + +For `pkg-build-reusable-workflow.yml`, callers should pass a `debian-ref` branch +name where the last two `/`-delimited fields are: + +- `/` + +Expected values: + +- `family`: `debian` or `ubuntu` +- `suite`: distro codename/suite such as `sid`, `bookworm`, `noble`, `resolute` + +Examples: + +- `qcom/debian/latest` (normalized to suite `sid`) +- `qcom/debian/bookworm` +- `qcom/ubuntu/resolute` +- `test/qcom/ubuntu/resolute` +- `ubuntu/resolute` +- `dev/whatever/yo/debian/trixie` + +Invalid examples: + +- `resolute` +- `ubuntu` +- `ubuntu-resolute` + +`pkg-build-reusable-workflow.yml` no longer takes a separate `suite` input for +routing; it resolves family/suite from `debian-ref`. +For PR validation where `debian-ref` is a transient branch (for example +`debian/pr/*`), routing falls back to `github.base_ref`. + ## Important Workflows - `.github/workflows/pkg-build-reusable-workflow.yml` - main hybrid package build/test entrypoint for package repos - `.github/workflows/pkg-release-reusable-workflow.yml` - - hybrid release entrypoint: Debian via Debusine, Ubuntu via pkg-builder + S3 flow + - hybrid release entrypoint: Debian via Debusine, Ubuntu via pkg-builder + direct apt upload - `.github/workflows/pkg-promote-reusable-workflow.yml` - upstream-to-packaging promotion flow - `.github/workflows/pkg-upstream-pr-build-reusable-workflow.yml` diff --git a/README.md b/README.md index e3f2b9d5..d0e08b13 100644 --- a/README.md +++ b/README.md @@ -74,13 +74,41 @@ Package repositories call these workflows from their own `.github/workflows/` di | **pkg-build-reusable-workflow** | Main package build workflow — routes Debian suites through Debusine and Ubuntu codenames through the local pkg-builder path. | | **pkg-promote-reusable-workflow** | Promotes a new upstream release into a package repo — merges upstream code, updates changelog, and creates a PR. | | **pkg-upstream-pr-build-reusable-workflow** | Validates that PRs in an upstream repo won't break the Debian package build. Called from the upstream repo. | -| **pkg-release-reusable-workflow** | Triggers a formal release — Debian suites use Debusine publish, Ubuntu codenames keep the local pkg-builder/S3 release path. | +| **pkg-release-reusable-workflow** | Triggers a formal release — Debian suites use Debusine publish; Ubuntu codenames prepare release source, build once, gate on environment approval, then publish provenance and upload the built artifacts to S3. | | **qcom-preflight-checks** | Security and quality gates — runs repolinter, semgrep, license checks, and dependency review. | `pkg-*` and `qcom-*` naming intentionally distinguish scope: - `pkg-*` workflows are package-lifecycle workflows used by `pkg-*` repositories. - `qcom-*` workflows are qcom-wide infrastructure/preflight workflows. +### Build Routing Convention (`pkg-build`) + +`pkg-build-reusable-workflow.yml` now resolves `family` and `suite` from the +`debian-ref` branch name instead of taking a separate `suite` input. + +Branch parsing rule: + +- take the last two `/`-delimited fields as `/` +- `family` must be `debian` or `ubuntu` + +Examples: + +- `qcom/debian/latest` (maps to `sid`) +- `qcom/debian/bookworm` +- `qcom/ubuntu/resolute` +- `test/qcom/ubuntu/resolute` +- `ubuntu/resolute` +- `dev/whatever/yo/debian/trixie` + +Invalid examples: + +- `resolute` +- `ubuntu` +- `ubuntu-resolute` + +For PR jobs that build transient heads (for example `debian/pr/*`), workflow +routing falls back to the PR base branch (`github.base_ref`). + ## Builder Images `qcom-build-utils` consumes two image families: @@ -133,14 +161,14 @@ See [pkg-example](https://github.com/qualcomm-linux/pkg-example) for a complete debian/qcom-next and tagged to staging repo │ ▼ - Release triggered ──▶ changelog finalized ──▶ upload to S3 + Release triggered ──▶ changelog finalized ──▶ build/test ──▶ approval/provenance ──▶ upload to S3 ``` 1. **Pre-merge**: A PR against `debian/qcom-next` triggers a build and ABI compatibility check. 2. **Post-merge**: On merge, the package is built, pushed to the staging APT repo, and tagged `debian/`. 3. **Upstream promotion**: When the upstream project tags a new release, the promote workflow merges it into the packaging branch and opens a PR. 4. **Upstream PR validation**: PRs in the upstream repo are validated against the package build to catch breakages early. -5. **Release**: A manual dispatch finalizes the changelog, builds the package, uploads artifacts to S3, and notifies [qcom-distro-images](https://github.com/qualcomm-linux/qcom-distro-images). +5. **Release**: A manual dispatch finalizes changelog state, builds and tests once, waits on `pkg-release-approval`, then pushes release git state, publishes provenance, uploads artifacts to S3, and notifies [qcom-distro-images](https://github.com/qualcomm-linux/qcom-distro-images). ## Build Infrastructure diff --git a/docs/reusable-workflows.md b/docs/reusable-workflows.md index 549400fe..953f68b4 100644 --- a/docs/reusable-workflows.md +++ b/docs/reusable-workflows.md @@ -43,11 +43,9 @@ flowchart TD |-----------|------|----------|---------|-------------| | `qcom-build-utils-ref` | string | Yes | - | The ref (branch/tag) of qcom-build-utils to use | | `debian-ref` | string | Yes | `debian/qcom-next` | The package-repository ref to check out and build | -| `suite` | string | No | `unstable` | Distribution codename or Debian suite | | `run-lintian` | boolean | No | `true` | Used by the Ubuntu/pkg-builder path | | `run-abi-checker` | boolean | No | `false` | Used by the Ubuntu/pkg-builder path | | `is-prebuilt` | string | No | `""` | Passed through to the Ubuntu/pkg-builder `build_package` action | -| `job-index` | string | No | `"0"` | Optional matrix index used to keep Debusine child workspace names unique | | `release` | boolean | No | `false` | Whether to prepare the release bundle before generating the Debian release source package | | `debusine-parent-workspace` | string | No | `ci` | Parent Debusine workspace used to create child CI workspaces for Debian builds | @@ -62,12 +60,32 @@ flowchart TD | Output | Description | |--------|-------------| -| `target_suite` | The resolved suite/codename actually used by the workflow | +| `family` | The resolved distro family (`debian` or `ubuntu`) | +| `suite` | The resolved suite/codename actually used by the workflow | | `workspace` | Debusine child workspace name | | `workspace_url` | Debusine web URL for that child workspace | | `srcpkg_name` | Source package name | | `srcpkg_version` | Source package version | +### Branch Parsing Rule (`debian-ref`) + +- The resolver splits the normalized branch name on `/`. +- It takes the last two fields as `/`. +- `family` must be `debian` or `ubuntu`. + +Valid examples: + +- `qcom/ubuntu/resolute` -> `family=ubuntu`, `suite=resolute` +- `test/qcom/ubuntu/resolute` -> `family=ubuntu`, `suite=resolute` +- `ubuntu/resolute` -> `family=ubuntu`, `suite=resolute` +- `dev/whatever/yo/debian/trixie` -> `family=debian`, `suite=trixie` + +Invalid examples: + +- `resolute` +- `ubuntu` +- `ubuntu-resolute` + ### Workflow Steps 1. **Resolve suite family**: Normalize the caller input and decide whether the run is Debian or Ubuntu @@ -86,7 +104,6 @@ jobs: with: qcom-build-utils-ref: development debian-ref: debian/qcom-next - suite: trixie debusine-parent-workspace: ${{ vars.DEBUSINE_PARENT_WORKSPACE }} ``` @@ -99,8 +116,6 @@ jobs: with: qcom-build-utils-ref: development debian-ref: refs/heads/${{ matrix.target_branch }} - suite: ${{ matrix.suite }} - job-index: ${{ strategy.job-index }} ``` This workflow is the low-level build/test primitive used directly by package @@ -112,21 +127,19 @@ repositories. **File**: `.github/workflows/pkg-release-reusable-workflow.yml` -**Purpose**: Release through a hybrid flow: Debian suites use Debusine build/test/publish, while Ubuntu codenames keep the older local `pkg-builder` + S3 release process. +**Purpose**: Release through a hybrid flow: Debian suites use Debusine build/test/publish, while Ubuntu codenames prepare release source locally, build once through `pkg-build-reusable-workflow`, pass through manual environment approval, then publish provenance and upload already-built artifacts to S3. ### Workflow Diagram ```mermaid flowchart TD - A[Workflow Called] --> B[Resolve suite family] - B -->|Debian| C[Prepare release bundle + build/test in Debusine] - C --> D{test-run?} - D -->|true| E[Stop after validation] - D -->|false| F[Publish CI workspace to Debusine prod] - F --> G[Push release tag and reopen development] - B -->|Ubuntu| H[Run local release/tag/provenance flow] - H --> I[Build in pkg-builder] - I --> J[Upload artifacts to S3] + A[Workflow Called] --> B[Prepare Ubuntu release source commit/tag local-only] + B --> C[Build and test via pkg-build-reusable-workflow] + C --> D[Manual environment approval + provenance generation] + D -->|Debian + !test-run| E[Publish to Debusine prod and push git state] + E --> F[Publish provenance] + D -->|Ubuntu| G[Push git state and publish provenance] + G --> H[Upload already-built artifacts to S3] ``` ### Inputs @@ -135,10 +148,13 @@ flowchart TD |-----------|------|----------|---------|-------------| | `qcom-build-utils-ref` | string | Yes | - | The ref (branch/tag) of qcom-build-utils to invoke | | `debian-branch` | string | No | `debian/qcom-next` | The packaging branch to release from | -| `suite` | string | No | `noble` | Distribution codename or Debian suite to build/test/release | -| `test-run` | boolean | No | `true` | Debian: stop after Debusine build/test. Ubuntu: keep the older release flow and upload to the test S3 location | +| `test-run` | boolean | No | `true` | Debian: stop after Debusine build/test. Ubuntu: still stateful, but upload to test/proposed destination based on this input | | `debusine-parent-workspace` | string | No | `ci` | Parent Debusine workspace passed through to the Debian build/test phase | +`debian-branch` uses the same parsing rule as `debian-ref` in the build +reusable workflow: split on `/`, take the last two fields as +`/`, and require `family` to be `debian` or `ubuntu`. + ### Secrets | Secret | Required | Description | @@ -160,14 +176,16 @@ flowchart TD ### Workflow Steps -1. **Resolve suite family**: Decide whether the release follows the Debian or Ubuntu branch -2. **Debian branch**: Reuse `pkg-build-reusable-workflow` with `release=true`, then optionally publish to Debusine prod and push git state -3. **Ubuntu branch**: Restore the earlier local release flow with changelog/tag handling, provenance generation, local `pkg-builder` build, and S3 upload +1. **Prepare release source**: For Ubuntu branches, finalize changelog and create local release commit/tag, then upload the prepared git repo as an artifact +2. **Build and test once**: Reuse `pkg-build-reusable-workflow` (`release=true`) with `debian-branch` as the build source ref +3. **Environment gate + provenance file**: Require `pkg-release-approval` after build/test and generate provenance from the exact built source state +4. **Debian release path**: For Debian and `test-run=false`, publish Debusine CI workspace to prod, push git state, then publish provenance +5. **Ubuntu release path**: Push git state atomically (branch + tag), publish provenance, and upload previously-built Docker artifacts to S3 ### Caller Requirements - Debian callers should pass `DEBUSINE_RELEASE_TOKEN` when they want the reusable workflow to publish to Debusine prod -- Ubuntu callers do not use the Debusine secrets, but still need `PAT` for the release git/S3/dispatch flow +- Ubuntu callers do not use the Debusine secrets, but still need `PAT` for prepared source, git push, provenance publish, S3 upload, and dispatch flow ### Usage Example @@ -177,8 +195,7 @@ jobs: uses: qualcomm-linux/qcom-build-utils/.github/workflows/pkg-release-reusable-workflow.yml@development with: qcom-build-utils-ref: development - suite: trixie - debian-branch: debian/qcom-next + debian-branch: qcom/ubuntu/resolute test-run: false debusine-parent-workspace: ${{ vars.DEBUSINE_PARENT_WORKSPACE }} secrets: diff --git a/scripts/README.md b/scripts/README.md index 62b1289e..3276543e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -132,6 +132,28 @@ Shell script for merging upstream changes into Debian packaging branch. ./scripts/merge_debian_packaging_upstream upstream/main ``` +### 4. resolve_branch_family_suite.sh + +Helper used by reusable workflows to resolve `family`/`suite` from branch-like refs. + +**Rule:** +- Normalize refs like `refs/heads/*`, `refs/remotes/*`, and `origin/*` +- Split by `/` +- Take the last two fields as `/` +- Require `family` to be `debian` or `ubuntu` + +**Usage:** +```bash +./scripts/resolve_branch_family_suite.sh +``` + +**Output:** +```text +normalized_ref= +family= +suite= +``` + ## Common Workflows ### Building a Single Package diff --git a/scripts/create_promotion_pr.py b/scripts/create_promotion_pr.py index d7a18b59..53140430 100755 --- a/scripts/create_promotion_pr.py +++ b/scripts/create_promotion_pr.py @@ -7,15 +7,17 @@ """ create_promotion_pr.py -Helper script to build a debian package using the container from the Dockerfile in the docker/ folder. +Helper script to generate and open the promotion PR. """ import subprocess import argparse +import sys +import traceback from color_logger import logger # This script is used to create the PR content of the promotion PR. Then, the actual PR is created using GitHub CLI def parse_arguments(): - parser = argparse.ArgumentParser(description="Craft the content for a promotion PR and opoen it using GitHub CLI.") + parser = argparse.ArgumentParser(description="Craft the content for a promotion PR and open it using GitHub CLI.") parser.add_argument("--base-branch", required=False, @@ -35,38 +37,31 @@ def parse_arguments(): return args def create_pr_title(normalized_version: str) -> str: - return f"Promotion to {normalized_version}" + return f"Promote to {normalized_version}" def create_pr_body(base_branch: str, upstream_tag: str, normalized_version: str) -> str: - pr_body = f""" -# This is an automated PR to test the promotion of this package repo to the upstream project version {normalized_version}. + return f""" +## Automated promotion PR -This PR merges the upstream changes from the upstream tag '{upstream_tag}' into the {base_branch} branch, and updated the debian/changelog version to reflect this new version. Whatever was the distro version before (the part after the '-' in a version x.y.z-a), it has been reset to -1. -The upstream tag '{upstream_tag}' has already been merged into the upstream/latest branch, and this PR merges that branch into {base_branch}. -In other words, this repo already contains the upstream changes in the upstream/latest branch, but the debian packaging is not yet updated to reflect this new upstream version. This is what this PR is doing. +This PR was generated by `pkg-promote` to move this package repo to upstream tag +`{upstream_tag}`. -The *pkg-build.yml* workflow is triggered automatically in this PR to test the promotion by building the Debian package with the updated upstream code and packaging. -If something breaks due to the promotion of the upstream sources to this new revision, this is the moment where you can checkout this branch locally, make changes and push additional commits to make the build pass. +### Summary -For example: you may need to add patches to the debian/patches/ folder to fix issues that were introduced upstream since the last version we were using, such as a new binary created upstream that needs to be packaged, or a build system change that requires updating the debian/rules file, etc. -Once you are satisfied with the changes, click the 'Merge' button below to finalize the promotion. +- Base branch: `{base_branch}` +- Upstream branch/tag prepared: `upstream/latest` / `upstream/{normalized_version}` +- PR branch: `debian/pr/{normalized_version}-1` +- Changelog update: `{normalized_version}-1` (UNRELEASED) -*Note: Due to the nature of the graph that is attempted to be merged, only a merge (and therefore the creation of a merge commit) with the 'Merge' button will work.* - Attempting second option 'Squash and Merge' or 'Rebase and Merge' will fail. This is because in both of these two cases, this head branch woule need to be cleanly rebasable onto the base branch, which is not the case here. +### Reviewer checklist +1. Review the upstream merge and packaging diff. +2. Confirm the `Build` workflow is green. +3. If needed, push follow-up packaging fixes to this PR branch. +4. Merge with **Merge commit** (do not squash or rebase). -This generated diagram attemps to illustrate what happened and what will happen when you click the 'Merge' button below.: - - The right most 'upstream-main' branch represents the upstream repo, where the {upstream_tag} was pulled from. - - To its left, the 'upstream/latest' branch lives is this repo, and represents a copy of the upstream repo (and it has already happened during the promotion workflow run). - The commit tagged 'upstream/{normalized_version}' is a merge from the upstream tag {upstream_tag} commit where in addition, - special git wizardry happened to perform a special filtering of any potential upstream .github/ and debian/ folders have been filtered out, - and only homonym folders from the debian/qcom-next branch have been kept. - - To its left, this 'debian/pr/{normalized_version}-1' branch was created during the promotion workflow and is the head branch of this PR. - It represents the merge of the upstream/latest branch into {base_branch}. - - Note that an extra commit for updating the debian/changelog file to reflect the new version {normalized_version}-1 has been added on top of that merge. -""" +### Flow - mermaid_diagram = f""" ```mermaid --- config: @@ -99,8 +94,6 @@ def create_pr_body(base_branch: str, upstream_tag: str, normalized_version: str) commit id: 'Changelog version update' type: HIGHLIGHT ``` """ - - return pr_body + mermaid_diagram def main(): args = parse_arguments() diff --git a/scripts/resolve_branch_family_suite.sh b/scripts/resolve_branch_family_suite.sh new file mode 100755 index 00000000..79e73609 --- /dev/null +++ b/scripts/resolve_branch_family_suite.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# +# SPDX-License-Identifier: BSD-3-Clause-Clear + +# Resolve distro family/suite from a branch-like ref. +# The last two '/'-delimited fields are interpreted as "/". + +set -euo pipefail + +normalize_ref() { + local ref="$1" + + if [[ "$ref" == refs/heads/* ]]; then + ref="${ref#refs/heads/}" + fi + if [[ "$ref" == refs/remotes/* ]]; then + ref="${ref#refs/remotes/}" + fi + if [[ "$ref" == origin/* ]]; then + ref="${ref#origin/}" + fi + + printf '%s\n' "$ref" +} + +resolve_from_ref() { + local ref="$1" + local -n out_family="$2" + local -n out_suite="$3" + local -a ref_parts + + IFS='/' read -r -a ref_parts <<< "$ref" + if (( ${#ref_parts[@]} < 2 )); then + return 1 + fi + + out_family="${ref_parts[$((${#ref_parts[@]} - 2))]}" + out_suite="${ref_parts[$((${#ref_parts[@]} - 1))]}" + + case "$out_family" in + debian|ubuntu) + return 0 + ;; + *) + return 1 + ;; + esac +} + +main() { + if (( $# != 1 )); then + echo "Usage: $0 " >&2 + exit 2 + fi + + local input_ref="$1" + local normalized_ref family suite + + normalized_ref="$(normalize_ref "$input_ref")" + if ! resolve_from_ref "$normalized_ref" family suite; then + exit 1 + fi + + printf 'normalized_ref=%s\n' "$normalized_ref" + printf 'family=%s\n' "$family" + printf 'suite=%s\n' "$suite" +} + +main "$@"