ci.yml

name: CI

# HOW ACTIONS ARE PINNED HERE, since the mix looks like an oversight and is not.
# actions/* ride moving major tags (checkout@v6, upload-artifact@v7): they are
# GitHub's own, published from the same place that runs them, and following them
# is what keeps the deprecation warnings away without a chore. Everything from a
# third party is pinned to a COMMIT (docker/*, tailscale/*, softwarity/*): a tag
# is a pointer its owner can move, so pinning is what a compromise of one of
# those repositories runs into. It buys nothing against a bad release, and is not
# meant to. The cost is that a pinned action never updates by itself, which is
# the trade taken deliberately for code we do not publish.

on:
  push:
    branches: ["**"]
  # Kept for FORK pull requests only, where no push reaches this repository.
  # A PR from a branch of this repository already has its push run, and the
  # pull_request run doubled it: two pipelines per commit, the second of which
  # skipped the clusters anyway. Every root job below carries the gate (a
  # trigger cannot filter on the head repository); what `needs` them follows.
  pull_request:
  workflow_dispatch: # manual e2e-over-the-mesh run

# One pipeline per branch: two close pushes used to run TWO full pipelines at
# once (12 dispatched clusters + 18 legs for 20 runner slots) and the starved
# serve jobs queued past the legs' 10-min cluster wait. Only the newest commit
# needs blessing — cancel the older run; its clusters die on their TTL
# backstop (that is what the backstop is for).
# Read-only by default. Thirteen jobs inherit this token, on three operating
# systems, and a checkout persists it in the runner's git config before the
# job goes on to run code pulled from Docker Hub. The jobs that need to write
# (the image publication, the cluster dispatch and its cancels) ask for it
# themselves, where the reason is visible; the e2e legs, which run pip, third
# party images and a binary installed from the cluster, do not persist it at all.
permissions:
  contents: read

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # plug's own code, built and unit-tested NATIVELY on all three OSes. Each job
  # records its real PASS/FAIL as an artifact the `platforms` job reads back.
  test:
    name: Test (${{ matrix.os }})
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-go@v6
        with:
          go-version: stable
          # On, and keyed on what actually decides the module set. It was off
          # since the org-convention alignment, with no reason recorded, so
          # every leg pulled gvisor cold from proxy.golang.org - three OSes,
          # every run. One bad answer from that proxy failed a green commit.
          cache: true
          cache-dependency-path: cli/go.sum
      - name: Warm the module cache (a flaky proxy must not fail a leg)
        shell: bash
        run: bash scripts/ci/go-mod-download.sh cli
      - name: Vet
        working-directory: cli
        run: go vet ./...
      - name: Test
        working-directory: cli
        shell: bash
        env:
          # -race, or the concurrency tests prove nothing: rearm_race_test.go is
          # written to fail OUTRIGHT under the detector and merely to be flaky
          # without it, so a plain `go test` would let a dropped mutex through.
          # Not on Windows: the detector needs cgo and a C toolchain the runner
          # does not guarantee, and none of this concurrency is OS-specific.
          # Written this way round on purpose: in GitHub expressions '' is FALSY,
          # so `windows && '' || '-race'` would hand Windows the -race it excludes.
          RACE: ${{ matrix.os != 'windows-latest' && '-race' || '' }}
        # -coverprofile alongside the run that was already happening: no extra
        # execution, and the profile is what the coverage job merges. Per OS,
        # because a file behind a build tag is not compiled on a runner that does
        # not need it - absent from the fraction rather than uncovered in it.
        # -v for what a quiet run hides: which tests skipped on this OS and why,
        # and what they logged. A test that skips where it should run passes
        # green and proves nothing. The lines that only say a test started or
        # passed are left out; the next step lists the skips on their own.
        run: |
          set -o pipefail
          go test $RACE -v -covermode=atomic -coverprofile=coverage.out ./... 2>&1 | tee test.log |
            { grep -vE '^ *(=== (RUN|PAUSE|CONT|NAME)|--- PASS)' || true; }
      - name: The tests that skipped here
        if: always()
        working-directory: cli
        shell: bash
        # With the line before each, which is the reason the test gave.
        run: grep -B1 -E '^ *--- SKIP' test.log | grep -v '^--$' || echo "none"
      - name: Keep the coverage profile for the merge
        if: always()
        working-directory: cli
        shell: bash
        run: test -s coverage.out && cp coverage.out "cli-${{ matrix.os }}.cov" || echo "no profile (the tests did not run)"
      # Not worth a release: this is a coverage report. GitHub's artifact service
      # answered FinalizeArtifact with a 403 once, the action calls that
      # non-retryable, the step failed, and everything after it in the job was
      # skipped.
      - uses: actions/upload-artifact@v7
        continue-on-error: true
        if: always()
        with:
          name: cov-cli-${{ matrix.os }}
          path: cli/cli-*.cov
          if-no-files-found: ignore
          retention-days: 1
      - name: Record result
        if: always()
        shell: bash
        run: mkdir -p plat && printf '%s' "${{ job.status }}" > "plat/test-${{ matrix.os }}.txt"
      # Retried once. Other jobs consume this artifact, so it cannot be optional,
      # but a transient 403 from GitHub's artifact service ("non-retryable" to the
      # action) must not cost a 25-minute release either. The second attempt
      # overwrites, since a failed finalize can leave the name taken.
      - uses: actions/upload-artifact@v7
        id: up_plat_test
        continue-on-error: true
        if: always()
        with:
          name: plat-test-${{ matrix.os }}
          path: plat/*.txt
          # One day: the only reader is the `platforms` job of THIS run (the
          # release is an approval on a green run, not a re-read of markers).
          retention-days: 1
      - uses: actions/upload-artifact@v7
        if: always() && steps.up_plat_test.outcome == 'failure'
        with:
          overwrite: true
          name: plat-test-${{ matrix.os }}
          path: plat/*.txt
          # One day: the only reader is the `platforms` job of THIS run (the
          # release is an approval on a green run, not a re-read of markers).
          retention-days: 1

  # The visible cross-platform proof: build plug natively on each OS and loop real
  # traffic through a real TUN device BY NAME (no agent, no Docker). macOS/Linux
  # run it under sudo and MUST pass. Windows is best-effort (WinTUN driver): the
  # step records the honest PASS/FAIL and never masquerades as a pass it isn't.
  # Not after `test`: the two answer different questions on different runners,
  # and the selftest is the longer one (3 minutes on windows-11-arm), so
  # waiting for the unit tests only pushed the e2e legs, which need both, back.
  selftest:
    name: TUN selftest (${{ matrix.os }})
    strategy:
      fail-fast: false
      matrix:
        # windows-11-arm is where the arm64 WinTUN driver meets a real arm64
        # kernel, which no cross-compile can answer. GA and free on public
        # repositories, which this one is. It runs the selftest only: the unit
        # tests already cover three OSes and are architecture-independent, while
        # opening a real device is exactly the half that is not.
        os: [ubuntu-latest, macos-latest, windows-latest, windows-11-arm]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-go@v6
        with:
          go-version: stable
          cache: true
          cache-dependency-path: cli/go.sum
      - name: Warm the module cache (a flaky proxy must not fail a leg)
        shell: bash
        run: bash scripts/ci/go-mod-download.sh cli
      - name: selftest
        shell: bash
        # No tolerance any more. It was granted while the WinTUN route config was
        # unfinished, and that has been done for a long time: the Windows selftest
        # has passed ten runs out of ten. Keeping it meant a job that went green on
        # a failure — the marker held the truth, and the release gate reads the
        # marker, so nothing shipped wrongly. But a green job that failed is the
        # exact shape of lie this suite spent a week removing everywhere else.
        env:
          PLAT_OS: ${{ matrix.os }}
        run: bash scripts/selftest.sh

      # Retried once. Other jobs consume this artifact, so it cannot be optional,
      # but a transient 403 from GitHub's artifact service ("non-retryable" to the
      # action) must not cost a 25-minute release either. The second attempt
      # overwrites, since a failed finalize can leave the name taken.
      - uses: actions/upload-artifact@v7
        id: up_plat_selftest
        continue-on-error: true
        if: always()
        with:
          name: plat-selftest-${{ matrix.os }}
          path: cli/plat/*.txt
          retention-days: 1 # read by this run's `platforms` job only; see plat-test above
      - uses: actions/upload-artifact@v7
        if: always() && steps.up_plat_selftest.outcome == 'failure'
        with:
          overwrite: true
          name: plat-selftest-${{ matrix.os }}
          path: cli/plat/*.txt
          retention-days: 1 # read by this run's `platforms` job only; see plat-test above

  # One-glance platform grid in the run Summary, built from the REAL per-OS
  # results each job recorded rather than from job statuses.
  platforms:
    name: Platform coverage
    needs: [test, selftest]
    if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository)
    runs-on: ubuntu-latest
    # The verdict the image gate reads, built from the MARKERS. Kept that way now
    # that no job may fail quietly: a marker records what happened even when a job
    # is skipped or cancelled, and 33f8c40 published an image because a status said
    # green where a marker said fail.
    outputs:
      verdict: ${{ steps.grid.outputs.verdict }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/download-artifact@v7
        with:
          path: plat
          pattern: plat-*
          merge-multiple: true
      - name: Render platform grid
        id: grid
        run: bash scripts/platform-grid.sh plat

  # The agent module's own vet and tests. It used to cross-compile the CLI for
  # the six platforms as well and upload them as plug-binaries, which nothing
  # downloaded: the image builds compile the same six (agent/Dockerfile), and
  # those are the binaries the e2e legs install from the cluster and the
  # release publishes. The compile check lives there.
  build:
    name: Vet and test the agent
    needs: test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-go@v6
        with:
          go-version: stable
          cache: true
          # Two modules build here, so both go.sums decide the key.
          cache-dependency-path: |
            cli/go.sum
            agent/go.sum
      - name: Warm the module cache (a flaky proxy must not fail a leg)
        shell: bash
        run: bash scripts/ci/go-mod-download.sh cli agent
      # The agent helper is its own module — vet/build/test it here (the e2e only
      # compiled it into the cluster image; nothing caught a vet issue or ran its
      # unit tests).
      - name: Vet & test the agent helper
        working-directory: agent
        run: go vet ./... && go build ./... && go test -race -covermode=atomic -coverprofile=agent.cov ./...
      # Not worth a release: this is a coverage report. GitHub's artifact service
      # answered FinalizeArtifact with a 403 once, the action calls that
      # non-retryable, the step failed, and everything after it in the job was
      # skipped.
      - uses: actions/upload-artifact@v7
        continue-on-error: true
        if: always()
        with:
          name: cov-agent
          path: agent/agent.cov
          if-no-files-found: ignore
          retention-days: 1

  # The number, and what it is worth. It runs after the tests rather than beside
  # them because it has nothing of its own to compute: it merges what they
  # already measured. `needs` names both producers so a missing profile is a
  # skipped job rather than a report quietly built from two thirds of the code.
  #
  # Nothing here fails a build. A floor would push someone to write unit tests
  # for the files sitting at 0%, and those are precisely the ones the e2e matrix
  # proves best - a daemon, a selftest, a swarm backend. See the header of
  # scripts/ci/coverage-report.py for what the number covers and what it cannot.
  coverage:
    name: Coverage
    needs: [test, build]
    if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-go@v6
        with:
          go-version: stable
          # No module cache here, and it has to be said or setup-go looks for a
          # go.mod at the root, finds none (they live in cli/ and agent/) and
          # warns on every run. This job downloads no dependencies: it merges
          # text profiles and calls `go tool cover`, which reads sources already
          # in the checkout.
          cache: false
      - uses: actions/download-artifact@v7
        with:
          pattern: cov-*
          path: cov
          merge-multiple: true
      - name: Merge the profiles and say what they mean
        shell: bash
        run: |
          shopt -s nullglob
          profiles=(cov/*.cov)
          if [ ${#profiles[@]} -eq 0 ]; then
            echo "no coverage profile was produced; the tests did not get far enough" >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          python3 scripts/ci/coverage-report.py summary.md "${profiles[@]}"
          cat summary.md >> "$GITHUB_STEP_SUMMARY"
      # One report per MODULE, not one for the repository: `go tool cover -html`
      # resolves each file through the module it is run from, so a profile mixing
      # cli and agent fails on whichever one is not the current module. Checked,
      # not assumed.
      - name: The line-by-line reports, one per module
        shell: bash
        run: |
          shopt -s nullglob
          mkdir -p html
          cli=(cov/cli-*.cov)
          if [ ${#cli[@]} -gt 0 ]; then
            (cd cli && { echo "mode: atomic"; tail -q -n +2 ../cov/cli-*.cov; } > /tmp/cli-merged.cov \
              && go tool cover -html=/tmp/cli-merged.cov -o ../html/cli.html)
          fi
          if [ -s cov/agent.cov ]; then
            (cd agent && go tool cover -html=../cov/agent.cov -o ../html/agent.html)
          fi
          ls -la html
      # Not worth a release: this is a coverage report. GitHub's artifact service
      # answered FinalizeArtifact with a 403 once, the action calls that
      # non-retryable, the step failed, and everything after it in the job was
      # skipped.
      - uses: actions/upload-artifact@v7
        continue-on-error: true
        with:
          name: coverage-html
          path: html/
          retention-days: 7

  ci-shape:
    name: The e2e chain, its cell files and the job names agree
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      # The chain is written once (_e2e.yml), so the three families cannot
      # drift from each other; what can still drift is the cell list against
      # its files, and the names the other scripts read (see the script).
      - run: bash scripts/ci/check-e2e-cells.sh
      # The cluster idle loop decides which caller job ends it from the corr id;
      # a wrong family or attempt there means six runners held to their TTL.
      - run: IDLE_SELFTEST=1 bash scripts/ci/idle-until-caller-done.sh

  # The e2e clients, built ONCE for every leg.
  #
  # Measured on the Windows swarm leg: 493s of a 769s setup went into building
  # them — and each of the three Windows legs paid it in full, for artefacts
  # that do not differ. What changes from one leg to the next is the CLUSTER in
  # front of the client, never the client. So: one Linux runner cross-compiles
  # the Go client and the two helpers (no cgo anywhere in them), builds the jar
  # once (bytecode), installs node_modules once (every driver here is pure JS),
  # and ships the lot. Python stays per-leg — its wheels are compiled.
  #
  # This does NOT depend on `test`: it compiles nothing plug ships, so making it
  # wait would only push the legs' start out.
  build-clients:
    name: Build the e2e clients (once, for every leg)
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-go@v6
        with:
          go-version: stable
          # Keyed on the e2e client's own go.sum — the module fetch is the slow
          # half. It is worth caching HERE, where it happens once, in a way it
          # never was on ten legs racing to write the same key.
          cache: true
          cache-dependency-path: e2e/clients/go/go.sum
      # Two attempts, because one failure here took a whole leg with it and had
      # nothing to do with this repository: setup-java downloads the JDK from
      # Adoptium and verifies its signature, and that verification failed once
      # ("Failed to verify signature for Temurin 21.0.12"), leaving the leg
      # without Java and every cell after it red. A download that fails its own
      # signature check is a transient upstream fault, not a state to reason
      # about, so it is retried once rather than diagnosed.
      - uses: actions/setup-java@v6
        id: java
        continue-on-error: true
        with:
          distribution: temurin
          java-version: '21'
          cache: maven
      - uses: actions/setup-java@v6
        if: steps.java.outcome == 'failure'
        with:
          distribution: temurin
          java-version: '21'
          cache: maven
      - uses: actions/cache@v6
        with:
          path: ~/.npm
          key: npm-Linux-${{ hashFiles('e2e/clients/node/package.json') }}
          restore-keys: npm-Linux-
      - name: Build them
        run: bash scripts/ci/build-clients.sh e2e-clients
      # Retried once. Other jobs consume this artifact, so it cannot be optional,
      # but a transient 403 from GitHub's artifact service ("non-retryable" to the
      # action) must not cost a 25-minute release either. The second attempt
      # overwrites, since a failed finalize can leave the name taken.
      - uses: actions/upload-artifact@v7
        id: up_clients
        continue-on-error: true
        with:
          name: e2e-clients
          path: e2e-clients/*
          if-no-files-found: error # a leg with no bundle rebuilds everything, silently slow
          retention-days: 1
      - uses: actions/upload-artifact@v7
        if: steps.up_clients.outcome == 'failure'
        with:
          overwrite: true
          name: e2e-clients
          path: e2e-clients/*
          if-no-files-found: error # a leg with no bundle rebuilds everything, silently slow
          retention-days: 1

  # Build FIRST, so the e2e legs run against the image we actually publish
  # instead of one they build themselves. It carries only `sha-<short>` at this
  # point: a digest nobody has exercised must never be what `latest` points at.
  #
  # No `needs`, and that is worth a minute and three quarters on EVERY leg. It
  # used to wait for `test` — all three OSes — so the image landed at 02:20:48 of
  # a run that started at 02:16:32, and no cluster can answer before it has
  # pulled. Ten legs sat on that.
  #
  # Nothing is loosened by it: this image only ever carries `sha-<short>`, and
  # the `image` job below still requires test, selftest, platforms, ci-shape and
  # all four e2e jobs before a single name is put on that digest. The e2e legs
  # keep their own `needs: test, selftest`, so a red test still skips them: the whole cost
  # of a failing run is one runner building an image nobody will publish.
  image-build:
    name: Build the standalone image (immutable tag only)
    if: github.event_name != 'pull_request'
    permissions:
      contents: read
      packages: write # the test image goes to GHCR (_docker.yml)
    uses: ./.github/workflows/_docker.yml
    with:
      immutable_only: true
    # The two _docker.yml declares, not the vault. `inherit` handed this build
    # every secret the repo had, a contents:write token on main included, to
    # run a job whose only credentialed act is a Docker Hub login. (The write
    # credential has since moved to the `release` environment, see below.)
    secrets:
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_RW: ${{ secrets.DOCKERHUB_RW }}
      PLUG_RELEASE_KEY: ${{ secrets.PLUG_RELEASE_KEY }}

  # The client for a gateway that embeds the agent: same code, a different verb
  # surface (keygen/pubkey, no update), stamped `-hosted` in the version string.
  #
  # NOT exercised by the e2e, deliberately. The legs test the datapath, the
  # tunnel and the name provisioning, none of which differ between the two
  # flavours - what differs is which verbs exist, and that is a unit test
  # (cli/flavour_test.go). It is published only when the standalone earns
  # publication, so the same green run blesses both.
  image-build-hosted:
    name: Build the hosted image (immutable tag only)
    if: github.event_name != 'pull_request'
    permissions:
      contents: read
      packages: write # the test image goes to GHCR (_docker.yml)
    uses: ./.github/workflows/_docker.yml
    with:
      immutable_only: true
      flavour: hosted
    secrets:
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_RW: ${{ secrets.DOCKERHUB_RW }}
      PLUG_RELEASE_KEY: ${{ secrets.PLUG_RELEASE_KEY }}

  image:
    name: Publish the tested images (standalone + hosted)
    # Publish ONLY once EVERY test has passed — unit tests, the TUN selftests on
    # all three OSes, and the full mesh e2e on the THREE cluster families
    # (compose, kind and swarm: the same chain, ×3 OS each, plus arm64). An
    # image must never ship from a commit the e2e didn't bless. A fully green
    # MANUAL dispatch publishes too ("a manual build is a whole build").
    # main publishes `latest` (+ the `main` tag); ANY OTHER branch publishes ONLY
    # its own `:<branch>` tag (metadata-action tags by ref and gates `latest` to
    # is_default_branch) — a throwaway image to test that branch on a real
    # cluster, never latest. Off PRs (secrets). Delete stale branch tags from
    # Docker Hub after merge.
    #
    # `needs: selftest` is NOT enough on its own. It was written when that job was
    # continue-on-error on Windows and reported success on a selftest that failed;
    # the tolerance is gone, but the reasoning stands — `needs` is satisfied by a
    # job that was skipped or cancelled too. The verdict from `platforms` reads the
    # markers those jobs wrote, which is the only record that cannot lie: without
    # it the image of 33f8c40 shipped with a red Windows selftest, past a comment
    # promising the opposite.
    needs: [image-build, image-build-hosted, build, selftest, platforms, ci-shape, e2e-compose, e2e-k8s, e2e-swarm, e2e-arm64, okd]
    if: success() && needs.platforms.outputs.verdict == 'pass' && github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    # id-token: write is cosign's keyless signature (see the last step): the
    # job proves it is ci.yml on main, and that identity is what gets signed.
    permissions:
      contents: read
      id-token: write
      packages: read # the tested digests are read from GHCR
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-tags: true # a release is recognised by the tag pointing at HEAD
      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
      - name: Log in to Docker Hub
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_RW }}
      # A new manifest list over the SAME digests the e2e legs just ran against.
      # Nothing is rebuilt, so "what we publish" and "what was tested" are the
      # same bytes — which was NOT true while the legs built their own image.
      #
      # Publishing is not a special act reserved for releases: it is one rule,
      # and the ref decides which names the tested digest earns.
      #
      #   any branch            → :<branch>
      #   the default branch    → :latest as well
      #   a v-tag on HEAD       → :x.y.z, :x.y and :x on top
      #
      # A release used to be a separate workflow, triggered by the tag, waiting
      # on this very run to finish before promoting. That existed because the tag
      # once REBUILT the image; it does not any more. Folding it back here
      # removes the gate that re-asks what this job already knows, the race
      # between release-flow's two pushes, and the manual re-run when either side
      # tripped — which is exactly what 2.10.0 cost.
      #
      # The one case it drops: a tag pushed onto a commit built EARLIER publishes
      # nothing, since this job has already run for it. Re-run the CI on that
      # commit and it lands.
      # The test images live on GHCR, not Docker Hub (see _docker.yml): signed
      # in with this run's own token.
      - name: Log in to GHCR
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}
      - name: Put the names this ref earns on both tested digests
        env:
          IMAGE: docker.io/softwarity/plug
          SRC: ${{ needs.image-build.outputs.image }}
          HSRC: ${{ needs.image-build-hosted.outputs.image }}
        run: |
          branch="${GITHUB_REF_NAME//\//-}"   # a docker tag cannot carry a slash
          tags="-t $IMAGE:$branch"
          if [ "$GITHUB_REF_NAME" = "${{ github.event.repository.default_branch }}" ]; then
            tags="$tags -t $IMAGE:latest"
          fi
          v=$(git tag --points-at HEAD --list 'v*' | sort -V | tail -1)
          if [ -n "$v" ]; then
            ver=${v#v}; maj=${ver%%.*}; min=${ver#*.}; min=${min%%.*}
            # The image itself is stamped with this version (see "Resolve
            # version" in _docker.yml), so the semver tags and the binaries
            # inside them cannot disagree: same commit, same build, one truth.
            tags="$tags -t $IMAGE:$ver -t $IMAGE:$maj.$min -t $IMAGE:$maj"
            echo "release $ver — publishing the semver tags too"
            # Asserted, not assumed. The comment above says the two cannot
            # disagree, and the reasoning holds only if the build saw the tag:
            # image-build has no `needs` and starts within seconds of the push,
            # while this job runs half an hour later. A release commit and its
            # tag arriving as two separate pushes puts that whole window between
            # them, and the image is then stamped dev+<sha> and published as
            # :2.13.0. Nothing downstream could tell, and every launcher fetching
            # that core would be told a version its bytes do not carry.
            #
            # Costs one pull of an image this job is already authenticated for.
            # Asked of the download account, as a launcher asks it: the image
            # has no shell and no cat.
            got=$(docker run --rm --entrypoint /usr/local/bin/plug-agent -e PLUG_ACCOUNT=get -e SSH_ORIGINAL_COMMAND=version "$SRC" 2>/dev/null || echo "?")
            if [ "$got" != "$ver" ]; then
              echo "::error::the image is stamped '$got' and is about to be published as $ver." >&2
              echo "         The build ran before the tag existed. Re-run the CI on this commit," >&2
              echo "         which rebuilds with the tag in place, rather than publishing this one." >&2
              exit 1
            fi
            echo "the image is stamped $got, which is the version it is being given"
          fi
          # From GHCR to Docker Hub: the tested digest is COPIED, byte for byte,
          # so the digest published is the digest the e2e ran.
          echo "promoting $SRC → $tags"
          docker buildx imagetools create $tags "$SRC"
          docker buildx imagetools inspect "$IMAGE:$branch" | head -5

          # The hosted client, promoted from the SAME green run. Every name it
          # earns carries the flavour, and the moving one on the default branch
          # is `hosted` rather than `latest-hosted`: it is the other product's
          # latest, not a variant of this one's.
          htags="-t $IMAGE:$branch-hosted"
          if [ "$GITHUB_REF_NAME" = "${{ github.event.repository.default_branch }}" ]; then
            htags="$htags -t $IMAGE:hosted"
          fi
          if [ -n "$v" ]; then
            htags="$htags -t $IMAGE:$ver-hosted -t $IMAGE:$maj.$min-hosted -t $IMAGE:$maj-hosted"
          fi
          echo "promoting $HSRC → $htags"
          docker buildx imagetools create $htags "$HSRC"
          docker buildx imagetools inspect "$IMAGE:$branch-hosted" | head -5
      - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
        if: github.ref == 'refs/heads/main'
      # Signed by digest, and the promotion above keeps the digest (one source,
      # an index, re-tagged as it is), so signing the tested build signs
      # `latest` and `hosted` with it. main only: a branch image is a throwaway,
      # and the identity users check names main. scripts/ci/signatures.sh says
      # what is signed and reads every signature back.
      - name: Sign both images, and check what was signed
        if: github.ref == 'refs/heads/main'
        # What is signed is what users pull: the Docker Hub names, resolved to
        # the digests the step above copied there. The test images on GHCR are
        # deleted when the run ends, and a signature there would go with them.
        run: |
          scripts/ci/signatures.sh sign docker.io/softwarity/plug:latest docker.io/softwarity/plug:hosted
          scripts/ci/signatures.sh verify docker.io/softwarity/plug:latest docker.io/softwarity/plug:hosted


  # THE RELEASE IS AN APPROVAL. Every green run on main ends on this job, which
  # waits on the `release` environment (a reviewer is required there) and does
  # nothing until somebody approves it. Approving releases THIS build: the
  # commit this run tested, with the bump written in the approval's comment
  # (empty is a patch, `minor` or `major` say so). Not approving costs nothing:
  # the next push to main cancels the waiting run (the concurrency above), so
  # only the latest green build is ever on offer. Before this job existed a
  # release was a workflow_dispatch that pushed a tag and had the WHOLE pipeline
  # run again on a commit differing by one Markdown file: 40 minutes, and one
  # flake away from a cancelled release (2.20.2 cost exactly that).
  #
  # REBUILT, NOT RE-TAGGED, and the reason is where plug keeps its version. It
  # is compiled into every binary (-X main.version): the launcher compares it
  # with the agent's, names its cache by it, and the release signature covers
  # the bytes that carry it. A layer stamped on top of the tested digest, which
  # is how a service with a runtime version does this, would ship binaries that
  # call themselves dev+<sha> under a tag that says 2.21.0. So the two images
  # are built again here, from the same commit, with the same toolchain and the
  # same Dockerfile; the version string is the one thing that differs from the
  # build the legs ran. The old release path rebuilt too, in a second pipeline.
  #
  # THE PUSH IS THE LAST THING THIS JOB DOES, and it says [skip ci]. The
  # version commit changes RELEASE_NOTES.md and nothing else; a pipeline on it
  # would rebuild and publish the same tags a second time, and its arrival
  # would cancel this very job while it ran its final steps. release-flow is
  # asked not to push; the commit message is amended, the tag moved onto the
  # amended commit, both pushed, and the GitHub Release created from the notes
  # release-flow extracted. The tag lands on the notes commit, one past the
  # tested one; the image's revision label names the tested commit.
  #
  # Two things can stand between a green run and its release, and both stop
  # the job before it touches anything: main moved by more than release notes
  # since this commit (a code push without a pipeline, or one that was
  # cancelled); or nobody approved, which means the environment is not
  # protected (GitHub creates an environment the moment a job names it) and a
  # release job would otherwise release every push.
  release:
    name: Release (approve to publish)
    needs: [image]
    if: >-
      !cancelled() && needs.image.result == 'success'
      && github.ref == 'refs/heads/main'
      && (github.event_name == 'workflow_dispatch'
          || (github.event_name == 'push' && !contains(github.event.head_commit.message, '[skip release]')))
    runs-on: ubuntu-latest
    timeout-minutes: 40
    environment: release
    permissions:
      contents: read
      # The approvals of this very run, where the bump is written.
      actions: read
      # cosign's keyless signature of the two images this job builds.
      id-token: write
    env:
      IMAGE: docker.io/softwarity/plug
    steps:
      - name: Read the approval
        id: bump
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -e
          approval=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/approvals" \
            --jq '[.[] | select(.state == "approved") | select(any(.environments[]; .name == "release"))] | last')
          if [ -z "$approval" ] || [ "$approval" = "null" ]; then
            echo "::error::nobody approved this release: the 'release' environment must require a reviewer (Settings > Environments > release)"
            exit 1
          fi
          comment=$(jq -r '.comment // ""' <<<"$approval")
          bump=$(printf '%s' "$comment" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')
          case "$bump" in
            "") bump="patch" ;;
            patch|minor|major) ;;
            *) echo "::error::the approval comment must be empty (patch), minor or major, not '$comment'"; exit 1 ;;
          esac
          echo "bump=$bump" >> "$GITHUB_OUTPUT"
          echo "approved as a $bump release"
      # The App that WRITES this repository: a token minted for this run, good
      # for an hour, from a private key that is a secret of the `release`
      # environment, so it reaches this job once approved and no other job of
      # the repository. An App rather than a personal token: nothing to renew,
      # nobody's account behind the version commits.
      - id: token
        uses: actions/create-github-app-token@v2
        with:
          app-id: ${{ vars.RELEASE_APP_ID }}
          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
          owner: softwarity
          repositories: plug
      # main, with its tags and its history, WITH the credential kept: the
      # version commit and the tag are pushed through this checkout's origin.
      - uses: actions/checkout@v6
        with:
          ref: main
          fetch-depth: 0
          token: ${{ steps.token.outputs.token }}
      # main must still be the commit this run tested, give or take release
      # notes: a [skip ci] push of notes alone is how notes are written here,
      # and notes are not compiled into anything (.dockerignore excludes *.md).
      # Anything else moved main without a pipeline, and this build does not
      # vouch for it.
      - name: main is still the tested commit
        id: main
        env:
          TESTED: ${{ github.sha }}
        run: |
          set -e
          head=$(git rev-parse HEAD)
          if [ "$head" != "$TESTED" ]; then
            moved=$(git diff --name-only "$TESTED" "$head")
            if [ -z "$moved" ] || [ "$moved" != "RELEASE_NOTES.md" ]; then
              echo "::error::main is at ${head::7} but this build tested ${TESTED::7}, and the difference is not release notes alone:"
              echo "$moved" | sed 's/^/         /'
              echo "         Run the pipeline on main again, then approve that one."
              exit 1
            fi
            echo "main moved from ${TESTED::7} to ${head::7} by release notes only; releasing ${head::7} with this build's green"
          fi
          echo "sha=$head" >> "$GITHUB_OUTPUT"
      # The same arithmetic release-flow does, from the same tags, computed
      # here first because the images need the version before the notes do.
      # release-flow is then TOLD this version, so the two cannot disagree.
      - name: Next version
        id: next
        env:
          BUMP: ${{ steps.bump.outputs.bump }}
        run: |
          set -e
          last=$(git tag --list 'v[0-9]*.[0-9]*.[0-9]*' | sed 's/^v//' | sort -t. -k1,1n -k2,2n -k3,3n | tail -1)
          last=${last:-0.0.0}
          maj=${last%%.*}; rest=${last#*.}; min=${rest%%.*}; pat=${rest#*.}
          case "$BUMP" in
            major) maj=$((maj + 1)); min=0; pat=0 ;;
            minor) min=$((min + 1)); pat=0 ;;
            patch) pat=$((pat + 1)) ;;
          esac
          {
            echo "version=$maj.$min.$pat"
            echo "minor=$maj.$min"
            echo "major=$maj"
            echo "date=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
          } >> "$GITHUB_OUTPUT"
          echo "$last -> $maj.$min.$pat ($BUMP)"
      - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
      # The base images (golang, alpine…) through mirror.gcr.io, Google's public
      # cache of Docker Hub, falling back to Docker Hub for what it does not hold.
      # Docker Hub counts every resolution of a base against one account's quota,
      # shared by every job of every run, and a day of CI ran it dry: the build
      # died resolving golang:1.26-alpine before compiling a line. Pushes still go
      # to Docker Hub; a mirror only serves pulls.
      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
        with:
          buildkitd-config-inline: |
            [registry."docker.io"]
              mirrors = ["mirror.gcr.io"]
      - name: Log in to Docker Hub
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_RW }}
      # The App's own identity on the version commit, as GitHub shows it. The
      # bot user's id is public, but `gh` in Actions refuses to run without a
      # token at all (exit 4), so it gets the run's own read-only one.
      - name: Commit as the App
        env:
          SLUG: ${{ steps.token.outputs.app-slug }}
          GH_TOKEN: ${{ github.token }}
        run: |
          id=$(gh api "users/${SLUG}[bot]" --jq .id)
          git config user.name "${SLUG}[bot]"
          git config user.email "${id}+${SLUG}[bot]@users.noreply.github.com"
      # Both architectures in one build: the Dockerfile's build stage runs on
      # the build platform (a Go cross-compile), so only the final stage's apk
      # runs under QEMU. Same build-args as _docker.yml's "Resolve version"
      # would pass for a tagged commit: the version, no GIT_REV.
      - name: Build and publish the standalone image as ${{ steps.next.outputs.version }}
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
        with:
          context: .
          file: agent/Dockerfile
          platforms: linux/amd64,linux/arm64
          push: true
          build-args: |
            VERSION=${{ steps.next.outputs.version }}
            GIT_REV=
            FLAVOUR=
          secrets: |
            release_key=${{ secrets.PLUG_RELEASE_KEY }}
          tags: |
            ${{ env.IMAGE }}:${{ steps.next.outputs.version }}
            ${{ env.IMAGE }}:${{ steps.next.outputs.minor }}
            ${{ env.IMAGE }}:${{ steps.next.outputs.major }}
            ${{ env.IMAGE }}:latest
            ${{ env.IMAGE }}:main
          labels: |
            org.opencontainers.image.source=https://github.com/${{ github.repository }}
            org.opencontainers.image.revision=${{ steps.main.outputs.sha }}
            org.opencontainers.image.version=${{ steps.next.outputs.version }}
            org.opencontainers.image.created=${{ steps.next.outputs.date }}
          cache-from: type=gha,scope=release-std
          cache-to: type=gha,scope=release-std,mode=min
      - name: Build and publish the hosted image as ${{ steps.next.outputs.version }}-hosted
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
        with:
          context: .
          file: agent/Dockerfile
          platforms: linux/amd64,linux/arm64
          push: true
          build-args: |
            VERSION=${{ steps.next.outputs.version }}
            GIT_REV=
            FLAVOUR=hosted
          secrets: |
            release_key=${{ secrets.PLUG_RELEASE_KEY }}
          tags: |
            ${{ env.IMAGE }}:${{ steps.next.outputs.version }}-hosted
            ${{ env.IMAGE }}:${{ steps.next.outputs.minor }}-hosted
            ${{ env.IMAGE }}:${{ steps.next.outputs.major }}-hosted
            ${{ env.IMAGE }}:hosted
            ${{ env.IMAGE }}:main-hosted
          labels: |
            org.opencontainers.image.source=https://github.com/${{ github.repository }}
            org.opencontainers.image.revision=${{ steps.main.outputs.sha }}
            org.opencontainers.image.version=${{ steps.next.outputs.version }}-hosted
            org.opencontainers.image.created=${{ steps.next.outputs.date }}
          cache-from: type=gha,scope=release-hosted
          cache-to: type=gha,scope=release-hosted,mode=min
      # Asserted, not assumed, as the image job does: the binaries inside must
      # say the version the tag says, and the signatures must be there (an
      # image built without the key would serve binaries no launcher runs).
      - name: The published images are stamped and signed
        env:
          VERSION: ${{ steps.next.outputs.version }}
        run: |
          set -e
          for suffix in "" "-hosted"; do
            ref="$IMAGE:$VERSION$suffix"
            # Asked of the download account, as a launcher asks it (the image
            # has no shell): the version it serves, and the signature it hands
            # out with the digest a launcher checks before running a core.
            get() { docker run --rm --platform linux/amd64 --entrypoint /usr/local/bin/plug-agent -e PLUG_ACCOUNT=get -e "SSH_ORIGINAL_COMMAND=$1" "$ref"; }
            got=$(get version)
            [ "$got" = "$VERSION$suffix" ] || { echo "::error::$ref is stamped '$got'"; exit 1; }
            digest=$(get "digest linux-amd64")
            grep -q '^sig=.' <<<"$digest" \
              || { echo "::error::$ref carries no release signature: was PLUG_RELEASE_KEY set?"; exit 1; }
            echo "$ref: stamped $got, signed"
          done
      - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
      # The release builds new digests (the version is compiled in, see the job
      # comment), so the image job's signature does not reach them: they are
      # signed here, before the tag exists, and read back the way a user checks
      # them. A release that cannot be signed stops before it is announced.
      - name: Sign the published images, and check what was signed
        env:
          VERSION: ${{ steps.next.outputs.version }}
        run: |
          scripts/ci/signatures.sh sign "$IMAGE:$VERSION" "$IMAGE:$VERSION-hosted"
          scripts/ci/signatures.sh verify "$IMAGE:$VERSION" "$IMAGE:$VERSION-hosted" "$IMAGE:latest" "$IMAGE:hosted"
      # Notes, version commit, tag: computed and written, pushed below by hand
      # so the commit can say [skip ci] (see the job comment). Given the same
      # bump, from the same tags: the step after checks it arrived at the
      # version the images were given (it takes no version as input).
      - name: Release notes, version commit and tag
        id: flow
        uses: softwarity/release-flow@58d6aa0216d97976a623edca2fe484eababf5b57
        with:
          bump: ${{ steps.bump.outputs.bump }}
          push: "false"
          create-release: "false"
      - name: Push the version commit and the tag, create the GitHub Release
        env:
          GH_TOKEN: ${{ steps.token.outputs.token }}
          VERSION: ${{ steps.next.outputs.version }}
          TAG: ${{ steps.flow.outputs.tag }}
          NOTES: ${{ steps.flow.outputs.notes }}
          TESTED: ${{ steps.main.outputs.sha }}
        run: |
          set -e
          [ "$TAG" = "v$VERSION" ] || { echo "::error::release-flow tagged $TAG, the images say $VERSION"; exit 1; }
          git commit --amend --reset-author -m "$VERSION [skip ci]"
          git tag -f "$TAG"
          git push origin HEAD:main "refs/tags/$TAG"
          printf '%s\n' "$NOTES" > "$RUNNER_TEMP/notes.md"
          gh release create "$TAG" --verify-tag --title "$TAG" --notes-file "$RUNNER_TEMP/notes.md"
          {
            echo "## Released $VERSION"
            echo
            echo "- images: \`$IMAGE:$VERSION\`, \`$IMAGE:$VERSION-hosted\` (also latest, hosted, and the x.y / x tags)"
            echo "- built from ${TESTED::7}, tagged $TAG on the notes commit $(git rev-parse --short HEAD)"
          } >> "$GITHUB_STEP_SUMMARY"
      # Meerkat hands out the hosted client and links the agent: it is told, and
      # its own workflow (plug-bump.yml) moves its pin, tests it and commits it.
      # Same App as the push above, a token for that one repository; a dispatch
      # needs no more than contents: write there. Told AFTER the release exists,
      # and never failing it: plug is released either way, and Meerkat's daily
      # run catches up with a dispatch that did not arrive.
      - id: meerkat
        uses: actions/create-github-app-token@v2
        continue-on-error: true
        with:
          app-id: ${{ vars.RELEASE_APP_ID }}
          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
          owner: softwarity
          repositories: meerkat
      - name: Tell Meerkat
        if: steps.meerkat.outcome == 'success'
        env:
          GH_TOKEN: ${{ steps.meerkat.outputs.token }}
          VERSION: ${{ steps.next.outputs.version }}
        run: |
          if gh api repos/softwarity/meerkat/dispatches -f event_type=plug-released -f "client_payload[version]=$VERSION"; then
            echo "- Meerkat told: plug-released $VERSION" >> "$GITHUB_STEP_SUMMARY"
          else
            echo "::warning::Meerkat was not told of $VERSION; its daily plug-bump run will catch up"
          fi

  # The clusters for the mesh e2e (see _e2e.yml for the e2e itself). THREE
  # FAMILIES, the same chain against each: compose-for.yml (docker compose),
  # k8s-for.yml (kind, real upstream Kubernetes, the agent from the published
  # deploy/plug-k8s.yaml) and swarm-for.yml (a single-node Swarm, the agent as
  # a Swarm service on a non-attachable overlay), all three facades over
  # _cluster.yml. Part of CI, not a separate workflow; runs on EVERY branch
  # push (so any branch gets its full mesh-blessed image). Off PRs: it is heavy
  # and needs the Tailscale/Docker secrets, absent on fork PRs.
  #
  # This job only DISPATCHES the cluster workflows; the e2e legs then wait for
  # those clusters to come up inside their own setup step. It waits for
  # nothing (see below for why): the e2e jobs carry their own `needs: test,
  # selftest`, so nothing is ever exercised against code whose tests are red,
  # and the kill jobs reap whatever was dispatched (they run on always() as long
  # as this job succeeded, e2e skipped or not).
  start-cluster:
    name: start clusters
    # NOT waiting for image-build. This job only DISPATCHES; what needs the image
    # is a job inside each cluster run, several steps later — and the reference
    # is predictable (sha-<commit>), so it can be handed over before it exists.
    # The cluster's pull waits for it while the runner boots, checks out and
    # joins the tailnet. Making the dispatch wait for the build serialised two
    # things that have no reason to be in sequence.
    #
    # Nor for the selftest, for the same reason: a cluster takes three to four
    # minutes to stand up, the tests and the selftest about as long, and the
    # legs used to wait for one and then the other (87s for kind to be up, after
    # the selftest had already taken its three minutes). The LEGS are what the
    # selftest gates, which is what the gate was for: a red selftest means the
    # image is never published, and the ten legs no longer run their 20
    # minutes for a run already lost. The clusters it costs are cancelled by
    # the kill jobs as soon as the legs are skipped, minutes into their boot.
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    permissions:
      actions: write # dispatch the cluster runs
      contents: read
    outputs:
      # The e2e legs read this to tell "the cluster is broken" from "the image
      # it runs was not published yet" — two states that look identical from a
      # leg, and only one of which is the cluster's fault.
      short_sha: ${{ steps.ref.outputs.short }}
      cluster_id_a: ${{ steps.dispatch.outputs.a }}
      cluster_id_b: ${{ steps.dispatch.outputs.b }}
      k8s_id_a: ${{ steps.dispatch.outputs.ka }}
      k8s_id_b: ${{ steps.dispatch.outputs.kb }}
      swarm_id_a: ${{ steps.dispatch.outputs.sa }}
      swarm_id_b: ${{ steps.dispatch.outputs.sb }}
    steps:
      - uses: actions/checkout@v6
      # The tag metadata-action will put on this build: sha-<7>. Computed here so
      # the dispatch does not have to wait for the build that produces it.
      - id: ref
        run: echo "short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
      # TWO clusters per family, so the e2e can assert multicluster: the same
      # service name must reach the RIGHT backend through each plug (each
      # cluster's `ident` answers its own corr id).
      # The six clusters, dispatched at once. Each dispatch costs about seven
      # seconds (the dispatch, then finding the run it made), and in a row they
      # put the last cluster thirty-five seconds behind the first: the two kind
      # clusters, the slowest to stand up, started fourteen and twenty-one
      # seconds late. Side by side they all start within the first dispatch's
      # seven. Titles differ per cluster, so the lookups cannot cross.
      - id: dispatch
        env:
          GH_TOKEN: ${{ github.token }}
          CLUSTER_IMAGE: ghcr.io/softwarity/plug:sha-${{ steps.ref.outputs.short }}-amd64
          CORR: ${{ github.run_id }}-${{ github.run_attempt }}
        run: |
          d="$RUNNER_TEMP/dispatch"; mkdir -p "$d"
          for c in a:compose-for.yml b:compose-for.yml ka:k8s-for.yml kb:k8s-for.yml sa:swarm-for.yml sb:swarm-for.yml; do
            bash scripts/ci/dispatch-cluster.sh "$CORR-${c%%:*}" "${c#*:}" > "$d/${c%%:*}" &
          done
          failed=0
          for p in $(jobs -p); do wait "$p" || failed=1; done
          for s in a b ka kb sa sb; do echo "$s=$(cat "$d/$s")" >> "$GITHUB_OUTPUT"; done
          # The clusters are runs of their own, not jobs of this one: listed
          # here with their links, they are one click from this run's page.
          {
            echo "### Clusters of this run"
            echo "| cluster | run |"
            echo "|---|---|"
            for c in a:compose-for b:compose-for ka:k8s-for kb:k8s-for sa:swarm-for sb:swarm-for; do
              s=${c%%:*} id="$(cat "$d/${c%%:*}")"
              if [ -n "$id" ]; then
                echo "| ${c#*:}-$CORR-$s | [$id]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$id) |"
              else
                echo "| ${c#*:}-$CORR-$s | not dispatched |"
              fi
            done
          } >> "$GITHUB_STEP_SUMMARY"
          [ "$failed" = 0 ] || { echo "::error::a cluster run could not be dispatched (see above)"; exit 1; }
      - name: Extract the latest published launchers
        run: |
          docker create --name pl docker.io/softwarity/plug:latest >/dev/null
          mkdir -p latest-launchers
          for b in plug-linux-amd64 plug-linux-arm64 plug-darwin-amd64 plug-darwin-arm64 plug-windows-amd64.exe; do
            docker cp "pl:/opt/plug/bin/$b" "latest-launchers/$b"
          done
          docker rm pl >/dev/null
          ls -la latest-launchers/
      # Retried once. Other jobs consume this artifact, so it cannot be optional,
      # but a transient 403 from GitHub's artifact service ("non-retryable" to the
      # action) must not cost a 25-minute release either. The second attempt
      # overwrites, since a failed finalize can leave the name taken.
      - uses: actions/upload-artifact@v7
        id: up_launchers
        continue-on-error: true
        with:
          name: latest-launchers
          path: latest-launchers/*
          retention-days: 1
      - uses: actions/upload-artifact@v7
        if: steps.up_launchers.outcome == 'failure'
        with:
          overwrite: true
          name: latest-launchers
          path: latest-launchers/*
          retention-days: 1

  # Cancels the run the moment an e2e leg FAILS, so the other legs and their
  # clusters stop within the minute instead of burning their full twenty. It is
  # a SEPARATE job on purpose: the failing leg used to cancel the run itself,
  # which turned ITS OWN conclusion from "failure" to "cancelled" - the culprit
  # vanished into a sea of cancelled jobs, and finding what broke meant digging
  # into step logs. Here the leg just fails (stays RED), and this watcher - whose
  # own colour nobody reads - issues the cancel. The kill-cluster jobs still run
  # (they complete during the cancellation grace, unchanged). It polls, because a
  # job cannot `needs:` a sibling that is still running; it exits on its own once
  # every e2e leg has an answer, run or skipped, so a green run is not held up.
  abort-on-fail:
    # The selftest too, like the legs it watches: started with the clusters and
    # no leg ever coming, it would poll for its whole timeout.
    needs: [start-cluster, selftest]
    runs-on: ubuntu-latest
    permissions:
      actions: write # cancel the run when a leg fails
    # Longer than the longest leg (30 minutes on k8s and swarm) plus the wait
    # for a runner: at 30 this watcher could expire before the legs it watches.
    timeout-minutes: 40
    steps:
      - name: Cancel the run when an e2e leg fails
        shell: bash
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          run="${{ github.run_id }}"; repo="${{ github.repository }}"
          corr="$run-${{ github.run_attempt }}-"
          while :; do
            # The clusters first. They run as runs of their own (<family>-for-
            # <corr>), started before anything else, so a cluster that fails
            # (an image it could not pull, a kind that would not come up) is
            # known minutes before the legs give up on reaching it, under a
            # message that points at the cluster rather than at the legs.
            # Cancelled is how every cluster normally ends; failure is not.
            broken=$(gh run list --repo "$repo" --limit 40 --json name,conclusion,url \
              --jq "[.[] | select((.name|contains(\"-for-$corr\")) and .conclusion==\"failure\")] | first | select(.) | \"\(.name) \(.url)\"") || broken=""
            if [ -n "$broken" ]; then
              echo "::error::cluster ${broken% *} failed (${broken##* }) -> cancelling the run: its legs could only wait for it and time out"
              gh run cancel "$run" --repo "$repo" || true
              exit 0
            fi
            # An API that answers 502 for a minute is weather, not a verdict:
            # ask again rather than end the watch, which used to leave a red
            # leg uncancelled and this job red for a reason of its own.
            legs=$(gh api "repos/$repo/actions/runs/$run/jobs?per_page=100" \
              --jq '[.jobs[] | select(.name|test("^e2e-"))]') || { sleep 15; continue; }
            failed=$(printf '%s' "$legs" | jq '[.[] | select(.conclusion=="failure")] | length')
            if [ "${failed:-0}" -gt 0 ]; then
              echo "an e2e leg failed -> cancelling the run (the failed leg keeps its red)"
              gh run cancel "$run" --repo "$repo" || true
              exit 0
            fi
            # Done once every leg listed has an answer. A leg whose Test or
            # client build failed never runs: it is listed as skipped, under
            # its caller's name, and never as pending, so waiting to have seen
            # one pending first kept this watch polling for its whole timeout
            # after a red Test. Nothing listed yet means the legs are still to
            # be created, not finished.
            total=$(printf '%s' "$legs" | jq 'length')
            pending=$(printf '%s' "$legs" | jq '[.[] | select(.conclusion==null)] | length')
            if [ "${total:-0}" -gt 0 ] && [ "${pending:-0}" -eq 0 ]; then
              ran=$(printf '%s' "$legs" | jq '[.[] | select(.conclusion!="skipped")] | length')
              if [ "${ran:-0}" -eq 0 ]; then
                echo "the e2e legs did not run (a job they need did not pass): nothing to watch"
              else
                echo "all e2e legs finished, none failed"
              fi
              exit 0
            fi
            sleep 15
          done

  # The mesh e2e, ONE chain written in _e2e.yml and run four times: the three
  # cluster families over the three OSes, and the arm64 client on the compose
  # clusters. The four jobs below differ only by what they pass. Each keeps
  # its own key because `needs` is how kill-<family> waits for EXACTLY the legs
  # that use its clusters, and `needs` cannot name a subset of a matrix.
  #
  # Off PRs: start-cluster is, so these are skipped with it (the secrets are
  # absent on a fork PR, and the run is heavy). Each carries `needs: test,
  # selftest`, so nothing is ever exercised against code whose tests are red.
  #
  # The job names the other readers rely on: abort-on-fail selects the legs by
  # `^e2e-` on the job name (a called workflow's jobs are named
  # "<caller> / <job>", so the key here is the prefix); image and the kill
  # jobs `needs` these keys; scripts/ci/check-e2e-cells.sh asserts all of it.
  e2e-compose:
    needs: [test, selftest, start-cluster, build-clients]
    permissions:
      contents: read
      packages: read # the image under test is on GHCR
    uses: ./.github/workflows/_e2e.yml
    with:
      family: compose
      label: by name over the mesh
      legs: '["ubuntu-latest", "macos-latest", "windows-latest"]'
      cluster_a: plug-compose-${{ github.run_id }}-${{ github.run_attempt }}-a
      cluster_b: plug-compose-${{ github.run_id }}-${{ github.run_attempt }}-b
      minutes: 25
      short_sha: ${{ needs.start-cluster.outputs.short_sha }}
    secrets:
      TAILSCALE_CLIENT_ID: ${{ secrets.TAILSCALE_CLIENT_ID }}
      TAILSCALE_CLIENT_SECRET: ${{ secrets.TAILSCALE_CLIENT_SECRET }}
  # compose's 25 + the extra wait for the kind rollout; a hung cell must still
  # not run the cluster for hours.
  e2e-k8s:
    needs: [test, selftest, start-cluster, build-clients]
    permissions:
      contents: read
      packages: read # the image under test is on GHCR
    uses: ./.github/workflows/_e2e.yml
    with:
      family: k8s
      label: the same chain on kind
      legs: '["ubuntu-latest", "macos-latest", "windows-latest"]'
      cluster_a: plug-k8s-${{ github.run_id }}-${{ github.run_attempt }}-ka
      cluster_b: plug-k8s-${{ github.run_id }}-${{ github.run_attempt }}-kb
      minutes: 30
      short_sha: ${{ needs.start-cluster.outputs.short_sha }}
    secrets:
      TAILSCALE_CLIENT_ID: ${{ secrets.TAILSCALE_CLIENT_ID }}
      TAILSCALE_CLIENT_SECRET: ${{ secrets.TAILSCALE_CLIENT_SECRET }}
  e2e-swarm:
    needs: [test, selftest, start-cluster, build-clients]
    permissions:
      contents: read
      packages: read # the image under test is on GHCR
    uses: ./.github/workflows/_e2e.yml
    with:
      family: swarm
      label: the same chain on swarm
      legs: '["ubuntu-latest", "macos-latest", "windows-latest"]'
      cluster_a: plug-swarm-${{ github.run_id }}-${{ github.run_attempt }}-sa
      cluster_b: plug-swarm-${{ github.run_id }}-${{ github.run_attempt }}-sb
      minutes: 30
      short_sha: ${{ needs.start-cluster.outputs.short_sha }}
    secrets:
      TAILSCALE_CLIENT_ID: ${{ secrets.TAILSCALE_CLIENT_ID }}
      TAILSCALE_CLIENT_SECRET: ${{ secrets.TAILSCALE_CLIENT_SECRET }}
  # The arm64 client. This asks an ARCHITECTURE question (does a NATIVE arm64
  # plug reach cluster services by name), not a cluster-family one. It used to
  # be a fourth leg of e2e-compose, which made three matrices that should read
  # as one comparison look like two identical blocks and an odd one out.
  #
  # linux/arm64 has been built and published on every release since the
  # multi-arch image existed, and nothing ever ran it: every leg was amd64.
  #
  # It borrows the compose clusters (the lightest) and owns its own names: the
  # cells that create names through -s derive them from $leg, which tells the
  # architectures apart (before that, this leg claimed exposed-linux and took
  # the amd64 leg down with it, measured). `full: false` is the subset it runs,
  # and _e2e.yml says which cells that leaves out and why.
  # OpenShift for real: MicroShift built from OKD, one node, on this runner. kind
  # admits any pod; OpenShift's default restricted-v2 SCC runs the agent as a uid
  # the cluster picks, never root, which is what this checks along with an
  # install from that agent and a session through it (scripts/ci/okd.sh). Its own
  # runner, no tailnet: the cluster is a container beside the client. Not named
  # e2e-*: it is not a cell chain of _e2e.yml, and abort-on-fail and
  # check-e2e-cells.sh read that prefix as one.
  okd:
    name: OpenShift (MicroShift, restricted-v2)
    needs: [test, image-build]
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    timeout-minutes: 45
    permissions:
      contents: read
      packages: read # the image under test is on GHCR
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      # The test images live on GHCR, not Docker Hub (see _docker.yml): signed
      # in with this run's own token.
      - name: Log in to GHCR
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}
      - name: plug on MicroShift
        run: scripts/ci/okd.sh "${{ needs.image-build.outputs.image }}"

  e2e-arm64:
    needs: [test, selftest, start-cluster, build-clients]
    permissions:
      contents: read
      packages: read # the image under test is on GHCR
    uses: ./.github/workflows/_e2e.yml
    with:
      family: compose
      label: a native arm64 client, on the compose clusters
      legs: '["ubuntu-24.04-arm"]'
      cluster_a: plug-compose-${{ github.run_id }}-${{ github.run_attempt }}-a
      cluster_b: plug-compose-${{ github.run_id }}-${{ github.run_attempt }}-b
      minutes: 25
      short_sha: ${{ needs.start-cluster.outputs.short_sha }}
      full: false
    secrets:
      TAILSCALE_CLIENT_ID: ${{ secrets.TAILSCALE_CLIENT_ID }}
      TAILSCALE_CLIENT_SECRET: ${{ secrets.TAILSCALE_CLIENT_SECRET }}
  kill-compose:
    name: kill compose clusters
    # e2e-arm64 runs on the SAME compose clusters and starts later, because the
    # ARM pool is scarce. Killing them on e2e-compose alone pulled the cluster
    # out from under it, and "it is still running when the clusters die" is the
    # normal case rather than the unlucky one.
    needs: [start-cluster, e2e-compose, e2e-arm64]
    if: always() && needs.start-cluster.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      actions: write # cancel the cluster runs
      contents: read
    steps:
      # A checkout for one script: the cancel, and what to say when it does not
      # cancel (scripts/ci/kill-clusters.sh, shared by the three kill jobs).
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - env:
          GH_TOKEN: ${{ github.token }}
          A: ${{ needs.start-cluster.outputs.cluster_id_a }}
          B: ${{ needs.start-cluster.outputs.cluster_id_b }}
        run: bash scripts/ci/kill-clusters.sh "$A:compose-a" "$B:compose-b"

  kill-k8s:
    name: kill k8s clusters
    needs: [start-cluster, e2e-k8s]
    if: always() && needs.start-cluster.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      actions: write
      contents: read
    steps:
      # Same shape as kill-compose, and for the same reason: a cancel that fails
      # must not look like a cancel that worked. These two idle for 2100s.
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - env:
          GH_TOKEN: ${{ github.token }}
          A: ${{ needs.start-cluster.outputs.k8s_id_a }}
          B: ${{ needs.start-cluster.outputs.k8s_id_b }}
        run: bash scripts/ci/kill-clusters.sh "$A:k8s-a" "$B:k8s-b"

  kill-swarm:
    name: kill swarm clusters
    needs: [start-cluster, e2e-swarm]
    if: always() && needs.start-cluster.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      actions: write
      contents: read
    steps:
      # Same shape as kill-compose, and for the same reason: a cancel that fails
      # must not look like a cancel that worked. These two idle for 2100s.
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - env:
          GH_TOKEN: ${{ github.token }}
          A: ${{ needs.start-cluster.outputs.swarm_id_a }}
          B: ${{ needs.start-cluster.outputs.swarm_id_b }}
        run: bash scripts/ci/kill-clusters.sh "$A:swarm-a" "$B:swarm-b"

  # The run's test images, deleted from GHCR once nothing reads them any more:
  # after the publish job (which copies the tested digest to Docker Hub), OKD,
  # and the kill jobs (each waits for its family's legs, which wait for the
  # clusters). always(): a red or cancelled run leaves its images exactly like
  # a green one, and they are nobody's either way. Leftovers of runs that died
  # before getting here go too, past two days (scripts/ci/ghcr-cleanup.sh).
  ghcr-cleanup:
    name: delete this run's test images from GHCR
    needs: [start-cluster, image-build, image-build-hosted, image, okd, kill-compose, kill-k8s, kill-swarm]
    if: always() && github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - name: Log in to GHCR
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}
      # The per-architecture digests the builds pushed (_docker.yml uploads one
      # file per digest): no tag names them, so this is how they are found.
      - uses: actions/download-artifact@v7
        continue-on-error: true # a run cancelled before its builds has none
        with:
          path: ${{ runner.temp }}/digests
          pattern: digests-*
          merge-multiple: true
      - env:
          GH_TOKEN: ${{ github.token }}
        run: bash scripts/ci/ghcr-cleanup.sh "${GITHUB_SHA::7}" "$RUNNER_TEMP/digests"