action: use post build hook to stream partial build results

.github/actions/atelier/action.yaml

@@ -23,13 +23,30 @@ inputs:
   rules:
     description: Atelier rule file whose substituters and trusted-public-keys are injected into nix.conf
     default: atelier.toml
+  post-build-hook:
+    description: Install the spooling post-build hook and name it in nix.conf
+    default: "false"
 
 runs:
   using: composite
   steps:
+    - name: Export Atelier Root
+      # publish the action checkout's repo root so later workflow steps can
+      # invoke files shipped with atelier (the runner downloads the whole
+      # repository for a subdirectory action)
+      shell: bash
+      run: | # zizmor: ignore[github-env] value is the runner-provided action path, not consumer input
+        # a failed strip must collapse to the empty value the push step guards
+        root="${GITHUB_ACTION_PATH%/.github/actions/atelier}"
+        if [ ! -f "${root}/src/atelier/stream.py" ]; then
+          echo "::warning::Atelier root not found under ${GITHUB_ACTION_PATH}, streaming disabled"
+          root=""
+        fi
+        printf 'ATELIER_ROOT=%s\n' "${root}" >> "$GITHUB_ENV"
+
     - name: Pre Install Hook
       # consumer-supplied command, passed via env (not interpolated into the
-      # script body) so it cannot break out of the run. it is the first step and
+      # script body) so it cannot break out of the run. it is the first consumer-visible step and
       # carries no install-command guard, so it fires before the built-in or a
       # custom installer alike, on a fresh runner before disk reclaim
       if: ${{ inputs.pre-install != '' }}
@@ -81,6 +98,7 @@ runs:
         SANDBOX: ${{ runner.os == 'macOS' && 'relaxed' || 'true' }}
         EXTRA_CONF: ${{ inputs.extra-conf }}
         RULES: ${{ inputs.rules }}
+        POST_BUILD_HOOK: ${{ inputs.post-build-hook }}
       run: |
         set -euo pipefail
         # the nix.conf lines that apply whichever installer ran, so a custom
@@ -113,6 +131,10 @@ runs:
           if [ -n "${keys}" ]; then
             echo "extra-trusted-public-keys = ${keys}"
           fi
+          # named before extra-conf so a consumer override wins (last wins)
+          if [ "${POST_BUILD_HOOK}" = "true" ]; then
+            echo "post-build-hook = /nix/var/atelier/hook.sh"
+          fi
           # explicit if, not `[ -n ] && printf`, so an empty value does not trip
           # set -e on the trailing conditional
           if [ -n "${EXTRA_CONF}" ]; then
@@ -178,6 +200,24 @@ runs:
       shell: bash
       run: sudo mkdir -p /nix/build
 
+    - name: Install Post Build Hook
+      # after the installer because /nix does not exist on macos before it
+      # creates the volume. no build can run between daemon start and this
+      # step (the steps in between invoke no nix commands and the consumer
+      # post-install hook runs later), so the hook path named in nix.conf
+      # always resolves by the time a build finishes
+      if: ${{ inputs.post-build-hook == 'true' }}
+      shell: bash
+      run: |
+        set -euo pipefail
+        # explicit modes, sudo inherits the caller's umask
+        # install -m /dev/null also truncates a spool surviving on a
+        # persistent /nix so old jobs' paths are not replayed
+        # the paths must match hook.sh and stream.py's ATELIER_SPOOL default
+        sudo mkdir -p -m 0755 /nix/var/atelier
+        sudo install -m 0755 "${GITHUB_ACTION_PATH}/hook.sh" /nix/var/atelier/hook.sh
+        sudo install -m 0644 /dev/null /nix/var/atelier/spool
+
     - name: Report Disk Space (Darwin)
       if: ${{ inputs.reclaim == 'true' && runner.os == 'macOS' }}
       uses: srz-zumix/post-run-action@42756f7452b9439d0365b7e087b2c364f54209c6 # v3.0.2

.github/actions/atelier/hook.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+# post-build hook, runs as root under the nix daemon
+# append every built output path to the spool and never fail the build loop
+# the env override exists for tests, the daemon never sets ATELIER_SPOOL
+# OUT_PATHS is deliberately unquoted so word splitting yields one path per line
+# set -f disables globbing so a metacharacter in a path never expands
+# an empty OUT_PATHS writes a blank line, the spool reader skips blank lines
+set -f
+{ printf '%s\n' $OUT_PATHS >> "${ATELIER_SPOOL:-/nix/var/atelier/spool}"; } 2>/dev/null || true

.github/workflows/build.yaml

@@ -117,22 +117,59 @@ jobs:
           rules: ${{ inputs.rules }}
           pre-install: ${{ inputs.pre-install }}
           post-install: ${{ inputs.post-install }}
+          post-build-hook: ${{ inputs.push && inputs.push-command == '' && github.event.pull_request.head.repo.fork != true && ((vars.ATTIC_SERVER != '' && vars.ATTIC_CACHE != '') || vars.CACHIX_CACHE != '' || vars.NIKS3_SERVER != '') }}
+
+      # the streamer runs as the runner user with the step env, which is what
+      # keeps the secrets out of the root-run hook. it drains the spool the
+      # hook fills and pushes while the build runs. its stdout lands in a log
+      # file the final drain replays into the step log
+      - name: Start Cache Streamer
+        if: ${{ inputs.push && inputs.push-command == '' && matrix.installable != '' && github.event.pull_request.head.repo.fork != true && ((vars.ATTIC_SERVER != '' && vars.ATTIC_CACHE != '') || vars.CACHIX_CACHE != '' || vars.NIKS3_SERVER != '') }}
+        shell: bash
+        env:
+          ATTIC_SERVER: ${{ vars.ATTIC_SERVER }}
+          ATTIC_CACHE: ${{ vars.ATTIC_CACHE }}
+          ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+          CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
+          CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+          CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }}
+          NIKS3_SERVER: ${{ vars.NIKS3_SERVER }}
+          NIKS3_TOKEN: ${{ secrets.NIKS3_TOKEN }}
+        run: |
+          set -euo pipefail
+          # a missing root means setup could not find the atelier tree
+          if [ -z "${ATELIER_ROOT:-}" ]; then
+            echo "::warning::Skipping cache streamer: setup did not complete"
+            exit 0
+          fi
+          # a stale sentinel or pidfile from a prior job on a persistent
+          # runner must not stop or stall this job's streamer
+          rm -f "${RUNNER_TEMP}/atelier-stream.done" "${RUNNER_TEMP}/atelier-stream.pid"
+          python3 "${ATELIER_ROOT}/src/atelier/stream.py" --mode stream \
+            > "${RUNNER_TEMP}/atelier-stream.log" 2>&1 &
+          printf '%s' "$!" > "${RUNNER_TEMP}/atelier-stream.pid" \
+            || echo "::warning::Could not record the streamer pid"
 
       - name: Build
+        id: build
         if: ${{ matrix.installable != '' }}
         shell: bash
         run: |
           set -o pipefail
           nix build "${INSTALLABLE}^*" --no-link --print-build-logs 2>&1 | tee build.log
 
       # the push phase brackets pre-push, the push (built-in or custom), and
-      # post-push under one guard: a push was requested for a real, non-fork
-      # build. it deliberately omits the "a backend is configured" check the
-      # built-in push carries, so push-command and the hooks can target a cache
-      # atelier has no native support for. each step is env-passed and not
-      # interpolated into the run body so it cannot break out of the run
+      # post-push. with built-in backends the streamer has already been
+      # pushing during the build, and the final drain below re-pushes the
+      # full spool plus the outputs' closure as the backstop, also on a
+      # failed build (never on a cancelled or setup-failed cell) so partial
+      # results land on the cache. the pre/post hooks keep their success-only
+      # timing unless the built-in drain runs, and only for a real build
+      # failure (not a setup failure, where no push happens). each step is
+      # env-passed and not interpolated into the run body so it cannot break
+      # out of the run
       - name: Pre Push Hook
-        if: ${{ inputs.push && matrix.installable != '' && github.event.pull_request.head.repo.fork != true && inputs.pre-push != '' }}
+        if: ${{ inputs.push && matrix.installable != '' && github.event.pull_request.head.repo.fork != true && inputs.pre-push != '' && (success() || (!cancelled() && steps.build.outcome == 'failure' && inputs.push-command == '' && ((vars.ATTIC_SERVER != '' && vars.ATTIC_CACHE != '') || vars.CACHIX_CACHE != '' || vars.NIKS3_SERVER != ''))) }}
         shell: bash
         env:
           PRE_PUSH: ${{ inputs.pre-push }}
@@ -149,9 +186,10 @@ jobs:
           bash -c "${PRE_PUSH}"
 
       - name: Push To Cache
-        if: ${{ inputs.push && inputs.push-command == '' && matrix.installable != '' && github.event.pull_request.head.repo.fork != true && ((vars.ATTIC_SERVER != '' && vars.ATTIC_CACHE != '') || vars.CACHIX_CACHE != '' || vars.NIKS3_SERVER != '') }}
+        if: ${{ (success() || (!cancelled() && steps.build.outcome == 'failure')) && inputs.push && inputs.push-command == '' && matrix.installable != '' && github.event.pull_request.head.repo.fork != true && ((vars.ATTIC_SERVER != '' && vars.ATTIC_CACHE != '') || vars.CACHIX_CACHE != '' || vars.NIKS3_SERVER != '') }}
         shell: bash
         env:
+          BUILD_OUTCOME: ${{ steps.build.outcome }}
           ATTIC_SERVER: ${{ vars.ATTIC_SERVER }}
           ATTIC_CACHE: ${{ vars.ATTIC_CACHE }}
           ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
@@ -161,90 +199,13 @@ jobs:
           NIKS3_SERVER: ${{ vars.NIKS3_SERVER }}
           NIKS3_TOKEN: ${{ secrets.NIKS3_TOKEN }}
         run: |
-          set -uo pipefail
-
-          # only the outputs we just built
-          paths="$(nix build "${INSTALLABLE}^*" --no-link --print-out-paths)"
-          [ -n "$paths" ] || exit 0
-
-          # push to every configured backend, best-effort: each runs in its own
-          # subshell so one cannot abort the others, and a failed push only warns
-          # (the artifact is already built, so a cache upload hiccup must not fail
-          # the build). drop -e here so a failing backend falls through to the
-          # next; each subshell still runs set -euo pipefail, effective only
-          # because it runs as a plain statement whose status we read with $? -
-          # a "( set -e ... ) || warn" would disable that inner set -e (bash
-          # ignores -e set inside a command on the left of ||)
-          set +e
-
-          if [ -n "${ATTIC_SERVER}" ] && [ -n "${ATTIC_CACHE}" ]; then
-            (
-              set -euo pipefail
-              # lix only has 'install', on cppnix it is a deprecated alias for
-              # 'add'. revert to 'add' once lix supports 'add'
-              nix profile install nixpkgs#attic-client
-              attic login default "${ATTIC_SERVER}" "${ATTIC_TOKEN}"
-              # a missing or unreachable cache is a soft skip, not a build failure
-              attic cache info "${ATTIC_CACHE}" || { echo "::warning::Attic cache unavailable, skipping"; exit 0; }
-              # shellcheck disable=SC2086
-              attic push "${ATTIC_CACHE}" $paths
-            )
-            # shellcheck disable=SC2181
-            [ $? -eq 0 ] || echo "::warning::Attic push failed"
-          fi
-
-          if [ -n "${CACHIX_CACHE}" ]; then
-            (
-              set -euo pipefail
-              # lix only has 'install', on cppnix it is a deprecated alias for
-              # 'add'. revert to 'add' once lix supports 'add'
-              nix profile install nixpkgs#cachix
-              [ -n "${CACHIX_SIGNING_KEY:-}" ] || unset CACHIX_SIGNING_KEY
-              # shellcheck disable=SC2086
-              printf '%s\n' $paths | cachix push "${CACHIX_CACHE}"
-            )
-            # shellcheck disable=SC2181
-            [ $? -eq 0 ] || echo "::warning::Cachix push failed"
-          fi
-
-          if [ -n "${NIKS3_SERVER}" ]; then
-            (
-              set -euo pipefail
-              # nixpkgs niks3 lags at 1.4.0 (no --auth-token-{path,script})
-              # use my pinned niks3
-              # lix only has 'install', on cppnix it is a deprecated alias for
-              # 'add'. revert to 'add' once lix supports 'add'
-              nix profile install github:stepbrobd/inc#niks3
-              if [ -n "${NIKS3_TOKEN:-}" ]; then
-                # token auth: write the secret to a private file, pass its path
-                tok="$(mktemp)"
-                (umask 077; printf '%s' "${NIKS3_TOKEN}" > "$tok")
-                # shellcheck disable=SC2086
-                niks3 push --server-url "${NIKS3_SERVER}" --auth-token-path "$tok" $paths
-              elif [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then
-                # oidc auth: the audience must match what the server validates the jwt against
-                aud="$(curl -sf "${NIKS3_SERVER}/api/cache-config?issuer=https://token.actions.githubusercontent.com" | jq -r '.oidc_audience // empty')"
-                if [ -z "$aud" ]; then
-                  echo "::warning::Skipping niks3 push: server advertises no oidc_audience for the GitHub issuer; configure a GitHub OIDC provider on the server"
-                else
-                  # niks3 reruns this script to refresh the token; leave the github
-                  # oidc env vars unexpanded so they are read at each mint, not now
-                  s="$(mktemp)"
-                  printf '%s\n' \
-                    '#!/bin/sh' \
-                    "exec curl -sf -H \"Authorization: Bearer \$ACTIONS_ID_TOKEN_REQUEST_TOKEN\" \"\$ACTIONS_ID_TOKEN_REQUEST_URL&audience=${aud}\" | jq '{token: .value, expires_at: ((now + 240) | todateiso8601)}'" \
-                    > "$s"
-                  chmod 700 "$s"
-                  # shellcheck disable=SC2086
-                  niks3 push --server-url "${NIKS3_SERVER}" --auth-token-script "$s" $paths
-                fi
-              else
-                echo "::warning::Skipping niks3 push: OIDC needs 'id-token: write' in the caller workflow"
-              fi
-            )
-            # shellcheck disable=SC2181
-            [ $? -eq 0 ] || echo "::warning::niks3 push failed"
+          set -euo pipefail
+          # a failed setup leaves no atelier checkout, skip rather than error
+          if [ -z "${ATELIER_ROOT:-}" ]; then
+            echo "::warning::Skipping cache push: setup did not complete"
+            exit 0
           fi
+          python3 "${ATELIER_ROOT}/src/atelier/stream.py" --mode final || echo "::warning::Cache push failed"
 
       # custom push replaces the built-in one (like install-command replaces the
       # installer): when set, the native Push To Cache above is skipped. the
@@ -269,7 +230,7 @@ jobs:
           bash -c "${PUSH_COMMAND}"
 
       - name: Post Push Hook
-        if: ${{ inputs.push && matrix.installable != '' && github.event.pull_request.head.repo.fork != true && inputs.post-push != '' }}
+        if: ${{ inputs.push && matrix.installable != '' && github.event.pull_request.head.repo.fork != true && inputs.post-push != '' && (success() || (!cancelled() && steps.build.outcome == 'failure' && inputs.push-command == '' && ((vars.ATTIC_SERVER != '' && vars.ATTIC_CACHE != '') || vars.CACHIX_CACHE != '' || vars.NIKS3_SERVER != ''))) }}
         shell: bash
         env:
           POST_PUSH: ${{ inputs.post-push }}

flake.nix

@@ -27,6 +27,7 @@
             # code
             ./src
             ./tests
+            ./.github/actions/atelier/hook.sh
             # meta
             ./license.txt
             ./pyproject.toml

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atelier"
-version = "2026.816.0"
+version = "2026.819.0"
 description = "nix atelier ;)"
 readme = "readme.md"
 requires-python = ">=3.14"

readme.md

@@ -169,6 +169,15 @@ Pushes happen on a push to your repository's default branch (or `master`) and on
 a run with `push: true`. Forked-PR runs never push. Caching is best-effort: a
 failed push to any backend is logged as a warning and never fails the build.
 
+Uploads start while the build runs: a post-build hook records every locally
+built store path, and a background uploader pushes them to every configured
+backend as they appear. After the build, a final drain re-pushes the full set
+plus the finished outputs' closure, so substituted dependencies missing from
+your cache still land there. A failed build pushes the paths it built before
+failing, and a cancelled or timed-out run keeps whatever was uploaded before the
+kill. On a self-hosted runner a cancelled cell can leave the background uploader
+running until the runner reaps job processes.
+
 ## Use it in your repo
 
 Atelier runs against whatever repository calls it. `actions/checkout` inside the
@@ -249,10 +258,13 @@ optional inputs change that without forking:
 Atelier separates installing Nix from configuring it. Whichever installer runs,
 Atelier applies its own required `nix.conf` afterwards (the GitHub access token,
 `experimental-features = nix-command flakes`, the build directory, the target
-`system`, and the sandbox mode), then appends your `extra-conf` last. So a
-custom installer still ends up with a correctly configured daemon, and adding
-settings is independent of the installer choice. Use Nix's `extra-` prefixes to
-add to a list setting rather than replace it.
+`system`, the sandbox mode, and the post-build hook), then appends your
+`extra-conf` last. So a custom installer still ends up with a correctly
+configured daemon, and adding settings is independent of the installer choice.
+Use Nix's `extra-` prefixes to add to a list setting rather than replace it.
+With a binary cache configured it also sets `post-build-hook` to record built
+paths for streaming. A `post-build-hook` of your own in `extra-conf` wins and
+replaces streaming.
 
 The inputs apply to every job, so discovery and every build cell use the same
 Nix. Install Lix instead of upstream Nix and enable the pipe operator:
@@ -323,6 +335,13 @@ jobs:
 `pre-push`, `post-push`, and `push-command` apply only to the build cells (the
 discovery job never pushes). They share the `push: true` guard.
 
+With built-in backends the push begins during the build, so `pre-push` runs
+before the final drain rather than before all push activity. Streamed batches
+that need a `pre-push` side effect fail soft and are re-pushed by the final
+drain after the hook has run. When a build fails and a built-in backend is
+configured, `pre-push` and `post-push` also run around the failure-path drain.
+With `push-command` set, hook timing is unchanged.
+
 `push-command` replaces the built-in push exactly like `install-command`
 replaces the installer (set it and the native Attic/Cachix/niks3 push is
 skipped). The attribute being built is exposed as `INSTALLABLE`, so a command