host-agent applies OS updates: install to the other slot, switch in the window, revert on its own (#563)

.github/workflows/ci-cloud-image.yml

@@ -560,7 +560,7 @@ jobs:
           BOOTS_INPUT: ${{ inputs.boots }}
         run: |
           set -euo pipefail
-          full="unseeded seeded access update ssh remap"
+          full="unseeded seeded access update ssh remap os-update os-revert"
           if [ "$EVENT" = "pull_request" ]; then
             # The compare API needs only `contents: read`, which this job has. The
             # PR files API would need `pull-requests: read`, and asking for it
@@ -587,11 +587,24 @@ jobs:
           for b in $list; do
             case "$b" in
               unseeded|seeded|frozen) case " $chain " in *" $b "*) ;; *) chain="${chain:+$chain }$b" ;; esac ;;
-              access|update|ssh|remap) case " ${groups[*]-} " in *" $b "*) ;; *) groups+=("$b") ;; esac ;;
+              access|update|ssh|remap|os-update|os-revert) case " ${groups[*]-} " in *" $b "*) ;; *) groups+=("$b") ;; esac ;;
               bios) echo "::error::'bios' is no longer a boot: every boot runs under both firmwares"; exit 1 ;;
-              *) echo "::error::unknown boot '$b' (known: unseeded seeded frozen access update ssh remap)"; exit 1 ;;
+              *) echo "::error::unknown boot '$b' (known: unseeded seeded frozen access update ssh remap os-update os-revert)"; exit 1 ;;
             esac
           done
+          # A run that publishes the OS bakes only the release root into the
+          # boot-proof image, so the throwaway-signed test bundle cannot install
+          # (#563). os-update then runs its refusal half only, and os-revert,
+          # which needs an install, is left out.
+          if [ "$SHOULD_PUBLISH_OS" = true ]; then
+            kept=(); for g in "${groups[@]}"; do [ "$g" = os-revert ] || kept+=("$g"); done; groups=("${kept[@]}")
+            echo "an OS release run: os-update runs its refusal half only, os-revert is left out"
+          fi
+          # The OS update boots need the test bundle; the build makes it only
+          # when one of them runs (about 1.5 min).
+          os_test=false
+          case " ${groups[*]} " in *" os-update "*|*" os-revert "*) os_test=true ;; esac
+          echo "os_test=${os_test}" >> "$GITHUB_OUTPUT"
           if [ -n "$chain" ]; then groups=("$chain" "${groups[@]}"); fi
           [ "${#groups[@]}" -gt 0 ] || { echo "::error::the boot list '${list}' names no boot"; exit 1; }
           matrix="$(printf '%s\n' "${groups[@]}" | jq -Rnc '[inputs] as $g | {include: [$g[] as $b | ("uefi", "bios") as $f | {boots: $b, firmware: $f}]}')"
@@ -673,6 +686,26 @@ jobs:
           qemu-img convert -f raw -O qcow2 .dev/cloud-boot/moose-cloud.raw .dev/cloud-boot/moose-cloud-boot.qcow2
           ls -l .dev/cloud-boot/moose-cloud-boot.qcow2
 
+      # The test-only OS bundle the os-update and os-revert boots install
+      # (#563): the boot-proof slot repacked fast (gzip level 1) with a
+      # host-agent one patch release up, signed with the throwaway key. Only
+      # when one of those boots runs.
+      - name: Build the OS test bundle
+        if: ${{ steps.boots.outputs.os_test == 'true' }}
+        env:
+          GO: ${{ steps.go.outputs.go-bin }}
+        run: sudo -E ./dev/cloud/test/build-os-test-bundle.sh .dev/cloud-boot/moose-cloud.raw .dev/cloud-boot/os-test
+      - name: Upload the OS test bundle
+        if: ${{ steps.boots.outputs.os_test == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          overwrite: true
+          name: cloud-image-os-test-bundle
+          path: .dev/cloud-boot/os-test
+          compression-level: 0
+          retention-days: 3
+          if-no-files-found: error
+
       # The squashfs inside is already xz, so zipping it again only costs time.
       # Three days is enough to re-run a failed boot or publish job.
       - name: Upload the boot-proof image
@@ -799,6 +832,12 @@ jobs:
         with:
           name: cloud-image-boot-qcow2
           path: .dev/cloud-boot
+      - name: Download the OS test bundle
+        if: ${{ matrix.boots == 'os-update' || matrix.boots == 'os-revert' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: cloud-image-os-test-bundle
+          path: .dev/cloud-boot/os-test
 
       # MOOSE_CLOUD_QCOW2 makes the lane boot the downloaded image and build
       # nothing. The boot list reaches the script through an env var, never
@@ -808,6 +847,8 @@ jobs:
           MOOSE_CLOUD_QCOW2: ${{ github.workspace }}/.dev/cloud-boot/moose-cloud-boot.qcow2
           MOOSE_CLOUD_BOOTS: ${{ matrix.boots }}
           MOOSE_CLOUD_FIRMWARES: ${{ matrix.firmware }}
+          MOOSE_CLOUD_OS_BUNDLE_DIR: ${{ github.workspace }}/.dev/cloud-boot/os-test
+          MOOSE_CLOUD_OS_REFUSE_ONLY: ${{ env.SHOULD_PUBLISH_OS }}
           GO: ${{ steps.go.outputs.go-bin }}
         run: make test-cloud-qemu
 

CLAUDE.md

@@ -92,7 +92,7 @@ Small set of rules. Codified now so we don't have to back them out later.
 - **Consumer-side interfaces.** Interfaces live in the package that *uses* them, not the package that implements them. `lifecycle.DockerDriver` lives in `internal/lifecycle/`, not in a hypothetical `internal/docker/`. Provider packages export concrete types only. Exception: a single interface shared by three or more consumers can move to the provider, but default to consumer-side until that's true.
 - **Layer boundaries.** `internal/lifecycle` is the transaction owner; only `cmd/brain` and `internal/api` may import it. `internal/store` is the persistence boundary; only `internal/lifecycle`, `internal/api`, `internal/auth`, `internal/audit`, and `cmd/brain` may import it. Anything else reaching in is breaking the model — push the call through the right seam instead.
 - **`log/slog` is the only logger.** No `"log"` imports, no `fmt.Println` for diagnostics. Structured fields, not interpolated strings: `slog.Info("app installed", "instance_id", id)`, not `slog.Info(fmt.Sprintf("installed %s", id))`. The default handler is set in `cmd/brain/main.go`; use `slog.Default()` (the package-level functions) — don't thread `*slog.Logger` through constructors.
-- **Standard structured fields.** Use these key names so journalctl/jq filters stay stable: `instance_id`, `manifest_id`, `slug`, `service`, `image`, `host`, `upstream`, `step`, `err`, `output`, `user_id`, `username`, `role`, `action`, `actor_user_id`, `target_kind`, `target_id`, `retry_after`, `retry_after_s`, `name`, `uid`, `iface`, `interfaces`, `src`, `dir`, `profile`, `box_id`, `zone`, `exposure`, `trusted_proxies`, `brain`, `ui`, `minimum_host_agent`, `keys`, `state_dir`, `job_id`, `window`, `url`, `from`, `count`, `dropped`, `tier`, `remap_base`, `gid`, `image_user`. `host` is a machine or upstream hostname only — a single network interface name is `iface`, a list of them is `interfaces` (never overload `host` for either). `src` is a source filesystem path (bind-source, folder-source); `dir` is a relative bind dir path. `profile` is the resolved environment profile (`appliance`|`hosted`, `ENVIRONMENT.md`). `box_id` is the hosted box's provisioned identity (`ENVIRONMENT.md` # Provisioning). `zone` is an IANA time-zone name (`TIME.md`, host-agent set-timezone). `exposure` is an app's per-instance access mode (`restricted`|`public`, `ENVIRONMENT.md` #306). `trusted_proxies` is the configured set of proxies whose `X-Forwarded-For` the brain reads when deriving a client IP (`BRAIN_UI_PROTOCOL.md` # Rate limiting & abuse). `brain` and `ui` are the two control-plane versions a release names, and `minimum_host_agent` the host-agent version it requires (`RELEASE_MANIFEST.md`); use them for versions, not for image refs — an image ref is `image`. `keys` is how many signing keys a build accepts, and `state_dir` a state directory path (`src` stays for a source path being read or bound). `job_id` is a host-agent job id (`internal/hostagent/jobs.go`), and `window` the configured update window (`UPDATES.md` # 8.4). `url` is an HTTP endpoint the box reads, and `from` says which of several configured sources a setting came from, one of `answer` (the control plane's update-target answer), `seed`, `env`, `default` (`UPDATES.md` # 8.4); where there are only two sources, the boolean `from_<source>` form is used instead (`from_ledger`). `dropped` lists what the box read but could not use, from catalog data it reads leniently (AI provider entries, manifest `role` and `requires` keys), and `count` is how many there were. `tier` is an instance's user-namespace tier (`default`|`caps`|`image`|`host`, `APP_ISOLATION.md` # User-namespace tiers), and `remap_base` the first host id of the Docker remap range (0 for none). `image_user` is the user an image sets (its `Config.User`, a name or a number, as the image wrote it), which the image tier resolves to ids. `name` is an app instance's display name (it rides alongside `instance_id`, never instead of it), and `uid` a numeric Unix user id — the allocated app-service identity, a resolved home owner, or an image user's id. `gid` is its numeric group id. `retry_after` and `retry_after_s` come from the two throttles that `AUTH.md` # Rate limiting keeps apart on purpose. Both are correct. Do not merge them. `retry_after` is the login backoff's wait, written as a duration string (`internal/api/auth.go`); that path sends no `Retry-After` header, by design. `retry_after_s` is the general request limiter's wait, written as a whole number of seconds; it matches that limiter's `retry_after_s` JSON field and the `Retry-After` header it sets (`internal/api/ratelimit.go`, `BRAIN_UI_PROTOCOL.md` # 429 contract). To find every throttled request you must search for both keys. That is the price of the split, not a bug. Adding a new recurring field? Add it here.
+- **Standard structured fields.** Use these key names so journalctl/jq filters stay stable: `instance_id`, `manifest_id`, `slug`, `service`, `image`, `host`, `upstream`, `step`, `err`, `output`, `user_id`, `username`, `role`, `action`, `actor_user_id`, `target_kind`, `target_id`, `retry_after`, `retry_after_s`, `name`, `uid`, `iface`, `interfaces`, `src`, `dir`, `profile`, `box_id`, `zone`, `exposure`, `trusted_proxies`, `brain`, `ui`, `minimum_host_agent`, `keys`, `state_dir`, `job_id`, `window`, `url`, `from`, `count`, `dropped`, `tier`, `remap_base`, `gid`, `image_user`, `os`, `slot`, `digest`. `host` is a machine or upstream hostname only — a single network interface name is `iface`, a list of them is `interfaces` (never overload `host` for either). `src` is a source filesystem path (bind-source, folder-source); `dir` is a relative bind dir path. `profile` is the resolved environment profile (`appliance`|`hosted`, `ENVIRONMENT.md`). `box_id` is the hosted box's provisioned identity (`ENVIRONMENT.md` # Provisioning). `zone` is an IANA time-zone name (`TIME.md`, host-agent set-timezone). `exposure` is an app's per-instance access mode (`restricted`|`public`, `ENVIRONMENT.md` #306). `trusted_proxies` is the configured set of proxies whose `X-Forwarded-For` the brain reads when deriving a client IP (`BRAIN_UI_PROTOCOL.md` # Rate limiting & abuse). `brain` and `ui` are the two control-plane versions a release names, and `minimum_host_agent` the host-agent version it requires (`RELEASE_MANIFEST.md`); use them for versions, not for image refs — an image ref is `image`. `keys` is how many signing keys a build accepts, and `state_dir` a state directory path (`src` stays for a source path being read or bound). `job_id` is a host-agent job id (`internal/hostagent/jobs.go`), and `window` the configured update window (`UPDATES.md` # 8.4). `url` is an HTTP endpoint the box reads, and `from` says which of several configured sources a setting came from, one of `answer` (the control plane's update-target answer), `seed`, `env`, `default` (`UPDATES.md` # 8.4); where there are only two sources, the boolean `from_<source>` form is used instead (`from_ledger`). `dropped` lists what the box read but could not use, from catalog data it reads leniently (AI provider entries, manifest `role` and `requires` keys), and `count` is how many there were. `tier` is an instance's user-namespace tier (`default`|`caps`|`image`|`host`, `APP_ISOLATION.md` # User-namespace tiers), and `remap_base` the first host id of the Docker remap range (0 for none). `image_user` is the user an image sets (its `Config.User`, a name or a number, as the image wrote it), which the image tier resolves to ids. `os` is an OS (moose) release version, as `brain` and `ui` are control-plane versions; `slot` is an A/B OS slot (`A`|`B`, `BUILD.md` # 1b); `digest` is an OS bundle's sha256 (`UPDATES.md` # 1). `name` is an app instance's display name (it rides alongside `instance_id`, never instead of it), and `uid` a numeric Unix user id — the allocated app-service identity, a resolved home owner, or an image user's id. `gid` is its numeric group id. `retry_after` and `retry_after_s` come from the two throttles that `AUTH.md` # Rate limiting keeps apart on purpose. Both are correct. Do not merge them. `retry_after` is the login backoff's wait, written as a duration string (`internal/api/auth.go`); that path sends no `Retry-After` header, by design. `retry_after_s` is the general request limiter's wait, written as a whole number of seconds; it matches that limiter's `retry_after_s` JSON field and the `Retry-After` header it sets (`internal/api/ratelimit.go`, `BRAIN_UI_PROTOCOL.md` # 429 contract). To find every throttled request you must search for both keys. That is the price of the split, not a bug. Adding a new recurring field? Add it here.
 - **Typed errors at boundaries, not everywhere.** Define a sentinel/typed error only when a *consumer* needs to discriminate (HTTP status, retry decision, UI text). `store.ErrNotFound` exists because the API maps it to 404. Don't pre-declare error types speculatively.
 - **No premature abstraction.** Don't introduce an interface, factory, or DI container until at least two concrete consumers exist. It bites hardest in Go where every extra interface is import-graph weight.
 - **`internal/` for everything except `cmd/`.** No `pkg/`. Anything inside `internal/` is private to this module by Go's own rules — no public API surface to maintain.

api/openapi.json

@@ -2888,6 +2888,96 @@
         ],
         "type": "object"
       },
+      "OSOutcomeDTO": {
+        "additionalProperties": false,
+        "properties": {
+          "at": {
+            "type": "string"
+          },
+          "from": {
+            "type": "string"
+          },
+          "id": {
+            "type": "string"
+          },
+          "outcome": {
+            "enum": [
+              "good",
+              "reverted"
+            ],
+            "type": "string"
+          },
+          "version": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "id",
+          "outcome",
+          "version",
+          "at"
+        ],
+        "type": "object"
+      },
+      "OSReleaseDTO": {
+        "additionalProperties": false,
+        "properties": {
+          "bundle_sha256": {
+            "type": "string"
+          },
+          "bundle_url": {
+            "type": "string"
+          },
+          "version": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "version",
+          "bundle_url",
+          "bundle_sha256"
+        ],
+        "type": "object"
+      },
+      "OSUpdateDTO": {
+        "additionalProperties": false,
+        "properties": {
+          "detail": {
+            "type": "string"
+          },
+          "last": {
+            "$ref": "#/components/schemas/OSOutcomeDTO"
+          },
+          "running": {
+            "type": "string"
+          },
+          "slot": {
+            "type": "string"
+          },
+          "state": {
+            "enum": [
+              "unsupported",
+              "none",
+              "refused",
+              "current",
+              "installing",
+              "installed",
+              "waiting",
+              "rebooting",
+              "held",
+              "failed"
+            ],
+            "type": "string"
+          },
+          "target": {
+            "$ref": "#/components/schemas/OSReleaseDTO"
+          }
+        },
+        "required": [
+          "state"
+        ],
+        "type": "object"
+      },
       "Parse-custom-overlayRequest": {
         "additionalProperties": false,
         "properties": {
@@ -3555,6 +3645,12 @@
           "host_agent_version": {
             "type": "string"
           },
+          "os_slot": {
+            "type": "string"
+          },
+          "os_version": {
+            "type": "string"
+          },
           "ui_image": {
             "type": "string"
           },
@@ -3656,6 +3752,9 @@
           "from": {
             "type": "string"
           },
+          "os": {
+            "$ref": "#/components/schemas/OSUpdateDTO"
+          },
           "profile": {
             "type": "string"
           },
@@ -6125,7 +6224,7 @@
             "description": "Error"
           }
         },
-        "summary": "What this box is running: brain version and commit, host-agent version, UI image"
+        "summary": "What this box is running: brain version and commit, host-agent version, UI image, OS version and slot"
       }
     },
     "/api/v1/users": {

api/openapi.yaml

@@ -2048,6 +2048,71 @@ components:
         - summary
         - read
       type: object
+    OSOutcomeDTO:
+      additionalProperties: false
+      properties:
+        at:
+          type: string
+        from:
+          type: string
+        id:
+          type: string
+        outcome:
+          enum:
+            - good
+            - reverted
+          type: string
+        version:
+          type: string
+      required:
+        - id
+        - outcome
+        - version
+        - at
+      type: object
+    OSReleaseDTO:
+      additionalProperties: false
+      properties:
+        bundle_sha256:
+          type: string
+        bundle_url:
+          type: string
+        version:
+          type: string
+      required:
+        - version
+        - bundle_url
+        - bundle_sha256
+      type: object
+    OSUpdateDTO:
+      additionalProperties: false
+      properties:
+        detail:
+          type: string
+        last:
+          $ref: "#/components/schemas/OSOutcomeDTO"
+        running:
+          type: string
+        slot:
+          type: string
+        state:
+          enum:
+            - unsupported
+            - none
+            - refused
+            - current
+            - installing
+            - installed
+            - waiting
+            - rebooting
+            - held
+            - failed
+          type: string
+        target:
+          $ref: "#/components/schemas/OSReleaseDTO"
+      required:
+        - state
+      type: object
     Parse-custom-overlayRequest:
       additionalProperties: false
       properties:
@@ -2523,6 +2588,10 @@ components:
           type: string
         host_agent_version:
           type: string
+        os_slot:
+          type: string
+        os_version:
+          type: string
         ui_image:
           type: string
         version:
@@ -2595,6 +2664,8 @@ components:
           type: string
         from:
           type: string
+        os:
+          $ref: "#/components/schemas/OSUpdateDTO"
         profile:
           type: string
         running:
@@ -4109,7 +4180,7 @@ paths:
               schema:
                 $ref: "#/components/schemas/ErrorModel"
           description: Error
-      summary: "What this box is running: brain version and commit, host-agent version, UI image"
+      summary: "What this box is running: brain version and commit, host-agent version, UI image, OS version and slot"
   /api/v1/users:
     get:
       operationId: list-users