A/B OS image: a slot that hangs, and a Debian major across /etc (#486)

cmd/host-agent-real/main.go

@@ -41,6 +41,7 @@ import (
 	"github.com/onmoose/os/internal/hostagent/brainlaunch"
 	"github.com/onmoose/os/internal/hostagent/controlplane"
 	"github.com/onmoose/os/internal/hostagent/cpupdate"
+	"github.com/onmoose/os/internal/hostagent/sshaccess"
 	"github.com/onmoose/os/internal/profile"
 	"github.com/onmoose/os/internal/protocol"
 	"github.com/onmoose/os/internal/version"
@@ -80,6 +81,25 @@ func main() {
 	a, cleanup := buildAgent()
 	defer cleanup()
 
+	// After a Debian major the /etc tidy-up keeps the sshd drop-in but drops the
+	// unit's enable links (BUILD.md # 1b, rule 4), and leaves a marker. Only
+	// then is sshd turned back on for the accounts the drop-in names. On any
+	// other start host-agent leaves sshd's run state alone.
+	if _, err := os.Stat(sshaccess.MajorTidiedMarker); err == nil {
+		if sm, ok := a.SSH.(*sshaccess.Manager); ok {
+			if on, err := sm.EnsureOnAtStart(); err != nil {
+				slog.Warn("could not turn sshd on after a Debian-major tidy-up; trying again at the next start", "err", err)
+			} else {
+				if on {
+					slog.Info("sshd turned on after a Debian-major tidy-up: accounts have SSH on")
+				}
+				if err := os.Remove(sshaccess.MajorTidiedMarker); err != nil {
+					slog.Warn("could not remove the tidy-up marker", "err", err)
+				}
+			}
+		}
+	}
+
 	// The brain's launch config is built once and used twice: to launch the
 	// brain at boot, and as the base of every control-plane update. Reusing it
 	// is the point — an updated brain has to be identical to a first-boot one

dev/cloud/build-bundle.sh

@@ -26,7 +26,9 @@
 #      never trust the throwaway signer;
 #   4. a wrong key: a bundle-shaped check against an unrelated CA is refused;
 #   5. a rehearsal of dev/release/sign-bundle.sh with a throwaway "release" CA,
-#      so the sign job's script runs on every build, not only on a release.
+#      so the sign job's script runs on every build, not only on a release;
+#   6. the image's account and group ids, read from the sysusers.d file the
+#      build generates, match dev/os-lock/cloud-accounts.lock.
 #
 # Writes to OUTDIR: moose-cloud.raucb, image-etc/{system.conf,keyring.pem} and
 # bundle-info.txt, and a size table to $GITHUB_STEP_SUMMARY when set.
@@ -96,6 +98,7 @@ rauc_run "$out" '
     # The image'"'"'s own RAUC config and keyring, read back out of the slot.
     unsquashfs -cat bundle/rootfs.img etc/rauc/system.conf > image-etc/system.conf
     unsquashfs -cat bundle/rootfs.img etc/rauc/keyring.pem > image-etc/keyring.pem
+    unsquashfs -cat bundle/rootfs.img usr/lib/sysusers.d/moose-image-accounts.conf > image-etc/accounts.conf
     compatible="$(sed -n "s/^compatible=//p" image-etc/system.conf | head -n1)"
     [ -n "$compatible" ] || { echo "the slot'"'"'s /etc/rauc/system.conf names no compatible" >&2; exit 1; }
     grep -qx "check-purpose=codesign" image-etc/system.conf || { echo "the slot'"'"'s system.conf does not ask for check-purpose=codesign" >&2; exit 1; }
@@ -145,6 +148,23 @@ echo "check 1: the slot carries the keyring this build staged (${mode})"
 cmp -s "$out/image-etc/keyring.pem" "$staged" || { echo "the slot's /etc/rauc/keyring.pem is not the ${mode} keyring this build staged" >&2; exit 1; }
 echo "ok: $(openssl x509 -in "$out/image-etc/keyring.pem" -noout -subject)"
 
+echo "check 6: the image's account ids match dev/os-lock/cloud-accounts.lock"
+# A box keeps an account's uid and gid for life: its /etc/passwd is in the
+# /etc upper layer, and its files on the state partition are owned by number.
+# So an image whose packages gave an account another id would leave a box's
+# files owned by the wrong account (BUILD.md # 1b, rule 3). The lock is edited
+# by hand: a new account is added, a changed id is a bug to fix in the build.
+awk '$1 == "g" { print "g", $2, $3 } $1 == "u" { split($3, id, ":"); print "u", $2, id[1], id[2] }' \
+    "$out/image-etc/accounts.conf" | LC_ALL=C sort > "$out/image-etc/cloud-accounts.lock"
+lock="${REPO_ROOT}/dev/os-lock/cloud-accounts.lock"
+if ! diff -u <(grep -v '^#' "$lock" 2>/dev/null | LC_ALL=C sort) "$out/image-etc/cloud-accounts.lock" > "$out/accounts.diff"; then
+    echo "the image's accounts differ from dev/os-lock/cloud-accounts.lock ('-' the lock, '+' the image):" >&2
+    cat "$out/accounts.diff" >&2
+    echo "A new account: add its line to the lock. A changed id: the image must keep the old id, because every box keeps it." >&2
+    exit 1
+fi
+echo "ok: $(grep -c . "$out/image-etc/cloud-accounts.lock") accounts and groups, ids as locked"
+
 echo "check 5: rehearse the sign job's signing with a throwaway release CA"
 mkdir -p "$out/rehearsal/image-etc"
 sed 's/^path=.*/path=keyring.pem/' "$out/image-etc/system.conf" > "$out/rehearsal/image-etc/system.conf"

dev/cloud/cloud-assertions.sh

@@ -151,7 +151,57 @@ fail() {
     # serial diag, then kill it and keep the run artifacts.
     exit 1
 }
+# Every file in the /etc upper layer must be one a Debian-major tidy-up knows
+# (BUILD.md # 1b, rule 4): on the keep list (/usr/lib/moose/etc-keep.list, and
+# this lane's own list in etc-keep.d/), an account file, a pinned file, or a
+# link sshd's run state makes, which host-agent makes again at start. A file
+# none of these covers would be lost at a major without anyone choosing that,
+# so it fails the boot here first.
+etc_upper_check() {
+    local up=/state/etc/upper pats p pat hit bad="" f
+    pats="$(cat /usr/lib/moose/etc-keep.list /usr/lib/moose/etc-keep.d/*.list 2>/dev/null | sed -e 's/#.*//' -e 's/[[:space:]]*$//' -e '/^$/d')"
+    pats="$pats
+passwd
+group
+shadow
+gshadow
+passwd-
+group-
+shadow-
+gshadow-
+subuid-
+subgid-
+.pwd.lock
+docker/daemon.json
+login.defs
+systemd/system/multi-user.target.wants/ssh.service
+systemd/system/sshd.service
+rc[0-6S].d/[SK][0-9][0-9]ssh"
+    while IFS= read -r p; do
+        [ -n "$p" ] || continue
+        hit=""
+        while IFS= read -r pat; do
+            # shellcheck disable=SC2053
+            if [[ $p == $pat || $p == $pat/* ]]; then hit=1; break; fi
+        done <<<"$pats"
+        [ -n "$hit" ] || bad="$bad $p"
+    done < <(cd "$up" && find . -mindepth 1 ! -type d -printf '%P\n' 2>/dev/null)
+    [ -z "$bad" ] || fail "the /etc upper layer holds files no tidy-up rule covers (add them to /usr/lib/moose/etc-keep.list or give them a rule):$bad"
+    # Every account and group the image made is on the box, with the image's
+    # id (BUILD.md # 1b, rule 3; systemd-sysusers adds a missing one at boot).
+    f=/usr/lib/sysusers.d/moose-image-accounts.conf
+    [ -s "$f" ] || fail "no $f in the slot"
+    while read -r kind name id _; do
+        case "$kind" in
+        u) [ "$(getent passwd "$name" | cut -d: -f3)" = "${id%%:*}" ] || bad="$bad user:$name(${id%%:*}/$(getent passwd "$name" | cut -d: -f3))" ;;
+        g) [ "$(getent group "$name" | cut -d: -f3)" = "$id" ] || bad="$bad group:$name($id/$(getent group "$name" | cut -d: -f3))" ;;
+        esac
+    done < "$f"
+    [ -z "$bad" ] || fail "accounts the image made are missing on the box or have another id (image/box):$bad"
+    echo "cloud-assertions: /etc upper layer: $(cd "$up" && find . -mindepth 1 ! -type d | wc -l) files, all covered by a tidy-up rule; the image's $(grep -c '^[ug] ' "$f") accounts and groups are on the box with their ids"
+}
 ok() {
+    etc_upper_check
     # Last gate, every boot (#561): the slot is read-only, so anything that
     # still writes to it fails. Catch it whether it failed a unit or only
     # logged. Container logs are left out: an app's own read-only filesystem
@@ -337,6 +387,21 @@ grep -qx "${BOOTED}_OK=1" <<<"$grubenv_now" || layout_fail "grubenv does not mar
 for u in emergency.service rescue.service; do
     systemctl cat "$u" 2>/dev/null | grep -q 'systemctl --no-block reboot' || layout_fail "$u has no reboot drop-in"
 done
+# A slot that hangs (#486 point 3, BUILD.md # 1b # As built): systemd feeds a
+# hardware watchdog, so a hung PID 1 or kernel resets the box. This lane is
+# QEMU q35, like a Hetzner Cloud VM, so it has the ICH9 TCO watchdog
+# (iTCO_wdt) that a real box has.
+wd_dev="$(systemctl show -p WatchdogDevice --value)"
+wd_sec="$(systemctl show -p RuntimeWatchdogUSec --value)"
+[ -n "$wd_dev" ] && [ -e /sys/class/watchdog/watchdog0 ] \
+    || layout_fail "no hardware watchdog in use (WatchdogDevice='$wd_dev', /sys/class/watchdog: $(ls /sys/class/watchdog 2>/dev/null | tr '\n' ' '))"
+[ "$wd_sec" = 1min ] || layout_fail "RuntimeWatchdogUSec is '$wd_sec', want 1min"
+# systemd logs this before journald runs, so it is in the kernel log (the
+# probe of run 37358065628 found it there and not under _PID=1).
+wd_log="$(dmesg 2>/dev/null; journalctl -k -b --no-pager -o cat 2>/dev/null; journalctl -b _PID=1 --no-pager -o cat 2>/dev/null)"
+grep -q 'Using hardware watchdog' <<<"$wd_log" \
+    || layout_fail "systemd does not feed the hardware watchdog (no 'Using hardware watchdog' in the boot's log)"
+echo "cloud-assertions: layout: systemd feeds the hardware watchdog $wd_dev ($(cat /sys/class/watchdog/watchdog0/identity 2>/dev/null)) with a $wd_sec timeout"
 dmesg 2>/dev/null | grep -q 'moose-state: bind mounts done' || journalctl -k -b --no-pager 2>/dev/null | grep -q 'moose-state: bind mounts done' \
     || layout_fail "no 'moose-state: bind mounts done' in the kernel log"
 # state-setup looked for the state partition on the boot disk only.
@@ -2419,6 +2484,7 @@ os-update|os-revert)
         echo "hostkey=$(ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub 2>/dev/null | awk '{print $2}')"
         echo "machineid=$(cat /etc/machine-id)"
         echo "data=$(cat "/home/${owner}/os-update-data.txt" 2>/dev/null)"
+        echo "groups=$(id -nG "$owner" 2>/dev/null | tr ' ' '\n' | sort | tr '\n' ' ')"
     }
 
     # 4. os-revert, last stage: slot B died before userspace (#575). The
@@ -2447,6 +2513,23 @@ os-update|os-revert)
         [ -n "$session_cookie" ] || fail "$MODE: no moose_session cookie from the SSO landing"
         owner="$(json_str_of "$(full_get /api/v1/me "$apex" "$session_cookie" 2>/dev/null || true)" username)"
         [ -n "$owner" ] && id -u "$owner" >/dev/null 2>&1 || fail "$MODE: the SSO owner '$owner' has no host account"
+        if [ "$MODE" = os-update ]; then
+            # SSH on for the owner, so the faked major below must bring sshd
+            # back on slot B (BUILD.md # 1b, rule 4: the tidy-up keeps the
+            # drop-in, drops the enable links, and host-agent enables sshd once).
+            # A known password first, for the elevation gate, as the ssh boot
+            # does (harness setup, not a product path).
+            printf '%s:%s\n' "$owner" 'moose-cloud-lane-owner-pw' | chpasswd || fail "$MODE: could not set a known password for '$owner'"
+            el="$(full_send POST /api/v1/auth/elevate "$apex" "$session_cookie" '{"password":"moose-cloud-lane-owner-pw"}' 2>/dev/null)"
+            grep -q ' 200' <<<"$(status_of "$el")" || fail "$MODE: elevate as the owner failed: status='$(status_of "$el")'"
+            rm -f /run/moose-os-update-key /run/moose-os-update-key.pub
+            ssh-keygen -t ed25519 -N '' -C 'moose-os-update' -f /run/moose-os-update-key >/dev/null 2>&1 || fail "$MODE: ssh-keygen failed"
+            on="$(full_send PUT /api/v1/me/ssh "$apex" "$session_cookie" \
+                "{\"enabled\":true,\"keys\":[{\"public_key\":\"$(tr -d '\n' < /run/moose-os-update-key.pub)\",\"label\":\"os-update\"}]}" 2>/dev/null)"
+            grep -q ' 200' <<<"$(status_of "$on")" || fail "$MODE: turning SSH on for the owner failed: status='$(status_of "$on")'"
+            [ "$(systemctl is-enabled ssh.service 2>/dev/null)" = enabled ] || fail "$MODE: ssh.service is not enabled after the owner turned SSH on"
+            echo "cloud-assertions: $MODE: SSH on for '$owner' (ssh.service enabled) before the faked major"
+        fi
         head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n' > "/home/${owner}/os-update-data.txt"
         chown "$owner" "/home/${owner}/os-update-data.txt"
         mkdir -p "$OS_STATE_DIR" && chmod 700 "$OS_STATE_DIR"
@@ -2546,6 +2629,16 @@ UNIT
         # 1c. THE SWITCH. An open window: host-agent puts slot B first and
         #     reboots. The next stage runs on slot B.
         boot_gate
+        if [ "$MODE" = os-update ]; then
+            # A faked Debian major (BUILD.md # 1b, rule 4): the state partition
+            # says the box last ran Debian 12, so slot B must tidy the /etc
+            # upper layer before it mounts it. An admin's own edit goes to the
+            # attic; the owner, the password hash, the host keys, machine-id and
+            # the sudo membership must come through (stage 2 checks them).
+            echo 12 > /state/etc/.moose-debian-major
+            echo "# an admin's own edit (os-update boot)" > /etc/moose-test-hand-edit.conf
+            sync
+        fi
         date +%s > "$OS_STATE_DIR/switch-at"
         set_os_stage 2
         write_os_target "$os_sum" "00:00-23:59"
@@ -2567,6 +2660,28 @@ UNIT
         [ "$(/usr/lib/moose/host-agent-real --version | awk '{print $2}')" = "$os_ver" ] || fail "os-update: slot B's host-agent is not $os_ver"
         want_outcome=good; want_state=current; want_ver="$os_ver"
         t_good="$(ha_log | grep 'the new slot is healthy and marked good' | tail -1 | awk '{print $1}')"
+        slot_major="$( . /usr/lib/os-release && echo "${VERSION_ID%%.*}" )"
+        [ "$(cat /state/etc/.moose-debian-major 2>/dev/null)" = "$slot_major" ] \
+            || fail "os-update: the state partition records Debian '$(cat /state/etc/.moose-debian-major 2>/dev/null)', want $slot_major after the tidy-up"
+        # dmesg first, as the layout checks read the moose-state lines: the
+        # journal's kernel log was empty here once (run 37374562648).
+        tidy_log="$(dmesg 2>/dev/null; journalctl -k -b --no-pager -o cat 2>/dev/null)"
+        grep -q "moose-state: tidied /etc for Debian 12 to $slot_major" <<<"$tidy_log" \
+            || fail "os-update: slot B did not tidy the /etc upper layer for the faked major: $(grep 'moose-state' <<<"$tidy_log" | tail -5 | tr '\n' ' ')"
+        [ ! -e /etc/moose-test-hand-edit.conf ] || fail "os-update: the admin's own /etc edit survived the faked major"
+        attic_edit="$(ls -d /state/etc/attic/*-debian-12-to-"$slot_major"/moose-test-hand-edit.conf 2>/dev/null | head -1)"
+        [ -n "$attic_edit" ] || fail "os-update: the admin's own /etc edit is not in the attic: $(ls /state/etc/attic 2>&1 | tr '\n' ' ')"
+        # sshd comes back: the drop-in was kept, its enable links were not, and
+        # host-agent enabled the unit once because of the tidy-up marker.
+        grep -qE "^AllowUsers .*\b${owner}\b" /etc/ssh/sshd_config.d/moose-allowed.conf 2>/dev/null \
+            || fail "os-update: the sshd drop-in does not name '$owner' after the faked major: $(cat /etc/ssh/sshd_config.d/moose-allowed.conf 2>&1 | tr '\n' ' ')"
+        for _i in $(seq 1 60); do [ "$(systemctl is-enabled ssh.service 2>/dev/null)" = enabled ] && break; sleep 1; done
+        [ "$(systemctl is-enabled ssh.service 2>/dev/null)" = enabled ] \
+            || fail "os-update: ssh.service is '$(systemctl is-enabled ssh.service 2>&1)' after the faked major, want enabled: $(ha_log | grep -i 'sshd\|tidy' | tail -3 | tr '\n' ' ')"
+        ha_log | grep -q "sshd turned on after a Debian-major tidy-up" || fail "os-update: host-agent did not log turning sshd on after the tidy-up"
+        [ ! -e /state/etc/.moose-major-tidied ] || fail "os-update: host-agent did not remove the tidy-up marker"
+        echo "cloud-assertions: os-update: SSHD BACK OK (drop-in names '$owner', ssh.service enabled again by host-agent after the tidy-up, marker removed)"
+        echo "cloud-assertions: os-update: MAJOR TIDY OK (faked Debian 12 to $slot_major: the upper layer was rebuilt, the admin's edit is in $(dirname "$attic_edit"), $(grep -o 'kept [0-9]* files' <<<"$tidy_log" | tail -1))"
         echo "cloud-assertions: os-update: SWITCH OK (booted slot B, marked good, grubenv ORDER='B A' B_OK=1 B_TRY=0); measured: switch to marked good $(awk -v a="$(cat "$OS_STATE_DIR/switch-at")" -v b="$t_good" 'BEGIN{printf "%.0f", b-a}') s, the reboot included"
         note_pat="updated its system to $os_ver"
     else

dev/cloud/mkosi.extra/etc/systemd/system.conf.d/10-moose-watchdog.conf

@@ -1,6 +1,7 @@
-# A slot that hangs must still revert (UPDATES.md # 1): where the machine has a
-# hardware watchdog, systemd feeds it, and a hung PID 1 gets a reset. On a
-# machine with none (the QEMU lane; Hetzner is not known yet, NEXT.md # A/B OS
-# image point 3), systemd logs that and carries on.
+# A slot that hangs must still revert (UPDATES.md # 1): systemd feeds the
+# hardware watchdog, and a hung PID 1 or kernel gets a reset. Hetzner Cloud VMs
+# (QEMU q35) have the ICH9 TCO watchdog, which the generic kernel's iTCO_wdt
+# drives; the reset comes about twice this value after the last feed
+# (BUILD.md # 1b # As built). The boot lane checks systemd uses it.
 [Manager]
 RuntimeWatchdogSec=60s

dev/cloud/mkosi.extra/usr/lib/moose/etc-keep.list

@@ -0,0 +1,26 @@
+# The files in the /etc upper layer that a Debian-major tidy-up keeps as they
+# are (BUILD.md # 1b, rule 4; state-setup step 3a). One shell glob per line,
+# relative to /etc; a directory keeps everything under it. Everything else in
+# the upper layer moves to the attic at a major, except the account files
+# (merged) and the pinned daemon.json and login.defs (taken again from the slot
+# when that keeps the box's remap). The boot lane fails a boot whose upper
+# layer holds a file this list, the account files, the pinned files and the
+# links sshd's run state makes do not cover (cloud-assertions.sh), so a new
+# file moose writes into /etc must be added here, or given a regeneration rule.
+#
+# The box's identity.
+machine-id
+ssh/ssh_host_*
+# What users set: SSH access (the drop-in is the enabled set) and keys, and
+# the time zone.
+ssh/sshd_config.d/moose-allowed.conf
+ssh/moose-authorized-keys
+localtime
+# The box's remap range, pinned at first boot (rule 2).
+subuid
+subgid
+# Appliance only (#564): the LUKS recovery key, the data-drive marker and the
+# network connections, WiFi included.
+moose/secrets
+moose/data-drive.enrolled
+NetworkManager/system-connections