diff --git a/.env.example b/.env.example index 1f4e8290..35d5da4b 100644 --- a/.env.example +++ b/.env.example @@ -16,7 +16,7 @@ DATABASE_URL="postgresql://exam:exam@localhost:15432/exam" # Redis (optional) — leave unset or empty to disable Redis. # Production (Docker Compose): the redis service REQUIRES REDIS_PASSWORD # (no default; Compose fails to expand without it) and runs with -# `requirepass` (P7 review P1-1). When enabling the optional redis profile, +# `requirepass`. When enabling the optional redis profile, # set REDIS_PASSWORD AND an AUTHENTICATED REDIS_URL: # REDIS_PASSWORD="" # REDIS_URL="redis://:@redis:6379" @@ -140,6 +140,27 @@ POSTGRES_USER=exam # Production: replace with a strong generated password. Required. POSTGRES_PASSWORD=exam POSTGRES_DB=exam + +# Launchpad first-install setup token. Unset/empty +# DISABLES the browser first-Admin setup form (POST /api/launchpad/bootstrap +# returns 403). Set it in .env BEFORE the first `docker compose up` to use +# the browser first-install page; generate with high entropy: +# LAUNCHPAD_SETUP_TOKEN="$(openssl rand -hex 32)" +# Body-only (never in a URL); never audit-logged in plaintext; rate-limited. +# Once the installation is initialized, /launchpad redirects to /login and +# never reopens — removing the last Admin does NOT reopen it (Admin recovery +# is operator CLI: bootstrap-admin / reset-admin-password). +# LAUNCHPAD_SETUP_TOKEN="" + +# WAL archive host path for PostgreSQL PITR. The +# mount is ALWAYS present on the db service but is INERT by default +# (archive_mode=off). To enable point-in-time recovery, run +# scripts/backup/postgres-enable-pitr.sh (ALTER SYSTEM). Production MUST +# point this at an INDEPENDENT failure domain (NAS / another server / a +# separate disk); the local default is for development/drills only and is +# NOT host-loss protection. +# EXAM_WAL_ARCHIVE_HOST_PATH="./data/wal-archive" + # JWT_SECRET — required in production (no default in the bundled Compose). # Generate a secure random string: openssl rand -base64 32 # JWT_SECRET="" @@ -151,7 +172,7 @@ POSTGRES_DB=exam # Set only when enabling the optional `redis` Compose profile # (`docker compose --profile redis up`). Production requires BOTH # REDIS_PASSWORD (required-expansion; the redis service runs with -# `requirepass`) and an AUTHENTICATED REDIS_URL (P7 review P1-1): +# `requirepass`) and an AUTHENTICATED REDIS_URL: # REDIS_PASSWORD="" # REDIS_URL="redis://:@redis:6379" # REDIS_URL="redis://:@redis:6379" diff --git a/.gitignore b/.gitignore index 686abb73..0bc1ee71 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ mutation-campaign-results/ # Unattended task execution evidence (baseline/test logs, morning reports) .artifacts/ /mvp/ +# P7-C1: operator-visible host persistence root for the production Compose +# topology (bind-mounted postgres/redis state). Never commit runtime data. +/data/ diff --git a/README.md b/README.md index 34d4aa28..a518a91b 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,12 @@ cp .env.example .env docker compose up -d --build # build + start app, db, email-worker docker compose logs -f app docker compose ps # verify app, db, email-worker are up -docker compose down -docker compose down -v # DANGEROUS: removes database data volumes +docker compose down # stops + removes containers (keeps ./data) +# NOTE: authoritative state lives in the operator-visible host bind +# mount ./data/postgres (EXAM_DATA_ROOT, default ./data). `docker compose down` +# retains it; `docker compose down -v` is a no-op for bind mounts. To destroy +# authoritative data you must explicitly delete ./data/postgres. See +# docs/deployment/backup-and-recovery.md. # (Optional) enable the Redis profile — the shared rate limiter reads/writes # Redis when the runtime is ready: @@ -207,6 +211,29 @@ The bootstrap: (1) locates or creates the internal default organization (4) writes an `admin.bootstrap` audit row. It refuses a second active Admin unless `--force` is supplied. It does NOT create Candidate accounts. +Alternatively, on a fresh installation you can use the **Launchpad** +first-install page: set `LAUNCHPAD_SETUP_TOKEN=` in +`.env`, start the stack, and navigate to `/launchpad` to complete the +first-Admin setup in the browser. The Launchpad and the CLI share one +canonical atomic mutation body (serialized by a transaction-scoped +PostgreSQL advisory lock so exactly one first installation may win); once +the installation is initialized, `/launchpad` redirects to `/login` (it +never reopens). See +[`docs/deployment/backup-and-recovery.md`](docs/deployment/backup-and-recovery.md) §11. + +#### Backup and recovery + +Authoritative state is the PostgreSQL data directory under +`./data/postgres`. **Host persistence is not backup** — see +[`docs/deployment/backup-and-recovery.md`](docs/deployment/backup-and-recovery.md) +for the full decision tree. There is exactly ONE production/operator Docker +Compose entry point (`docker-compose.yml`); optional capabilities such as +PITR are PostgreSQL database configuration +(`scripts/backup/postgres-enable-pitr.sh`), not an alternate Docker +topology. The supported paths are: stopped-directory relocation (C1), +cold-filesystem backup/restore (C1), C2 logical `pg_dump` online backup + +clean restore, and C3 physical `pg_basebackup` + WAL archive / PITR. + ## Docker Files Reference | File | Purpose | diff --git a/apps/api/openapi.json b/apps/api/openapi.json index af9269e7..e42a6178 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -671,6 +671,173 @@ } } }, + "/api/launchpad/status": { + "get": { + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "initialized": { + "type": "boolean" + } + }, + "required": ["initialized"], + "additionalProperties": false + } + } + } + } + } + } + }, + "/api/launchpad/bootstrap": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "organizationName": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "organizationDisplayName": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "adminUsername": { + "type": "string", + "minLength": 3, + "maxLength": 50 + }, + "adminPassword": { + "type": "string", + "minLength": 8, + "maxLength": 100 + }, + "adminName": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "setupToken": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + }, + "required": [ + "organizationName", + "adminUsername", + "adminPassword", + "adminName", + "setupToken" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean", + "enum": [true] + }, + "organizationSlug": { + "type": "string" + }, + "adminUsername": { + "type": "string" + } + }, + "required": ["ok", "organizationSlug", "adminUsername"], + "additionalProperties": false + } + } + } + }, + "403": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "details": {}, + "requestId": { + "type": "string", + "minLength": 1 + } + }, + "required": ["code", "message", "requestId"], + "additionalProperties": false + } + }, + "required": ["error"], + "additionalProperties": false + } + } + } + }, + "409": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "details": {}, + "requestId": { + "type": "string", + "minLength": 1 + } + }, + "required": ["code", "message", "requestId"], + "additionalProperties": false + } + }, + "required": ["error"], + "additionalProperties": false + } + } + } + } + } + } + }, "/api/settings/branding": { "get": { "parameters": [ diff --git a/apps/api/src/audit/auditPolicy.ts b/apps/api/src/audit/auditPolicy.ts index 0321c8dd..111434aa 100644 --- a/apps/api/src/audit/auditPolicy.ts +++ b/apps/api/src/audit/auditPolicy.ts @@ -126,7 +126,10 @@ export const AUDIT_ACTION_DEFINITIONS = { .object({ username: z.string().max(50), name: z.string().max(100), - source: z.literal("local_script"), + // The bootstrap adapter that invoked the canonical mutation: the + // operator CLI (local_script) or the HTTP Launchpad first-install + // adapter (launchpad). + source: z.enum(["local_script", "launchpad"]), }) .strict(), ), diff --git a/apps/api/src/authz/routeRegistryConformanceWholeApp.test.ts b/apps/api/src/authz/routeRegistryConformanceWholeApp.test.ts index 67b76ffd..7de4e9f4 100644 --- a/apps/api/src/authz/routeRegistryConformanceWholeApp.test.ts +++ b/apps/api/src/authz/routeRegistryConformanceWholeApp.test.ts @@ -325,8 +325,14 @@ describe("P4-C1 whole-application authorization route regression lock", () => { ["GET", "/api/settings/branding"], ["GET", "/api/system/info"], ["GET", "/api/system/public-config"], + // P7-C1 Launchpad: initial-installation-only public routes. GET status + // reveals only "is the default org initialized" (login UX already + // implies it); POST bootstrap refuses once initialized, so neither is + // a token oracle nor a completed-installation oracle. + ["GET", "/api/launchpad/status"], + ["POST", "/api/launchpad/bootstrap"], ]; - return set.some(([m, u]) => m === method && url === u); + return set.some(([m, u]) => m === method && u === url); } it("the authenticate-only + public route set is exactly the documented closed set (no drift)", () => { @@ -347,7 +353,7 @@ describe("P4-C1 whole-application authorization route regression lock", () => { ).toEqual([]); }); - it("the full composition reconciles to 113 primary routes (99 protected + 14 non-protected)", () => { + it("the full composition reconciles to 115 primary routes (99 protected + 16 non-protected)", () => { const protectedCount = capturedRoutes.filter( (r) => categorize(r) === "protected", ).length; @@ -362,7 +368,9 @@ describe("P4-C1 whole-application authorization route regression lock", () => { // Admin Recovery Center read routes (queue + aggregate detail + attempt // operations context) → 112 primary = 98 protected + 14 non-protected. // J5-I1B4 adds the Exam Recovery Context read route → 113 primary = 99 - // protected + 14 non-protected. This is a regression anchor, not a + // protected + 14 non-protected. P7-C1 adds 2 public Launchpad routes + // (status + bootstrap) → 115 primary = 99 protected + 16 non-protected. + // This is a regression anchor, not a // hard-coded PASS: if a route is added/removed the counts move and the // failure message names the delta so the regression is triaged, not // silently swallowed. @@ -371,9 +379,9 @@ describe("P4-C1 whole-application authorization route regression lock", () => { "protected (capability/ownership-gated) routes", ).toBe(99); expect(nonProtectedCount, "non-protected (auth-only + public) routes").toBe( - 14, + 16, ); - expect(capturedRoutes.length, "total primary routes").toBe(113); + expect(capturedRoutes.length, "total primary routes").toBe(115); }); it("every protected route's capability gate carries a valid catalog permission (no ad-hoc permission strings)", () => { diff --git a/apps/api/src/config/runtimeConfig.ts b/apps/api/src/config/runtimeConfig.ts index 0fdb3fe9..4dd35247 100644 --- a/apps/api/src/config/runtimeConfig.ts +++ b/apps/api/src/config/runtimeConfig.ts @@ -196,6 +196,27 @@ export interface EmailWorkerConfig { concurrency: number; } +/** + * Launchpad first-install configuration (P7-C1). + * + * The setup token is a deployment bootstrap secret: high entropy, body-only + * (never in a URL), never audit-logged in plaintext, and required for the + * initial first-Admin setup via `/api/launchpad/bootstrap`. An unset/empty + * value means launchpad is refused (no token-validation oracle). The token + * is read from the `LAUNCHPAD_SETUP_TOKEN` env var and is intentionally + * NOT fail-fast at boot: an unset token simply disables launchpad, so a + * bare `docker compose up` without launchpad configured starts normally. + */ +export interface LaunchpadConfig { + /** + * The configured setup token, or an empty string when not configured. + * Comparison against a request token MUST be constant-time and MUST be + * preceded by the installation-initialized check so a completed + * installation cannot become a token-validity oracle. + */ + setupToken: string; +} + export interface AppRuntimeConfig { app: { mode: AppMode; @@ -221,6 +242,7 @@ export interface AppRuntimeConfig { email: EmailConfig; emailWorker: EmailWorkerConfig; publicWebOrigin: PublicWebOriginConfig; + launchpad: LaunchpadConfig; } const DEFAULT_JWT_SECRET = "development-only-change-me"; @@ -872,6 +894,13 @@ export function loadRuntimeConfig( email, emailWorker: resolveEmailWorkerConfig(env, email), publicWebOrigin: { origin: resolvePublicWebOrigin(env, mode) }, + launchpad: { + // P7-C1: unset/empty LAUNCHPAD_SETUP_TOKEN disables launchpad (the + // bootstrap endpoint refuses). NOT fail-fast — a bare `docker compose + // up` without launchpad configured must start normally. Trimmed to + // treat a whitespace-only value as unset. + setupToken: (env.LAUNCHPAD_SETUP_TOKEN ?? "").trim(), + }, }; } diff --git a/apps/api/src/routes/launchpad.test.ts b/apps/api/src/routes/launchpad.test.ts new file mode 100644 index 00000000..dd92ca13 --- /dev/null +++ b/apps/api/src/routes/launchpad.test.ts @@ -0,0 +1,316 @@ +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; +import { eq, sql } from "drizzle-orm"; +import launchpadRoutes from "./launchpad.js"; +import { buildTestApp, type TestContext } from "./testHelpers.js"; +import { resetRuntimeConfigForTest } from "../config/runtimeConfig.js"; +import { schema } from "@exam/db/src/schema/pg.js"; + +const VALID_TOKEN = "test-launchpad-setup-token-DO-NOT-USE-IN-PROD"; + +function basePayload(overrides: Record = {}) { + return { + organizationName: "Launchpad Test Org", + adminUsername: `lpadmin-${crypto.randomUUID().slice(0, 8)}`, + adminPassword: "Launchpad-Admin-123!", + adminName: "Launchpad Admin", + setupToken: VALID_TOKEN, + ...overrides, + }; +} + +/** + * Reset the worker test DB to an "uninitialized" state (no default + * organization, no users, no audit rows) by truncating every public table + * except the drizzle migration-metadata tables. Mirrors the + * `truncateBusinessTables` helper but runs through the Drizzle handle the + * TestContext exposes (so we do not need the raw `postgres` Sql client). + */ +async function resetToUninitialized(ctx: TestContext): Promise { + await ctx.db.execute( + sql.raw(` + TRUNCATE + organizations, + organization_settings, + candidate_fields, + users, + candidate_profiles, + user_role_assignments, + courses, + questions, + exams, + exam_enrollments, + exam_attempts, + attempt_grading_entries, + notifications, + email_outbox, + worker_heartbeats, + attempt_interruptions, + attempt_interruption_events, + attempt_time_adjustments, + exam_incidents, + exam_incident_events, + exam_incident_actions, + exam_incident_attempts, + exam_incident_interruption_links, + exam_proctor_assignments, + exam_proctor_assignment_events, + attempt_command_receipts, + audit_logs, + client_events, + import_job_logs + RESTART IDENTITY CASCADE + `), + ); +} + +describe("launchpad routes", () => { + let ctx: TestContext; + + beforeAll(async () => { + ctx = await buildTestApp(launchpadRoutes, { prefix: "/api" }); + }); + + afterAll(async () => { + // If a test left the DB in the uninitialized state, restore the seeded + // default org so subsequent test files in the same worker see the + // expected seeded state. best-effort; cleanup() closes the app. + try { + const rows = await ctx.db + .select({ id: schema.organizations.id }) + .from(schema.organizations) + .where(eq(schema.organizations.slug, "default")) + .limit(1); + if (rows.length === 0) { + const now = new Date(); + await ctx.db.insert(schema.organizations).values({ + id: crypto.randomUUID(), + name: "Default Organization", + displayName: "Default Organization", + slug: "default", + createdAt: now, + updatedAt: now, + }); + } + } catch { + // ignore — cleanup is best-effort + } + await ctx.cleanup(); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + resetRuntimeConfigForTest(); + }); + + // ── GET /api/launchpad/status — initialized (default seeded org exists) ── + it("GET /api/launchpad/status returns initialized=true when default org exists", async () => { + const res = await ctx.app.inject({ + method: "GET", + url: "/api/launchpad/status", + }); + expect(res.statusCode).toBe(200); + expect(res.json()).toEqual({ initialized: true }); + }); + + // ── POST /api/launchpad/bootstrap — refuses when initialized ───────────── + it("POST /api/launchpad/bootstrap returns 409 when already initialized (no token oracle)", async () => { + // Configure a token so the only reason for refusal is the init gate. + vi.stubEnv("LAUNCHPAD_SETUP_TOKEN", VALID_TOKEN); + resetRuntimeConfigForTest(); + + const res = await ctx.app.inject({ + method: "POST", + url: "/api/launchpad/bootstrap", + payload: basePayload({ setupToken: VALID_TOKEN }), + }); + expect(res.statusCode).toBe(409); + expect(res.json().error.code).toBe("LAUNCHPAD_ALREADY_INITIALIZED"); + }); + + it("POST /api/launchpad/bootstrap returns 409 with the WRONG token too (init gate first, no oracle)", async () => { + // The installation-initialized check MUST run before token validation, + // so an initialized installation returns 409 regardless of token + // validity — it never reveals whether the token is correct. + vi.stubEnv("LAUNCHPAD_SETUP_TOKEN", VALID_TOKEN); + resetRuntimeConfigForTest(); + + const res = await ctx.app.inject({ + method: "POST", + url: "/api/launchpad/bootstrap", + payload: basePayload({ setupToken: "wrong-token" }), + }); + expect(res.statusCode).toBe(409); + expect(res.json().error.code).toBe("LAUNCHPAD_ALREADY_INITIALIZED"); + }); + + // ── POST /api/launchpad/bootstrap — token disabled when unset ──────────── + it("POST /api/launchpad/bootstrap returns 403 when setup token is unset (launchpad disabled)", async () => { + vi.stubEnv("LAUNCHPAD_SETUP_TOKEN", ""); + resetRuntimeConfigForTest(); + + // Use an uninitialized database state so the init gate does NOT fire + // first; this isolates the token-disabled behavior. + await resetToUninitialized(ctx); + + const res = await ctx.app.inject({ + method: "POST", + url: "/api/launchpad/bootstrap", + payload: basePayload({ setupToken: "anything" }), + }); + expect(res.statusCode).toBe(403); + expect(res.json().error.code).toBe("LAUNCHPAD_INVALID_SETUP_TOKEN"); + }); + + it("POST /api/launchpad/bootstrap returns 403 when token mismatches (uninitialized)", async () => { + vi.stubEnv("LAUNCHPAD_SETUP_TOKEN", VALID_TOKEN); + resetRuntimeConfigForTest(); + + // Uninitialized state so the init gate does not fire first. + await resetToUninitialized(ctx); + + const res = await ctx.app.inject({ + method: "POST", + url: "/api/launchpad/bootstrap", + payload: basePayload({ setupToken: "wrong-token" }), + }); + expect(res.statusCode).toBe(403); + expect(res.json().error.code).toBe("LAUNCHPAD_INVALID_SETUP_TOKEN"); + }); + + // ── POST /api/launchpad/bootstrap — success on uninitialized + valid token + it("POST /api/launchpad/bootstrap creates first Admin on uninitialized install with valid token", async () => { + vi.stubEnv("LAUNCHPAD_SETUP_TOKEN", VALID_TOKEN); + resetRuntimeConfigForTest(); + + // Uninitialized state. + await resetToUninitialized(ctx); + const statusBefore = await ctx.app.inject({ + method: "GET", + url: "/api/launchpad/status", + }); + expect(statusBefore.json()).toEqual({ initialized: false }); + + const username = `lpok-${crypto.randomUUID().slice(0, 8)}`; + const res = await ctx.app.inject({ + method: "POST", + url: "/api/launchpad/bootstrap", + payload: basePayload({ + adminUsername: username, + organizationName: "Fresh Install Org", + organizationDisplayName: "Fresh Install Display", + }), + }); + expect(res.statusCode).toBe(200); + const body = res.json(); + expect(body).toEqual({ + ok: true, + organizationSlug: "default", + adminUsername: username, + }); + + // The canonical mutation body landed: default org + one active Admin + + // admin.bootstrap audit row, atomically. + const orgRows = await ctx.db + .select() + .from(schema.organizations) + .where(eq(schema.organizations.slug, "default")); + expect(orgRows).toHaveLength(1); + expect(orgRows[0]!.name).toBe("Fresh Install Org"); + expect(orgRows[0]!.displayName).toBe("Fresh Install Display"); + + const adminRows = await ctx.db + .select() + .from(schema.users) + .where(eq(schema.users.username, username)); + expect(adminRows).toHaveLength(1); + expect(adminRows[0]!.role).toBe("Admin"); + expect(adminRows[0]!.isActive).toBe(true); + + const assignmentRows = await ctx.db + .select() + .from(schema.userRoleAssignments) + .where(eq(schema.userRoleAssignments.userId, adminRows[0]!.id)); + expect( + assignmentRows.some( + (a) => a.role === "Admin" && a.isPrimary && a.isActive, + ), + ).toBe(true); + + const auditRows = await ctx.db + .select() + .from(schema.auditLogs) + .where(eq(schema.auditLogs.action, "admin.bootstrap")); + expect(auditRows.length).toBeGreaterThanOrEqual(1); + // The HTTP adapter's canonical mutation records the launchpad source. + const auditMetadata = auditRows[0]!.metadata as Record; + expect(auditMetadata.source).toBe("launchpad"); + // Token must NEVER appear in the audit row. + const auditJson = JSON.stringify(auditRows); + expect(auditJson).not.toContain(VALID_TOKEN); + + // After bootstrap, status flips to initialized and a second bootstrap + // is refused — even with the correct token (no oracle). + const statusAfter = await ctx.app.inject({ + method: "GET", + url: "/api/launchpad/status", + }); + expect(statusAfter.json()).toEqual({ initialized: true }); + + const second = await ctx.app.inject({ + method: "POST", + url: "/api/launchpad/bootstrap", + payload: basePayload({ + adminUsername: `lpsecond-${crypto.randomUUID().slice(0, 8)}`, + setupToken: VALID_TOKEN, + }), + }); + expect(second.statusCode).toBe(409); + }); + + it("concurrent HTTP bootstrap attempts: exactly one 200, one 409, never 500", async () => { + // Two simultaneous first-install requests race through the freshness + // gate (both may pass it), then serialize on the transaction-scoped + // advisory lock inside the canonical mutation. The loser must map to + // the same 409 as the freshness gate — never an internal 500. + vi.stubEnv("LAUNCHPAD_SETUP_TOKEN", VALID_TOKEN); + resetRuntimeConfigForTest(); + await resetToUninitialized(ctx); + + const [ra, rb] = await Promise.all([ + ctx.app.inject({ + method: "POST", + url: "/api/launchpad/bootstrap", + payload: basePayload({ + adminUsername: `lprace1-${crypto.randomUUID().slice(0, 8)}`, + }), + }), + ctx.app.inject({ + method: "POST", + url: "/api/launchpad/bootstrap", + payload: basePayload({ + adminUsername: `lprace2-${crypto.randomUUID().slice(0, 8)}`, + }), + }), + ]); + + const codes = [ra.statusCode, rb.statusCode].sort(); + expect(codes).toEqual([200, 409]); + const loser = ra.statusCode === 409 ? ra : rb; + expect(loser.json().error.code).toBe("LAUNCHPAD_ALREADY_INITIALIZED"); + + // Exactly one Admin authority remains after the race. + const adminRows = await ctx.db + .select() + .from(schema.users) + .where(eq(schema.users.role, "Admin")); + expect(adminRows).toHaveLength(1); + }); +}); diff --git a/apps/api/src/routes/launchpad.ts b/apps/api/src/routes/launchpad.ts new file mode 100644 index 00000000..095577ed --- /dev/null +++ b/apps/api/src/routes/launchpad.ts @@ -0,0 +1,205 @@ +import { FastifyPluginAsync } from "fastify"; +import { timingSafeEqual } from "node:crypto"; +import { + LaunchpadStatusResponseSchema, + LaunchpadBootstrapRequestSchema, + LaunchpadBootstrapResponseSchema, + ErrorResponseSchema, +} from "@exam/contracts"; +import { AdminAlreadyExistsError } from "@exam/domain"; +import type { PublicBrandingContext } from "@exam/domain"; +import { createOrganizationRepo } from "@exam/db/src/repository/organizationRepo.js"; +import { bootstrapAdminOnFreshDb } from "../scripts/bootstrap-admin.js"; +import { getRuntimeConfig } from "../config/runtimeConfig.js"; +import { buildErrorResponse } from "../lib/errorResponse.js"; + +/** + * Constant-time equality check for two secret strings. + * + * Compares the UTF-8 byte representations only after confirming equal + * length, so a caller cannot learn the configured token length from a + * timing side channel. Returns false (not throw) on length mismatch — + * `crypto.timingSafeEqual` throws on Buffer length mismatch, so we guard + * it and return false instead to keep the call site branch-free on the + * secret. + */ +function constantTimeEqual(a: string, b: string): boolean { + const aBuf = Buffer.from(a, "utf8"); + const bBuf = Buffer.from(b, "utf8"); + if (aBuf.length !== bBuf.length) { + // Still do a comparison to keep wall-clock time independent of length. + timingSafeEqual(bBuf, bBuf); + return false; + } + return timingSafeEqual(aBuf, bBuf); +} + +/** + * Fastify plugin that registers the launchpad first-install routes. + * + * Launchpad is INITIAL INSTALLATION ONLY: it creates the first Admin and + * the internal default organization. It is NOT signup, NOT login, and NOT + * Admin recovery. Once the installation is initialized (the default + * organization exists), launchpad bootstrap is refused and the status + * endpoint redirects behavior to the normal login flow. Removing/disabling + * the last Admin does NOT reopen launchpad — Admin-loss recovery is + * operator CLI territory. + * + * The canonical mutation body is `bootstrapAdminOnFreshDb`, shared with the + * `bootstrap-admin` CLI (P6-008): organization + Admin + primary Admin role + * assignment + `admin.bootstrap` audit commit atomically in one + * transaction. The HTTP adapter is a thin shim that performs the + * installation-initialized gate and setup-token check before delegating to + * that canonical body — it does NOT duplicate the irreversible mutation + * logic. + * + * Setup-token contract (P7-C1): + * - high entropy (operator-generated, e.g. `openssl rand -hex 32`) + * - body only, never URL — validated from the JSON request body + * - never audit-logged in plaintext (the audit row written by the + * canonical body carries username/name/source only, not the token) + * - rate limited (max 5 attempts / minute per IP) + * - a completed installation MUST NOT become a token-validity oracle: + * the installation-initialized check runs FIRST; once initialized, + * bootstrap returns 409 regardless of whether the token is correct + */ +const launchpadRoutes: FastifyPluginAsync = async (fastify) => { + /** + * Internal helper: is this installation initialized? Returns true once the + * internal default organization (slug "default") exists. This is the + * FIRST-INSTALL gate only — it is deliberately NOT `activeAdminCount == 0` + * (removing the last Admin must not reopen launchpad). Delegates to the + * organization repository so the route never imports DB schema directly + * (architecture lint: routes use repositories, not raw queries). + */ + async function isInstallationInitialized(): Promise { + return createOrganizationRepo(fastify.db).defaultOrganizationExists({ + purpose: "public_branding", + } as PublicBrandingContext); + } + + fastify.get( + "/launchpad/status", + { + schema: { + response: { + 200: LaunchpadStatusResponseSchema, + }, + }, + }, + /** + * GET /launchpad/status — public installation-status probe. + * + * Reveals only whether the installation has been initialized (the + * default organization exists). This is NOT a token-validity oracle + * and never reveals token state. The frontend uses it to decide + * whether to render the first-Admin setup form or redirect to /login. + */ + async () => { + const initialized = await isInstallationInitialized(); + return { initialized }; + }, + ); + + fastify.post( + "/launchpad/bootstrap", + { + config: { rateLimit: { max: 5, timeWindow: 60 * 1000 } }, + schema: { + body: LaunchpadBootstrapRequestSchema, + response: { + 200: LaunchpadBootstrapResponseSchema, + 403: ErrorResponseSchema, + 409: ErrorResponseSchema, + }, + }, + }, + /** + * POST /launchpad/bootstrap — first-Admin setup (first-install only). + * + * Ordering is security-critical: + * 1. Check installation NOT initialized FIRST. If already initialized, + * return 409 without distinguishing token validity — a completed + * installation must not become a token-validation oracle. + * 2. Constant-time compare the body setupToken against the configured + * token. An unset/empty configured token means launchpad is + * disabled → 403. + * 3. Delegate to the canonical `bootstrapAdminOnFreshDb` atomic + * mutation body (shared with the bootstrap-admin CLI). + * + * The role is NOT selectable: the server always creates role = Admin. + */ + async (request, reply) => { + const data = LaunchpadBootstrapRequestSchema.parse(request.body); + + // 1. Installation-initialized gate FIRST (no token oracle). + const initialized = await isInstallationInitialized(); + if (initialized) { + return reply + .code(409) + .send( + buildErrorResponse(request.id, "LAUNCHPAD_ALREADY_INITIALIZED"), + ); + } + + // 2. Setup-token check (constant-time; disabled if unset/empty). + const configuredToken = getRuntimeConfig().launchpad.setupToken; + if ( + !configuredToken || + !constantTimeEqual(data.setupToken, configuredToken) + ) { + return reply + .code(403) + .send( + buildErrorResponse(request.id, "LAUNCHPAD_INVALID_SETUP_TOKEN"), + ); + } + + // 3. Canonical atomic mutation (org + Admin + assignment + audit in + // one transaction). Refuses a second active Admin internally. + const orgOptions: { + organizationName: string; + organizationDisplayName?: string; + } = { organizationName: data.organizationName }; + if (data.organizationDisplayName) { + orgOptions.organizationDisplayName = data.organizationDisplayName; + } + let result: Awaited>; + try { + result = await bootstrapAdminOnFreshDb( + fastify.db, + { + username: data.adminUsername, + password: data.adminPassword, + name: data.adminName, + }, + orgOptions, + "launchpad", + ); + } catch (err) { + // First-install race loser: another bootstrap (HTTP or CLI) won the + // advisory-lock serialization and committed the first Admin while + // this request was in flight. Both requests passed the freshness + // gate before the lock, so the loser discovers the winner inside + // the canonical mutation. This is an EXPECTED outcome — map it to + // the same 409 as the freshness gate, never an internal 500. + if (err instanceof AdminAlreadyExistsError) { + return reply + .code(409) + .send( + buildErrorResponse(request.id, "LAUNCHPAD_ALREADY_INITIALIZED"), + ); + } + throw err; + } + + return { + ok: true as const, + organizationSlug: result.organization.slug, + adminUsername: result.user.username, + }; + }, + ); +}; + +export default launchpadRoutes; diff --git a/apps/api/src/routes/registerApiRoutes.ts b/apps/api/src/routes/registerApiRoutes.ts index d0f572ff..116aeaad 100644 --- a/apps/api/src/routes/registerApiRoutes.ts +++ b/apps/api/src/routes/registerApiRoutes.ts @@ -1,6 +1,7 @@ import type { FastifyInstance } from "fastify"; import authRoutes from "./auth.js"; +import launchpadRoutes from "./launchpad.js"; import settingsRoutes from "./settings.js"; import candidateFieldRoutes from "./candidateField.js"; import userRoutes from "./user.js"; @@ -39,6 +40,7 @@ export async function registerApiRoutes( const prefix = opts.prefix ?? "/api"; await app.register(authRoutes, { prefix: `${prefix}/auth` }); + await app.register(launchpadRoutes, { prefix }); await app.register(settingsRoutes, { prefix }); await app.register(candidateFieldRoutes, { prefix }); await app.register(userRoutes, { prefix }); diff --git a/apps/api/src/scripts/bootstrap-admin.test.ts b/apps/api/src/scripts/bootstrap-admin.test.ts index 8f1e3a50..7b55db7a 100644 --- a/apps/api/src/scripts/bootstrap-admin.test.ts +++ b/apps/api/src/scripts/bootstrap-admin.test.ts @@ -6,6 +6,7 @@ import { schema } from "@exam/db/src/schema/pg.js"; import { setupIsolatedTestDb } from "@exam/db/src/testIsolation.js"; import { and, eq, sql } from "drizzle-orm"; import { verifyPassword } from "@exam/auth/src/password.js"; +import { AdminAlreadyExistsError } from "@exam/domain"; import { bootstrapAdmin, bootstrapAdminOnFreshDb, @@ -444,6 +445,131 @@ describe("bootstrapAdminOnFreshDb (production bootstrap path)", () => { // The fresh schema started empty; bootstrap never adds Candidates. expect(candidates).toHaveLength(0); }); + + it("records the adapter source in the admin.bootstrap audit metadata", async () => { + // The canonical mutation accepts an explicit source so the audit + // reflects the real entry point (CLI = local_script, HTTP launchpad = + // launchpad) instead of a single hardcoded value. + const iso = await setupIsolatedTestDb({ + namespace: "script-bootstrap-source", + databaseUrl: resolveTestDbUrl(), + }); + try { + const conn = await createDatabase(resolveTestDbUrl(), iso.schemaName); + const sourceDb = conn.db; + await migratePostgres(sourceDb, { migrationsSchema: iso.schemaName }); + try { + const username = `source-${Date.now()}`; + await bootstrapAdminOnFreshDb( + sourceDb, + { + username, + password: "StrongPass123!", + name: "Source Admin", + }, + { organizationName: "Source Org" }, + "launchpad", + ); + const audits = await sourceDb + .select() + .from(schema.auditLogs) + .where(eq(schema.auditLogs.action, "admin.bootstrap")); + expect(audits).toHaveLength(1); + const metadata = audits[0]!.metadata as Record; + expect(metadata.source).toBe("launchpad"); + } finally { + await conn.sql.end(); + } + } finally { + await iso.cleanup(); + } + }); + + it("serializes concurrent first-install attempts: exactly one winner, one Admin, one audit", async () => { + // Two non-force bootstrap attempts race on a migrated-but-empty schema. + // The transaction-scoped advisory lock makes the serialization domain + // explicit: exactly one attempt commits; the loser re-reads the Admin + // authority inside its own (now-serialized) transaction and refuses + // with the typed AdminAlreadyExistsError — never a silent second Admin. + const iso = await setupIsolatedTestDb({ + namespace: "script-bootstrap-race", + databaseUrl: resolveTestDbUrl(), + }); + try { + const conn = await createDatabase(resolveTestDbUrl(), iso.schemaName); + const raceDb = conn.db; + await migratePostgres(raceDb, { migrationsSchema: iso.schemaName }); + try { + const ts = Date.now(); + const [a, b] = await Promise.allSettled([ + bootstrapAdminOnFreshDb( + raceDb, + { + username: `race-a-${ts}`, + password: "StrongPass123!", + name: "Race Admin A", + }, + { organizationName: "Race Org" }, + "local_script", + ), + bootstrapAdminOnFreshDb( + raceDb, + { + username: `race-b-${ts}`, + password: "StrongPass123!", + name: "Race Admin B", + }, + { organizationName: "Race Org" }, + "launchpad", + ), + ]); + + const winner = a.status === "fulfilled" ? a : b; + const loser = a.status === "rejected" ? a : b; + expect(winner.status).toBe("fulfilled"); + if (loser.status === "rejected") { + expect(loser.reason).toBeInstanceOf(AdminAlreadyExistsError); + } else { + expect.unreachable("the race loser must reject"); + } + + // Exactly one Admin authority. + const admins = await raceDb + .select() + .from(schema.users) + .where(eq(schema.users.role, "Admin")); + expect(admins).toHaveLength(1); + + // Exactly one primary Admin assignment. + const assignments = await raceDb + .select() + .from(schema.userRoleAssignments); + const primaryAdmins = assignments.filter( + (x) => x.role === "Admin" && x.isPrimary && x.isActive, + ); + expect(primaryAdmins).toHaveLength(1); + + // Exactly one first-bootstrap audit, sourced from the winner's + // adapter (the loser's transaction rolled back). + const audits = await raceDb + .select() + .from(schema.auditLogs) + .where(eq(schema.auditLogs.action, "admin.bootstrap")); + expect(audits).toHaveLength(1); + const metadata = audits[0]!.metadata as Record; + const winnerSource = + winner.status === "fulfilled" && + winner.value.user.username === `race-a-${ts}` + ? "local_script" + : "launchpad"; + expect(metadata.source).toBe(winnerSource); + } finally { + await conn.sql.end(); + } + } finally { + await iso.cleanup(); + } + }); }); describe("resolveOrCreateDefaultOrganization", () => { diff --git a/apps/api/src/scripts/bootstrap-admin.ts b/apps/api/src/scripts/bootstrap-admin.ts index 9402f09d..c81217b7 100644 --- a/apps/api/src/scripts/bootstrap-admin.ts +++ b/apps/api/src/scripts/bootstrap-admin.ts @@ -41,10 +41,31 @@ import type { Database } from "@exam/db/src/types.js"; import { schema } from "@exam/db/src/schema/pg.js"; import { createUserRepo } from "@exam/db/src/repository/userRepo.js"; import { createUserRoleAssignmentRepo } from "@exam/db/src/repository/userRoleAssignmentRepo.js"; +import { AdminAlreadyExistsError } from "@exam/domain"; import { recordAtomicSystemAudit } from "../audit/auditWriter.js"; import { loadRootEnv } from "../config/loadRootEnv.js"; import { resolveDatabaseUrlFromEnv } from "../config/runtimeConfig.js"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; + +/** + * Stable PostgreSQL advisory-lock key for first-install bootstrap + * serialization (P7-C corrective pass §6). + * + * `pg_advisory_xact_lock(bigint)` is transaction-scoped: acquired on BEGIN, + * auto-released on COMMIT/ROLLBACK. It serializes the "exactly one first + * installation may win" invariant across BOTH adapters (HTTP Launchpad and + * the bootstrap-admin CLI) so that ordinary RR-isolation + ON CONFLICT is + * not the only thing standing between two concurrent first-Admin writes. + * + * The value is an arbitrary fixed 64-bit integer (must be the same in every + * caller for the lock to coordinate). It is documented here as the single + * source of truth; do not derive it at runtime. Two concurrent bootstrap + * calls (HTTP-vs-HTTP, HTTP-vs-CLI, CLI-vs-CLI) enter the SAME serialization + * domain and are serviced strictly one at a time; the loser re-reads the + * initialized state / Admin authority inside its own (now-serialized) txn + * and refuses. + */ +export const BOOTSTRAP_ADVISORY_XACT_LOCK_KEY = -7095023170042848127n; /** Default slug for the internal organization (single-tenant Phase 1). */ export const DEFAULT_ORG_SLUG = "default"; @@ -62,6 +83,15 @@ export interface BootstrapAdminParams { force?: boolean; } +/** + * Which adapter invoked the canonical bootstrap mutation. Recorded in the + * `admin.bootstrap` audit metadata so the audit reflects the real entry + * point instead of a single hardcoded value: + * - `local_script` — the bootstrap-admin CLI (operator shell); + * - `launchpad` — the HTTP Launchpad first-install adapter. + */ +export type BootstrapAdminSource = "local_script" | "launchpad"; + export interface BootstrapAdminOrganizationOptions { /** * Explicit organization name. When omitted and the default organization @@ -104,6 +134,7 @@ export async function bootstrapAdmin( db: Database, organizationId: string, params: BootstrapAdminParams, + source: BootstrapAdminSource = "local_script", ): Promise<{ user: BootstrapAdminResult["user"]; }> { @@ -126,7 +157,7 @@ export async function bootstrapAdmin( "Admin", ); if (activeAdminCount > 0 && !params.force) { - throw new Error( + throw new AdminAlreadyExistsError( `An active Admin already exists in this organization. ` + `Use --force to create an additional Admin.`, ); @@ -160,7 +191,7 @@ export async function bootstrapAdmin( metadata: { username: user.username, name: user.name, - source: "local_script", + source, }, }, ); @@ -269,6 +300,14 @@ async function resolveOrCreateDefaultOrganizationInTx( * atomically. If any step fails, none of them land (no orphan org, no * orphan user, no orphan assignment, no orphan audit). * + * Concurrency (P7-C corrective pass §6): the transaction opens by taking a + * transaction-scoped PostgreSQL advisory lock + * ({@link BOOTSTRAP_ADVISORY_XACT_LOCK_KEY}) so HTTP Launchpad and the CLI + * enter the SAME serialization domain. Under a true first-install race + * (HTTP-vs-HTTP, HTTP-vs-CLI, CLI-vs-CLI) exactly one caller commits; the + * loser re-reads the initialized state / Admin authority inside its own + * (now-serialized) transaction and refuses. + * * This is the canonical production bootstrap path (P6-008). The baseline * dev/test seed (`packages/db/src/seed.ts`) is NOT the production path. */ @@ -276,10 +315,23 @@ export async function bootstrapAdminOnFreshDb( db: Database, params: BootstrapAdminParams, options: BootstrapAdminOrganizationOptions = {}, + source: BootstrapAdminSource = "local_script", ): Promise { const passwordHash = await hashPassword(params.password); const result = await executeInTransaction(db, async (tx) => { + // Serialize the first-install "exactly one winner" invariant across BOTH + // adapters (HTTP Launchpad AND this CLI) via a transaction-scoped + // PostgreSQL advisory lock. RR-isolation + ON CONFLICT already protects + // correctness; this makes the domain explicit and removes reliance on + // retry semantics alone. The lock is auto-released at COMMIT/ROLLBACK. + // See BOOTSTRAP_ADVISORY_XACT_LOCK_KEY. + await tx.execute( + sql.raw( + `SELECT pg_advisory_xact_lock(${BOOTSTRAP_ADVISORY_XACT_LOCK_KEY.toString()})`, + ), + ); + // 1. Resolve/create the default org INSIDE this transaction so the // org, Admin, assignment, and audit commit atomically. const organization = await resolveOrCreateDefaultOrganizationInTx( @@ -302,7 +354,7 @@ export async function bootstrapAdminOnFreshDb( "Admin", ); if (activeAdminCount > 0 && !params.force) { - throw new Error( + throw new AdminAlreadyExistsError( `An active Admin already exists in this organization. ` + `Use --force to create an additional Admin.`, ); @@ -337,7 +389,7 @@ export async function bootstrapAdminOnFreshDb( metadata: { username: user.username, name: user.name, - source: "local_script", + source, }, }, ); @@ -417,7 +469,12 @@ async function main() { if (params.organizationDisplayName) { orgOptions.organizationDisplayName = params.organizationDisplayName; } - const result = await bootstrapAdminOnFreshDb(conn.db, params, orgOptions); + const result = await bootstrapAdminOnFreshDb( + conn.db, + params, + orgOptions, + "local_script", + ); const orgVerb = result.organization.created ? "Created" : "Resolved"; process.stdout.write( `Organization ${orgVerb}.\n` + diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 2832a1fe..62fd5003 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -16,6 +16,7 @@ import { DateTimeProvider } from "@/contexts/DateTimeContext"; import { ErrorBoundary } from "@/components/shared/ErrorBoundary"; import { getDocumentTitle } from "@/lib/pageMeta"; import { LoginPage } from "@/pages/LoginPage"; +import { LaunchpadPage } from "@/pages/LaunchpadPage"; import { PlaceholderPage } from "@/pages/PlaceholderPage"; import { SettingsPage } from "@/pages/admin/SettingsPage"; import { CandidateFieldsPage } from "@/pages/admin/CandidateFieldsPage"; @@ -68,6 +69,7 @@ export function AppRoutes() { return ( } /> + } /> }> } /> } /> diff --git a/apps/web/src/i18n/locales/zh-CN.ts b/apps/web/src/i18n/locales/zh-CN.ts index 39d288d1..c9196c50 100644 --- a/apps/web/src/i18n/locales/zh-CN.ts +++ b/apps/web/src/i18n/locales/zh-CN.ts @@ -2205,6 +2205,40 @@ const zhCN = { }, }, + /** + * Launchpad first-install page copy (P7-C1). Initial installation only — + * creates the first Admin and the internal default organization. NOT + * signup, NOT login, NOT Admin recovery. Once initialized the page + * redirects to /login. + */ + launchpad: { + title: "系统初始化", + subtitle: "创建首个管理员账户以完成系统初始化", + organizationNameLabel: "组织名称", + organizationDisplayNameLabel: "组织显示名称（可选）", + adminNameLabel: "管理员姓名", + adminUsernameLabel: "管理员用户名", + adminPasswordLabel: "管理员密码", + setupTokenLabel: "初始化令牌", + organizationNamePlaceholder: "请输入组织名称", + organizationDisplayNamePlaceholder: "请输入组织显示名称（可选）", + adminNamePlaceholder: "请输入管理员姓名", + adminUsernamePlaceholder: "请输入管理员用户名", + adminPasswordPlaceholder: "请输入管理员密码", + setupTokenPlaceholder: "请输入初始化令牌", + organizationNameRequired: "请输入组织名称", + adminNameRequired: "请输入管理员姓名", + adminUsernameRequired: "请输入管理员用户名", + adminPasswordRequired: "请输入管理员密码", + setupTokenRequired: "请输入初始化令牌", + submit: "完成初始化", + submitting: "正在初始化...", + errors: { + loadStatusFailed: "无法检测初始化状态，请稍后重试", + bootstrapFailed: "初始化失败，请检查输入或初始化令牌后重试", + }, + }, + /** StartExamPage (candidate pre-exam page) copy. */ startExam: { errors: { @@ -2281,6 +2315,7 @@ const zhCN = { fallbackPageTitle: "页面", static: { login: "登录", + launchpad: "系统初始化", dashboard: "仪表盘", users: "用户管理", candidates: "考生管理", diff --git a/apps/web/src/lib/pageMeta.ts b/apps/web/src/lib/pageMeta.ts index 44c3239f..6e3f3aac 100644 --- a/apps/web/src/lib/pageMeta.ts +++ b/apps/web/src/lib/pageMeta.ts @@ -19,6 +19,7 @@ interface RouteTitleRule { /** Static mapping from route paths to i18n keys. */ const staticRouteTitleKeys = new Map([ [routes.login, "pageMeta.static.login"], + [routes.launchpad, "pageMeta.static.launchpad"], [routes.admin.dashboard, "pageMeta.static.dashboard"], [routes.admin.users, "pageMeta.static.users"], [routes.admin.candidates, "pageMeta.static.candidates"], diff --git a/apps/web/src/lib/routes.ts b/apps/web/src/lib/routes.ts index ce92d34b..3f98d3b6 100644 --- a/apps/web/src/lib/routes.ts +++ b/apps/web/src/lib/routes.ts @@ -1,6 +1,7 @@ /** Centralized route path constants and path-builder functions. */ export const routes = { login: "/login", + launchpad: "/launchpad", admin: { root: "/admin", dashboard: "/admin/dashboard", diff --git a/apps/web/src/pages/LaunchpadPage.test.tsx b/apps/web/src/pages/LaunchpadPage.test.tsx new file mode 100644 index 00000000..8d8a8193 --- /dev/null +++ b/apps/web/src/pages/LaunchpadPage.test.tsx @@ -0,0 +1,157 @@ +import { act, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { MemoryRouter, Route, Routes, useLocation } from "react-router"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { LaunchpadPage } from "./LaunchpadPage"; + +const { apiGet, apiPost } = vi.hoisted(() => ({ + apiGet: vi.fn(), + apiPost: vi.fn(), +})); + +vi.mock("@/lib/api", () => ({ + api: { + get: (...args: unknown[]) => apiGet(...args), + post: (...args: unknown[]) => apiPost(...args), + }, + setNavigate: () => {}, +})); + +vi.mock("sonner", () => ({ + toast: { success: vi.fn(), error: vi.fn() }, +})); + +function LocationProbe() { + const location = useLocation(); + return {location.pathname}; +} + +function renderLaunchpad() { + return render( + + + } /> + } /> + + , + ); +} + +describe("LaunchpadPage", () => { + beforeEach(() => { + apiGet.mockReset(); + apiPost.mockReset(); + }); + + it("redirects to /login when the installation is already initialized", async () => { + apiGet.mockResolvedValueOnce({ initialized: true }); + + renderLaunchpad(); + + await waitFor(() => { + expect(screen.getByTestId("current-path")).toHaveTextContent("/login"); + }); + }); + + it("renders the first-Admin setup form when uninitialized", async () => { + apiGet.mockResolvedValueOnce({ initialized: false }); + + renderLaunchpad(); + + await waitFor(() => { + expect(screen.getByLabelText("组织名称")).toBeInTheDocument(); + }); + expect(screen.getByLabelText("管理员姓名")).toBeInTheDocument(); + expect(screen.getByLabelText("管理员用户名")).toBeInTheDocument(); + expect(screen.getByLabelText("管理员密码")).toBeInTheDocument(); + expect(screen.getByLabelText("初始化令牌")).toBeInTheDocument(); + expect( + screen.getByRole("button", { name: "完成初始化" }), + ).toBeInTheDocument(); + }); + + it("shows field validation errors and does not submit when fields are empty", async () => { + apiGet.mockResolvedValueOnce({ initialized: false }); + const user = userEvent.setup(); + + renderLaunchpad(); + await waitFor(() => + expect(screen.getByLabelText("组织名称")).toBeInTheDocument(), + ); + + await user.click(screen.getByRole("button", { name: "完成初始化" })); + + expect(screen.getByText("请输入组织名称")).toBeInTheDocument(); + expect(screen.getByText("请输入管理员姓名")).toBeInTheDocument(); + expect(screen.getByText("请输入管理员用户名")).toBeInTheDocument(); + expect(screen.getByText("请输入管理员密码")).toBeInTheDocument(); + expect(screen.getByText("请输入初始化令牌")).toBeInTheDocument(); + expect(apiPost).not.toHaveBeenCalled(); + }); + + it("submits the bootstrap payload and redirects to /login on success", async () => { + apiGet.mockResolvedValueOnce({ initialized: false }); + apiPost.mockResolvedValueOnce({ + ok: true, + organizationSlug: "default", + adminUsername: "newadmin", + }); + const user = userEvent.setup(); + + renderLaunchpad(); + await waitFor(() => + expect(screen.getByLabelText("组织名称")).toBeInTheDocument(), + ); + + await user.type(screen.getByLabelText("组织名称"), "Fresh Org"); + await user.type(screen.getByLabelText("管理员姓名"), "New Admin"); + await user.type(screen.getByLabelText("管理员用户名"), "newadmin"); + await user.type(screen.getByLabelText("管理员密码"), "Strong-Admin-1!"); + await user.type(screen.getByLabelText("初始化令牌"), "the-setup-token"); + + await user.click(screen.getByRole("button", { name: "完成初始化" })); + + // The payload excludes organizationDisplayName when left blank and sends + // the canonical field names the backend expects. + await waitFor(() => { + expect(apiPost).toHaveBeenCalledWith("/api/launchpad/bootstrap", { + organizationName: "Fresh Org", + adminName: "New Admin", + adminUsername: "newadmin", + adminPassword: "Strong-Admin-1!", + setupToken: "the-setup-token", + }); + }); + await waitFor(() => { + expect(screen.getByTestId("current-path")).toHaveTextContent("/login"); + }); + }); + + it("shows an error banner and stays on /launchpad when bootstrap fails", async () => { + apiGet.mockResolvedValueOnce({ initialized: false }); + apiPost.mockRejectedValueOnce(new Error("初始化令牌无效或未配置")); + const user = userEvent.setup(); + + renderLaunchpad(); + await waitFor(() => + expect(screen.getByLabelText("组织名称")).toBeInTheDocument(), + ); + + await user.type(screen.getByLabelText("组织名称"), "Fresh Org"); + await user.type(screen.getByLabelText("管理员姓名"), "New Admin"); + await user.type(screen.getByLabelText("管理员用户名"), "newadmin"); + await user.type(screen.getByLabelText("管理员密码"), "Strong-Admin-1!"); + await user.type(screen.getByLabelText("初始化令牌"), "wrong-token"); + + await user.click(screen.getByRole("button", { name: "完成初始化" })); + + await waitFor(() => { + expect(screen.getByRole("alert")).toHaveTextContent( + "初始化令牌无效或未配置", + ); + }); + // Still on /launchpad (no redirect). + expect(screen.queryByTestId("current-path")).not.toBeInTheDocument(); + await act(async () => {}); + }); +}); diff --git a/apps/web/src/pages/LaunchpadPage.tsx b/apps/web/src/pages/LaunchpadPage.tsx new file mode 100644 index 00000000..1d9e04f1 --- /dev/null +++ b/apps/web/src/pages/LaunchpadPage.tsx @@ -0,0 +1,291 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Navigate } from "react-router"; +import { BrandHeader } from "@/components/layout/BrandHeader"; +import { Card, CardContent, CardHeader } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { FieldError } from "@/components/shared/FieldError"; +import { InlineErrorBanner } from "@/components/shared/InlineErrorBanner"; +import { FieldGroup, Field } from "@/components/shared/FieldGroup"; +import { PageContainer } from "@/components/shared/PageContainer"; +import { api } from "@/lib/api"; +import { getApiErrorMessage } from "@/lib/apiErrors"; +import type { + LaunchpadStatusResponse, + LaunchpadBootstrapRequest, +} from "@exam/contracts"; + +/** + * Launchpad first-install page (P7-C1). + * + * Initial installation ONLY: creates the first Admin and the internal + * default organization via POST /api/launchpad/bootstrap. NOT signup, NOT + * login, NOT Admin recovery. The role is NOT selectable — the server always + * creates role = Admin. + * + * On mount it probes GET /api/launchpad/status. If the installation is + * already initialized, it redirects to /login (a completed installation + * never renders a Launchpad "completed" page). After a successful bootstrap + * it also redirects to /login so the new Admin can log in normally. + */ +export function LaunchpadPage() { + const { t } = useTranslation(); + + const [statusState, setStatusState] = useState< + "loading" | "uninitialized" | "initialized" | "error" + >("loading"); + + const [organizationName, setOrganizationName] = useState(""); + const [organizationDisplayName, setOrganizationDisplayName] = useState(""); + const [adminName, setAdminName] = useState(""); + const [adminUsername, setAdminUsername] = useState(""); + const [adminPassword, setAdminPassword] = useState(""); + const [setupToken, setSetupToken] = useState(""); + const [fieldErrors, setFieldErrors] = useState>({}); + const [submitError, setSubmitError] = useState(""); + const [submitting, setSubmitting] = useState(false); + const [done, setDone] = useState(false); + + useEffect(() => { + let cancelled = false; + (async () => { + try { + const status = await api.get( + "/api/launchpad/status", + ); + if (cancelled) return; + setStatusState(status.initialized ? "initialized" : "uninitialized"); + } catch { + if (cancelled) return; + setStatusState("error"); + } + })(); + return () => { + cancelled = true; + }; + }, []); + + // Once initialized (or after a successful bootstrap), redirect to /login. + if (statusState === "initialized" || done) { + return ; + } + + const validate = () => { + const errors: Record = {}; + if (!organizationName.trim()) + errors.organizationName = t("launchpad.organizationNameRequired"); + if (!adminName.trim()) errors.adminName = t("launchpad.adminNameRequired"); + if (!adminUsername.trim()) + errors.adminUsername = t("launchpad.adminUsernameRequired"); + if (!adminPassword.trim()) + errors.adminPassword = t("launchpad.adminPasswordRequired"); + if (!setupToken.trim()) + errors.setupToken = t("launchpad.setupTokenRequired"); + setFieldErrors(errors); + return Object.keys(errors).length === 0; + }; + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + if (!validate()) return; + setSubmitting(true); + setSubmitError(""); + try { + const body: LaunchpadBootstrapRequest = { + organizationName: organizationName.trim(), + ...(organizationDisplayName.trim() + ? { organizationDisplayName: organizationDisplayName.trim() } + : {}), + adminName: adminName.trim(), + adminUsername: adminUsername.trim(), + adminPassword, + setupToken, + }; + await api.post("/api/launchpad/bootstrap", body); + setDone(true); + } catch (err) { + setSubmitError( + getApiErrorMessage(err, t("launchpad.errors.bootstrapFailed")), + ); + } finally { + setSubmitting(false); + } + }; + + // While the installation status is unknown, render the same shell so the + // page does not flash empty before the redirect resolves. + if (statusState === "loading") { + return ( +
+ + + + + + + + +
+ ); + } + + return ( +
+ + + + + + +

{t("launchpad.title")}

+

{t("launchpad.subtitle")}

+ {statusState === "error" && ( + + {t("launchpad.errors.loadStatusFailed")} + + )} +
+ + + + { + setOrganizationName(e.target.value); + if (fieldErrors.organizationName) + setFieldErrors((prev) => ({ + ...prev, + organizationName: "", + })); + }} + disabled={submitting} + /> + {fieldErrors.organizationName} + + + + setOrganizationDisplayName(e.target.value)} + disabled={submitting} + /> + + + + { + setAdminName(e.target.value); + if (fieldErrors.adminName) + setFieldErrors((prev) => ({ ...prev, adminName: "" })); + }} + disabled={submitting} + /> + {fieldErrors.adminName} + + + + { + setAdminUsername(e.target.value); + if (fieldErrors.adminUsername) + setFieldErrors((prev) => ({ + ...prev, + adminUsername: "", + })); + }} + disabled={submitting} + /> + {fieldErrors.adminUsername} + + + + { + setAdminPassword(e.target.value); + if (fieldErrors.adminPassword) + setFieldErrors((prev) => ({ + ...prev, + adminPassword: "", + })); + }} + disabled={submitting} + /> + {fieldErrors.adminPassword} + + + + { + setSetupToken(e.target.value); + if (fieldErrors.setupToken) + setFieldErrors((prev) => ({ ...prev, setupToken: "" })); + }} + disabled={submitting} + /> + {fieldErrors.setupToken} + + {submitError && ( + {submitError} + )} + + +
+ + + +
+ ); +} diff --git a/docker-compose.yml b/docker-compose.yml index 0aa78826..d30ebac8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,14 @@ # Production Compose contract. # +# ONE-COMPOSE MODEL: this file is the ONLY +# production/operator Docker Compose entry point. There is no +# docker-compose.pitr.yml / backup.yml / restore.yml / production.yml. +# Optional PostgreSQL capabilities such as PITR are DATABASE configuration, +# not an alternate Docker topology: enabling PITR is +# `scripts/backup/postgres-enable-pitr.sh` (ALTER SYSTEM), not a second +# Compose file. Development/test Compose files (docker-compose.dev.yml, +# docker-compose.test.yml) are development infrastructure and may remain. +# # Security invariants enforced here and by the # scripts/repository-contract/deployment-topology-contract.mjs guard: # @@ -18,11 +27,21 @@ # bare `docker compose up` needs no Redis configuration. When the # profile IS enabled it MUST be authenticated: the redis container # refuses to start without REDIS_PASSWORD (startup guard) and the -# server runs with `requirepass` (P7 review P1-1 — Redis now owns the +# server runs with `requirepass` (Redis now owns the # shared rate-limit state, so an open instance is not acceptable). The # API reads REDIS_URL; when the profile is enabled the operator must set # it to the authenticated URL, e.g. redis://:@redis:6379. # See docs/deployment/mvp-deployment-runbook.md §10. +# +# Portable persistence: authoritative state lives in operator-visible +# host bind mounts under ${EXAM_DATA_ROOT:-./data}. Containers are disposable; +# the declared host data directory is not. See +# docs/deployment/backup-and-recovery.md. The official PostgreSQL image owns +# its internal PGDATA layout (/var/lib/postgresql/18/docker); the host +# abstraction is simply ${EXAM_DATA_ROOT}/postgres — operators do not need to +# understand the internal subdirectories. Scripts that launch this topology +# for tests/smoke MUST set EXAM_DATA_ROOT to an isolated temp directory so they +# never share the repo-root ./data/ (isolation rule). services: app: build: . @@ -52,6 +71,13 @@ services: # change business-time comparison semantics). APP_TIMEZONE: ${APP_TIMEZONE:-Asia/Shanghai} TZ: ${TZ:-Asia/Shanghai} + # Forward the Launchpad first-install setup + # token explicitly (NOT via a broad env_file, to keep explicit + # environment ownership). Unset/empty disables Launchpad (the + # bootstrap endpoint refuses) so a bare `docker compose up` still + # starts normally. Setting it in .env makes the browser first-Admin + # setup form usable. See backup-and-recovery.md §11. + LAUNCHPAD_SETUP_TOKEN: ${LAUNCHPAD_SETUP_TOKEN:-} depends_on: db: condition: service_healthy @@ -87,7 +113,20 @@ services: TZ: ${TZ:-Asia/Shanghai} PGTZ: ${APP_TIMEZONE:-Asia/Shanghai} volumes: - - pgdata:/var/lib/postgresql + # Authoritative state is an operator-visible host bind mount. + # Containers are disposable; this host directory is not. The official + # postgres image owns its internal PGDATA layout under this path. + - ${EXAM_DATA_ROOT:-./data}/postgres:/var/lib/postgresql + # The WAL archive mount is ALWAYS present on the + # canonical db service, but it is inert by default (archive_mode stays + # off until an operator enables PITR via + # scripts/backup/postgres-enable-pitr.sh). The mount existing != PITR + # enabled. A normal operator never needs to understand WAL/PITR just + # to run Exam. For real disaster recovery, point + # EXAM_WAL_ARCHIVE_HOST_PATH at an INDEPENDENT failure domain (NAS / + # another server / a separate disk); the local default path is for + # development/drills only and is NOT host-loss protection. + - ${EXAM_WAL_ARCHIVE_HOST_PATH:-${EXAM_DATA_ROOT:-./data}/wal-archive}:/wal-archive healthcheck: test: [ @@ -107,7 +146,7 @@ services: # Redis is OPTIONAL in the implemented MVP (ADR-001 / P6-010), but it owns # the shared rate-limit state when enabled (P7) — so an ENABLED production - # Redis MUST be authenticated (P7 review P1-1; ADR-001 security + # Redis MUST be authenticated (ADR-001 security # considerations). The service is gated behind the `redis` profile: it is # NOT started by a bare `docker compose up`. The password check lives at # CONTAINER STARTUP (not Compose expansion), so the profile stays truly @@ -137,14 +176,18 @@ services: exec redis-server --appendonly yes --requirepass "$$REDIS_PASSWORD" profiles: ["redis"] environment: - # P7 review P1: the empty default keeps Redis optional at Compose + # The empty default keeps Redis optional at Compose # parse time (a bare `docker compose up` needs no Redis # configuration); the startup guard above enforces the password when # the redis profile actually runs. REDIS_PASSWORD: ${REDIS_PASSWORD:-} TZ: ${TZ:-Asia/Shanghai} volumes: - - redisdata:/data + # Redis persistence is OPTIONAL for correctness. It holds only + # rate-limit counters; losing it resets operational history but never + # affects PostgreSQL authority. Restoring Redis is NOT a condition for + # restoring Exam authority (C1.2). + - ${EXAM_DATA_ROOT:-./data}/redis:/data healthcheck: # An unauthenticated ping must fail (NOAUTH) — the healthcheck itself # proves the server enforces the password. @@ -235,9 +278,9 @@ services: - exam-net restart: unless-stopped -volumes: - pgdata: - redisdata: +# No top-level named volumes: authoritative state lives in +# operator-visible host bind mounts under ${EXAM_DATA_ROOT:-./data} +# (see the db and redis service volumes above). networks: exam-net: diff --git a/docs/audits/P7-C-PORTABLE-BACKUP-RECOVERY-CLOSEOUT.md b/docs/audits/P7-C-PORTABLE-BACKUP-RECOVERY-CLOSEOUT.md new file mode 100644 index 00000000..35d3989f --- /dev/null +++ b/docs/audits/P7-C-PORTABLE-BACKUP-RECOVERY-CLOSEOUT.md @@ -0,0 +1,285 @@ +# P7-C — Portable Persistence, Backup & PostgreSQL Disaster Recovery (Closeout) + +**Status:** READY FOR HUMAN REVIEW +**Program:** P7-C REBUILD (portable persistence / backup / PostgreSQL DR) +**Baseline (`origin/master`):** `2a1a9eb30fc40a10d119571d4ad3befb5b52e26e` +**Verification implementation head:** see Git history / PR (the commit on +which the deployment suite in §5 was executed) +**Branch:** `feat/p7-c-portable-backup-recovery` + +This document is the single source of truth for the FINAL P7-C shape: what +the program shipped, what it deliberately does NOT ship, and what remains +for P7-E. Authoritative procedures live in +`docs/deployment/backup-and-recovery.md`; this closeout is the audit / +decision record. Historical development mistakes and earlier architecture +states are recorded in `docs/audits/*` and Git history, not repeated here. + +--- + +## 1. Final architecture + +```text +ONE production docker-compose.yml + +C1 portable persistence (operator-visible host bind mounts) + + stopped filesystem backup/restore + +C2 pg_dump -Fc (online logical backup) + + clean pg_restore (DROP + template0, --no-owner --exit-on-error) + +C3 pg_basebackup (physical base backup) + + pg_verifybackup (physical integrity) + + optional PostgreSQL-native WAL archiving / PITR + +Launchpad first-install only (HTTP adapter over the canonical bootstrap) + + operator CLI fallback +``` + +**One-Compose model.** `docker-compose.yml` is the ONLY production/operator +Compose entry point. There is no `docker-compose.pitr.yml`, no custom +postgres-pitr image, no backup/restore Compose variant. A repository guard +(`scripts/repository-contract/deployment-topology-contract.mjs`) forbids +production Compose variants while allowing intentional dev/test files +(`docker-compose.dev.yml`, `docker-compose.test*.yml`). + +**PITR is a PostgreSQL cluster capability, not a topology.** Enabling it is +`scripts/backup/postgres-enable-pitr.sh` — `ALTER SYSTEM SET archive_mode = +on` (plus idempotent `archive_command` and `archive_timeout = 60s`), a +db-only restart, then real archiver evidence from `pg_stat_archiver`. +`ALTER SYSTEM` persists into `postgresql.auto.conf` inside PGDATA, so the +configuration survives `docker compose down` / `up` and host relocation of +the same PGDATA. + +**Routine vs. recovery backup.** + +```text +routine backup = pg_dump -Fc (C2, online) +physical backup = pg_basebackup -X stream -Fp --manifest-checksums SHA256 (C3) +physical check = pg_verifybackup (per-file checksums + manifest integrity) +PITR = WAL archive + explicit recovery target (C3, optional) +Redis = non-authoritative (rate-limit counters only) +restore = operator-owned (no browser restore, ever) +``` + +--- + +## 2. Authority stores + +| Store | Authority | Durability | +| --- | --- | --- | +| PostgreSQL | **sole authoritative** durable store | `pgdata` → `${EXAM_DATA_ROOT}/postgres` (bind-mounted, operator-visible) | +| Redis | **non-authoritative** | rate-limit counters only, TTL-bounded, optional profile | +| App filesystem | **no durable writes** | no durable state outside PostgreSQL | + +--- + +## 3. Backup / recovery matrix + +| Path | Online? | Scope | Replaces history? | Operator command | Verification | +| --- | --- | --- | --- | --- | --- | +| Container restart / recreation | — | — | No — same history | `docker compose down` / `up -d` | `persistence-and-cold-restore.sh` | +| Stopped-directory relocation (C1) | No (stop) | Whole data dir | **No** — same history | `cp -a` / `rsync -aHAX` while stopped | `persistence-and-cold-restore.sh` | +| Cold-filesystem backup/restore (C1) | No (stop) | Whole data dir | Yes | `cold-filesystem-backup.sh` / `-restore.sh` | `persistence-and-cold-restore.sh` | +| Logical backup (C2) | **Yes** | One DB (`exam`) | Yes (clean restore) | `postgres-logical-backup.sh` / `-restore.sh` | `logical-backup-restore.sh` | +| Physical base backup (C3) | **Yes** | Whole cluster | Yes | `pg-basebackup.sh` | `pitr.sh` (base backup + verify) | +| WAL archive + PITR (C3) | **Yes** | WAL replay to target | Yes (replay + promote) | `postgres-enable-pitr.sh` + recovery procedure | `pitr.sh` (happy + failure modes) | + +--- + +## 4. Operator commands + +```text +scripts/backup/ +├── cold-filesystem-backup.sh # stopped PGDATA copy (refuses a running source) +├── cold-filesystem-restore.sh # restore into a fresh data root (explicit confirm) +├── postgres-logical-backup.sh # online pg_dump -Fc --no-owner +├── postgres-logical-restore.sh # DROP + template0 + pg_restore (CLEAN target) +├── pg-basebackup.sh # online physical base backup + pg_verifybackup +└── postgres-enable-pitr.sh # ALTER SYSTEM archive_* + archiver proof +``` + +The **online PostgreSQL scripts** (`postgres-logical-backup.sh`, +`postgres-logical-restore.sh`, `pg-basebackup.sh`, +`postgres-enable-pitr.sh`) derive the deployment's `POSTGRES_USER` / +`POSTGRES_DB` (and password where needed) from the RUNNING db container — +no hardcoded credentials, no password on argv. The **cold-filesystem +scripts** (`cold-filesystem-backup.sh`, `cold-filesystem-restore.sh`) +never touch a running database container: they validate source/destination +paths, refuse a live `postmaster.pid` (backup) or a populated destination +(restore), and copy via a throwaway helper container that preserves +ownership/mode/symlinks. + +Verification lives separately under `tests/deployment/` (capability-named, +phase-free): + +```text +tests/deployment/ +├── lib.sh # mechanical shared helpers (polling, temp roots, probes) +├── compose-smoke.sh # production Compose parses/starts; env contract; worker/redis wiring +├── launchpad-bootstrap.sh # first-install deployment contract (token wiring, 409, register disabled) +├── persistence-and-cold-restore.sh # container recreation + relocation + cold round-trip +├── logical-backup-restore.sh # online dump; A present, B absent after clean restore +└── pitr.sh # archive idempotency; happy PITR; missing-WAL / corrupt / invalid-target +``` + +Entry points: `pnpm test:deployment` (full suite) and per-capability +`test:deployment:compose|launchpad|persistence|logical|pitr`. + +--- + +## 5. Verification evidence + +### 5.1 Deployment suite (deterministic Docker drills) + +All tests run the canonical `docker-compose.yml` against isolated Compose +projects and isolated temp `EXAM_DATA_ROOT`s — the repo-root `./data/` and +any human/dev stack are never touched. No test generates a temporary +Compose override; recovery clusters are started with the canonical Compose +plus environment (`EXAM_DATA_ROOT`, `EXAM_WAL_ARCHIVE_HOST_PATH`, +`POSTGRES_PASSWORD`, `COMPOSE_PROJECT_NAME`). Readiness is bounded polling +(`pg_isready`, `/api/health`, `pg_stat_archiver`, archived-segment +presence), never arbitrary fixed sleeps. + +| Suite | Proves | +| --- | --- | +| `compose-smoke.sh` | `POSTGRES_PASSWORD` required; Redis optional at parse, authenticated when enabled; app/db/worker ordering; migrations exactly once; bootstrap-admin one Admin; login; seed refusal; SIGTERM clean shutdown | +| `launchpad-bootstrap.sh` | token reaches app via compose interpolation; wrong token 403; first Admin 200; re-bootstrap 409 (no token oracle); register disabled; unset token disables launchpad | +| `persistence-and-cold-restore.sh` | container recreation persistence; stopped-directory relocation; cold backup/restore round-trip — identical invariants each time | +| `logical-backup-restore.sh` | online `pg_dump -Fc`; State A → mutate B → clean restore → A present, B absent, org/admin/audit + Admin password hash match A | +| `pitr.sh` | archive idempotency (absent → OK, identical retry → OK, byte collision → FAIL); happy PITR (A + A1 + B present, C absent, promoted); F1 missing REQUIRED WAL (explicit target unreachable — recovery stays in recovery, replay LSN < target, post-missing state absent, restore_command failures visible); F2 corrupt base backup (`pg_verifybackup` rejects); F3 invalid recovery target (refused loudly) | + +### 5.2 Repository gates + +| Gate | Result | +| --- | --- | +| `pnpm verify:static` | PASS (lint, architecture, env-contract, repo-contract incl. one-compose guard, migration journal) | +| `pnpm test` | PASS (see PR for final counts) | +| `pnpm build` | PASS | +| CI | see PR (do not merge automatically) | + +--- + +## 6. Known limitations + +1. **Retention is manual.** P7-C does NOT automate PITR retention/pruning. + A promised recovery window requires: a usable base backup old enough to + precede the earliest desired target, every required WAL segment from + that base backup through the target window, and required timeline + history. Do not manually prune backup/WAL history unless the retained + chain has been deliberately validated. (§8.5 of `backup-and-recovery.md`.) +2. **Physical backups are PG-major-tied.** Cross-major migration requires + the C2 logical path. +3. **`archive_timeout` (60s) bounds archival freshness for active + workloads.** The archiver switches and archives a segment at least every + 60s, so at most 60s of WAL can await archival on a busy cluster. + Recovery precision is NOT bounded by `archive_timeout`: recovery + depends on the available archived WAL and the selected recovery target + (`recovery_target_time`, `recovery_target_lsn`, + `recovery_target_name`, or `recovery_target_xid`). +4. **C3 replication uses the configured `POSTGRES_USER` superuser over the + db container's loopback namespace.** A hardened deployment should create + a narrowly scoped replication role (documented in `pg-basebackup.sh`); + the API itself never gets replication authority. +5. **The WAL archive default path is `${EXAM_DATA_ROOT}/wal-archive`.** + Production MUST override `EXAM_WAL_ARCHIVE_HOST_PATH` to an independent + failure domain; the default is for development/drills only and is NOT + host-loss protection. +6. **No Admin backup visibility.** Operators run scripts from the host; + there is no in-product backup dashboard. Restore is operator-only by + design. +7. **Same-host relocation proof.** The automated relocation regression uses + two temp dirs on the same host. The product contract is ordinary + filesystem relocation while PostgreSQL is stopped; per-OS filesystem + proof is out of scope. +8. **`recovery_target_lsn` is the recommended target type.** Time-based + targets need clock alignment; xid-based targets need a 32-bit xid (the + 64-bit xid8 from `pg_current_xact_id()` is rejected). Documented in + `backup-and-recovery.md` §8.3. + +**Future boundary (explicit, not started):** if Exam later requires +automatic off-host WAL shipping, S3/MinIO, encryption, compression, +incremental backup chains, automated pruning/retention, large backup sets, +or low-RPO operational automation, evaluate **WAL-G** or **pgBackRest** +rather than growing Exam's shell scripts indefinitely. + +--- + +## 7. Same-history vs. history-replacement boundary (ADR-016) + +| Event | Same authoritative history? | +| --- | --- | +| Container restart / recreation | Yes | +| Stopped-directory relocation (C1) | Yes — same files, same timeline, new host | +| Cold-filesystem restore (C1) | **No** — snapshot from a past moment replaces the live history | +| Logical restore (C2) | **No** — fresh-clean database from a dump | +| Physical restore / PITR (C3) | **No** — base backup + WAL replay to target, then promote (new timeline) | + +No schema change is introduced to mark history-replacement events. The +exam system's authoritative state is whatever PostgreSQL currently holds; +it does not need to know HOW it got there. Any future offline-client +`recoveryEpoch` concern is a Phase 4 platformization concern and is NOT +implemented here. + +--- + +## 8. Scope discipline (what was NOT built) + +- NO second production Compose; NO PITR Docker image; NO migration + preflight; NO custom backup format; NO backup manifest protocol; NO + custom WAL manager; NO retention engine; NO backup scheduler; NO backup + UI; NO restore UI. +- NO WAL-G; NO pgBackRest; NO Kubernetes; NO Patroni; NO HA; NO startup + reconciler; NO P7-E. +- Launchpad remains FIRST-INSTALL ONLY. The canonical bootstrap mutation + (`bootstrapAdminOnFreshDb`) is shared verbatim by the HTTP Launchpad + adapter and the `bootstrap-admin` CLI; both serialize on the same + transaction-scoped PostgreSQL advisory lock (`pg_advisory_xact_lock`), so + HTTP-vs-HTTP, HTTP-vs-CLI, and CLI-vs-CLI races have exactly one winner. + The audit records the real adapter (`source: "local_script" | "launchpad"`); + the HTTP race loser maps to `409 LAUNCHPAD_ALREADY_INITIALIZED` (never an + internal 500). Covered by deterministic concurrency tests + (`bootstrap-admin.test.ts`, `launchpad.test.ts`). + +--- + +## 9. P7-E handoff + +The following are explicitly deferred to a future P7-E control-plane +program (NOT started, NOT scheduled by this closeout): + +1. **RPO/RTO profile automation.** Today's scheduling is cron-on-host. + P7-E could define named profiles (small-internal / standard / + high-stakes) and automate the schedule + retention per profile. +2. **Retention automation.** A control plane that keeps the base-backup + + WAL-chain invariant (§8.5 of `backup-and-recovery.md`) without operator + manual discipline. +3. **Admin backup visibility surface.** A read-only Admin view of backup + history / manifest status / last-verified timestamp. (Restore stays + operator-owned — no Admin restore button, ever.) +4. **Files/settings backup beyond PostgreSQL.** Attachments, generated + exports, and organization settings currently live in PostgreSQL (in-DB + authority). A separate files/settings backup is future only if/when + durable state appears outside PostgreSQL. +5. **Cross-PG-major upgrade playbook.** A documented + drilled + `pg_dump`-on-old → `pg_restore`-on-new procedure (the C2 primitives + exist; the playbook is P7-E). + +None of the above is a safety regression in the current shape. They are all +capability extensions. + +--- + +## 10. Findings + +| Severity | Count | Notes | +| --- | --- | --- | +| **P0** (blocks release) | 0 | — | +| **P1** (must fix before merge, safety) | 0 | — | +| **P2** (should fix, tracked) | 0 | — | +| **P3** (nice-to-have) | 0 | — | + +--- + +## 11. Verdict + +P7-C PORTABLE PERSISTENCE / BACKUP / POSTGRESQL RECOVERY READY FOR HUMAN REVIEW diff --git a/docs/audits/P7-C-REBUILD-ADVERSARIAL-PG-BACKUP-CONFIG-AUDIT.md b/docs/audits/P7-C-REBUILD-ADVERSARIAL-PG-BACKUP-CONFIG-AUDIT.md new file mode 100644 index 00000000..f3b9600c --- /dev/null +++ b/docs/audits/P7-C-REBUILD-ADVERSARIAL-PG-BACKUP-CONFIG-AUDIT.md @@ -0,0 +1,408 @@ +# P7-C Rebuild — Adversarial Audit: PG Backup Simplicity & Config Sync + +Repository: `jnhu76/exam` +Branch: `feat/p7-c-portable-backup-recovery` +Audit target: P7-C rebuild (C1 portable + cold + Launchpad, C2 logical, C3 physical + PITR) +Commits audited (`2a1a9eb3..0ebdf6eb`): + +``` +0ebdf6eb docs(p7-c): close portable backup and recovery program +0df4ba72 feat(p7-c3): physical backup and PITR +e164fd30 feat(p7-c2): logical backup and verified clean restore +df2d07df feat(p7-c1): portable persistence, cold backup and launchpad +``` + +Audit date: 2026-08-10 +Audit method: adversarial code/proof review focused on (a) whether the +PostgreSQL backup/restore tooling follows PostgreSQL's documented contract +and is as simple as it can be, and (b) whether the configuration surface +(`.env`* / `docker-compose*.yml` / scripts / docs) is internally +synchronized. PostgreSQL behavior was verified against the official +PostgreSQL 18 documentation via Context7; the `docker-entrypoint.sh` +extension handling was verified against the official postgres image source. + +Scope note: this is **not** a re-run of the earlier PR #273 audit +(`docs/audits/P7-C1-ADVERSARIAL-PORTABLE-DEPLOYMENT-AUDIT.md`, which targets +a different, abandoned PR). That document is retained as history. This audit +covers the rebuilt C1/C2/C3 on this branch only. + +--- + +## 1. Executive verdict + +**Request changes** — the backup/restore *mechanics* are sound and follow +PostgreSQL's documented contract, but two configuration-synchronization +defects make a documented operator path silently inert: + +1. **Critical (config sync) — PITR initdb seed is silently dead.** + `docker-compose.pitr.yml` mounts `docker/pitr/wal-archive.conf` into + `/docker-entrypoint-initdb.d/99-pitr-wal-archive.conf`. The official + postgres image **ignores `.conf` files** in that directory (verified + against the image's `docker-entrypoint.sh`: only `.sh`, `.sql`, + `.sql.gz`, `.sql.xz`, `.sql.zst` are processed; `.conf` hits the `*)` + branch → `ignoring`). The documented fresh-start path + (`docker compose -f docker-compose.yml -f docker-compose.pitr.yml up -d`) + therefore **does not enable WAL archiving at first init**. The drills + bypass this by applying the same settings via `ALTER SYSTEM`, so the + broken compose path is never exercised. An operator following the docs + for a fresh PITR-enabled deployment will believe archiving is on when it + is not — the first base backup taken will have **no WAL chain forward** + and PITR will silently not be possible. (§3.1) + +2. **Critical (config sync) — Launchpad token is never forwarded to the app + container (P2-1, carried over from the prior audit).** `docker-compose.yml` + has **no** `LAUNCHPAD_SETUP_TOKEN` in the `app` service `environment:` + block and no `env_file:`. The README, the runbook, and + `docs/deployment/backup-and-recovery.md` §11.1 all instruct the operator + to "set `LAUNCHPAD_SETUP_TOKEN` in `.env`". Compose uses `.env` for + **interpolation only** — a value there is never injected into a container + unless an `environment:` entry references it. With the token in `.env`, + the app container sees `LAUNCHPAD_SETUP_TOKEN` unset, so + `runtimeConfig.ts` disables launchpad (`setupToken: ""`), and + `POST /api/launchpad/bootstrap` always returns 403 + `LAUNCHPAD_INVALID_SETUP_TOKEN`. The headline C1.6 first-install UX is + **inert in the bundled deployment**; only the `bootstrap-admin` CLI path + works. (§4.1) + +The backup scripts themselves (C1 cold copy, C2 `pg_dump -Fc` + clean +restore, C3 `pg_basebackup` + `pg_verifybackup`) are faithful to PostgreSQL's +documented patterns and are simpler than the rebuilt program claims to be — +see §2. Several lower-severity config-sync and PG-convention issues are in +§3/§4. + +--- + +## 2. PostgreSQL backup simplicity & PG-convention review + +### 2.1 What is genuinely good (and simple) + +| Tool | PG-contract adherence | Notes | +| --- | --- | --- | +| `cold-filesystem-backup.sh` / `-restore.sh` | Correct | Refuses a live copy; validates `PG_VERSION` + `postgresql.conf`; container-assisted `cp -a` to preserve uid-999 ownership; refuses to overwrite; path-guarded. Matches PG's "cold physical backup = stopped server + complete PGDATA copy" contract. | +| `postgres-logical-backup.sh` (`pg_dump -Fc`) | Correct | `-Fc` (custom format) + `-X stream`-equivalent self-consistency + `--no-owner`; never puts the password on argv; verifies artifact with non-empty + `PGDMP` magic + `pg_restore --list`. This is the routine online backup PG documents. | +| `postgres-logical-restore.sh` | Correct & notably well-reasoned | `DROP DATABASE` + `CREATE DATABASE ... TEMPLATE template0` before `pg_restore` is the PG-documented way to get an **exact** match of the dump (avoids the `--clean --if-exists` merge trap where dump-absent objects survive). `--exit-on-error` fails fast. Refuses `postgres`/`template0`/`template1`. | +| `pg-basebackup.sh` | Correct | `-X stream -c fast -Fp --manifest-checksums SHA256`; `pg_verifybackup` on the manifest before success; `--no-sync` deliberately not used. | +| WAL `archive_command` | Correct | `test ! -f /wal-archive/%f && cp %p /wal-archive/%f` is the PG-documented idiom that **refuses to silently overwrite** a colliding segment. | +| PITR recovery config | Correct | `recovery.signal` + `postgresql.auto.conf` (`restore_command` + `recovery_target_lsn` + `recovery_target_inclusive = on` + `recovery_target_action = 'promote'`). `recovery_target_lsn` is the clock-skew-independent choice PG documents. Verified against PG18 docs. | + +The program is appropriately **simple**: it uses PG-native tools, adds no +custom protocol, and the negative-space contract (§9 of the closeout) is +honest. This is the right shape. + +### 2.2 PG-convention issues (ordered by severity) + +**(Required) — `pg-basebackup.sh` claims "no replication password needed" +but connects over TCP (`-h 127.0.0.1`), which the official image authenticates +with `scram-sha-256` by default.** `scripts/backup/pg-basebackup.sh:96-100, +119-122`: + +``` +# The local connection uses peer/trust auth for the bootstrap superuser +# (POSTGRES_USER), so no replication password is needed for the bundled +# single-node path. +... +docker run --rm \ + --network "container:${DB_CONTAINER}" \ + -e PGPASSWORD="${PGPASSWORD:-}" \ + postgres:18.4-bookworm \ + pg_basebackup -h 127.0.0.1 -U "${PGUSER:-exam}" ... +``` + +The comment is wrong. The official postgres image's default `pg_hba.conf` +authenticates TCP connections (`host all all all scram-sha-256`); **trust** +applies only to Unix-socket local connections. `-h 127.0.0.1` is TCP, so +`pg_basebackup` **will** be challenged for a password. The script reads +`PGPASSWORD` from the **host** environment (`${PGPASSWORD:-}`); if the +operator has not `export PGPASSWORD=` on the host, the +connection fails with `password authentication failed`. The drill works only +because the drill exports `POSTGRES_PASSWORD` and the operator would have to +also export `PGPASSWORD` — but the script header explicitly tells them they +do not need to. Two clean fixes, either is fine: + +- connect over the Unix socket (`-h /var/run/postgresql`, no `-h`, or + `PGHOST=/var/run/postgresql`) inside the db container's network namespace + — then `trust`/`peer` actually applies and the comment becomes true; or +- drop the "no password needed" claim and document that the operator must + `export PGPASSWORD=` on the host (and use + `PGUSER="${PGUSER:-$POSTGRES_USER}"` — see next item). + +Note: `pg_basebackup` requires a role with `REPLICATION` or `SUPERUSER` +(verified against PG18 `app-pgbasebackup.html` + `warm-standby.html`). The +`POSTGRES_USER` superuser satisfies this, so authority is fine; the defect is +purely the auth-method/password claim. + +**(Nit) — `PGUSER` default `exam` diverges from `POSTGRES_USER` when the +operator customizes the latter.** `pg-basebackup.sh:122` hardcodes +`-U "${PGUSER:-exam}"`. The C2 backup script correctly reads +`-U "$POSTGRES_USER"` from inside the container. If the deployment uses +`POSTGRES_USER=appdb`, `pg-basebackup.sh` connects as the wrong/nonexistent +role. Mirror the C2 pattern or default `PGUSER="${PGUSER:-exam}"` and +document that `PGUSER` must equal `POSTGRES_USER`. + +**(Nit) — cosmetic `POSTGRES_INITDB_ARGS: ""` in the PITR override is dead +config.** `docker-compose.pitr.yml:34` sets `POSTGRES_INITDB_ARGS: ""`. The +base compose does not set it, so the official image default (empty) already +applies. Setting `""` is a no-op that reads as if an arg (e.g. +`--data-checksums`) was intended and dropped. Remove it or document why it is +there. (This is not the cause of the `.conf`-ignored defect in §3.1 — that is +the file extension.) + +**(FYI) — cold backup does not explicitly check `postmaster.pid` absence.** +A clean `docker compose down` removes it, and the restore script refuses a +populated destination, so this is not reachable in the supported flow. PG +will refuse to start on a stale `postmaster.pid` anyway (loud). Recorded as a +known-safe gap; no action required. + +--- + +## 3. Critical config-sync defect: PITR initdb seed is silently ignored + +### 3.1 The defect + +`docker-compose.pitr.yml:45`: + +```yaml +- ./docker/pitr/wal-archive.conf:/docker-entrypoint-initdb.d/99-pitr-wal-archive.conf:ro +``` + +`docker/pitr/wal-archive.conf` contains `ALTER SYSTEM SET archive_mode = 'on';` +etc. The official postgres image's `docker-entrypoint.sh` processes +`/docker-entrypoint-initdb.d/` files with this `case` (verified verbatim +against `docker-library/postgres` master): + +```bash +case "$f" in + *.sh) ... ;; + *.sql) ... docker_process_sql -f "$f" ... ;; + *.sql.gz) ... gunzip -c "$f" | docker_process_sql ... ;; + *.sql.xz) ... xzcat "$f" | docker_process_sql ... ;; + *.sql.zst) ... zstd -dc "$f" | docker_process_sql ... ;; + *) printf '%s: ignoring %s\n' "$0" "$f" ;; +esac +``` + +A `.conf` file matches `*)` and is **ignored** with a log line. `archive_mode` +is **not** enabled. The override's own header even hedges: "For a +fresh start with this override, the entrypoint-init WAL config below seeds it +at first init." — but it does not, because of the extension. + +### 3.2 Why the drills do not catch it + +Both PITR drills (`p7-c3-pitr-drill.sh`, `p7-c3-pitr-failure-drill.sh`) +**ignore the compose override entirely** and instead apply archiving via +`ALTER SYSTEM` against the running cluster: + +```bash +psql_src -c "ALTER SYSTEM SET archive_mode = 'on';" +psql_src -c "ALTER SYSTEM SET archive_command = 'test ! -f /wal-archive/%f && cp %p /wal-archive/%f';" +``` + +So the documented operator path (`docker compose -f docker-compose.yml -f +docker-compose.pitr.yml up -d`) is **never exercised by any test or drill**. +The closeout's §7 "5/5 drills PASS" claim is true but the drills prove a +different code path than the one the docs tell operators to use. + +### 3.3 Impact (operator-visible) + +1. Operator sets up a fresh PITR-enabled deployment per the docs. +2. `archive_mode` is silently `off`. `SHOW archive_mode` would reveal `off`, + but nothing prompts the operator to check. +3. Operator takes a base backup via `pg-basebackup.sh`. `-X stream` makes the + base backup internally consistent, so `pg_verifybackup` passes. +4. Some time later, operator needs PITR. They restore the base backup + point + `restore_command` at the WAL archive. The archive is **empty** (archiving + was never on). Recovery cannot replay forward and PITR fails — or, worse, + recovers only to the base-backup checkpoint and the operator does not + immediately notice the data loss. + +### 3.4 Fix (smallest acceptable) + +Rename the seed file so the extension is processed: + +```bash +git mv docker/pitr/wal-archive.conf docker/pitr/wal-archive.sql +# update the mount in docker-compose.pitr.yml: +# ./docker/pitr/wal-archive.sql:/docker-entrypoint-initdb.d/99-pitr-wal-archive.sql:ro +``` + +`ALTER SYSTEM ... ;` is valid SQL and runs fine through `docker_process_sql`. +Then add a **fresh-start drill** (or extend the existing one) that actually +boots `docker compose -f docker-compose.yml -f docker-compose.pitr.yml up -d` +on an empty data root and asserts `SHOW archive_mode` returns `on`. Until +that drill exists, the documented PITR init path is unverified. + +--- + +## 4. Critical config-sync defect: Launchpad token never reaches the container + +### 4.1 The defect (P2-1, carried over and still present) + +`docker-compose.yml` `app` service `environment:` block (lines 41-64) has +**no** `LAUNCHPAD_SETUP_TOKEN` entry and the service has no `env_file:`. Yet: + +- `README.md:215`: "set `LAUNCHPAD_SETUP_TOKEN=` in + `.env` ... navigate to `/launchpad`". +- `docs/deployment/backup-and-recovery.md:524`: "Set `LAUNCHPAD_SETUP_TOKEN` + in `.env` before the first `docker compose up`". +- `docs/deployment/mvp-deployment-runbook.md:180`: "LAUNCHPAD_SETUP_TOKEN=... + in .env BEFORE step 4". +- `apps/api/src/config/runtimeConfig.ts:902`: + `setupToken: (env.LAUNCHPAD_SETUP_TOKEN ?? "").trim()` — empty → disabled. + +Compose reads `.env` for **variable substitution** only. A value in `.env` +is injected into a container **only** when an `environment:` entry references +it (e.g. `LAUNCHPAD_SETUP_TOKEN: ${LAUNCHPAD_SETUP_TOKEN:-}`). With no such +entry, the token stays on the host and the container sees nothing. + +### 4.2 Runtime consequence + +- `LAUNCHPAD_SETUP_TOKEN` set in `.env` → app container env unset → + `runtimeConfig.launchpad.setupToken === ""` → launchpad disabled. +- `GET /api/launchpad/status` → `{ initialized: false }` (renders the form). +- `POST /api/launchpad/bootstrap` with the correct token → + `!configuredToken` is true → **403 LAUNCHPAD_INVALID_SETUP_TOKEN**. + +The documented first-install UX cannot succeed. The operator must either +hand-edit the compose (undocumented) or fall back to the `bootstrap-admin` +CLI. The C1.6 deliverable is inert in the bundled deployment. + +### 4.3 Fix (one line + contract awareness) + +In `docker-compose.yml` `app` service `environment:`: + +```yaml +LAUNCHPAD_SETUP_TOKEN: ${LAUNCHPAD_SETUP_TOKEN:-} +``` + +The empty default keeps launchpad disabled for a bare `docker compose up` +(preserves the "not fail-fast at boot" contract in `runtimeConfig.ts`). Then +add a smoke assertion that a token-bearing `.env` yields a working +`/launchpad/bootstrap`. Also add `LAUNCHPAD_SETUP_TOKEN` (commented, with the +`openssl rand -hex 32` guidance) to `.env.example` so the single source of +configuration truth actually lists it — today `.env.example` does not mention +it at all, while the README/runbook/backup-guide do. + +**Check the topology contract** (`scripts/repository-contract/...`) does not +reject new `environment:` keys before/after this change; if it does, extend +the allowlist rather than weakening the guard. + +--- + +## 5. Other config-sync & doc-consistency findings + +**(Required) — `.env.example` is missing `LAUNCHPAD_SETUP_TOKEN` while three +docs reference it.** `.env.example` is the documented single source of +runtime configuration (`AGENTS.md` local-DB-discipline: "copy from +`.env.example`"). Its absence there while the README, runbook, and +backup-guide all instruct setting it is a config-surface desynchronization. +Add it (commented, with entropy guidance), alongside the fix in §4.3. + +**(Nit) — `pg_isready -U exam -d exam` is hardcoded in restore/backup scripts +while the real connection uses `$POSTGRES_USER`/`$POSTGRES_DB`.** +`postgres-logical-restore.sh:83`, `postgres-logical-backup.sh:89`, +`pg-basebackup.sh:85`, and the PITR drills. `pg_isready` does not actually +authenticate (it only checks the postmaster accepts connections), so this is +cosmetic, not functional — but it misleads readers about which user/db the +script targets. Either parameterize (`-U "${POSTGRES_USER:-exam}"`) or add a +comment that the ready-check user is arbitrary. + +**(Nit) — PITR drills hardcode `-U exam -d exam` for source and recovery +probes.** `p7-c3-pitr-drill.sh:72,78,228,240-242` and the failure drill. The +drills set `POSTGRES_PASSWORD` but never `POSTGRES_USER`, so the default +`exam` is correct for the bundled path — but the recovery cluster's +`POSTGRES_USER` is inherited from the base compose (`${POSTGRES_USER:-exam}`) +and would diverge if an operator customized it. Low priority (drills are +throwaway), but worth a note. + +**(Nit) — `archive_timeout` differs between the compose seed (60s), the docs +(§8.2: 60s), and the drills (30s).** Not a correctness issue (both satisfy +"bounded archive window for low-write clusters"), but the drills do not +exercise the documented value. Pick one and align, or document that the drill +uses an aggressive value for speed. + +**(FYI) — `docker-compose.pitr.yml` mounts the postgres bind a second time +identically to the base.** Line 43 (`${EXAM_DATA_ROOT:-./data}/postgres:/var/lib/postgresql`) +duplicates the base `db` volume. Compose merges these (same target path), so +it is harmless, but it is noise. The only additions the override needs are +the WAL archive mount and the initdb seed. Consider dropping the duplicated +postgres line. + +--- + +## 6. Drill-vs-documented-path gap (process finding) + +The five drills are well-constructed and path-isolated, but **none of them +boots the documented operator compose invocation**. The drills: + +- boot `docker compose -p -f docker-compose.yml up -d` (base only); +- enable archiving via `ALTER SYSTEM` + a temp WAL-mount override; +- never apply `-f docker-compose.pitr.yml`. + +This is why §3.1 went undetected. The closeout's "5/5 drills PASS" is +accurate for what the drills test, but the drills test a **different PITR +activation path** than the one operators are told to use. Recommendation: +add a sixth drill (or extend the happy-path drill) that boots the literal +`-f docker-compose.yml -f docker-compose.pitr.yml` on an empty data root and +asserts `archive_mode = on`. This is the single highest-leverage process fix. + +--- + +## 7. Summary table + +| Area | Verdict | Severity | +| --- | --- | --- | +| C1 cold backup/restore PG contract | Sound, simple | — | +| C2 `pg_dump`/`pg_restore` clean-target contract | Sound, well-reasoned | — | +| C3 `pg_basebackup` + `pg_verifybackup` PG contract | Sound | — | +| C3 WAL `archive_command` (non-overwrite idiom) | Correct | — | +| C3 PITR `recovery_target_lsn` + `recovery.signal` | Correct (PG18-verified) | — | +| `pg-basebackup.sh` "no password needed" claim vs TCP/scram | Misleading / will fail without host `PGPASSWORD` | **Required** | +| `wal-archive.conf` mounted into initdb.d is **ignored** (`.conf` extension) | Documented fresh-PITR path silently dead | **Critical** | +| `LAUNCHPAD_SETUP_TOKEN` not forwarded to app container | Documented first-install UX inert | **Critical** | +| `LAUNCHPAD_SETUP_TOKEN` absent from `.env.example` | Config-surface desync | **Required** | +| `PGUSER` default `exam` vs `POSTGRES_USER` | Diverges on customization | Nit | +| `POSTGRES_INITDB_ARGS: ""` dead config | Noise | Nit | +| `pg_isready -U exam -d exam` hardcoded | Cosmetic | Nit | +| `archive_timeout` 60s (docs/compose) vs 30s (drills) | Align or document | Nit | +| Duplicated postgres bind in PITR override | Noise | Nit | +| No drill exercises the documented PITR compose path | Process gap (masks the `.conf` bug) | **Required** | + +--- + +## 8. Required corrective actions + +1. **(Critical)** Rename `docker/pitr/wal-archive.conf` → `.sql` (or `.sh`) + and update the mount in `docker-compose.pitr.yml`. Add a drill that boots + `-f docker-compose.yml -f docker-compose.pitr.yml` on an empty root and + asserts `SHOW archive_mode = on`. (§3) +2. **(Critical)** Add `LAUNCHPAD_SETUP_TOKEN: ${LAUNCHPAD_SETUP_TOKEN:-}` to + the `app` service `environment:` in `docker-compose.yml`; verify the + topology contract still passes. Add a smoke assertion that a token-bearing + `.env` yields a working `/launchpad/bootstrap`. (§4) +3. **(Required)** Add `LAUNCHPAD_SETUP_TOKEN` (commented, entropy guidance) to + `.env.example`. (§5) +4. **(Required)** Fix `pg-basebackup.sh`: either connect over the Unix socket + (so the "trust/no password" claim becomes true) or correct the comment and + require the operator to `export PGPASSWORD`. (§2.2) +5. **(Required)** Add a drill that exercises the documented PITR compose + invocation (closes the drill-vs-doc gap that hid #1). (§6) + +Nits (§5) are optional cleanup; address at author discretion. + +--- + +## 9. Verification notes + +- PostgreSQL behavior (`archive_command`, `restore_command`, `recovery_target_*`, + `recovery.signal`, `pg_basebackup` replication authority, `template0` + restore contract) verified against the PostgreSQL 18 documentation via + Context7 (`/websites/postgresql_18`, `continuous-archiving.html`, + `runtime-config-wal.html`, `app-pgbasebackup.html`, `warm-standby.html`). +- `docker-entrypoint.sh` extension handling verified against the official + `docker-library/postgres` master `docker-entrypoint.sh` source — only + `.sh/.sql/.sql.gz/.sql.xz/.sql.zst` are processed; `*)` logs `ignoring`. +- No runtime mutations were performed. All findings are from source/config + inspection + official-doc cross-check. The two Critical findings are + statically determinable and do not require a live cluster to confirm + (though a live check of `SHOW archive_mode` after the documented compose up + would refute or confirm §3.1 directly). diff --git a/docs/audits/P7-C1-ADVERSARIAL-PORTABLE-DEPLOYMENT-AUDIT.md b/docs/audits/P7-C1-ADVERSARIAL-PORTABLE-DEPLOYMENT-AUDIT.md new file mode 100644 index 00000000..f7564c0f --- /dev/null +++ b/docs/audits/P7-C1-ADVERSARIAL-PORTABLE-DEPLOYMENT-AUDIT.md @@ -0,0 +1,421 @@ +# P7-C1 / PR #273 — Adversarial Reality Audit + +Repository: `jnhu76/exam` +Target PR: [#273](https://github.com/jnhu76/exam/pull/273) +Branch: `feat/p7-c1-portable-single-node-deployment` +Audit date: 2026-08-10 +Audit method: adversarial attack/proof per the P7-C1 audit mission (do not assume PR text, tests, green CI, or comments are correct; falsify the claimed invariants with isolated runtimes; do not fix findings during the pass). + +--- + +## 1. Baseline / PR head / CI state + +| Item | Value | +|---|---| +| PR base SHA | `2a1a9eb30fc40a10d119571d4ad3befb5b52e26e` (`origin/master`) | +| Head at audit start (expected) | `11ff7d05268394e95d4d5dabae2537f975d6c523` | +| Head at audit end (HEAD moved — recorded per mission) | **`3bd2cab781398f055978da6c16276bc951589b47`** | +| Head movement | `11ff7d05 → 5c2b88aa (fix: drill seed-refusal assertion) → 3bd2cab7 (fix: uid-999 PGDATA pack/extract via sudo)` | +| Working tree at audit end | reverted to `master` by an external process during the audit (see §20.1); all product evidence was read via `git show :` and built containers | +| Changed files (base..final head) | 43 | +| Main CI on final head | Static checks ✅ Build ✅ API/Web/Package coverage ✅ E2E shards 1/2 ✅ (all SUCCESS) | +| P7-C1 relocation drill (clean-host) on final head | **A: SUCCESS (all 9 steps), B: SUCCESS (all 10 steps)** — run `31324409704` | +| Drill on intermediate heads | `11ff7d05`: FAILURE (seed-refusal assertion, pipefail bug); `5c2b88aa`: FAILURE (pack tar permission, missing sudo) — both deterministic workflow bugs, fixed in-branch | + +CI state note: the clean-host workflow is **red on the two earlier heads and green only on the final head**. The final head's green run is the only clean-host evidence; it is real and passed every step (see §12). + +--- + +## 2. Scope and non-goals + +Audited: C1.1 canonical data root, C1.2 image-only contract, C1.3 migration preflight, C1.4 relocation drills, C1.5 Redis non-authority, C1.6 launchpad, C1.7 docs/roadmap, plus P6 regression preservation. Not in scope (not implemented, correctly absent): C2 logical backup, C3 historical restore, PITR, retention, P7-E settings, Admin backup UI, Desktop/offline-first, HA, Kubernetes, generic job system. No production code was modified during the audit (temporary mutations were restored; §26). + +## 3. Executive verdict + +**DO NOT MERGE** — two P1 findings (A/B-class) must be corrected first: + +1. **P1-1 — FRESH_INSTALL oracle is false for untracked non-fresh databases** (HYP-1 proven at runtime). Any DB with business tables/data but a missing or empty `drizzle.__drizzle_migrations` journal is classified `FRESH_INSTALL` (exit 0), and the subsequent migrate **silently applies nothing** (`42P07` from migration `0000`'s bare `CREATE TABLE` is swallowed by `migratePostgres`'s concurrent-worker tolerance), leaving the DB permanently untracked — every restart re-reports `FRESH_INSTALL` and future image upgrades will silently never apply (the exact C0 P2-1 "silently misbehaves" class the preflight was built to close). +2. **P1-2 — Relocation drill / Redis proof inherit `EXAM_DATA_ROOT` from the environment and can seed a real deployment** (demonstrated at runtime). The p6 smoke sets its own isolated root; the two C1 scripts do not. With `EXAM_DATA_ROOT` exported (the deployment docs' own layout), `pnpm drill:p7-c1-relocation` boots compose A against `${EXAM_DATA_ROOT}/postgres` and the entrypoint's `RUN_SEED=e2e` seed runs against the real deployment DB (default-credential accounts + demo data) before the drill's `FRESH_INSTALL` assertion fails. + +A P2 finding (launchpad unusable through the documented ordinary path; see P2-1) and several P3s are also recorded. Clean-host relocation, image-only deployment, Redis non-authority, and first-Admin single-winner concurrency are otherwise proven (details below). + +## 4. Claimed invariants — verdicts + +| Invariant | Verdict | +|---|---| +| INV-C1-1 Canonical durable root = `${EXAM_DATA_ROOT:-./data}/postgres` | PARTIAL — bind-mount design and contract guards hold; two drill scripts can redirect the root via inherited env (P1-2) | +| INV-C1-2 Ordinary relocation recovers same state | PROVEN (local clean-root drill + clean-host CI run, byte-identical invariants + admin login) | +| INV-C1-3 Image authority | PROVEN (image-only compose, `EXAM_IMAGE:?` required, guards, identity layers documented) | +| INV-C1-4 Migration compatibility (fresh/normal/upgrade proceed; stale/divergent refuse) | FAILED for the FRESH_INSTALL branch (P1-1); NORMAL/FORWARD_UPGRADE/STALE/DIVERGENT branches verified correct on the real journal | +| INV-C1-5 Redis non-authority | PROVEN at runtime (proof script, Redis enabled, counters reset, business state identical) | +| INV-C1-6 Launchpad = first-install handoff, exactly one first Admin | PARTIAL — single-winner authority proven (incl. HTTP-vs-CLI); the launchpad itself is inert in the bundled compose (token never forwarded) | +| INV-C1-7 Launchpad never reopens | PARTIAL — holds for all supported operations (last-Admin removal/disable verified); reopens after hard-deleting org+users (operator corruption only) | + +## 5. Canonical-data-root audit (Attack Area A) + +Compose facts (SOURCE_FACT): production topology has zero named volumes; db binds `${EXAM_DATA_ROOT:-./data}/postgres`, redis binds `${EXAM_DATA_ROOT:-./data}/redis`; the topology contract refuses named volumes and missing binds (mutation tests M7/M8). + +Script isolation matrix: + +| Script | Overrides EXAM_DATA_ROOT? | Verdict | +|---|---|---| +| `p6-corr1-compose-smoke.sh` | Yes — sets `${REPO_ROOT}/.tmp-p6-smoke-data--` with strict prefix guard + export | SAFE | +| `p7-c1-relocation-drill.sh` | **No** (exports only POSTGRES_PASSWORD/JWT_SECRET/CORS_ORIGIN/PUBLIC_WEB_ORIGIN) | HAZARD (P1-2) | +| `p7-c1-redis-nonauthority-proof.sh` | **No** (same export set) | HAZARD (P1-2) | +| `.github/workflows/p7-c1-relocation.yml` | No, but runners have no EXAM_DATA_ROOT → default `./data` under RUNNER_TEMP project dirs | SAFE | + +Demonstration (RUNTIME_PROOF): + +```bash +# with EXAM_DATA_ROOT inherited: +EXAM_DATA_ROOT=/opt/real-deploy/data docker compose -f docker-compose.yml config +# → volumes: - type: bind source: /opt/real-deploy/data/postgres +# unset: +docker compose -f docker-compose.yml config +# → source: /tmp/demo-drill/data/postgres +``` + +Attack scenario (static reasoning, non-destructive): operator exports `EXAM_DATA_ROOT=/opt/real-deploy/data` (the documented deployment layout) and runs `pnpm drill:p7-c1-relocation` from that shell. Drill A copies compose + writes `.env` (no EXAM_DATA_ROOT), boots with the seed override; compose resolves the bind to the **real** root; the real postgres starts on its existing PGDATA; entrypoint preflight classifies NORMAL (proceeds), migrate is a no-op, then `RUN_SEED=e2e` runs the canonical E2E seed **against the real deployment DB** (baseline admin/admin123 + candidate accounts + demo courses/exams/attempts). Only afterwards does the drill's `FRESH_INSTALL` assertion fail. Cleanup removes only the drill's temp base. Result: real deployment polluted with demo data and default-credential accounts; drill fails confusingly. The compose comment claims "smoke/drills override it to a unique temp dir for isolation" — false for these two scripts. + +Dangerous-values check: `EXAM_DATA_ROOT=""` → `${VAR:-./data}` fallback applies (compose semantics), `EXAM_DATA_ROOT=/` and `./`-relative values are not validated by any script; `rm -rf` in both scripts is guarded by strict `.tmp-p7c1-` / `.tmp-p6-smoke-data-` prefix checks — no unvalidated `rm -rf`, no `docker system prune`. Empty-value handling: UNSAFE_BUT_OPERATOR_OWNED (the scripts' own cleanup is guarded; the hazard is the inherited-root bind). + +## 6. Image/version authority audit (Attack Areas B, C) + +Mutation matrix (all mutations restored): + +| # | Mutation | Topology contract | Image/version contract | Caught? | +|---|---|---|---|---| +| M1 | `app` gains `build: .` | FAIL | FAIL | ✅ | +| M2 | `email-worker` gains `build: .` | FAIL | FAIL | ✅ | +| M3 | `EXAM_IMAGE:?...` → literal tag | FAIL | PASS | ✅ | +| M4 | postgres `18.4-bookworm` → `19.4-bookworm` | PASS | **PASS** | ❌ **no static oracle** | +| M5 | postgres → `latest` | PASS | FAIL | ✅ | +| M6 | redis `7.4.10-alpine` → `7-alpine` | PASS | FAIL | ✅ | +| M7 | db → named volume `pgdata` | FAIL | PASS | ✅ | +| M8 | db bind removed | FAIL | PASS | ✅ | + +Identity semantics (SOURCE_FACT): docs distinguish OCI version label (metadata) / OCI revision (source provenance) / `EXAM_IMAGE` reference (the actual identity), recommend digest pinning for relocation, and the drill records `RepoDigests`. Docs never equate a mutable tag with immutable identity. PASS. + +HYP-6 (PG-major drift): **TRUE** (mutation M4). `EXPECTED_PG_MAJOR = 18` in `apps/api/src/scripts/preflight.ts` and the compose `postgres:18.4-bookworm` are separate authorities with no CI/static synchronization oracle. The runtime preflight fails closed (`SHOW server_version_num` major ≠ 18 → refuse), so drift is survivable but only discovered at boot. Redis: exact-patch pin enforced by the image/version contract (M6 caught). P3 finding. + +## 7. Migration-preflight adversarial audit (Attack Areas D, E) + +### 7.1 FRESH_INSTALL oracle (D1–D4) — HYP-1: PROVEN TRUE + +`runPreflight()` sets `isFreshInstall = true` when `to_regclass('drizzle.__drizzle_migrations')` is NULL **or** the journal has 0 rows — with no check on business-schema presence. Isolated PostgreSQL states (all via docker exec against throwaway DBs; never touched dev/prod data): + +| State | Constructed as | Preflight outcome | migrate.js outcome | +|---|---|---|---| +| D1 truly fresh (no tables, no journal) | new DB | FRESH_INSTALL ✅ | applies 29 migrations ✅ | +| D2 business schema + data, journal dropped | migrate fully, `DROP SCHEMA drizzle CASCADE` | **FRESH_INSTALL (WRONG)** | **silent no-op: exit 0, "Migrations complete.", journal 0 rows** | +| D3 journal exists but empty + business data | same DB after D2 | **FRESH_INSTALL (WRONG)** | silent no-op | +| D4 partial restore (organizations/users/exam_attempts only, no journal) | new DB + 3 tables | **FRESH_INSTALL (WRONG)** | silent no-op (0 journal rows) | + +Root cause (RUNTIME_PROOF + SOURCE_FACT): migration `0000_cultured_fantastic_four.sql` uses bare `CREATE TABLE` (no `IF NOT EXISTS`). On a tables-present DB, drizzle's per-file transaction throws `42P07 duplicate_table`; `migratePostgres` swallows `42P07` as "concurrent worker already applied" (`isDuplicateTableDuringMigration`); `migrate.js` prints "Migrations complete." and exits 0; the journal stays empty forever. + +Consequence chain (first link runtime-proven, later links mechanical): every restart re-reports FRESH_INSTALL; a future image with new migrations will fail at 0000 again, so **new migrations never apply** while the app keeps starting — the silent-schema-drift hazard (C0 P2-1) the preflight was created to close. The docs claim ("an incompatible DB/image combination refuses to start instead of being silently mutated") is overstated: this dangerous state is approved and then silently NOT migrated. Classified: P1 (matches the P1 example "supported startup incorrectly classifies dangerous DB as fresh and mutates it" — here the mutation is a silent non-tracking no-op that permanently defeats future upgrades). + +How D2-D4 states can arise: operator mistake, journal corruption, partial/dirty restore, old/manual deployment. The preflight's stated purpose is to refuse dangerous states, so these are in scope even though abnormal. + +### 7.2 Membership/frontier algorithm (Attack E) — 13-case matrix on the REAL journal + +Ran `classifyMigrationCompatibility` against the real bundled journal (29 migrations; verified backward `when`s for 0022/0024; no duplicate `when`s; allowlist tags match real tags — note the synthetic unit-test tag `0027_convergence` does NOT exist; the real tag is `0027_converge_skipped_migrations`): + +| # | Case | Outcome | Expected | +|---|---|---|---| +| 1 | fully current DB | NORMAL | ✅ | +| 2 | forward upgrade (last missing) | FORWARD_UPGRADE | ✅ | +| 3 | DB ahead (future row) | STALE_IMAGE_DB_AHEAD | ✅ | +| 4 | hash mismatch at max when | DIVERGENT | ✅ | +| 5 | unknown older row | DIVERGENT | ✅ | +| 6 | missing non-allowlisted below frontier (0001) | DIVERGENT | ✅ | +| 7 | missing 0004 (allowlisted) | NORMAL | ✅ | +| 8 | missing 0022 (allowlisted) | NORMAL | ✅ | +| 9 | missing 0024 (allowlisted) | NORMAL | ✅ | +| 10 | historical omissions, converged (missing only 0004/0022/0024) | NORMAL | ✅ | +| 11 | converged after 0027 | NORMAL | ✅ | +| 12 | 0027 absent but 0028 present | **DIVERGENT** | ✅ | +| 13 | forged combo (0022 + 0003 missing) | DIVERGENT | ✅ | + +Key adversarial question answered: allowlist holes are tolerated **only when the convergence migration (0027) is present** — a DB missing 0027 is DIVERGENT (case 12), so the "repair evidence" requirement holds implicitly. Docs do not claim schema-effect integrity (they describe journal-history compatibility), so no overclaim here. PASS. + +## 8. Historical-migration exception audit (part of Attack E) + +The `HISTORICAL_OMISSION_TAGS = {0004_wide_phantom_reporter, 0022_engine_policy_seam, 0024_breezy_tigra}` allowlist matches the real journal tags (verified). 0022/0024 `when`s genuinely predate 0021/0023 (real journal: 0022=1785253697471 < 0021=1787200000000; 0024=1785621462155 < 0023=1787600000000), so drizzle would skip them on converged DBs — the exception is justified by journal mechanics, not asserted by comment. Convergence evidence (0027) is implicitly required (case 12). PASS. + +## 9. Launchpad authority audit (Attack Areas H, J, K, M, N) + +### 9.1 Two command bodies (HYP-2): CONFIRMED — duplication without demonstrated divergence + +`bootstrapInitialAdmin` (CLI adapter) and `bootstrapInitialAdminWithLock` (HTTP) are two ~60-line copies of the irreversible mutation (org resolve/create → first Admin → primary assignment → `admin.bootstrap` audit). Differences: the CLI path checks "active Admin count" (refuses unless `--force`); the HTTP path checks org/user existence under an advisory lock and never re-checks after its org INSERT resolves a conflict. Runtime concurrency evidence (§10) shows no correctness divergence under the supported adapters; the duplication is an authority hazard for future edits. P3 (per severity model: duplication without demonstrated divergence). + +### 9.2 Crash atomicity (Attack M) + +SOURCE_FACT: org → user → assignment → audit run in ONE `executeInTransaction` (default **repeatable read** + serialization-failure retry, `packages/db/src/types.ts`); the advisory lock is transaction-scoped (auto-release on commit/rollback). Any failure rolls back all four writes; no orphan states. The concurrency runs also demonstrate the loser leaves no partial state (single org/user/assignment/audit). Not fault-injected (would require code instrumentation); classified structurally proven. + +### 9.3 Permanent completion (Attack J) — HYP-5: PARTIALLY FALSE + +Runtime on a completed install: +- delete all users (org remains) → `/status` = COMPLETED; POST with correct token → 409. ✅ (also covered by `launchpad.test.ts`) +- **hard-delete org AND users (manual SQL) → `/status` = READY — launchpad reopens and bootstraps a new first Admin (runtime-proven: 201 after reopening).** + +`isInstallationFresh` derives "ever existed" from **current** org/user existence (`SELECT 1 ... LIMIT 1` on each table). The docs' "permanently COMPLETED once any org/user has ever existed" is stronger than the persisted evidence. Reachable via supported runtime: NO (the only supported hard-delete is `DELETE /users/:id`; no organization delete endpoint exists; org persists → COMPLETED holds). Reachable via manual DB manipulation / future historical-restore interactions: yes. Classified: OPERATOR_CORRUPTION / FUTURE_INTERACTION → P3 (docs wording). + +### 9.4 Setup-token oracle after completion (Attack K) — HYP-4: PROVEN TRUE + +On a completed installation (`{"state":"COMPLETED"}`): + +```text +POST /api/launchpad/bootstrap wrong token → 403 LAUNCHPAD_SETUP_TOKEN_INVALID +POST /api/launchpad/bootstrap correct token → 409 LAUNCHPAD_ALREADY_COMPLETED +``` + +The route validates the token **before** the freshness check, so a completed installation discloses whether a guessed deployment secret is correct (distinct status codes; timing not meaningfully distinguishable over loopback). Token comparison is constant-time (`tokensMatch`), the token never enters logs/audit/OpenAPI (only the schema property `setupToken` exists in openapi.json, no example literal) or the frontend URL (body-only, component state only). Practical impact: the setup token is high-entropy and re-bootstrap is impossible, so the leak is not exploitable without a weak token — but the safe property ("completed installation should not validate/disclose deployment-secret correctness") is violated. Severity: P3 (cheap fix: freshness check before token comparison). + +### 9.5 Rate limiting / Redis-required (Attack L) + +- Route-level limit `{max: 10, timeWindow: 60s}` verified with Redis **disabled**: burst of 12 POSTs → 429s after the budget (in-memory store). ✅ +- `REDIS_MODE=required` with unavailable Redis: the rate-limit plugin fails closed (`503 RATE_LIMIT_UNAVAILABLE`, DelegatingRateLimitStore) — a fresh install in that mode cannot reach bootstrap until Redis is healthy; this is the declared P7 contract (fail-closed), not a C1 regression. NOT_PROVEN by a live run; declared behavior per `plugins/rateLimit.ts` (SOURCE_FACT). +- `GET /api/launchpad/status`: two `LIMIT 1` probes per call, no amplification → P3 non-issue (recorded, no action). + +### 9.6 Launchpad UI is not authority (Attack N) + +`LaunchpadPage.tsx`: freshness from `GET /status` (server-authoritative), no role/organizationId fields, no auto-login (navigates to /login after success), token in body only, not persisted. `/register` → `403 AUTH_REGISTER_DISABLED` (runtime-proven). LoginPage does not advertise registration. PASS. + +## 10. Launchpad concurrency matrix (Attack I) + +| Pair | Deterministic seam | Result (4 runs) | Verdict | +|---|---|---|---| +| HTTP vs HTTP (different usernames) | unit test (`launchpad.test.ts` P2-5) | one 201 + one 409, exactly 1 user/org/audit | PROVEN | +| HTTP vs CLI | fresh DB + `BEFORE INSERT` trigger with `pg_sleep` on `organizations` as a deterministic barrier (polled `pg_stat_activity` for the in-flight INSERT before firing the HTTP request) | exactly one winner in every run; loser gets a coherent refusal (CLI: "An active Admin already exists"; HTTP: 409 ALREADY_COMPLETED or 201) | PROVEN — **HYP-3 FALSIFIED** | +| CLI vs CLI | same mechanism | single winner | PROVEN by mechanism (see below) | + +Mechanism (SOURCE_FACT + RUNTIME_PROOF): the advisory lock is NOT what serializes cross-adapter races — `bootstrapInitialAdminWithLock` takes it, the CLI does not. The actual serialization is `executeInTransaction`'s **repeatable-read** isolation + the **`ON CONFLICT DO UPDATE` on `organizations.slug`**: a concurrent org INSERT against a row committed after the snapshot raises `40001 serialization_failure`; the retry loop re-runs the body, whose fresh/count check then observes the committed first Admin and refuses. Both directions were observed (HTTP wins → CLI refuses; CLI in-flight org INSERT → HTTP still wins the insert race → CLI refuses), always exactly one first Admin, never two. Note the invariant therefore rests on RR+retry semantics, not on the advertised "one lock" — worth a comment/test, but the property holds. + +## 11. Launchpad secret-boundary audit + +- `LAUNCHPAD_SETUP_TOKEN` never logged (no logging statement), never in audit metadata (audit stores username/name/source only), never in OpenAPI examples, never in frontend URL/query (body-only), not persisted by the page. PASS. +- **P2-1 (runtime-proven): the token never reaches the app container.** `docker-compose.yml` has no `LAUNCHPAD_SETUP_TOKEN` in any `environment:` block and no `env_file:`; the image has no `/app/.env`. Docs (portable guide §6/§7, runbook §5/§7, `.env.example`) instruct setting it **in `.env`** — Compose uses `.env` for interpolation only, so the container sees nothing. Verified: with the token in `.env`, container `LAUNCHPAD_SETUP_TOKEN=[UNSET]`, `/status` = OPERATOR_ACTIVATION_REQUIRED, POST = 403 LAUNCHPAD_SETUP_REQUIRED. The documented recommended first-install path cannot work; the operator must hand-edit the compose (undocumented) or use the CLI. P2 (material C1.6 readiness blocker; the launchpad deliverable is inert in the bundled deployment). + +## 12. Relocation proof audit (Attacks O, P) + +### 12.1 Clean-host (CI) + +Final head `3bd2cab7` run `31324409704`: Job A steps 1–9 all success (build → boot fresh A with seed override → FRESH_INSTALL + seed-refusal assertions → record invariants → `down` (data preserved) → pack via **sudo tar** → upload); Job B steps 1–10 all success (download → extract via **sudo tar** → `docker load` → `compose pull db` → boot B ordinary path → preflight NORMAL, no seed ran → **invariants byte-identical** (migration count + per-table counts/md5) → seeded admin login works → teardown). Clean-host relocation: **PROVEN** at the final head. + +Workflow history (important): the same workflow FAILED on `11ff7d05` (seed-refusal assertion under `set -euo pipefail` — the guard throws by design so `docker exec | grep -q` fails even when grep matches; reproduced locally) and on `5c2b88aa` (pack tar without `sudo`: PGDATA is 700/uid-999, runner uid 1000 → `Permission denied`; reproduced from the log). Both were fixed in-branch (`5c2b88aa`, `3bd2cab7`). At the audited HEAD the proof was therefore NOT_PROVEN; at the final head it is PROVEN. The two bugs are resolved but demonstrate the drill's fragility. + +### 12.2 Clean-host without checkout (HYP-7) + +Job B's checkout step is unused by the deployment: all B steps after download operate on `${RUNNER_TEMP}/bundle` + the extracted dir; no step reads `${GITHUB_WORKSPACE}`. The passing run on a fresh runner with only the bundle proves the deployment does not depend on checkout (deployment dependency: NONE; the checkout is verification-tooling-only in name and unused in practice). HYP-7: FALSIFIED (deployment independent of checkout). + +### 12.3 Transport metadata (Attack P) + +Pack: `sudo tar -czf` (root) on runner A; extract: `sudo tar -xzf` (root) on runner B → uid/gid (999), modes (700), and directory layout are preserved; the relocated postgres (uid 999) can read PGDATA — evidenced by B's successful boot + identical md5s (which also exercise the app, not just file presence). One passing run is evidence, not a universal guarantee — supported-host assumption documented: GitHub-hosted ubuntu runners with sudo + `docker compose pull db` for the pinned image. `docker save`/`load` transports the exact image bytes (tag + RepoDigests recorded in A). + +## 13. Redis non-authority proof (Attack R) + +`scripts/deployment/p7-c1-redis-nonauthority-proof.sh` executed end-to-end on the audited code (from a pristine /tmp export): **ALL CHECKS PASSED** — Redis profile ON with authenticated counters, business state captured (counts/md5), shutdown, relocation WITHOUT Redis state, restart with Redis ON, rate-limit counters reset, business state byte-identical, seeded admin login OK on first attempt. Redis non-authority: PROVEN. (Stale-Redis restoration was not separately executed; conclusion is limited to the proven reset behavior — labeled honestly.) + +## 14. Regression-guard mutation tests (Attack T) + +- Static guards: 7 of 8 mutations caught (§6); M4 (PG 18→19) not caught statically — P3. +- `verify:static` wiring: `verify-static-includes-guards.mjs` asserts `lint:repo-contract` ∈ `verify:static`; `package.json` chains both contract scripts into `lint:repo-contract`. Verified. +- Unit tests: `preflight.test.ts` (12 cases), `launchpad.test.ts` (8 + concurrency). CI API coverage SUCCESS on final head. +- Clean-host workflow triggers: `workflow_dispatch` + push to `feat/p7-c1-portable-single-node-deployment` only — **after merge the clean-host proof never runs automatically (HYP-8 TRUE)**; the permanent gate is the local clean-root drill (`pnpm drill:p7-c1-relocation`, manual) + static contracts. Classified: P3 coverage debt (deliberate, documented; but note the workflow required 3 commits to pass, so its "manual drill" value depends on it being run — recommend at least a scheduled/manual-on-master trigger). + +## 15. Existing P6 deployment regression check (Attack S) + +`p6-corr1-compose-smoke.sh` re-run against the PR-head compose (isolated temp root, own image build; local run needed `APP_PORT=3901` because the developer's `pnpm dev` holds host port 3000): **RUN #2: ALL CHECKS PASSED** — POSTGRES_PASSWORD required-expansion; Redis optional at parse; redis profile authenticated (requirepass, healthcheck PONG, startup guard without password fails); db→app→email-worker ordering; migration-once; worker heartbeat; bootstrap-admin single explicit Admin; login; no default Candidate accounts; production seed refusal; Redis absence does not block startup. Old P6 invariants: **PRESERVED**. + +## 16. Documentation / operator usability + +Wording bans: no "relocation == backup" (portable guide: "Do not use the relocation procedure as a backup/restore procedure"); no "raw PGDATA == version-independent backup" (PG-major caveat stated); no "Redis == authority" (both guides mark Redis non-authoritative); no "Launchpad == registration" (`/register` stays 403, launchpad first-install only); no "mutable tag == immutable identity" (§4 distinguishes the layers). Operator Q&A: data location ✅; down/down -v semantics ✅ (down preserves, down -v no-op on data, `rm -rf ${EXAM_DATA_ROOT}` destructive); relocation procedure ✅ (stop stack first — "PG stopped or fully flushed"); image identity ✅; Redis loss ✅; "./data copying ≠ historical backup" ✅ ("does not go back in time"); "recover yesterday's state" — no (C2/C3 not implemented) ✅; PITR — no ✅; first Admin ✅; setup-token holder ✅; launchpad after Admin removal ✅; preserve besides ./data — `.env` (compose + secrets) ✅. + +Doc errors found: +- `portable-deployment.md` §5: "surfaced in `/api/system/diagnostics` as `preflightBypassed: true`" — **no such field exists in any code** (grep across apps/packages: only the WARN log line in `preflight.ts`). P3. +- "smoke/drills override it [EXAM_DATA_ROOT] to a unique temp dir for isolation" (docker-compose.yml comment) — false for the C1 drill and Redis proof (P1-2 companion). +- "permanently COMPLETED once any org/user has ever existed" — overstated vs. current-existence semantics (P3, §9.3). + +## 17. Scope-leak check + +No `pg_dump`/`pg_restore`/`pg_basebackup`/`archive_mode`/`archive_command`/PITR/retention/scheduler/backup artifact/restore command in runtime code (grep of apps/packages/compose/Dockerfile/entrypoint; hits are pre-existing "swallow/restore-command" attempt-recovery code and comments). Docs mention C2–C7/PITR strictly as future phases. No generic startup reconciler, job queue, Admin restore button, DB-backed backup settings, or Desktop code entered C1. **C2+ scope leak: NONE.** + +## 18. Findings + +### P0 +None. + +### P1 + +**P1-1 — FRESH_INSTALL oracle false for untracked non-fresh databases (HYP-1).** +- Invariant violated: INV-C1-4 (dangerous non-fresh DB must not be classified FRESH_INSTALL). +- Exact code: `apps/api/src/scripts/preflight.ts` (`isFreshInstall = table absent || 0 rows`); `packages/db/src/postgres.ts` (`isDuplicateTableDuringMigration` swallows 42P07); `packages/db/migrations/postgres/0000_*.sql` (bare `CREATE TABLE`). +- Reproduction: migrate a DB fully, `DROP SCHEMA drizzle CASCADE` (or truncate the journal; or create a 3-table subset) → run preflight → FRESH_INSTALL (exit 0) → run `node dist/scripts/migrate.js` → "Migrations complete." exit 0, journal 0 rows. +- Observed: approval of a non-fresh DB; silent no-op migration; app starts; every restart re-reports FRESH_INSTALL; future image upgrades will silently never apply. +- Expected: refusal (or a distinct "UNTRACKED_DB" outcome) for any DB with business-schema evidence and no journal. +- Impact: defeats the preflight's stated safety purpose (C0 P2-1); permanent untracked state + silent future upgrade drift. +- Smallest acceptable correction boundary: fresh-install oracle must include business-schema evidence (e.g., any known business relation present ⇒ not FRESH_INSTALL); classify as refuse until an operator decision (docs must not claim "refuses instead of being silently mutated" for this class). +- Required regression test: preflight integration test over D2/D3/D4 states asserting non-FRESH refusal; migrate test asserting a journal-less tables-present DB is not silently "complete". + +**P1-2 — C1 drill / Redis proof inherit `EXAM_DATA_ROOT` and can seed a real deployment.** +- Invariant violated: INV-C1-1 (no authoritative state may secretly depend on a host-global root) / shared-test-data hazard. +- Exact code: `scripts/deployment/p7-c1-relocation-drill.sh` and `scripts/deployment/p7-c1-redis-nonauthority-proof.sh` (no EXAM_DATA_ROOT handling; compose uses `${EXAM_DATA_ROOT:-./data}`). +- Reproduction: `EXAM_DATA_ROOT=/opt/real-deploy/data bash scripts/deployment/p7-c1-relocation-drill.sh` → compose resolves db bind to `/opt/real-deploy/data/postgres` (demonstrated via `compose config`); drill A's entrypoint then runs preflight (NORMAL) + migrate + RUN_SEED=e2e seed against that DB before the FRESH_INSTALL assertion fails. +- Observed: real-root bind resolution (runtime-demonstrated); seed-before-assert ordering (code). +- Expected: drill forces its own isolated root (like the p6 smoke) or refuses to run when EXAM_DATA_ROOT is set to a non-temp path. +- Impact: default-credential accounts (admin/admin123) + demo data injected into a real deployment by a documented verification command; drill result meaningless. +- Smallest acceptable correction boundary: `EXAM_DATA_ROOT="$(mktemp -d ...)"` with a strict prefix guard (mirror the smoke) or an explicit `unset EXAM_DATA_ROOT` + assertion it is unset. +- Required regression test: drill run with EXAM_DATA_ROOT exported must fail fast (or isolate) and must never write outside its temp root. + +### P2 + +**P2-1 — Launchpad unusable via the documented ordinary path (token never forwarded).** `docker-compose.yml` omits `LAUNCHPAD_SETUP_TOKEN` from the app environment (no `env_file:`); docs/`.env.example` say "set it in .env". Runtime: token in `.env` → container env unset → status OPERATOR_ACTIVATION_REQUIRED → POST 403 LAUNCHPAD_SETUP_REQUIRED. The C1.6 headline first-install path is inert in the bundled deployment; operator must hand-edit compose (undocumented) or use the CLI. Fix: forward `${LAUNCHPAD_SETUP_TOKEN:-}` in the app service environment (guarded by the topology contract), or document a required override file; add a smoke assertion that a token-bearing `.env` yields READY. + +**P2-2 — Clean-host proof is historical-only after merge (HYP-8).** Workflow triggers: `workflow_dispatch` + the feature branch. Post-merge there is no automatic clean-host gate; a later PR can break relocation while ordinary CI stays green. Combined with the workflow's demonstrated fragility (3 commits to pass), the "permanent guard" claim should be re-scoped: either accept as documented manual drill (P3) or add a schedule/manual-master trigger. Recorded as P2 for the PR's "permanently guarded" wording; smallest fix is a docs statement + optional scheduled trigger. + +### P3 + +- **P3-1** PG-major drift (HYP-6): compose tag ↔ `EXPECTED_PG_MAJOR` have no static oracle; runtime preflight fails closed (mutation M4). +- **P3-2** Docs claim `/api/system/diagnostics` surfaces `preflightBypassed` — field does not exist (only a log WARN). +- **P3-3** Launchpad token oracle (HYP-4): completed install distinguishes correct/wrong setup token (403 vs 409); constant-time compare, no logging; impact limited to weak tokens — fix by checking freshness before token validation. +- **P3-4** "Permanently completed once ever existed" (HYP-5) overstated: reopen after hard-deleting org+users (operator corruption; no supported path) — align docs wording with current-existence semantics. +- **P3-5** Duplicated bootstrap command bodies (HYP-2): no demonstrated divergence, but a maintainability/authority hazard. +- **P3-6** Clean-host gate is a manual drill post-merge (see P2-2 for the re-scoped classification if accepted as documented). +- **P3-7** `/api/launchpad/status` unauthenticated DB probes: two LIMIT-1 queries, no amplification — recorded, no action. + +## 19. Unknowns / not-proven items + +- Attack M crash injection (kill mid-transaction) not executed — structural proof only; the retry/409 behavior under real concurrency is runtime-verified. +- Stale-Redis restoration (restore old counters, observe over/under-limit) not executed — only the reset direction is proven. +- P1-2's full drill-run-against-real-root was NOT executed (deliberately non-destructive); the bind resolution is runtime-demonstrated and the seed-before-assert ordering is code-verified. +- D2-state "future upgrade silently never applies" is a mechanical inference from the runtime-proven silent no-op + the migrator loop semantics. +- `REDIS_MODE=required` + unavailable Redis at bootstrap: classified from the declared fail-closed store behavior, not a live run. + +## 20. Exact commands and runtime evidence + +All experiments ran on a local Docker engine with an image built from the audited tree (`exam-p7c1-probe:latest`), isolated temp data roots, and throwaway DB names (`p7c1_audit_d2/d4`, `p7c1_race3_db`, etc.). The developer's `exam`/`exam_test`/`exam_e2e` databases were never touched (the `exam` dev project running on port 3000 was left untouched; the p6 smoke was re-run with `APP_PORT=3901` for that reason). + +Key evidence lines (see report body for full command context): +- `git rev-parse HEAD` before/after: `11ff7d05…` → `3bd2cab7…` (PR head moved; recorded). +- `gh pr view 273 --json statusCheckRollup`: final head all SUCCESS incl. "A — build + seed + pack" and "B — consume bundle + verify". +- `gh run view 31324409704 --json jobs`: A/B step-by-step SUCCESS (9/10 steps). +- Reproduction of CI seed-refusal failure at `11ff7d05`: `docker exec -e APP_MODE=production … node dist/seed.js 2>&1 | grep -q …` → pipeline exit 1 while `grep -c` finds the message (pipefail + expected non-zero exit). +- D2/D3/D4: preflight output `{"preflight":"FRESH_INSTALL",…}` on all three; `migrate.js` "Migrations complete." + `SELECT count(*) FROM drizzle.__drizzle_migrations` → `0`. +- `EXAM_DATA_ROOT=/opt/real-deploy/data docker compose config` → `source: /opt/real-deploy/data/postgres`. +- HYP-4: POST wrong → `403 LAUNCHPAD_SETUP_TOKEN_INVALID`; POST correct → `409 LAUNCHPAD_ALREADY_COMPLETED` (completed install). +- HYP-3: 4 concurrent HTTP-vs-CLI runs, always exactly 1 user/1 org/1 assignment/1 audit. +- HYP-5 hard case: `DELETE FROM organizations; DELETE FROM users;` → `/status` READY. +- P2-1: `.env` token set → `docker exec … printenv LAUNCHPAD_SETUP_TOKEN` → unset; status OPERATOR_ACTIVATION_REQUIRED; POST 403. +- `pnpm proof:p7-c1-redis-nonauthority` → ALL CHECKS PASSED; local drill → ALL CHECKS PASSED (clean-root); p6 smoke → ALL CHECKS PASSED (run #2). +- Mutation matrix §6 (all mutations restored; `git status` clean apart from pre-existing untracked logs). + +### 20.1 Environment note +During the audit the repository working tree was reset to `master` twice by an external process (a concurrent local run of the drill/CI tooling left `.tmp-ci-fail*.log` in the tree). All evidence above is anchored to commit SHAs (`git show`), built images, and isolated containers — the tree resets do not affect the findings. The audit report itself is a new untracked file. + +## 21. Merge recommendation + +**DO NOT MERGE** until P1-1 and P1-2 are corrected with their required regression tests (P2-1's compose forwarding is a one-line fix that should ride along; P2-2 wording re-scope recommended). Everything else audited is proven or acceptable debt. + +--- + +## 22. Required corrective actions (for the author, after human review) + +1. **P1-1**: extend the fresh-install oracle with business-schema evidence; refuse untracked non-fresh DBs (new outcome or DIVERGENT); regression test D2/D3/D4. Optionally make the `42P07` swallow distinguish "concurrent worker" from "untracked DB" (fail loudly instead of "Migrations complete."). +2. **P1-2**: force an isolated EXAM_DATA_ROOT in both C1 scripts (strict temp prefix, mirror the p6 smoke) or refuse when a non-temp EXAM_DATA_ROOT is inherited; regression test with the env exported. +3. **P2-1**: forward `${LAUNCHPAD_SETUP_TOKEN:-}` in the app service environment (or documented override); smoke assert READY with token set. +4. **P3 fast-follow**: fix the `preflightBypassed` diagnostics doc claim (implement or reword); align "permanently COMPLETED" wording; consider freshness-before-token ordering on POST /bootstrap; document the clean-host gate's post-merge trigger semantics. + +--- + +## Experiment matrix (required form) + +| Attack | Expected result | Observed result | Evidence | Verdict | Severity | +|---|---|---|---|---|---| +| fresh DB | FRESH_INSTALL | FRESH_INSTALL, 29 migrations applied | drill A / D1 run | PASS | — | +| non-fresh DB without journal | refuse | **FRESH_INSTALL (exit 0)** | D2 preflight | **FAIL** | P1 | +| empty journal + business tables | refuse | **FRESH_INSTALL** | D3 preflight | **FAIL** | P1 | +| healthy current DB | NORMAL | NORMAL (29/29, frontier match) | audit-a preflight | PASS | — | +| forward upgrade | FORWARD_UPGRADE | FORWARD_UPGRADE | real-journal matrix #2 | PASS | — | +| DB ahead | STALE_IMAGE_DB_AHEAD | STALE_IMAGE_DB_AHEAD | matrix #3 | PASS | — | +| hash divergence | DIVERGENT | DIVERGENT | matrix #4 | PASS | — | +| historical 0022/0024 omission | NORMAL | NORMAL | matrix #8/9/10 | PASS | — | +| bad historical omission combo | DIVERGENT | DIVERGENT | matrix #6/13 | PASS | — | +| 0027 absent + 0028 present | DIVERGENT | DIVERGENT | matrix #12 | PASS | — | +| HTTP vs HTTP launchpad | 1 winner | 1 winner (201+409) | unit test + code | PASS | — | +| HTTP vs CLI bootstrap | 1 winner | 1 winner ×4 runs | race3 runs | PASS (HYP-3 falsified) | — | +| CLI vs CLI bootstrap | 1 winner | 1 winner (mechanism) | RR+retry analysis | PASS | — | +| wrong token (fresh) | 403 INVALID | 403 INVALID | runtime | PASS | — | +| correct token (fresh) | 201 | 201 + admin | runtime | PASS | — | +| completed + wrong token | no oracle | 403 INVALID (distinguishable) | runtime | **FAIL (oracle)** | P3 | +| completed + correct token | 409 | 409 ALREADY_COMPLETED | runtime | PASS | — | +| last Admin disabled/removed | stays COMPLETED | COMPLETED, 409 on POST | runtime + unit test | PASS | — | +| hard-delete org+users | stays COMPLETED (claimed) | **READY — reopens** | runtime | **FAIL (claim)** | P3 | +| bootstrap crash before commit | rollback, no orphans | structural (single tx, RR+retry); 409 on loser | code + concurrency runs | PASS (structural) | — | +| container recreation | same state | NORMAL on re-boot | drill B / audit-a re-runs | PASS | — | +| local clean-root relocation | identical | ALL CHECKS PASSED | drill run | PASS | — | +| clean-host relocation | identical | ALL CHECKS PASSED | CI run 31324409704 | PASS | — | +| clean-host without checkout | succeeds | succeeds (no checkout file consumed) | workflow inspection + pass | PASS | — | +| Redis-enabled relocation w/o Redis state | counters reset, state identical | ALL CHECKS PASSED | proof run | PASS | — | +| old P6 Compose smoke | P6 invariants hold | ALL CHECKS PASSED | smoke run #2 | PASS | — | +| compose app gains build: | guard fails | FAIL/FAIL | M1 | PASS | — | +| email-worker gains build: | guard fails | FAIL/FAIL | M2 | PASS | — | +| EXAM_IMAGE requirement removed | guard fails | FAIL | M3 | PASS | — | +| PG 18→19 (minor pin) | guard fails | **PASS/PASS** | M4 | **FAIL (no static oracle)** | P3 | +| PG floats to latest | guard fails | FAIL | M5 | PASS | — | +| Redis floats | guard fails | FAIL | M6 | PASS | — | +| db returns to named pgdata | guard fails | FAIL | M7 | PASS | — | +| canonical data bind removed | guard fails | FAIL | M8 | PASS | — | + +## Special hypotheses + +| Hypothesis | Verdict | Evidence | +|---|---|---| +| HYP-1 journal-less DB classified FRESH_INSTALL | **TRUE** | D2/D3/D4 runtime | +| HYP-2 two copies of the mutation body | **TRUE** (duplication) | code; no divergence demonstrated → P3 | +| HYP-3 HTTP serialized but CLI not (cross-adapter race) | **FALSE** | RR+retry serialization; 4 runtime runs single-winner | +| HYP-4 completed launchpad validates token first (oracle) | **TRUE** | 403 vs 409 runtime | +| HYP-5 "ever existed" stronger than evidence | **TRUE** | hard-delete reopen runtime | +| HYP-6 EXPECTED_PG_MAJOR vs compose tag unguarded | **TRUE** | mutation M4 | +| HYP-7 clean-host uses checkout in B | **FALSE** | no checkout file consumed; run passed | +| HYP-8 clean-host proof not a permanent gate after merge | **TRUE** | workflow triggers branch-only | + +--- + +``` +PR #273 P7-C1 ADVERSARIAL AUDIT — DO NOT MERGE + +P0: none +P1: P1-1 FRESH_INSTALL oracle false for untracked non-fresh DBs (HYP-1, runtime) + P1-2 C1 drill/Redis proof inherit EXAM_DATA_ROOT → can seed a real deployment +P2: P2-1 launchpad token never forwarded by compose (documented path inert) + P2-2 clean-host proof historical-only after merge (HYP-8) +P3: P3-1..P3-7 (PG drift oracle, preflightBypassed doc claim, token oracle, + "ever existed" wording, duplicated bootstrap bodies, manual-drill gate, status probes) + +Portable data root: + PARTIAL +Image-only deployment: + PROVEN +Migration preflight: + FAILED +Clean-host relocation: + PROVEN (final head only) +Redis non-authority: + PROVEN +Launchpad first-Admin authority: + PROVEN +Launchpad cross-adapter concurrency: + PROVEN +Launchpad permanent-close semantics: + PARTIAL +Secret boundary: + PARTIAL +Old P6 deployment invariants: + PRESERVED +C2+ scope leak: + NONE + +Merge blockers: + [P1-1 FRESH_INSTALL oracle, P1-2 EXAM_DATA_ROOT drill hazard] +``` diff --git a/docs/deployment/backup-and-recovery.md b/docs/deployment/backup-and-recovery.md new file mode 100644 index 00000000..9c250184 --- /dev/null +++ b/docs/deployment/backup-and-recovery.md @@ -0,0 +1,706 @@ +# Backup and Recovery Guide + +> **Authority:** canonical operator guide for portable persistence, cold +> filesystem backup, and disaster recovery of a single-node Exam deployment. +> Companion to [`mvp-deployment-runbook.md`](./mvp-deployment-runbook.md). +> +> Scope: LAN/on-premise, single-tenant. Do NOT use this guide for any +> multi-tenant, cloud, or Phase 4 deployment — those modes are not +> implemented. + +--- + +## One-Compose model (read first) + +There is **exactly ONE production/operator Docker Compose entry point**: + +```text +docker-compose.yml +``` + +Normal operations are always: + +```bash +docker compose up -d +docker compose down +``` + +There is **no** alternative production startup command involving another +Compose file. Optional PostgreSQL capabilities such as PITR are **database +configuration**, not an alternate Docker topology: + +```text +Docker/container topology != PostgreSQL backup policy +PITR = optional PostgreSQL cluster capability +PITR != alternate Exam deployment topology +``` + +Development/test Compose files (`docker-compose.dev.yml`, +`docker-compose.test.yml`) are development infrastructure and may remain; +they are NOT operator entry points. + +--- + +## 0. Read this first — what is authoritative + +```text +PostgreSQL = authoritative durable Exam state + (attempts, answers, grading, results, audit, …) +Redis = non-authoritative (rate-limit counters only; + TTL-bounded; may be lost without consequence) +application filesystem = no durable application writes +``` + +The PostgreSQL data directory is the **only** bytes you must preserve to +keep the same Exam system. Everything else — Redis, container writable +layers, `/app/data`, logs, browser storage — is disposable. + +> **Host persistence is not backup.** The live `./data/postgres` directory +> keeps the system running, but a copy on the same failing disk is a weak +> local copy, **not** disaster recovery. A real backup lives on an +> **independent failure domain** (NAS, another server, a separate disk). + +--- + +## 1. Where data is stored + +The production Compose topology (`docker-compose.yml`) uses operator-visible +host bind mounts under `${EXAM_DATA_ROOT:-./data}`: + +```text +exam/ +├── docker-compose.yml +├── .env +└── data/ + ├── postgres/ ← authoritative: the PostgreSQL data directory (PGDATA) + ├── redis/ ← non-authoritative: rate-limit AOF/RDB (may be lost) + └── wal-archive/ ← PITR archive (only meaningful when PITR is enabled) +``` + +- `data/postgres/` is **required**. Deleting it destroys authoritative Exam + state. The official PostgreSQL image owns its internal layout + (`data/postgres/18/docker/...`); the operator only needs the + `data/postgres` parent. +- `data/redis/` is **optional for correctness**. It holds rate-limit state; + losing it resets operational history but never affects Exam authority. +- `data/wal-archive/` is the default WAL archive path. The mount is ALWAYS + present on the db service but is **inert by default** (`archive_mode = off`). + It only matters once PITR is enabled + (`scripts/backup/postgres-enable-pitr.sh`). A normal operator never needs + to understand WAL/PITR just to run Exam. + +Set `EXAM_DATA_ROOT` to relocate the whole data root (e.g. to a mounted +NAS volume). The default is `./data` relative to the Compose file. Set +`EXAM_WAL_ARCHIVE_HOST_PATH` to point the WAL archive at an **independent +failure domain** for real disaster recovery (the local default is for +development/drills only and is NOT host-loss protection). + +--- + +## 2. What deleting things means + +| What you delete | Result | +| --- | --- | +| `docker compose down` | Containers + network removed; **`data/` retained**. | +| `docker compose down -v` | With bind mounts this is a no-op for `data/` (it only removes named volumes; there are none). `data/` is retained. | +| `rm -rf data/postgres` | **Authoritative state destroyed.** Nothing left to restore from unless you have a backup. | +| `rm -rf data/redis` | Rate-limit counters reset; no Exam truth change. | +| Delete the `app`/`email-worker` containers | No state in them; recreate with `docker compose up -d`. | +| Delete the `db` container | `data/postgres` is retained; recreate with `docker compose up -d`. | + +--- + +## 3. Stop and start + +```bash +# Stop (graceful — SIGTERM propagates, drains audit writes, closes DB pool): +docker compose down + +# Start again (same data root, fresh containers): +docker compose up -d + +# Confirm authoritative state survived: +docker compose exec db sh -c \ + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT count(*) FROM organizations;"' +``` + +--- + +## 4. Decision tree (C1 + C2 + C3) + +```text +I am moving the server to a new host + → §5 stopped-directory relocation (C1) + +I want the simplest full backup + → §6 cold-filesystem backup (C1) + +I want routine backups without shutting Exam down (online) + → §7 pg_dump logical backup (C2) + +I need exact PostgreSQL physical backup / faster full-cluster recovery + → §8 pg_basebackup physical backup (C3) + +I deleted something at 14:32 and need 14:31 + → §8 WAL archive + PITR (C3) +``` + +Quick comparison: + +| Path | Online? | Scope | Replaces history? | Best for | +| --- | --- | --- | --- | --- | +| §5 stopped relocation (C1) | **No** (stop server) | Whole data dir | **No** — same authoritative history | Moving the deployment to new hardware | +| §6 cold-filesystem backup (C1) | **No** (stop server) | Whole data dir | Yes (restore replaces history) | Simplest full snapshot when Exam can stop | +| §7 pg_dump logical (C2) | **Yes** | One database (`exam`) | Yes (clean restore into a fresh DB) | Routine online backups; cross-PG-major portability | +| §8 pg_basebackup physical (C3) | **Yes** | Whole cluster | Yes (physical cluster restore) | Full-cluster recovery; consistent snapshot at scale | +| §8 WAL archive + PITR (C3) | **Yes** | WAL replay to target | Yes (replay to target then promote) | Recover a specific past point; undo a destructive change | + +Restore replaces the authoritative history in all backup/restore paths EXCEPT +stopped relocation (§5), which carries the SAME history forward to a new +host. See §9 for the boundary. + +--- + +## 5. Cold directory relocation (C1) + +Move the entire Exam deployment to a new host by copying the stopped data +directory. This preserves the **same authoritative history** (same Exam +system, new machine) — it is NOT a historical restore and NOT PITR. + +```text +deployment A (host A) + → docker compose down (PostgreSQL stopped cleanly) + → copy the COMPLETE data/ directory (rsync -aHAX / tar while stopped) + → deployment B (host B): place the copy at $EXAM_DATA_ROOT + → docker compose up -d (same PostgreSQL major, same creds) + → same Exam state +``` + +The PGDATA files are owned by the container's postgres user and are not +readable by the host user, so copy as root (or via a helper container): + +```bash +# Stop Exam on host A first. +docker compose down + +# Copy the COMPLETE data root to host B. On host A (as root, or via a +# throwaway container that preserves ownership/mode/symlinks): +rsync -aHAX /path/on/hostA/data/ /path/on/hostB/data/ +# Equivalent: tar -C /path/on/hostA -cf - data | tar -C /path/on/hostB -xf - + +# On host B, point EXAM_DATA_ROOT at the copied directory and start with +# the SAME PostgreSQL major version and the SAME DB credentials the +# volume was initialized with: +export EXAM_DATA_ROOT=/path/on/hostB/data +export POSTGRES_PASSWORD= +docker compose up -d +``` + +> **Raw PostgreSQL directory copying is supported only as a complete +> stopped-server filesystem copy in the compatible PostgreSQL environment.** +> The PGDATA is tied to the PostgreSQL major version (currently 18). A +> `pg_dump` restore (C2) is portable across majors; a raw-PGDATA copy is +> not. Do not live-copy PostgreSQL's active data directory with ordinary +> `cp`/`tar`; do not partial-copy PostgreSQL relation files. + +--- + +## 6. Cold-filesystem backup and restore (C1) + +Treats a stopped copy of the complete PostgreSQL persistent directory as a +same-version/same-major cold physical backup. Simplest full backup option; +requires downtime while PostgreSQL is stopped. + +### 6.1 Backup + +```bash +# 1. Stop Exam cleanly (PostgreSQL must be STOPPED — a live copy is corrupt-prone): +docker compose down + +# 2. Run the backup helper (copies the COMPLETE postgres tree to a fresh +# destination with ownership/mode/symlinks preserved). The source is the +# deployment's EXAM_DATA_ROOT (the production Compose default is ./data): +scripts/backup/cold-filesystem-backup.sh \ + "${EXAM_DATA_ROOT:-./data}" \ + /mnt/nas/exam-backups/2026-08-10 + +# 3. Restart Exam: +docker compose up -d +``` + +Store the destination on an **independent failure domain** (NAS / another +server / a separate disk). A copy on the same disk as the live data is a +weak local copy, not disaster recovery. + +### 6.2 Restore + +```bash +# Restore into a FRESH data root (the script refuses to overwrite a populated +# one). Start Exam afterwards with the same PostgreSQL major and the same DB +# credentials the backup was taken with. +mkdir -p /opt/exam/data-fresh +scripts/backup/cold-filesystem-restore.sh /mnt/nas/exam-backups/2026-08-10 /opt/exam/data-fresh + +export EXAM_DATA_ROOT=/opt/exam/data-fresh +export POSTGRES_PASSWORD= +docker compose up -d + +# Run your Exam business-invariant checks after start. +``` + +This is filesystem-level cold restore. It is **not** `pg_restore`, **not** +PITR, and **not** a cross-major PostgreSQL upgrade. The restored PGDATA is +tied to the PostgreSQL major version of the backup. The official postgres +image fixes ownership/permissions of the PGDATA on container start; no host +`chmod` is required. + +### 6.3 Permissions + +The PGDATA files are owned by the container's postgres user with the +ownership/mode the official PostgreSQL image produces. The operator +contract is: **preserve** that ownership/mode — do not `chmod` PGDATA +broadly. Relocation tools (`rsync -aHAX`, `tar`, or the helper-container +`cp -a` used by the scripts) preserve owner/group/mode. Do **not** run +broad host `chmod 777` on the data directory. The official postgres image +chowns the PGDATA on container start, so a fresh empty bind mount is also +initialized correctly. + +--- + +## 7. Logical online backup and clean restore (C2) + +Takes an internally consistent PostgreSQL backup while Exam is running, using +`pg_dump` custom format (`-Fc`), and restores it into a CLEAN target +database. This is the routine backup users are most likely to want: +PostgreSQL stays online and the dump is internally consistent. Prefer this +path for routine backups unless cold-copy simplicity is preferred. + +### 7.1 Backup (online) + +```bash +# PostgreSQL stays ONLINE. The API may be down; only PostgreSQL must be up. +# is the Compose project name (for the default production +# stack started from the repo root it's the directory name, usually "exam"). +scripts/backup/postgres-logical-backup.sh exam /mnt/nas/exam-logical/$(date +%Y%m%d).dump +``` + +The helper: connects via the `db` container; produces a timestamped +custom-format dump (`-Fc`, `--no-owner`); fails non-zero on error; never +puts the DB password on the argv (uses `PGPASSWORD` env); refuses to claim +success for an empty/partial artifact (non-empty + `PGDMP` magic + +`pg_restore --list` OK). Store the artifact on an **independent failure +domain**. + +### 7.2 Clean restore (exact historical replacement) + +```bash +# 1. Stop the API + worker (avoid writes during restore): +docker compose stop app email-worker + +# 2. Restore into a CLEAN target (DROP + recreate from template0, then +# pg_restore). No target-only schema/data from the previous database +# survives (clean logical reconstruction of the dumped state — NOT a +# merge). The script requires you to type the target DB name to confirm. +scripts/backup/postgres-logical-restore.sh exam /mnt/nas/exam-logical/.dump exam + +# 3. Restart the API + worker to use the restored database: +docker compose up -d app email-worker + +# 4. Run your Exam business-invariant checks after restart. +``` + +The clean-target contract fixes the exact-historical-replacement gap: the +runbook's older `pg_dump --clean --if-exists | psql` path does **not** remove +objects that exist in the target DB yet are absent from an older dump. The +C2 restore script enforces `DROP DATABASE ... WITH (FORCE)` (terminates any +lingering connections; stop the API + worker first per §7.2) + +`CREATE DATABASE ... TEMPLATE template0` (a truly empty database) before +`pg_restore`, so no target-only +schema/data from the previous database survives — the restored database is a +clean logical reconstruction of the dumped Exam database state. This is NOT a +claim of physical byte identity; a logical dump reconstructs the dumped +database's logical schema/data under the supported deployment contract. This was validated by an +automated suite +(`tests/deployment/logical-backup-restore.sh`) that proves a fresh +working Exam with State A is produced from a State-A dump, and State-B-only +data is correctly absent. Restore is **operator-only** (no browser restore +button; the Phase 1 rule). + +### 7.3 Cluster globals (not required for the bundled path) + +Exam does **not** require `pg_dumpall --globals-only` for the bundled +single-node Compose path: + +- The `db` service creates the application role and database at image init + from `POSTGRES_USER` / `POSTGRES_DB` / `POSTGRES_PASSWORD`. Restoring the + dump into a database created by the same Compose stack therefore finds the + role/database already present — they are **recreated by Docker/bootstrap + configuration, not required in the dump**. +- No PostgreSQL cluster-level roles, tablespaces, or other globals are + application-defined. The application owns only objects inside its database. + +`pg_dumpall --globals-only` is therefore **not** included in the default +backup. If you run against an external PostgreSQL cluster where the role is +not created by Docker init, recreate the role/database manually before +restore (this is already the runbook's external-Postgres stance). + +--- + +## 8. Physical backup, WAL archive, and PITR (C3) + +C3 ships **PostgreSQL-native physical backup and point-in-time recovery**: +`pg_basebackup` of the running cluster, continuous WAL archiving, and PITR +restore to an explicit target. This is for full-cluster recovery and for +recovering a specific past point — NOT for routine backups. Routine backups +should use the C2 logical backup (§7). + +C3 requires PostgreSQL to be at `wal_level=replica` (already the default in +the bundled image; confirmed at runtime — `replica` is sufficient, `minimal` +is the only level that blocks PITR). It does NOT raise `wal_level` to +`logical`, which would add WAL overhead without changing the PITR capability. + +### 8.1 pg_basebackup — physical base backup + +`scripts/backup/pg-basebackup.sh` takes a physical base backup of the +running PostgreSQL cluster without stopping it: + +```bash +# Online physical base backup into /mnt/backup-exam/base-$(date +%FT%H%M): +bash scripts/backup/pg-basebackup.sh exam-project-name /mnt/backup-exam/base-$(date +%FT%H%M) +``` + +Properties: + +- `pg_basebackup -X stream -c fast -Fp --manifest-checksums SHA256` — the + required WAL is **streamed at backup time** (`-X stream`), so the base + backup is internally consistent on its own. +- A `backup_manifest` is produced and **verified** with `pg_verifybackup` + before the script returns success. `pg_verifybackup` verifies the backup + contents against the PostgreSQL backup manifest — file presence and + size, the configured per-file SHA256 checksums, and the manifest's own + checksum. Manifest verification is backup-integrity + evidence (the manifest uses checksums; it is not a digital-signature + system); it is NOT proof that Exam can start and satisfy business + invariants after restore. A restore drill is still required. +- The backup target must be OUTSIDE the live PGDATA (never write a base + backup into the directory PostgreSQL is running from). +- The replication connection uses the configured PostgreSQL superuser + (`POSTGRES_USER`) over the loopback network namespace of the db container, + authenticated with the deployment password (`POSTGRES_PASSWORD`) passed via + `PGPASSWORD` (never argv). `pg_basebackup` requires a SUPERUSER or + REPLICATION-capable role; the bootstrap superuser satisfies this for the + bundled single-node path. A narrowly scoped replication-only role is NOT + provisioned today (future hardening / P7-E); the API itself never gets + replication authority. + +> **PITR base-backup rule:** WAL archiving MUST be active BEFORE the base +> backup that will anchor PITR. The sequence is: +> ```text +> enable WAL archiving (postgres-enable-pitr.sh) +> → verify the archiver actually works +> → take pg_basebackup +> → continue archiving WAL +> → PITR can target later history +> ``` +> A base backup taken BEFORE WAL archiving was established is NOT a valid +> anchor for later continuous PITR in the documented procedure. + +A base backup is a **whole-cluster** snapshot — it is NOT a per-database +restore. To restore only the `exam` database (or cross PG-major), use the +C2 logical backup (§7). + +### 8.2 WAL archive — continuous archiving for PITR + +Point-in-time recovery requires a **continuously archived WAL chain** that +starts BEFORE the first base backup. PostgreSQL's documented contract: +"the WAL archiving procedure must be active before the first base backup is +taken." + +PITR is enabled by ONE canonical operator command — there is no PITR Compose +file: + +```bash +scripts/backup/postgres-enable-pitr.sh [COMPOSE_PROJECT] [COMPOSE_FILE] +``` + +The script: + +1. locates the canonical db container and requires PostgreSQL healthy; +2. makes `/wal-archive` writable by the postgres user with **restrictive** + permissions (NEVER `chmod 777` — WAL contains database contents); +3. checks `wal_level != minimal` (`replica` is already sufficient); +4. `ALTER SYSTEM SET archive_mode = 'on'` (postmaster-level — restart required); +5. sets an **idempotent** `archive_command` (see §8.2.1); +6. `ALTER SYSTEM SET archive_timeout = '60s'`; +7. restarts ONLY the db service; +8. waits deterministically for PostgreSQL readiness; +9. verifies `archive_mode` / `archive_command` / `archive_timeout`; +10. forces a WAL switch and polls `pg_stat_archiver` for REAL archive evidence + (not a fixed sleep) — it reports success only after the archiver has + actually archived a segment. + +Because the mechanism is `ALTER SYSTEM` (persisted into +`postgresql.auto.conf` inside PGDATA), the configuration survives +`docker compose down` / `docker compose up` / host relocation of the same +PGDATA. No separate configuration topology is needed. + +**The WAL archive MUST be on an INDEPENDENT failure domain** from the +database (NAS / another server / a separate disk). Set +`EXAM_WAL_ARCHIVE_HOST_PATH` to the independent-storage path in production +(the local default `${EXAM_DATA_ROOT}/wal-archive` is for +development/drills only and is NOT host-loss protection). + +#### 8.2.1 Idempotent `archive_command` + +PostgreSQL may retry archiving the same WAL segment. The canonical +`archive_command` is correct for all three cases (proved by +`tests/deployment/pitr.sh`): + +```text +test ! -f /wal-archive/%f && cp %p /wal-archive/%f || cmp -s %p /wal-archive/%f + +target absent → cp succeeds → exit 0 +target exists + identical bytes → cmp -s succeeds → exit 0 +target exists + different bytes → cmp -s fails → exit non-zero (FAILURE) +``` + +This replaces the older `test ! -f target && cp source target` form, which +would fail forever on an identical retry (target already exists → non-zero). +A byte collision under the same WAL filename is a visible stuck archive, NOT +a silent overwrite. + +### 8.3 PITR — recover to an explicit target + +To recover to a specific point: + +1. Stop the source cluster (`docker compose down`). +2. Restore a **base backup** (§8.1) into the recovery cluster's PGDATA at + `${EXAM_DATA_ROOT}/postgres/18/docker` (the PG18 image's PGDATA layout). +3. Copy the WAL archive (§8.2) into the recovery cluster. +4. Write `recovery.signal` into the PGDATA and append to + `postgresql.auto.conf`: + ```text + restore_command = 'cp /wal-archive/%f %p' + recovery_target_lsn = '0/50176E8' # OR recovery_target_time / xid + recovery_target_inclusive = on + recovery_target_action = 'promote' + ``` + Choose ONE target: + - `recovery_target_lsn = ''` — preferred for deterministic drills + (clock-skew-independent); capture with `SELECT pg_current_wal_lsn();` + immediately after the change you want to include. + - `recovery_target_time = 'YYYY-MM-DD HH:MM:SS'` — natural for "undo the + 14:32 mistake"; needs clock alignment between client and server. + - `recovery_target_xid = '<32-bit xid>'` — needs a 32-bit xid from + `txid_current()::text::integer`-style extraction; do NOT pass the + 64-bit xid8 from `pg_current_xact_id()` directly. +5. Start the recovery cluster. PostgreSQL replays WAL up to the target, + then `promote`s. The recovered cluster is now the new authoritative + history. + +PITR recovery replaces the authoritative history up to the target. See §9 +for the boundary. + +### 8.4 What C3 does NOT do + +- **Not a routine backup strategy.** Use C2 pg_dump (§7) for routine daily + backups. C3 base backups are heavier; C3 PITR is for disaster recovery. +- **Not a per-database restore.** `pg_basebackup` is a whole-cluster + snapshot. To restore just the `exam` database, use C2. +- **Not cross-PG-major portable.** A physical base backup is tied to the + PostgreSQL major version that produced it. To cross PG majors (upgrade), + use C2 `pg_dump`/`pg_restore`. +- **No PG18 incremental base backups.** Exam scale today does not justify + them. Re-evaluate at P7-E if measured scale demands it. +- **No retention engine.** Retention of base backups + WAL is the + operator's responsibility (§8.5). P7-E may add a control plane; not + started. +- **No automatic desktop client `recoveryEpoch`.** Per ADR-016, C2 logical + restore and C3 PITR are authoritative-history REPLACEMENT events (not + same-history). Any offline-client recovery-epoch concern is a future + Phase 4 concern; no schema change is introduced here. + +### 8.5 Retention (operator-owned; no automation shipped) + +> **P7-C3 does NOT ship automatic PITR retention/pruning.** Retention is +> operator discipline. Future retention automation belongs in later +> operations / P7-E work, or a mature PostgreSQL backup system (§8.7). + +A base backup can only recover **forward** from its own history. A common +but **incorrect** rule is: *"For an N-day PITR window, retain only the most +recent base backup plus WAL."* That is wrong — the most recent base backup +may have been taken INSIDE the window, so it cannot anchor recovery to any +point before itself. + +For an earliest desired recovery point `T`, retain at least: + +```text +a usable base backup whose completion/history PRECEDES T ++ +all WAL required from that base backup through the desired recovery window ++ +required timeline history files when timelines exist +``` + +Conservative guidance: + +```text +Do not manually delete base backups or archived WAL that may be required +for the promised recovery window. +``` + +You MAY delete: + +- older base backups, AS LONG AS you keep at least one base backup whose + history precedes the earliest point in your recovery window, plus the + complete unbroken WAL chain forward from its checkpoint to the end of the + window; +- archived WAL segments OLDER than the retained base backup's start + checkpoint (these cannot be replayed against any retained base backup). + +Do NOT delete: + +- the base backup(s) anchoring your recovery window; +- any WAL segment between the retained base backup and the current end of + the PITR window — a single missing segment breaks the chain. + +There is no retention automation today; retention is the operator's +discipline. P7-E may add a control plane; it is NOT started. + +### 8.6 Verification evidence + +The deterministic suite in `tests/deployment/` proves the contracts +(`pnpm test:deployment:pitr`, or the whole suite with `pnpm test:deployment`): + +- **Happy PITR** — enable archiving via the canonical + `postgres-enable-pitr.sh` → base backup → pre-base marker → post-base + State A → State B (capture LSN) → destructive State C → recover to the + captured LSN → assert A/A1/B present, C absent, promoted. PASS. +- **F1 missing required WAL** — an UNTOUCHED base backup plus a + complete archive MINUS the one segment that must be replayed to reach an + explicit `recovery_target_lsn`. The assertion is about failing to REACH + the target: the cluster stays in archive recovery (`pg_controldata` + reports `in archive recovery`), restore_command failures for the missing + segment are visible, and the server never completes recovery (no + promotion) within a bounded window. Note: `restore_command` returning + file-not-found for a missing segment is NORMAL at the end of any archive + — routine recovery routinely asks for files that do not exist. Only a + recovery target that requires replay through a missing segment proves + "required WAL missing". +- **F2 corrupt base backup** — tamper one backed-up file → + `pg_verifybackup` rejects it loudly (per-file checksum mismatch). +- **F3 invalid recovery target** — malformed `recovery_target_lsn` → + PostgreSQL refuses recovery loudly. +- **Archive idempotency** — the idempotent `archive_command` is correct + for all three cases: empty target → success; identical retry → success; + byte collision under the same name → non-zero failure. + +> **Product path == test path.** The operator path and the test path are +> the SAME enable-PITR script. No test privately configures PostgreSQL +> through a second hidden method, and no test generates a temporary Compose +> override — recovery clusters start from the same `docker-compose.yml` +> with isolated `EXAM_DATA_ROOT` / `EXAM_WAL_ARCHIVE_HOST_PATH` / +> `COMPOSE_PROJECT_NAME`. + +### 8.7 Future boundary: WAL-G / pgBackRest + +P7-C deliberately does NOT introduce WAL-G or pgBackRest. But if Exam later +requires any of: + +```text +automatic off-host WAL shipping +S3/MinIO +encryption +compression +incremental physical backups +automated retention +large backup chains +low operational RPO +``` + +then evaluate **WAL-G** or **pgBackRest** instead of growing Exam's own +shell scripts into a bespoke PostgreSQL backup product. Current scope stays +PostgreSQL-native and small. This is an explicit future boundary, not +near-term work. + +--- + +## 9. What Redis loss means + +Redis holds only shared rate-limit counters with mandatory TTL. Losing +`data/redis/` (or the whole Redis instance): + +- resets rate-limit windows; +- has **no** effect on PostgreSQL Exam authority; +- never creates durable Exam corruption. + +Restoring Redis is **not** a condition for restoring Exam authority. Stale +Redis counters only ever cause brief over-limiting; they never +under-limit an irreversible fact. + +--- + +## 10. Restore boundary — same history vs. history replacement (ADR-016) + +The recovery procedures fall into exactly two categories: + +- **Same authoritative history** — §5 stopped-directory relocation. This + carries the SAME PostgreSQL history forward to a new host (same files, + same timeline, same exam state). It is NOT a restore in the + history-replacement sense; it is the same Exam system on new hardware. + +- **Authoritative history replacement** — §6 cold-filesystem restore, + §7 logical restore, §8 physical restore / PITR. All of these REPLACE the + authoritative history: the cluster comes back with a different timeline + (PITR promotes), or a fresh-clean database (C2 `DROP DATABASE` + + `template0`), or a snapshot from a past moment (C1 cold restore). The + pre-restore PostgreSQL history is gone after the restore. + +Per ADR-016, **no schema change is introduced** to mark these events. The +exam system's authoritative state is whatever PostgreSQL currently holds; +the system does not need to know HOW it got there. Any future offline-client +`recoveryEpoch` concern (e.g. a desktop client noticing the server's history +was replaced) is a Phase 4 platformization concern and is NOT implemented +here. Container restarts and §5 relocations are NOT history-replacement +events — they preserve the timeline. + +--- + +## 11. First Admin (Launchpad) and Admin recovery + +### 11.1 Launchpad (initial installation only) + +If the installation has never been initialized (the internal default +organization does not exist), navigate to `/launchpad` and complete the +first-Admin setup form. Set `LAUNCHPAD_SETUP_TOKEN` in `.env` before the +first `docker compose up`; it is the deployment bootstrap secret (high +entropy, e.g. `openssl rand -hex 32`). The role is not selectable — the +server always creates role = Admin. + +Once initialized, `/launchpad` redirects to `/login` (it never renders a +"completed" page and never reopens). Removing/disabling the last Admin +does **not** reopen launchpad — Admin-loss recovery is operator CLI +territory. + +### 11.2 Operator CLI fallback + +The first Admin can also be created via the bootstrap CLI (equivalent +canonical mutation body, atomic): + +```bash +docker compose exec app \ + node dist/scripts/bootstrap-admin.js \ + --username admin --password '' \ + --name 'System Admin' --organization-name 'My Organization' +``` + +### 11.3 Reset an Admin's password + +```bash +docker compose exec app \ + node dist/scripts/reset-admin-password.js \ + --username admin --password '' +``` + +This script can ONLY reset Admin passwords. Candidate passwords are reset +by an Admin through the API. diff --git a/docs/deployment/mvp-deployment-runbook.md b/docs/deployment/mvp-deployment-runbook.md index 8c8b8ad1..10c62d3b 100644 --- a/docs/deployment/mvp-deployment-runbook.md +++ b/docs/deployment/mvp-deployment-runbook.md @@ -168,10 +168,19 @@ docker compose ps # 7. Bootstrap the first Admin (production path — see §5). This also # creates the internal default organization, which unblocks the worker. +# Two equivalent paths share one canonical atomic mutation body: +# +# (a) CLI fallback (operator path): docker compose exec app \ node dist/scripts/bootstrap-admin.js \ --username admin --password '' \ --name 'System Admin' --organization-name 'My Organization' +# +# (b) Launchpad first-install page (browser path): set +# LAUNCHPAD_SETUP_TOKEN= in .env BEFORE step 4, +# then navigate to http://:/launchpad and complete +# the first-Admin setup form. Once initialized, /launchpad redirects +# to /login and never reopens. See backup-and-recovery.md §8. # 8. The same worker container detects the new organization, resolves it, # and enters its poll loop without restarting. Verify: @@ -775,18 +784,71 @@ and §24 (deferred capabilities). Highlights: ## 17. Backup / export (operator-supplied) -The currently documented backup procedure is `pg_dump` against the `pgdata` volume. +> **Canonical backup & recovery authority:** see +> [`docs/deployment/backup-and-recovery.md`](./backup-and-recovery.md). That +> guide documents the supported C1 cold-filesystem backup/restore and +> relocation procedures in full. The summary below is retained for +> runbook-local context. -> **CURRENT PROCEDURE UNVALIDATED — do not treat as a proven exact historical -> restore until P7-C restore drills close this gap.** The P7-C0 durability -> audit (`docs/audits/P7-C0-DURABILITY-PERSISTENCE-REALITY-AUDIT.md` §16/§16.1) -> classified this path as **documented-only, UNVALIDATED**: the P6 audit -> verified the migrate-from-zero path but never executed a live -> pg_dump/restore cycle, and exact historical state replacement is NOT proven -> (restoring an older dump into an already-newer database may leave objects -> absent from the dump unless the target is recreated/cleaned under an -> explicit restore contract). Validate on first production deploy before -> relying on it. +Authoritative state is the PostgreSQL data directory under +`${EXAM_DATA_ROOT:-./data}/postgres` (a host bind mount since P7-C1; the +former `pgdata` named volume is gone). **Host persistence is not backup** — +a copy on the same failing disk is a weak local copy, not disaster recovery. + +### C1 cold-filesystem backup (validated) + +The simplest full backup: stop Exam cleanly, copy the COMPLETE postgres +directory to an off-host destination, restart. Use the helper scripts, +which preserve ownership/mode/symlinks and refuse unsafe paths: + +```bash +# Stop Exam first (PostgreSQL must be STOPPED — a live copy is corrupt-prone). +# The source is the deployment's EXAM_DATA_ROOT (default ./data): +docker compose down +scripts/backup/cold-filesystem-backup.sh \ + "${EXAM_DATA_ROOT:-./data}" \ + /mnt/nas/exam-backups/$(date +%Y%m%d) +docker compose up -d +``` + +Restore into a fresh data root, then start Exam with the same PostgreSQL +major version and the same DB credentials: + +```bash +scripts/backup/cold-filesystem-restore.sh /mnt/nas/exam-backups/ /opt/exam/data-fresh +EXAM_DATA_ROOT=/opt/exam/data-fresh POSTGRES_PASSWORD= docker compose up -d +``` + +Both procedures were validated by an automated suite +(`tests/deployment/persistence-and-cold-restore.sh`) that proves a +fresh working Exam deployment with identical authoritative state is +produced from the backup. See backup-and-recovery.md §6. + +### pg_dump logical backup and clean restore (validated by C2) + +The C2 logical path is the recommended routine backup (PostgreSQL stays +online). It was validated by an automated suite +(`tests/deployment/logical-backup-restore.sh`) that proves a fresh +working Exam with State A is produced from a State-A dump, with State-B-only +data correctly absent — closing the P7-C0 P2-2/P2-3 gaps. The clean-restore +contract (DROP + recreate from template0, then `pg_restore`) is enforced by +`scripts/backup/postgres-logical-restore.sh`. See +`docs/deployment/backup-and-recovery.md` §7. + +```bash +# Online logical backup (PostgreSQL stays ONLINE; API may be down): +scripts/backup/postgres-logical-backup.sh exam /mnt/nas/exam-logical/$(date +%Y%m%d).dump + +# Clean restore (STOP API + worker first; script requires typing target DB name): +docker compose stop app email-worker +scripts/backup/postgres-logical-restore.sh exam /mnt/nas/exam-logical/.dump exam +docker compose up -d app email-worker +``` + +The older `pg_dump --clean --if-exists | psql` one-liner is retained below +for reference, but the clean-target contract above is the supported path +(`--clean --if-exists` into a dirty target does NOT remove dump-absent +objects): ```bash # Backup (online, consistent). $POSTGRES_USER / $POSTGRES_DB are expanded @@ -805,7 +867,7 @@ tail -5 backup_*.sql # should contain 'PostgreSQL database dump complete' # NOTE: --clean --if-exists drops objects present in the dump, but does NOT # remove objects that exist in the target DB yet are absent from an older # dump. For an EXACT historical replacement, recreate/clean the target -# database under an explicit restore contract (P7-C restore drills). +# database under an explicit restore contract (C2 restore drill). docker compose stop app email-worker docker compose exec -T db sh -c \ 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB"' \ @@ -813,13 +875,14 @@ docker compose exec -T db sh -c \ docker compose up -d app email-worker ``` -For larger deployments, consider `pg_basebackup` for physical backups or -continuous WAL archiving. Schedule backups via cron on the Docker host — the -MVP does not ship a backup scheduler. Note that `wal_level=replica` is already -sufficient for continuous archiving / PITR; the actual missing pieces for PITR -are `archive_mode=on`, an `archive_command`, a WAL archive destination/retention -contract, a base-backup/recovery procedure, and a recovery drill (P7-C0 §7). - -> **Note:** the P6 audit verified the migrate-from-zero path (§9 of the -> audit) but did not execute a live pg_dump/restore cycle. Validate the -> backup/restore procedure on first production deploy before relying on it. +For larger deployments, prefer the C2 logical backup (`scripts/backup/postgres-logical-backup.sh`, +which produces a `pg_dump -Fc` artifact and is verified by a clean-restore +drill) for routine backups. Physical `pg_basebackup`, continuous WAL +archiving, and PITR are now implemented (P7-C3): see +[`docs/deployment/backup-and-recovery.md`](backup-and-recovery.md) §8 for +the pg_basebackup script, the canonical `scripts/backup/postgres-enable-pitr.sh` +WAL-archiving command (there is no PITR Compose file — PITR is a database +capability configured via `ALTER SYSTEM`, not an alternate Docker topology), +the PITR procedure, retention contract, and drill evidence. Schedule backups +via cron on the Docker host — the platform does not ship a backup scheduler +(a control plane is a P7-E concern, not started). diff --git a/docs/roadmap/P7-system-readiness-and-exam-modes.md b/docs/roadmap/P7-system-readiness-and-exam-modes.md index 3798f925..b5e25973 100644 --- a/docs/roadmap/P7-system-readiness-and-exam-modes.md +++ b/docs/roadmap/P7-system-readiness-and-exam-modes.md @@ -262,12 +262,48 @@ authority for attempts, answers, grading, audit, and business configuration. Moving one of those responsibilities is possible but requires a separate accepted ADR. -## 6. Workstream C — Backup and restore - -### Current gap - -The MVP runbook delegates backups to an operator-supplied `pg_dump` schedule. -That is a minimum deployment note, not a complete backup/recovery capability. +## 6. Workstream C — Portable persistence, backup, and PostgreSQL DR + +> **Rebuilt 2026-08-10 (P7-C).** This workstream was rebuilt from a +> config-taxonomy framing to the as-shipped portable-persistence + backup + +> PostgreSQL disaster-recovery program. The current authority is +> `docs/deployment/backup-and-recovery.md` and the closeout +> `docs/audits/P7-C-PORTABLE-BACKUP-RECOVERY-CLOSEOUT.md`. The phase shape +> is: +> +> - **C0** reality audit (CLOSED) — PostgreSQL is the sole authoritative +> store; Redis is non-authoritative; app filesystem has no durable +> writes. +> - **C1** portable persistence — bind-mount `${EXAM_DATA_ROOT}/postgres` +> (operator-visible, relocatable), cold-filesystem backup/restore, and +> the Launchpad first-install surface. +> - **C2** logical backup — online `pg_dump -Fc` + verified clean restore +> (`DROP DATABASE` + `template0` + `pg_restore --no-owner +> --exit-on-error`), no `--clean --if-exists` into a dirty DB. +> - **C3** physical backup + PITR — `pg_basebackup -X stream` + +> `pg_verifybackup` manifest, PostgreSQL-native WAL continuous +> archiving (`archive_mode=on`, non-overwriting `archive_command`), and +> PITR to an explicit `recovery_target_lsn`/`time`/`xid`. +> +> All four phases are backed by deterministic Docker suites under +> `tests/deployment/` (`compose-smoke.sh`, `launchpad-bootstrap.sh`, +> `persistence-and-cold-restore.sh`, `logical-backup-restore.sh`, +> `pitr.sh`). Scope discipline: +> NO Admin restore button, NO retention engine, NO Desktop recoveryEpoch, +> NO schema change for history-replacement marking (see ADR-016). A +> future P7-E control plane (RPO/RTO profiles, retention automation, +> Admin backup visibility) is NOT started. + +### Current gap (post-rebuild) + +The rebuilt C0–C3 covers the PostgreSQL authority end-to-end. Remaining +work is explicitly P7-E control-plane territory: + +- RPO/RTO profile automation and scheduling (cron-only today); +- Admin backup visibility surface (restore stays operator-owned); +- backup of files/settings beyond the PostgreSQL authority (attachments, + exports, organization settings are in-DB today; a separate + files/settings backup is future). ### Recovery objectives @@ -564,9 +600,14 @@ P7-B1 Backup/RPO/RTO design → P7-B3 PITR/retention/verification → P7-B4 Admin backup surface + restore drill evidence -P7-C1 Configuration taxonomy + schema - → P7-C2 Settings service/version/audit - → P7-C3 Admin settings UI +P7-C Portable persistence, backup, PostgreSQL DR ✅ REBUILT & SHIPPED + (C0 reality audit closed → C1 portable + cold + Launchpad → C2 logical → + C3 physical + PITR; deterministic drills; ADR-016 boundary). The + pre-rebuild C1=config-taxonomy / C2=settings-service / C3=settings-UI + framing is superseded; those config-control-plane items now live under + Workstream E and are NOT started. + → P7-E RPO/RTO profiles, retention automation, Admin backup surface + (control plane; not started) → P7-M1 Exam policy schema + conflict validator → P7-M2 Profile templates + snapshot resolution → P7-M3 Exam creation wizard diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 2e2c498b..1148347b 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -12,7 +12,7 @@ | Phase 1 — Minimal Deliverable | ✅ COMPLETE | Admin + Candidate reliable exam loop. | | Phase 2 — Exam Operation | ✅ GATE ITEMS IMPLEMENTED | `timed_sync` / `deadline` / `untimed` and queue admission remain open. | | Phase 3 — Collaboration / Permissions | 🟡 PARTIALLY IMPLEMENTED | MVP role model and implemented product subset are closed; broader Phase 3 work remains. | -| P7 — System Readiness and Exam Modes | 🟡 IN PROGRESS | P7-D1 Redis decision accepted (2026-08-08); shared rate limit shipped (PR #265, P7-D2/D3). State/backup/config-control-plane/exam-modes/UI workstreams remain open. | +| P7 — System Readiness and Exam Modes | 🟡 IN PROGRESS | P7-D1 Redis decision accepted + shared rate limit shipped (PR #265). P7-C portable persistence + backup + PostgreSQL DR rebuilt & shipped (C1/C2/C3 + drills). State-machine, config-control-plane (P7-E), exam-modes, and UI workstreams remain open. | | Phase 4 — Platformization | ⬜ NOT STARTED | pass-to-proceed, service tokens, webhooks, optional multiTenant. | See [`docs/status/implementation-status.md`](../status/implementation-status.md) @@ -116,6 +116,13 @@ P7 does not redefine M11; M11 remains resource-relationship authorization. - add validation, clean-host restore, and restore drills; - provide CLI and Admin visibility. + > **P7-C rebuild status (2026-08-10):** the portable-persistence + + > backup + PostgreSQL DR core is shipped (C1 cold path + Launchpad, + > C2 logical, C3 physical + PITR), with deterministic drills. The + > remaining items here (RPO/RTO profile automation, Admin backup + > surface, settings/files backup beyond the PostgreSQL authority) + > are P7-E control-plane work, NOT started. + 5. **Crash and outage recovery** - define API/host/PostgreSQL/Redis/worker/scanner failure behavior; - make committed operations safely repeatable; @@ -156,7 +163,13 @@ P7-S1 → crash recovery / startup reconciliation P7-D1 (accepted: shared rate limit only) → Redis lifecycle hardening → shared rate limit ✅ SHIPPED (PR #265, P7-D2/D3) -backup design → backup/restore CLI → PITR/verification → Admin surface +P7-C portable persistence, backup, PostgreSQL DR ✅ REBUILT & SHIPPED + (C0 reality audit closed; C1 portable bind-mounts + cold backup + + Launchpad; C2 logical pg_dump + verified clean restore; C3 physical + pg_basebackup + WAL archive + PITR; all backed by deterministic Docker + drills). The Admin backup surface (formerly P7-B4) is explicitly OUT of + scope here — restore is operator-owned; no browser restore button. + configuration schema → versioned service → Admin settings UI → exam policy schema → profiles → creation wizard UI pilot → controlled family-by-family UI closeout diff --git a/package.json b/package.json index 3da04c28..e6a5fcd6 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,12 @@ "verify": "pnpm verify:static && TEST_DB_ISOLATION=worker-database API_TEST_MAX_WORKERS=4 pnpm coverage && pnpm build", "test:api": "TEST_DB_ISOLATION=worker-database API_TEST_MAX_WORKERS=4 pnpm --filter @exam/api test", "test:db": "pnpm --filter @exam/db test", + "test:deployment:compose": "bash tests/deployment/compose-smoke.sh", + "test:deployment:launchpad": "bash tests/deployment/launchpad-bootstrap.sh", + "test:deployment:persistence": "bash tests/deployment/persistence-and-cold-restore.sh", + "test:deployment:logical": "bash tests/deployment/logical-backup-restore.sh", + "test:deployment:pitr": "bash tests/deployment/pitr.sh", + "test:deployment": "pnpm test:deployment:compose && pnpm test:deployment:launchpad && pnpm test:deployment:persistence && pnpm test:deployment:logical && pnpm test:deployment:pitr", "coverage:db": "pnpm --filter @exam/db coverage", "coverage:api": "TEST_DB_ISOLATION=worker-database API_TEST_MAX_WORKERS=4 pnpm --filter @exam/api coverage", "verify:nodb-tests": "turbo coverage --filter=!@exam/db --filter=!@exam/api", diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index ee98cd8e..ab15dd17 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -1,5 +1,6 @@ export * from "./common.js"; export * from "./auth.js"; +export * from "./launchpad.js"; export * from "./settings.js"; export * from "./organization.js"; export * from "./user.js"; diff --git a/packages/contracts/src/launchpad.ts b/packages/contracts/src/launchpad.ts new file mode 100644 index 00000000..a332ecce --- /dev/null +++ b/packages/contracts/src/launchpad.ts @@ -0,0 +1,64 @@ +import { z } from "zod"; +import { passwordField } from "./passwordPolicy.js"; + +// ── Launchpad (first-install only, P7-C1) ────────────────────────── + +/** + * Response schema for the installation-status probe. + * + * `initialized` is true once the internal default organization exists + * (slug "default"). This is the FIRST-INSTALL gate only: removing/disabling + * the last Admin does NOT reopen launchpad. The endpoint reveals only the + * init state (which `/login` UX implies anyway) — it is NOT a + * token-validity oracle and never reveals token state. + */ +export const LaunchpadStatusResponseSchema = z.object({ + initialized: z.boolean(), +}); + +/** Type for the launchpad installation-status response. */ +export type LaunchpadStatusResponse = z.infer< + typeof LaunchpadStatusResponseSchema +>; + +/** + * Request schema for the first-Admin bootstrap via launchpad. + * + * The role is NOT selectable — the server always creates role = Admin. The + * `setupToken` is the deployment bootstrap secret (body-only, never URL, + * never audit-logged). Field names mirror the canonical + * `bootstrapAdminOnFreshDb` parameters so the HTTP adapter is a thin shim + * over the same atomic mutation body the CLI uses. + */ +export const LaunchpadBootstrapRequestSchema = z.object({ + organizationName: z.string().min(1).max(200), + organizationDisplayName: z.string().min(1).max(200).optional(), + adminUsername: z.string().min(3).max(50), + adminPassword: passwordField(), + adminName: z.string().min(1).max(100), + setupToken: z.string().min(1).max(1024), +}); + +/** Type for a launchpad bootstrap request. */ +export type LaunchpadBootstrapRequest = z.infer< + typeof LaunchpadBootstrapRequestSchema +>; + +/** + * Response schema returned after a successful first-Admin bootstrap. + * + * Mirrors the minimal public subset of the canonical + * `bootstrapAdminOnFreshDb` result: organization slug + the created Admin's + * username. No secrets, no internal ids beyond what a login flow needs to + * hint the operator toward `/login`. + */ +export const LaunchpadBootstrapResponseSchema = z.object({ + ok: z.literal(true), + organizationSlug: z.string(), + adminUsername: z.string(), +}); + +/** Type for a launchpad bootstrap response. */ +export type LaunchpadBootstrapResponse = z.infer< + typeof LaunchpadBootstrapResponseSchema +>; diff --git a/packages/contracts/src/messageRegistry.ts b/packages/contracts/src/messageRegistry.ts index 9dec4330..341b3dbd 100644 --- a/packages/contracts/src/messageRegistry.ts +++ b/packages/contracts/src/messageRegistry.ts @@ -33,6 +33,7 @@ export const errorMessages = { INTERNAL_ERROR: "服务器内部错误", CURRENT_PASSWORD_INVALID: "当前密码不正确", USER_ALREADY_EXISTS: "用户名已存在", + ADMIN_ALREADY_EXISTS: "已存在启用的管理员", CANDIDATE_IDENTITY_CONFLICT: "身份信息已存在", CANDIDATE_FIELD_IN_USE: "该身份字段正在使用，无法删除", CANDIDATE_IDENTITY_FIELD_CONFLICT: "只能设置一个唯一身份字段", @@ -64,6 +65,8 @@ export const errorMessages = { INCIDENT_ACTION_ALREADY_LINKED: "该操作已关联到其他事件", CSRF_ORIGIN_REJECTED: "请求来源不被允许", AUTH_REGISTER_DISABLED: "Phase 1 不支持公开注册", + LAUNCHPAD_ALREADY_INITIALIZED: "系统已完成初始化，请直接登录", + LAUNCHPAD_INVALID_SETUP_TOKEN: "初始化令牌无效或未配置", PASSWORD_RESET_TARGET_ROLE_NOT_ALLOWED: "不能重置该角色用户的密码", AUTHZ_UNAVAILABLE: "授权服务暂不可用，请稍后重试", RATE_LIMIT_UNAVAILABLE: "限流服务暂不可用，请稍后重试", diff --git a/packages/db/src/repository/organizationRepo.ts b/packages/db/src/repository/organizationRepo.ts index e94db629..f6066594 100644 --- a/packages/db/src/repository/organizationRepo.ts +++ b/packages/db/src/repository/organizationRepo.ts @@ -107,5 +107,23 @@ export function createOrganizationRepo(db: Database) { } return organization; }, + /** + * Returns true when the internal default organization (slug "default") + * exists. This is the P7-C1 launchpad FIRST-INSTALL gate only: once the + * default organization exists the installation is considered initialized + * and launchpad bootstrap is refused. It is deliberately NOT + * `activeAdminCount == 0` (removing the last Admin must not reopen + * launchpad). + */ + async defaultOrganizationExists( + _ctx: PublicBrandingContext, + ): Promise { + const rows = await db + .select({ id: organizations.id }) + .from(organizations) + .where(eq(organizations.slug, "default")) + .limit(1); + return rows.length > 0; + }, }; } diff --git a/packages/domain/src/errors.ts b/packages/domain/src/errors.ts index 4d81ded9..186b8df7 100644 --- a/packages/domain/src/errors.ts +++ b/packages/domain/src/errors.ts @@ -109,6 +109,21 @@ export class UserAlreadyExistsError extends AppError { } } +/** + * A first-install bootstrap lost the "exactly one first Admin" race: an + * active Admin already exists in the organization (HTTP 409). + * + * Thrown by the canonical bootstrap mutation when the transaction-scoped + * advisory lock serialization reveals a winner already committed. The + * Launchpad HTTP adapter maps this expected loser to + * `LAUNCHPAD_ALREADY_INITIALIZED`; the CLI surfaces the message as-is. + */ +export class AdminAlreadyExistsError extends AppError { + constructor(message = "An active Admin already exists in this organization") { + super(message, "ADMIN_ALREADY_EXISTS", 409); + } +} + /** Candidate identity field value conflicts with an existing candidate (HTTP 409). */ export class CandidateIdentityConflictError extends AppError { constructor(message = "Candidate identity already exists") { diff --git a/scripts/backup/cold-filesystem-backup.sh b/scripts/backup/cold-filesystem-backup.sh new file mode 100755 index 00000000..4864a858 --- /dev/null +++ b/scripts/backup/cold-filesystem-backup.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# Cold-filesystem backup helper. +# +# Treats a STOPPED copy of PostgreSQL's complete persistent directory as a +# simple same-version/same-major cold physical backup. This is the simplest +# full backup option; it requires downtime while PostgreSQL is stopped. +# +# Supported cold-backup flow (C1.6): +# stop Exam / PostgreSQL cleanly +# → copy the COMPLETE PostgreSQL persistent directory +# → store the copy OUTSIDE the primary host failure domain +# → restart the service +# +# What this script does and does NOT do: +# - It copies the COMPLETE ${EXAM_DATA_ROOT}/postgres tree with ownership, +# mode, and symlinks preserved (container-assisted, because the files are +# owned by the container postgres user and not readable by the host user). +# - It refuses unsafe source/dest paths and refuses to overwrite an +# existing destination. +# - It validates the source looks like a PGDATA (presence of +# PG_VERSION/postgresql.conf under the postgres major-version subdir). +# - It refuses an obviously RUNNING source: if a live `postmaster.pid` is +# present in the actual PGDATA, it aborts before copying. Do not merely +# print "make sure PostgreSQL is stopped" and copy anyway. The operator +# MUST `docker compose down` first. +# - It does NOT start, stop, or restart the deployment for you. The +# operator must stop PostgreSQL cleanly BEFORE running this script and +# restart it AFTER. A live copy of an active PGDATA is corrupt-prone and +# is explicitly NOT supported. +# - It does NOT verify PostgreSQL major-version compatibility with any +# restore target. Raw PGDATA is tied to the postgres major version; see +# docs/deployment/backup-and-recovery.md. +# +# Distinction (do not conflate): +# ./data/postgres on the live host = persistence +# copy on the same failing disk = weak local copy, NOT DR +# copy on NAS / another server / independent disk = disaster backup +# Store the destination on an INDEPENDENT failure domain. This script does +# not enforce that — it cannot know which disk a path lives on — so the +# operator is responsible for choosing an off-host destination. +# +# Usage: +# ./cold-filesystem-backup.sh +# Example: +# # 1. Stop Exam first: docker compose down +# # 2. Run this script: ./cold-filesystem-backup.sh "${EXAM_DATA_ROOT:-./data}" /mnt/nas/exam-backups/2026-08-10 +# # 3. Restart Exam: docker compose up -d +set -euo pipefail + +# Helper container: the deployment's OWN postgres image (pinned by +# docker-compose.yml and already present on any host that runs this +# deployment). It provides sh/cp/find for PGDATA validation and the +# container-assisted copy. There is deliberately NO separate helper image +# dependency. +HELPER_IMAGE="postgres:18.4-bookworm" + +print_usage() { + cat >&2 <<'EOF' +Usage: cold-filesystem-backup.sh + + EXAM_DATA_ROOT the host data root whose postgres/ subtree holds PGDATA + (the production Compose default is ./data). + DEST_DIR the backup destination. Must not exist yet (the script + creates it). Place it on an INDEPENDENT failure domain + (NAS / another server / a separate disk). + +IMPORTANT: stop PostgreSQL cleanly (docker compose down) BEFORE running this +script, and restart it (docker compose up -d) AFTER. A live copy of an +active PGDATA is NOT supported. +EOF +} + +if [ "$#" -ne 2 ]; then + print_usage + exit 2 +fi + +SRC_ROOT="$1" +DEST="$2" + +# ── Path safety: refuse empty, relative-in-a-bad-way, or unsafe inputs ── +# Require non-empty arguments that are not the filesystem root and do not +# traverse outside via "..". This is a guard, not a full sandbox. +validate_path() { + local name="$1" + local path="$2" + if [ -z "${path}" ]; then + echo "FAIL: ${name} is empty." >&2 + exit 2 + fi + case "${path}" in + /|/etc|/usr|/bin|/sbin|/boot|/proc|/sys|/dev) + echo "FAIL: ${name} '${path}' is a system path; refusing." >&2 + exit 2 + ;; + esac +} +validate_path "EXAM_DATA_ROOT" "${SRC_ROOT}" +validate_path "DEST_DIR" "${DEST}" + +SRC_PG="${SRC_ROOT}/postgres" +if [ ! -d "${SRC_PG}" ]; then + echo "FAIL: source postgres directory not found at ${SRC_PG}." >&2 + echo " Expected \${EXAM_DATA_ROOT}/postgres to exist (the bind-mounted PGDATA)." >&2 + exit 2 +fi +if [ -e "${DEST}" ]; then + echo "FAIL: destination '${DEST}' already exists; refusing to overwrite." >&2 + echo " Choose a fresh destination path for each backup." >&2 + exit 2 +fi + +# Validate the source looks like a PGDATA via a helper container (the files +# are owned by the container postgres user and not readable by the host +# user). Locate the major-version subdir (e.g. 18/docker) and check for +# PG_VERSION + postgresql.conf. +PGDATA_SUBDIR="$(docker run --rm -v "${SRC_PG}:/pg:ro" "${HELPER_IMAGE}" \ + sh -c 'find /pg -maxdepth 3 -name PG_VERSION -print -quit 2>/dev/null || true')" +if [ -z "${PGDATA_SUBDIR}" ]; then + echo "FAIL: no PG_VERSION found under ${SRC_PG}; does not look like a PGDATA tree." >&2 + exit 2 +fi +PGDATA_DIR="$(dirname "${PGDATA_SUBDIR}")" +if ! docker run --rm -v "${SRC_PG}:/pg:ro" "${HELPER_IMAGE}" \ + sh -c "test -f '${PGDATA_DIR}/postgresql.conf'" 2>/dev/null; then + echo "FAIL: postgresql.conf not found next to PG_VERSION at ${PGDATA_DIR}." >&2 + exit 2 +fi + +# ── Refuse an obviously RUNNING source. ── +# A live copy of an active PGDATA is corrupt-prone and NOT supported. The +# smallest SOURCE-SPECIFIC evidence is a live `postmaster.pid` present in the +# actual PGDATA being backed up. (A broad `docker ps | grep db-1` check is +# NOT used: it would false-positive against any unrelated db container +# running on the same host from a different Compose project — the running +# source is identified by its own PGDATA, not by a name pattern.) A clean +# `docker compose down` removes postmaster.pid; its presence therefore means +# a postmaster is (or believes it is) still owning THIS cluster. We do NOT +# build a process detector framework. The supported flow is +# `docker compose down` THEN this script. +if docker run --rm -v "${SRC_PG}:/pg:ro" "${HELPER_IMAGE}" \ + sh -c "test -f '${PGDATA_DIR}/postmaster.pid'" 2>/dev/null; then + echo "FAIL: postmaster.pid present at ${PGDATA_DIR}." >&2 + echo " A postmaster appears to still own this PGDATA. Run" >&2 + echo " 'docker compose down' and ensure PostgreSQL is fully stopped" >&2 + echo " before a cold-filesystem copy." >&2 + exit 2 +fi + +# Create the destination (parent must exist). The backup mirrors the +# deployment data-root layout: PGDATA lands at ${DEST}/postgres/18/docker/... +# so cold-filesystem-restore.sh can target ${DEST_EXAM_DATA_ROOT} directly. +DEST_PARENT="$(dirname "${DEST}")" +if [ ! -d "${DEST_PARENT}" ]; then + echo "FAIL: destination parent '${DEST_PARENT}' does not exist." >&2 + exit 2 +fi +mkdir -p "${DEST}/postgres" + +echo "Cold-filesystem backup:" +echo " source PGDATA: ${SRC_PG} (PGDATA at ${PGDATA_DIR#/pg})" +echo " destination: ${DEST}/postgres" +echo "" +echo " IMPORTANT: PostgreSQL must be STOPPED before this copy (the" +echo " postmaster.pid check above is the safety gate). A live copy of an" +echo " active PGDATA is corrupt-prone and is NOT supported." + +# Container-assisted copy preserves ownership/mode/symlinks. The PGDATA files +# are owned by the container postgres user and not readable by the host user, +# so a host-side cp -a would fail with EACCES. Equivalent to running +# `rsync -aHAX` or `tar | tar` as root on the host. Copy the COMPLETE postgres +# tree (never partial relation files) into ${DEST}/postgres. +echo "Copying COMPLETE postgres tree..." +docker run --rm \ + -v "${SRC_PG}:/from:ro" \ + -v "${DEST}/postgres:/to" \ + "${HELPER_IMAGE}" \ + sh -c 'cp -a /from/. /to/' + +# Verify the copy landed and looks like a PGDATA (use find so we do not +# depend on path arithmetic between the /pg source mount and the /to dest +# mount). The backup layout mirrors the deployment data root, so PGDATA is +# at ${DEST}/postgres/18/docker/PG_VERSION (depth 4 from ${DEST}). +if ! docker run --rm -v "${DEST}:/to:ro" "${HELPER_IMAGE}" \ + sh -c 'find /to -maxdepth 4 -name PG_VERSION -print -quit 2>/dev/null | grep -q .'; then + echo "FAIL: copy verification failed — PG_VERSION missing in destination." >&2 + exit 1 +fi + +echo "" +echo "Cold-filesystem backup COMPLETE." +echo " destination: ${DEST}" +echo " Remember: store this on an INDEPENDENT failure domain (NAS / another" +echo " server / a separate disk). A copy on the same disk as the live data" +echo " is a weak local copy, NOT disaster recovery." +echo " Raw PGDATA is tied to the PostgreSQL major version; restore only with" +echo " a compatible postgres image. See cold-filesystem-restore.sh and" +echo " docs/deployment/backup-and-recovery.md." diff --git a/scripts/backup/cold-filesystem-restore.sh b/scripts/backup/cold-filesystem-restore.sh new file mode 100755 index 00000000..28f6c90b --- /dev/null +++ b/scripts/backup/cold-filesystem-restore.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Cold-filesystem restore helper. +# +# Restores a cold-filesystem backup (produced by cold-filesystem-backup.sh) +# into a fresh/isolated host data root, then lets the compatible PostgreSQL +# image start from it. This is filesystem-level cold restore — it is NOT +# pg_restore, NOT PITR, and NOT a cross-major PostgreSQL upgrade. Keep those +# concepts separate. +# +# Supported cold-restore flow (C1.7): +# fresh isolated host/root +# → compatible PostgreSQL runtime (same major version as the backup) +# → restore the COMPLETE stopped PostgreSQL directory +# → preserve/fix ownership and permissions +# → docker compose up +# → Exam business invariants match the backup state +# +# Safety: +# - The destination must NOT already exist or must be empty. This script +# refuses to overwrite an existing populated data root. +# - It requires explicit confirmation. +# - It copies the COMPLETE postgres tree (never partial relation files). +# +# Usage: +# ./cold-filesystem-restore.sh +# Example: +# # Restore into a fresh data root, then start Exam from it: +# EXAM_DATA_ROOT=/opt/exam/data-fresh +# ./cold-filesystem-restore.sh /mnt/nas/exam-backups/2026-08-10 "${EXAM_DATA_ROOT}" +# EXAM_DATA_ROOT="${EXAM_DATA_ROOT}" docker compose up -d +set -euo pipefail + +# Helper container: the deployment's OWN postgres image (pinned by +# docker-compose.yml and already present on any host that runs this +# deployment). It provides sh/cp/find for PGDATA validation and the +# container-assisted copy. There is deliberately NO separate helper image +# dependency. +HELPER_IMAGE="postgres:18.4-bookworm" + +print_usage() { + cat >&2 <<'EOF' +Usage: cold-filesystem-restore.sh + + BACKUP_DIR the cold backup produced by cold-filesystem-backup.sh + (contains the postgres/ tree). + DEST_EXAM_DATA_ROOT a FRESH destination data root. Must not exist or be + empty; the script creates it. Start Exam afterwards + with EXAM_DATA_ROOT pointing here. + +This restores the COMPLETE postgres directory; it does NOT perform partial +relation-file restore, pg_restore, PITR, or a cross-major PostgreSQL upgrade. +The restored PGDATA is tied to the PostgreSQL major version of the backup — +start it with a compatible postgres image. +EOF +} + +if [ "$#" -ne 2 ]; then + print_usage + exit 2 +fi + +SRC="$1" +DEST_ROOT="$2" + +validate_path() { + local name="$1" + local path="$2" + if [ -z "${path}" ]; then + echo "FAIL: ${name} is empty." >&2 + exit 2 + fi + case "${path}" in + /|/etc|/usr|/bin|/sbin|/boot|/proc|/sys|/dev) + echo "FAIL: ${name} '${path}' is a system path; refusing." >&2 + exit 2 + ;; + esac +} +validate_path "BACKUP_DIR" "${SRC}" +validate_path "DEST_EXAM_DATA_ROOT" "${DEST_ROOT}" + +SRC_PG="${SRC}/postgres" +if [ ! -d "${SRC_PG}" ]; then + echo "FAIL: backup postgres directory not found at ${SRC_PG}." >&2 + echo " '${SRC}' does not look like a cold-filesystem backup." >&2 + exit 2 +fi +# Validate the backup still looks like a PGDATA (helper container; the files +# are owned by the container postgres user and may not be host-readable). +if ! docker run --rm -v "${SRC_PG}:/from:ro" "${HELPER_IMAGE}" \ + sh -c 'find /from -maxdepth 3 -name PG_VERSION -print -quit 2>/dev/null | grep -q .'; then + echo "FAIL: no PG_VERSION found under ${SRC_PG}; does not look like a PGDATA backup." >&2 + exit 2 +fi + +# Refuse to overwrite an existing populated destination. +if [ -e "${DEST_ROOT}" ]; then + if [ -d "${DEST_ROOT}" ] && [ -z "$(ls -A "${DEST_ROOT}" 2>/dev/null || true)" ]; then + : # empty existing directory is OK + else + echo "FAIL: destination '${DEST_ROOT}' already exists and is non-empty." >&2 + echo " Refusing to overwrite a populated data root. Choose a fresh path." >&2 + exit 2 + fi +else + DEST_PARENT="$(dirname "${DEST_ROOT}")" + if [ ! -d "${DEST_PARENT}" ]; then + echo "FAIL: destination parent '${DEST_PARENT}' does not exist." >&2 + exit 2 + fi +fi +mkdir -p "${DEST_ROOT}" + +echo "Cold-filesystem restore:" +echo " backup source: ${SRC_PG}" +echo " destination: ${DEST_ROOT}/postgres" +echo "" +echo " This restores the COMPLETE postgres directory. It is NOT pg_restore," +echo " NOT PITR, and NOT a cross-major PostgreSQL upgrade. The restored" +echo " PGDATA must be started with a compatible (same-major) postgres image." +echo " Continue? [type RESTORE to confirm]" +read -r confirm +if [ "${confirm}" != "RESTORE" ]; then + echo "Aborted." + exit 1 +fi + +DEST_PG="${DEST_ROOT}/postgres" +mkdir -p "${DEST_PG}" + +# Container-assisted copy preserves ownership/mode/symlinks (the PGDATA files +# are owned by the container postgres user). Equivalent to `rsync -aHAX` or +# `tar | tar` as root. +echo "Copying COMPLETE postgres tree from backup to destination..." +docker run --rm \ + -v "${SRC_PG}:/from:ro" \ + -v "${DEST_PG}:/to" \ + "${HELPER_IMAGE}" \ + sh -c 'cp -a /from/. /to/' + +# Verify the restore landed. +if ! docker run --rm -v "${DEST_PG}:/to:ro" "${HELPER_IMAGE}" \ + sh -c 'find /to -maxdepth 3 -name PG_VERSION -print -quit 2>/dev/null | grep -q .'; then + echo "FAIL: restore verification failed — PG_VERSION missing in destination." >&2 + exit 1 +fi + +echo "" +echo "Cold-filesystem restore COMPLETE." +echo " restored to: ${DEST_PG}" +echo " Next: start Exam from this data root with the SAME PostgreSQL major" +echo " version and the SAME DB credentials the volume was initialized with:" +echo " EXAM_DATA_ROOT='${DEST_ROOT}' \\" +echo " POSTGRES_PASSWORD= \\" +echo " docker compose up -d" +echo " The official postgres image fixes ownership/permissions of the PGDATA" +echo " on container start; no host chmod is required." +echo " Run your Exam business-invariant checks after start." diff --git a/scripts/backup/pg-basebackup.sh b/scripts/backup/pg-basebackup.sh new file mode 100755 index 00000000..54bc211f --- /dev/null +++ b/scripts/backup/pg-basebackup.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# PostgreSQL physical online base backup helper (pg_basebackup). +# +# Takes a physical online backup of a RUNNING PostgreSQL server using +# PostgreSQL-native pg_basebackup. The base backup is a complete PostgreSQL +# cluster (PGDATA) captured consistently while the server stays online. It is +# the foundation for physical recovery and, combined with archived WAL, for +# PITR (point-in-time recovery). +# +# Requirements (C3.3): +# - running server supported (PostgreSQL stays ONLINE) +# - complete PostgreSQL cluster +# - required WAL included/streamed (-X stream) +# - backup target OUTSIDE the live PGDATA +# - no unsafe --no-sync in the production path +# +# Authentication truth: pg_basebackup runs in a +# sibling container that shares the db container's NETWORK namespace and +# connects over loopback TCP (-h 127.0.0.1). The official postgres image +# authenticates TCP connections with scram-sha-256 by default (trust applies +# only to Unix-socket local connections), so a password IS required. This +# script derives the actual deployment's POSTGRES_USER and POSTGRES_PASSWORD +# from the RUNNING db container's environment and passes the password via +# PGPASSWORD (never argv). PGUSER/PGPASSWORD defaults therefore follow the +# deployment; an operator does not need to maintain a separate backup +# credential namespace for the bundled single-node path. +# +# Replication privilege: pg_basebackup requires a +# SUPERUSER or REPLICATION-capable role. For the bundled single-node +# deployment, this script uses the bootstrap PostgreSQL superuser +# (POSTGRES_USER), which satisfies that requirement. A narrowly scoped +# replication-only role is NOT provisioned by this script; future hardening +# (a dedicated REPLICATION role with no other authority) belongs in a later +# operations / P7-E pass and is documented separately. The comment and the +# implementation agree: the bundled path uses the superuser over loopback TCP +# with the deployment password. +# +# Manifest verification (C3.4): after the base backup, run pg_verifybackup on +# the manifest as an integrity check. NOTE the documented limitation: manifest +# verification is backup-integrity evidence (backup contents match the +# manifest's per-file checksums and the manifest's own checksum verifies), NOT +# proof that Exam can successfully start and satisfy business invariants after +# restore. A restore drill is still required (see the PITR drill / +# physical-restore path). +# +# PITR base-backup rule (§21): WAL archiving MUST be active BEFORE the base +# backup that will anchor PITR. Run scripts/backup/postgres-enable-pitr.sh +# FIRST, confirm the archiver is producing evidence, THEN take this base +# backup. A base backup taken before WAL archiving was established is NOT a +# valid anchor for later continuous PITR. +# +# Usage: +# ./pg-basebackup.sh +# Example: +# ./pg-basebackup.sh exam /mnt/nas/exam-basebackups/2026-08-10 +set -euo pipefail + +print_usage() { + cat >&2 <<'EOF' +Usage: pg-basebackup.sh + + COMPOSE_PROJECT the Compose project name (addresses -db-1). + DEST_DIR the base-backup destination (created by the script; must + not exist yet). Place it on an INDEPENDENT failure domain. + +Takes a physical online base backup of the running PostgreSQL server +(complete cluster + streamed WAL via -X stream), then verifies the backup +manifest with pg_verifybackup. PostgreSQL stays ONLINE. The backup target +is OUTSIDE the live PGDATA. --no-sync is NOT used in this production path. + +The connection uses the deployment's POSTGRES_USER over loopback TCP with +the deployment password (read from the db container's environment, passed +via PGPASSWORD). For PITR, WAL archiving must already be active (run +postgres-enable-pitr.sh first). +EOF +} + +if [ "$#" -ne 2 ]; then + print_usage + exit 2 +fi + +PROJECT="$1" +DEST="$2" + +if [ -z "${PROJECT}" ] || [ -z "${DEST}" ]; then + echo "FAIL: COMPOSE_PROJECT and DEST_DIR must be non-empty." >&2 + exit 2 +fi +case "${DEST}" in + /|/etc|/usr|/bin|/sbin|/boot|/proc|/sys|/dev) + echo "FAIL: DEST_DIR '${DEST}' is a system path; refusing." >&2 + exit 2 + ;; +esac +if [ -e "${DEST}" ]; then + echo "FAIL: destination '${DEST}' already exists; refusing to overwrite." >&2 + exit 2 +fi +DEST_PARENT="$(dirname "${DEST}")" +if [ ! -d "${DEST_PARENT}" ]; then + echo "FAIL: destination parent '${DEST_PARENT}' does not exist." >&2 + exit 2 +fi + +DB_CONTAINER="${PROJECT}-db-1" +if ! docker inspect "${DB_CONTAINER}" >/dev/null 2>&1; then + echo "FAIL: db container '${DB_CONTAINER}' not found (project '${PROJECT}')." >&2 + exit 2 +fi + +# Derive the actual deployment's POSTGRES_USER / POSTGRES_PASSWORD from the +# RUNNING db container (NOT hardcoded). The bundled Compose seeds these; an +# operator that customized POSTGRES_USER=appdb is honored automatically. +DEPLOY_PG_USER="$(docker inspect "${DB_CONTAINER}" \ + --format '{{range .Config.Env}}{{println .}}{{end}}' \ + | sed -n 's/^POSTGRES_USER=//p' | head -1)" +DEPLOY_PG_DB="$(docker inspect "${DB_CONTAINER}" \ + --format '{{range .Config.Env}}{{println .}}{{end}}' \ + | sed -n 's/^POSTGRES_DB=//p' | head -1)" +DEPLOY_PG_PASSWORD="$(docker inspect "${DB_CONTAINER}" \ + --format '{{range .Config.Env}}{{println .}}{{end}}' \ + | sed -n 's/^POSTGRES_PASSWORD=//p' | head -1)" +DEPLOY_PG_USER="${DEPLOY_PG_USER:-exam}" +DEPLOY_PG_DB="${DEPLOY_PG_DB:-exam}" + +if ! docker exec "${DB_CONTAINER}" pg_isready -U "${DEPLOY_PG_USER}" -d "${DEPLOY_PG_DB}" >/dev/null 2>&1; then + echo "FAIL: PostgreSQL is not ready in ${DB_CONTAINER}." >&2 + exit 2 +fi + +# PGUSER/PGPASSWORD precedence (§18): an explicit host export wins (operators +# with a separate backup credential), otherwise fall back to the deployment's +# POSTGRES_USER / POSTGRES_PASSWORD read from the container. Never hardcode +# PGUSER=exam when POSTGRES_USER may vary. +EFFECTIVE_PGUSER="${PGUSER:-${DEPLOY_PG_USER}}" +EFFECTIVE_PGPASSWORD="${PGPASSWORD:-${DEPLOY_PG_PASSWORD}}" +if [ -z "${EFFECTIVE_PGPASSWORD}" ]; then + echo "FAIL: no PostgreSQL password available." >&2 + echo " The connection is over loopback TCP (scram-sha-256), which" >&2 + echo " requires a password. Export PGPASSWORD= or" >&2 + echo " ensure the db container exposes POSTGRES_PASSWORD." >&2 + exit 2 +fi + +echo "Physical online base backup (pg_basebackup):" +echo " source: ${DB_CONTAINER} (PG user: ${EFFECTIVE_PGUSER})" +echo " destination: ${DEST}" +echo " PostgreSQL stays ONLINE; required WAL streamed (-X stream)." +echo " auth: loopback TCP + scram-sha-256, password via PGPASSWORD (never argv)." + +# Run pg_basebackup in a sibling container that shares the db container's +# network namespace and connects over loopback TCP. -D points at a path +# INSIDE the container; we bind-mount the host destination as the backup +# target. +mkdir -p "${DEST}" + +# pg_basebackup options: +# -D backup target (inside container, bind-mounted from host) +# -X stream stream required WAL segments (not fetch) so the backup is +# self-consistent without relying on archive_command +# -c fast checkpoint type (fast) to start promptly +# -Fp plain format (directory tree, like a real PGDATA) +# -l