ci: Change host tests to use cached index

.github/workflows/ci-host.yaml

@@ -33,6 +33,10 @@ permissions:
   # Lets the warm-test-images step pull the private GHCR mirror packages.
   packages: read
   pull-requests: read
+  # Lets scripts/import-cached-index.sh (via `gh run list` / `gh run
+  # download`) fetch the boxel-index-cache artifact produced by main runs
+  # of ci.yaml.
+  actions: read
 
 jobs:
   check-percy:
@@ -78,6 +82,101 @@ jobs:
             echo "Percy will be skipped — no UI-relevant changes"
           fi
 
+  check-index-cache:
+    # The boxel-index-cache artifact (produced by main runs of ci.yaml)
+    # snapshots the index tables, letting shards boot on it instead of
+    # spending ~2-3 min indexing base/skills/openrouter from scratch.
+    #
+    # Changed realm *content* needs no gate: mtimes are normalized to a
+    # function of file content on both sides (scripts/normalize-realm-mtimes.mjs),
+    # so the boot index skips files this checkout hasn't touched and
+    # re-renders the ones it has. A PR editing packages/base therefore
+    # still uses the cache and still tests its own cards.
+    #
+    # What can't be detected per-file is a change to how indexing or
+    # rendering *behaves*: the cached rows for untouched files were
+    # produced by main's code, and nothing about those files' content
+    # signals that they'd index differently now. Those paths fall back to
+    # from-scratch indexing.
+    #
+    # Push-to-main and workflow_dispatch runs never use the cache: the
+    # freshest artifact predates the commit under test, and main runs feed
+    # the memory-baseline update, which shouldn't be measured against a
+    # differently-seeded index.
+    name: Check if cached index is usable
+    runs-on: ubuntu-latest
+    outputs:
+      use_cache: ${{ steps.check.outputs.use_cache }}
+    steps:
+      - name: Check for index-behavior changes
+        id: check
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ "${{ github.event_name }}" != "pull_request" ]]; then
+            echo "use_cache=false" >> "$GITHUB_OUTPUT"
+            echo "Not a pull request — realms will index from scratch"
+            exit 0
+          fi
+
+          CHANGED_FILES=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files --paginate --jq '.[].filename')
+
+          # packages/realm-server, packages/runtime-common: how files are
+          #   indexed and prerendered.
+          # packages/boxel-ui: baked into prerendered card HTML — the
+          #   deployed post-deploy reindex keys on a boxel-ui checksum for
+          #   this same reason (lib/boxel-ui-change-checker.ts).
+          # packages/postgres: schema of the dumped tables.
+          # package.json / pnpm-lock.yaml: a dependency change can alter
+          #   render or index output.
+          #
+          # The host render path gets the same treatment. `prerendered_html`
+          # rows are produced by rendering cards through the host bundle, and a
+          # card this PR did not touch keeps its cached row — so a change to
+          # the code that emits that HTML would leave tests reading markup
+          # built by main's host, and a rendering regression could pass. The
+          # test to apply when adding a path here: does this file shape the
+          # HTML the prerenderer writes? Client-side consumers of that HTML
+          # (renderable-search-entries, markdown rendering) do not qualify —
+          # they read it, and they read freshly rendered HTML the same way.
+          #
+          # Deliberately not all of packages/host: it is this workflow's main
+          # path trigger, so that would return nearly every run to
+          # from-scratch indexing. Ordinary host UI — operator mode, the AI
+          # assistant, code submode — never appears in a card's prerendered
+          # HTML, which is why the deployed equivalent keys on boxel-ui rather
+          # than on host-bundle identity.
+          use_cache=true
+          while IFS= read -r file; do
+            case "$file" in
+              packages/realm-server/*|packages/runtime-common/*|packages/postgres/*|packages/boxel-ui/*|package.json|pnpm-lock.yaml)
+                use_cache=false
+                echo "Index-behavior change detected: $file"
+                break
+                ;;
+              packages/host/app/components/card-prerender.gts |\
+              packages/host/app/components/card-renderer.gts |\
+              packages/host/app/lib/isolated-render.gts |\
+              packages/host/app/lib/prerender-* |\
+              packages/host/app/routes/render.ts |\
+              packages/host/app/routes/render/* |\
+              packages/host/app/services/render-* |\
+              packages/host/app/templates/render.gts |\
+              packages/host/app/utils/render-*)
+                use_cache=false
+                echo "Host render-path change detected: $file"
+                break
+                ;;
+            esac
+          done <<< "$CHANGED_FILES"
+
+          echo "use_cache=$use_cache" >> "$GITHUB_OUTPUT"
+          if [ "$use_cache" = "true" ]; then
+            echo "Cached index will be imported — the boot index will reconcile any changed realm content"
+          else
+            echo "Realms will index from scratch"
+          fi
+
   test-web-assets:
     name: Build test web assets
     uses: ./.github/workflows/test-web-assets.yaml
@@ -98,7 +197,7 @@ jobs:
     name: Live Tests (realm)
     if: github.event.action != 'ready_for_review'
     runs-on: ubuntu-latest
-    needs: test-web-assets
+    needs: [test-web-assets, check-index-cache]
     concurrency:
       group: boxel-live-test-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
@@ -186,6 +285,20 @@ jobs:
       - name: Warm test Docker images from the GHCR mirror
         continue-on-error: true
         uses: ./.github/actions/warm-test-images
+      # Seed the index tables from a recent main boxel-index-cache artifact,
+      # leaving the boot index to reconcile whatever this checkout changed
+      # rather than indexing every realm from scratch. Runs before the
+      # services start, not inside them: the import replays a large dump, and
+      # doing that ahead of `start-server-and-test` inside the service task
+      # would push Synapse's startup past the budget `create realm users`
+      # waits with. GH_TOKEN is what the `gh run download` inside
+      # scripts/import-cached-index.sh authenticates with.
+      - name: Import cached realm index
+        if: ${{ needs.check-index-cache.outputs.use_cache == 'true' }}
+        run: mise run ci:import-index
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Start test services (icons + host dist + realm servers)
         run: mise run test-services:host | tee -a /tmp/server.log &
 
@@ -295,7 +408,7 @@ jobs:
   host-test:
     name: Host Tests
     runs-on: ubuntu-latest
-    needs: [test-web-assets, check-percy]
+    needs: [test-web-assets, check-percy, check-index-cache]
     strategy:
       fail-fast: false
       matrix:
@@ -422,6 +535,20 @@ jobs:
       - name: Warm test Docker images from the GHCR mirror
         continue-on-error: true
         uses: ./.github/actions/warm-test-images
+      # Seed the index tables from a recent main boxel-index-cache artifact,
+      # leaving the boot index to reconcile whatever this checkout changed
+      # rather than indexing every realm from scratch. Runs before the
+      # services start, not inside them: the import replays a large dump, and
+      # doing that ahead of `start-server-and-test` inside the service task
+      # would push Synapse's startup past the budget `create realm users`
+      # waits with. GH_TOKEN is what the `gh run download` inside
+      # scripts/import-cached-index.sh authenticates with.
+      - name: Import cached realm index
+        if: ${{ needs.check-index-cache.outputs.use_cache == 'true' }}
+        run: mise run ci:import-index
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Start test services (icons + host dist + realm servers)
         run: mise run test-services:host | tee -a /tmp/server.log &
         env:
@@ -459,10 +586,13 @@ jobs:
           skills_info="https://realm-server.ci.localhost/skills/${info}"
           base_info="https://realm-server.ci.localhost/base/${info}"
 
-          # base/_readiness-check returns 200 only after the from-scratch
-          # index of the base realm finishes. The per-slug Postgres is fresh
-          # every run so this always runs, ~60s. Tests that load base cards
-          # block on this being indexed.
+          # base/_readiness-check returns 200 only after the base realm's
+          # boot index finishes — the ~60s from-scratch index when the
+          # per-slug Postgres started empty, or just the files this checkout
+          # changed when a cached index was imported (see check-index-cache).
+          # It awaits the in-flight index either way, so tests never start
+          # against a half-reconciled realm. Tests that load base cards block
+          # on this being indexed.
           ok=0
           for i in $(seq 1 120); do
             code=$(curl -sS -o /tmp/base_ready.json -w '%{http_code}' \

mise-tasks/ci/import-index

@@ -0,0 +1,41 @@
+#!/bin/bash
+#MISE description="Seed this environment's index tables from the cached CI index"
+#MISE dir="packages/realm-server"
+
+set -euo pipefail
+
+export PATH="./node_modules/.bin:$PATH"
+REPO_ROOT="$(cd "../.." && pwd)"
+
+# Brings the database up to the point where scripts/import-cached-index.sh can
+# load a snapshot into it: Postgres running, the per-environment database
+# created, migrations applied. All three are idempotent — the service stack
+# runs them again on its own way up (start:pg, infra:ensure-db, main.ts
+# --migrateDB).
+#
+# This is a separate step from starting the services on purpose. The import
+# downloads and replays a large dump, and anything that runs before
+# `start-server-and-test` delays every service it launches — including
+# Synapse, which the caller's `register-realm-users` step waits on with a
+# fixed budget. Keeping the import in its own step means the services start
+# promptly once it is done, and its cost shows up as its own line in the CI
+# timing rather than hiding inside service startup.
+./scripts/start-pg.sh
+echo "Waiting for Postgres to accept connections…"
+until docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1; do sleep 1; done
+"$REPO_ROOT/scripts/ensure-branch-db.sh"
+pnpm migrate
+
+# Ask for the host-scoped snapshot: a test stack serves base, skills and
+# openrouter, and the full snapshot's other realms are a few hundred MB this
+# job would download and replay for rows it never reads. The import falls back
+# to the full snapshot when the scoped one is missing or expired.
+export INDEX_CACHE_ARTIFACT="${INDEX_CACHE_ARTIFACT:-boxel-index-cache-host}"
+
+# A cache miss is not a failure: the realms simply index from scratch, which
+# is what they did before this task existed.
+if "$REPO_ROOT/scripts/import-cached-index.sh"; then
+  echo "[ci:import-index] index cache imported — the boot index will reconcile only what this checkout changed"
+else
+  echo "[ci:import-index] no index cache available — realms will index from scratch"
+fi

mise-tasks/test-services/host

@@ -25,6 +25,54 @@ NODE_TEST_REALM_READY="${REALM_TEST_READY_SCHEME}://${REALM_TEST_URL#*://}/node-
 HOST_TEST_LOG_LEVELS="${HOST_TEST_LOG_LEVELS:-*=info,realm:requests=warn}"
 SKIP_CATALOG="${SKIP_CATALOG:-}"
 
+REPO_ROOT="$(cd "../.." && pwd)"
+
+# Give every file in the realms this stack boots a content-derived mtime, so
+# they match the mtimes recorded in a cached index built on a different
+# runner (see scripts/normalize-realm-mtimes.mjs for the mechanism). Three
+# realms, matching the `--path` args services/realm-server passes that
+# aren't switched off by the SKIP_* flags below: base, skills, openrouter.
+# Keep this list in step with the one in ci/cache-index — a realm normalized
+# on only one side never matches and re-indexes wholesale.
+#
+# Runs on every CI shard, not only the ones importing a cache: it keeps
+# mtimes identical between cache and from-scratch runs, so `last-modified`
+# on these realms' source files — and the "last saved" text the host renders
+# from it — reads the same either way. Percy compares PR snapshots against
+# baselines from main runs, which never import the cache, so normalizing
+# only the importing runs would diff every snapshot that surfaces a file's
+# modified time.
+#
+# Confined to CI because it rewrites mtimes in the checkout it runs against.
+# On a developer's machine that would restamp their working tree and leave
+# their own dev-server index looking stale on its next boot; a local host-test
+# run indexes from scratch anyway, so it has nothing to gain here.
+#
+# skills:setup first because the skills realm is a separate clone that may
+# not be on disk yet; it no-ops when the content is already there.
+if [ -n "${CI:-}" ]; then
+  pnpm --dir=../skills-realm skills:setup
+  node "$REPO_ROOT/scripts/normalize-realm-mtimes.mjs" \
+    ../base ../skills-realm/contents ../openrouter-realm
+fi
+
+# Seeding the index tables from a cached snapshot is `mise run ci:import-index`,
+# which the caller runs as its own step *before* this one. It deliberately does
+# not happen here: this task's first act is to hand control to
+# `start-server-and-test`, and any work in front of that delays every service
+# it launches — Synapse included, which callers wait on with a fixed budget.
+#
+# When a snapshot has been imported, the boot index is what reconciles it
+# against this checkout, so no REALM_SERVER_FULL_INDEX_ON_STARTUP override is
+# set here: `discoverInvalidations` skips every file whose mtime matches its
+# indexed row and revisits the rest, so this checkout's own changes — and
+# whatever the invalidation fan-out reaches from them — get re-rendered while
+# untouched files keep their cached rows. `applyBatchUpdates` upserts only the
+# invalidated URLs, so rows the pass never visits survive, and
+# `_readiness-check` awaits the in-flight index, so tests can't race the
+# reconcile. With no snapshot imported the same pass simply finds nothing to
+# skip and indexes from scratch.
+
 # START_SERVER_AND_TEST_INSECURE=1 scopes to the wait-on readiness probe.
 # The realm-server / vite now serve the self-signed mkcert leaf on
 # https://localhost:4201|4202|4200; start-server-and-test forces

scripts/import-cached-index.sh

@@ -11,7 +11,18 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
 DB_NAME="${PGDATABASE:-boxel}"
 REPO="cardstack/boxel"
-ARTIFACT_NAME="boxel-index-cache"
+# Artifacts to try, in order. `boxel-index-cache` carries every realm the
+# export job indexes; `boxel-index-cache-host` carries only the realms a host
+# test stack serves and is a fraction of the size, so a caller that serves
+# just those asks for it via INDEX_CACHE_ARTIFACT. The full snapshot stays the
+# fallback: a caller that wants the scoped one still works when it is missing
+# or expired, just with more rows than it needs. Each artifact holds a single
+# `<artifact-name>.sql.gz`.
+DEFAULT_ARTIFACT="boxel-index-cache"
+ARTIFACT_NAMES="${INDEX_CACHE_ARTIFACT:-$DEFAULT_ARTIFACT}"
+if [ "$ARTIFACT_NAMES" != "$DEFAULT_ARTIFACT" ]; then
+  ARTIFACT_NAMES="$ARTIFACT_NAMES $DEFAULT_ARTIFACT"
+fi
 DOWNLOAD_DIR="/tmp/boxel-index-cache-$$"
 
 cleanup() {
@@ -33,27 +44,49 @@ if ! command -v gh >/dev/null 2>&1; then
   exit 1
 fi
 
-# Find the latest successful CI run on main that produced the cache artifact.
+# Find a recent successful CI run on main that produced the cache artifact.
+# Not every successful main run has one — the cache-index job is skipped
+# when change-check decides nothing boxel-related changed — so walk the
+# most recent runs until a download succeeds instead of pinning to the
+# single latest run.
 echo "Looking for cached index from CI..."
-RUN_ID=$(gh run list -w ci.yaml -b main -s success -L 1 \
-  --json databaseId -q '.[0].databaseId' -R "$REPO" 2>/dev/null) || RUN_ID=""
-if [ -z "$RUN_ID" ]; then
-  echo "No CI run with index cache found, skipping."
+RUN_IDS=$(gh run list -w ci.yaml -b main -s success -L 10 \
+  --json databaseId -q '.[].databaseId' -R "$REPO" 2>/dev/null) || RUN_IDS=""
+if [ -z "$RUN_IDS" ]; then
+  echo "No successful CI runs on main found, skipping."
   exit 1
 fi
 
-# Download the artifact
-echo "Downloading index cache from CI run $RUN_ID..."
-if ! gh run download "$RUN_ID" -n "$ARTIFACT_NAME" -D "$DOWNLOAD_DIR" -R "$REPO" 2>/dev/null; then
-  echo "Failed to download index cache artifact, skipping."
-  exit 1
-fi
+# Phase timings: this script is on the critical path of every CI shard that
+# uses it, and "the import took a while" is not actionable — the download and
+# the replay have different fixes (a smaller artifact vs. a cheaper load).
+# Report them separately, with the size that explains both.
+DOWNLOAD_START=$(date +%s)
+CACHE_FILE=""
+for ARTIFACT_NAME in $ARTIFACT_NAMES; do
+  CANDIDATE="$DOWNLOAD_DIR/${ARTIFACT_NAME}.sql.gz"
+  for RUN_ID in $RUN_IDS; do
+    echo "Trying artifact ${ARTIFACT_NAME} from CI run $RUN_ID..."
+    if gh run download "$RUN_ID" -n "$ARTIFACT_NAME" -D "$DOWNLOAD_DIR" -R "$REPO" 2>/dev/null \
+      && [ -f "$CANDIDATE" ]; then
+      echo "Downloaded ${ARTIFACT_NAME} from CI run $RUN_ID."
+      CACHE_FILE="$CANDIDATE"
+      break
+    fi
+    rm -rf "$DOWNLOAD_DIR"
+  done
+  if [ -n "$CACHE_FILE" ]; then
+    break
+  fi
+done
 
-CACHE_FILE="$DOWNLOAD_DIR/boxel-index-cache.sql.gz"
-if [ ! -f "$CACHE_FILE" ]; then
-  echo "Cache file not found in artifact, skipping."
+if [ -z "$CACHE_FILE" ] || [ ! -f "$CACHE_FILE" ]; then
+  echo "No index cache artifact found in recent CI runs, skipping."
   exit 1
 fi
+echo "Index cache download took $(( $(date +%s) - DOWNLOAD_START ))s ($(du -h "$CACHE_FILE" | cut -f1) compressed)."
+
+REPLAY_START=$(date +%s)
 
 # Clear any partial data before importing.
 echo "Truncating index tables..."
@@ -87,4 +120,5 @@ else
     | docker exec -i boxel-pg psql $PSQL_OPTS
 fi
 
+echo "Index cache replay took $(( $(date +%s) - REPLAY_START ))s."
 echo "Index cache imported successfully."