diff --git a/.github/workflows/ci-host.yaml b/.github/workflows/ci-host.yaml index ece8e67e526..9314000bd12 100644 --- a/.github/workflows/ci-host.yaml +++ b/.github/workflows/ci-host.yaml @@ -33,6 +33,10 @@ permissions: # Lets the warm-test-images step pull the private GHCR mirror packages. packages: read pull-requests: read + # Lets scripts/import-cached-index.sh (via `gh run list` / `gh run + # download`) fetch the boxel-index-cache artifact produced by main runs + # of ci.yaml. + actions: read jobs: check-percy: @@ -78,6 +82,101 @@ jobs: echo "Percy will be skipped — no UI-relevant changes" fi + check-index-cache: + # The boxel-index-cache artifact (produced by main runs of ci.yaml) + # snapshots the index tables, letting shards boot on it instead of + # spending ~2-3 min indexing base/skills/openrouter from scratch. + # + # Changed realm *content* needs no gate: mtimes are normalized to a + # function of file content on both sides (scripts/normalize-realm-mtimes.mjs), + # so the boot index skips files this checkout hasn't touched and + # re-renders the ones it has. A PR editing packages/base therefore + # still uses the cache and still tests its own cards. + # + # What can't be detected per-file is a change to how indexing or + # rendering *behaves*: the cached rows for untouched files were + # produced by main's code, and nothing about those files' content + # signals that they'd index differently now. Those paths fall back to + # from-scratch indexing. + # + # Push-to-main and workflow_dispatch runs never use the cache: the + # freshest artifact predates the commit under test, and main runs feed + # the memory-baseline update, which shouldn't be measured against a + # differently-seeded index. + name: Check if cached index is usable + runs-on: ubuntu-latest + outputs: + use_cache: ${{ steps.check.outputs.use_cache }} + steps: + - name: Check for index-behavior changes + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "use_cache=false" >> "$GITHUB_OUTPUT" + echo "Not a pull request — realms will index from scratch" + exit 0 + fi + + CHANGED_FILES=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files --paginate --jq '.[].filename') + + # packages/realm-server, packages/runtime-common: how files are + # indexed and prerendered. + # packages/boxel-ui: baked into prerendered card HTML — the + # deployed post-deploy reindex keys on a boxel-ui checksum for + # this same reason (lib/boxel-ui-change-checker.ts). + # packages/postgres: schema of the dumped tables. + # package.json / pnpm-lock.yaml: a dependency change can alter + # render or index output. + # + # The host render path gets the same treatment. `prerendered_html` + # rows are produced by rendering cards through the host bundle, and a + # card this PR did not touch keeps its cached row — so a change to + # the code that emits that HTML would leave tests reading markup + # built by main's host, and a rendering regression could pass. The + # test to apply when adding a path here: does this file shape the + # HTML the prerenderer writes? Client-side consumers of that HTML + # (renderable-search-entries, markdown rendering) do not qualify — + # they read it, and they read freshly rendered HTML the same way. + # + # Deliberately not all of packages/host: it is this workflow's main + # path trigger, so that would return nearly every run to + # from-scratch indexing. Ordinary host UI — operator mode, the AI + # assistant, code submode — never appears in a card's prerendered + # HTML, which is why the deployed equivalent keys on boxel-ui rather + # than on host-bundle identity. + use_cache=true + while IFS= read -r file; do + case "$file" in + packages/realm-server/*|packages/runtime-common/*|packages/postgres/*|packages/boxel-ui/*|package.json|pnpm-lock.yaml) + use_cache=false + echo "Index-behavior change detected: $file" + break + ;; + packages/host/app/components/card-prerender.gts |\ + packages/host/app/components/card-renderer.gts |\ + packages/host/app/lib/isolated-render.gts |\ + packages/host/app/lib/prerender-* |\ + packages/host/app/routes/render.ts |\ + packages/host/app/routes/render/* |\ + packages/host/app/services/render-* |\ + packages/host/app/templates/render.gts |\ + packages/host/app/utils/render-*) + use_cache=false + echo "Host render-path change detected: $file" + break + ;; + esac + done <<< "$CHANGED_FILES" + + echo "use_cache=$use_cache" >> "$GITHUB_OUTPUT" + if [ "$use_cache" = "true" ]; then + echo "Cached index will be imported — the boot index will reconcile any changed realm content" + else + echo "Realms will index from scratch" + fi + test-web-assets: name: Build test web assets uses: ./.github/workflows/test-web-assets.yaml @@ -98,7 +197,7 @@ jobs: name: Live Tests (realm) if: github.event.action != 'ready_for_review' runs-on: ubuntu-latest - needs: test-web-assets + needs: [test-web-assets, check-index-cache] concurrency: group: boxel-live-test-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -186,6 +285,20 @@ jobs: - name: Warm test Docker images from the GHCR mirror continue-on-error: true uses: ./.github/actions/warm-test-images + # Seed the index tables from a recent main boxel-index-cache artifact, + # leaving the boot index to reconcile whatever this checkout changed + # rather than indexing every realm from scratch. Runs before the + # services start, not inside them: the import replays a large dump, and + # doing that ahead of `start-server-and-test` inside the service task + # would push Synapse's startup past the budget `create realm users` + # waits with. GH_TOKEN is what the `gh run download` inside + # scripts/import-cached-index.sh authenticates with. + - name: Import cached realm index + if: ${{ needs.check-index-cache.outputs.use_cache == 'true' }} + run: mise run ci:import-index + env: + GH_TOKEN: ${{ github.token }} + - name: Start test services (icons + host dist + realm servers) run: mise run test-services:host | tee -a /tmp/server.log & @@ -295,7 +408,7 @@ jobs: host-test: name: Host Tests runs-on: ubuntu-latest - needs: [test-web-assets, check-percy] + needs: [test-web-assets, check-percy, check-index-cache] strategy: fail-fast: false matrix: @@ -422,6 +535,20 @@ jobs: - name: Warm test Docker images from the GHCR mirror continue-on-error: true uses: ./.github/actions/warm-test-images + # Seed the index tables from a recent main boxel-index-cache artifact, + # leaving the boot index to reconcile whatever this checkout changed + # rather than indexing every realm from scratch. Runs before the + # services start, not inside them: the import replays a large dump, and + # doing that ahead of `start-server-and-test` inside the service task + # would push Synapse's startup past the budget `create realm users` + # waits with. GH_TOKEN is what the `gh run download` inside + # scripts/import-cached-index.sh authenticates with. + - name: Import cached realm index + if: ${{ needs.check-index-cache.outputs.use_cache == 'true' }} + run: mise run ci:import-index + env: + GH_TOKEN: ${{ github.token }} + - name: Start test services (icons + host dist + realm servers) run: mise run test-services:host | tee -a /tmp/server.log & env: @@ -459,10 +586,13 @@ jobs: skills_info="https://realm-server.ci.localhost/skills/${info}" base_info="https://realm-server.ci.localhost/base/${info}" - # base/_readiness-check returns 200 only after the from-scratch - # index of the base realm finishes. The per-slug Postgres is fresh - # every run so this always runs, ~60s. Tests that load base cards - # block on this being indexed. + # base/_readiness-check returns 200 only after the base realm's + # boot index finishes — the ~60s from-scratch index when the + # per-slug Postgres started empty, or just the files this checkout + # changed when a cached index was imported (see check-index-cache). + # It awaits the in-flight index either way, so tests never start + # against a half-reconciled realm. Tests that load base cards block + # on this being indexed. ok=0 for i in $(seq 1 120); do code=$(curl -sS -o /tmp/base_ready.json -w '%{http_code}' \ diff --git a/mise-tasks/ci/import-index b/mise-tasks/ci/import-index new file mode 100755 index 00000000000..9d5a419b86f --- /dev/null +++ b/mise-tasks/ci/import-index @@ -0,0 +1,41 @@ +#!/bin/bash +#MISE description="Seed this environment's index tables from the cached CI index" +#MISE dir="packages/realm-server" + +set -euo pipefail + +export PATH="./node_modules/.bin:$PATH" +REPO_ROOT="$(cd "../.." && pwd)" + +# Brings the database up to the point where scripts/import-cached-index.sh can +# load a snapshot into it: Postgres running, the per-environment database +# created, migrations applied. All three are idempotent — the service stack +# runs them again on its own way up (start:pg, infra:ensure-db, main.ts +# --migrateDB). +# +# This is a separate step from starting the services on purpose. The import +# downloads and replays a large dump, and anything that runs before +# `start-server-and-test` delays every service it launches — including +# Synapse, which the caller's `register-realm-users` step waits on with a +# fixed budget. Keeping the import in its own step means the services start +# promptly once it is done, and its cost shows up as its own line in the CI +# timing rather than hiding inside service startup. +./scripts/start-pg.sh +echo "Waiting for Postgres to accept connections…" +until docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1; do sleep 1; done +"$REPO_ROOT/scripts/ensure-branch-db.sh" +pnpm migrate + +# Ask for the host-scoped snapshot: a test stack serves base, skills and +# openrouter, and the full snapshot's other realms are a few hundred MB this +# job would download and replay for rows it never reads. The import falls back +# to the full snapshot when the scoped one is missing or expired. +export INDEX_CACHE_ARTIFACT="${INDEX_CACHE_ARTIFACT:-boxel-index-cache-host}" + +# A cache miss is not a failure: the realms simply index from scratch, which +# is what they did before this task existed. +if "$REPO_ROOT/scripts/import-cached-index.sh"; then + echo "[ci:import-index] index cache imported — the boot index will reconcile only what this checkout changed" +else + echo "[ci:import-index] no index cache available — realms will index from scratch" +fi diff --git a/mise-tasks/test-services/host b/mise-tasks/test-services/host index 29e5d22065b..f23e1a64cbf 100755 --- a/mise-tasks/test-services/host +++ b/mise-tasks/test-services/host @@ -25,6 +25,54 @@ NODE_TEST_REALM_READY="${REALM_TEST_READY_SCHEME}://${REALM_TEST_URL#*://}/node- HOST_TEST_LOG_LEVELS="${HOST_TEST_LOG_LEVELS:-*=info,realm:requests=warn}" SKIP_CATALOG="${SKIP_CATALOG:-}" +REPO_ROOT="$(cd "../.." && pwd)" + +# Give every file in the realms this stack boots a content-derived mtime, so +# they match the mtimes recorded in a cached index built on a different +# runner (see scripts/normalize-realm-mtimes.mjs for the mechanism). Three +# realms, matching the `--path` args services/realm-server passes that +# aren't switched off by the SKIP_* flags below: base, skills, openrouter. +# Keep this list in step with the one in ci/cache-index — a realm normalized +# on only one side never matches and re-indexes wholesale. +# +# Runs on every CI shard, not only the ones importing a cache: it keeps +# mtimes identical between cache and from-scratch runs, so `last-modified` +# on these realms' source files — and the "last saved" text the host renders +# from it — reads the same either way. Percy compares PR snapshots against +# baselines from main runs, which never import the cache, so normalizing +# only the importing runs would diff every snapshot that surfaces a file's +# modified time. +# +# Confined to CI because it rewrites mtimes in the checkout it runs against. +# On a developer's machine that would restamp their working tree and leave +# their own dev-server index looking stale on its next boot; a local host-test +# run indexes from scratch anyway, so it has nothing to gain here. +# +# skills:setup first because the skills realm is a separate clone that may +# not be on disk yet; it no-ops when the content is already there. +if [ -n "${CI:-}" ]; then + pnpm --dir=../skills-realm skills:setup + node "$REPO_ROOT/scripts/normalize-realm-mtimes.mjs" \ + ../base ../skills-realm/contents ../openrouter-realm +fi + +# Seeding the index tables from a cached snapshot is `mise run ci:import-index`, +# which the caller runs as its own step *before* this one. It deliberately does +# not happen here: this task's first act is to hand control to +# `start-server-and-test`, and any work in front of that delays every service +# it launches — Synapse included, which callers wait on with a fixed budget. +# +# When a snapshot has been imported, the boot index is what reconciles it +# against this checkout, so no REALM_SERVER_FULL_INDEX_ON_STARTUP override is +# set here: `discoverInvalidations` skips every file whose mtime matches its +# indexed row and revisits the rest, so this checkout's own changes — and +# whatever the invalidation fan-out reaches from them — get re-rendered while +# untouched files keep their cached rows. `applyBatchUpdates` upserts only the +# invalidated URLs, so rows the pass never visits survive, and +# `_readiness-check` awaits the in-flight index, so tests can't race the +# reconcile. With no snapshot imported the same pass simply finds nothing to +# skip and indexes from scratch. + # START_SERVER_AND_TEST_INSECURE=1 scopes to the wait-on readiness probe. # The realm-server / vite now serve the self-signed mkcert leaf on # https://localhost:4201|4202|4200; start-server-and-test forces diff --git a/scripts/import-cached-index.sh b/scripts/import-cached-index.sh index c8ccd575690..41c9ab67bd2 100755 --- a/scripts/import-cached-index.sh +++ b/scripts/import-cached-index.sh @@ -11,7 +11,18 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" DB_NAME="${PGDATABASE:-boxel}" REPO="cardstack/boxel" -ARTIFACT_NAME="boxel-index-cache" +# Artifacts to try, in order. `boxel-index-cache` carries every realm the +# export job indexes; `boxel-index-cache-host` carries only the realms a host +# test stack serves and is a fraction of the size, so a caller that serves +# just those asks for it via INDEX_CACHE_ARTIFACT. The full snapshot stays the +# fallback: a caller that wants the scoped one still works when it is missing +# or expired, just with more rows than it needs. Each artifact holds a single +# `.sql.gz`. +DEFAULT_ARTIFACT="boxel-index-cache" +ARTIFACT_NAMES="${INDEX_CACHE_ARTIFACT:-$DEFAULT_ARTIFACT}" +if [ "$ARTIFACT_NAMES" != "$DEFAULT_ARTIFACT" ]; then + ARTIFACT_NAMES="$ARTIFACT_NAMES $DEFAULT_ARTIFACT" +fi DOWNLOAD_DIR="/tmp/boxel-index-cache-$$" cleanup() { @@ -33,27 +44,49 @@ if ! command -v gh >/dev/null 2>&1; then exit 1 fi -# Find the latest successful CI run on main that produced the cache artifact. +# Find a recent successful CI run on main that produced the cache artifact. +# Not every successful main run has one — the cache-index job is skipped +# when change-check decides nothing boxel-related changed — so walk the +# most recent runs until a download succeeds instead of pinning to the +# single latest run. echo "Looking for cached index from CI..." -RUN_ID=$(gh run list -w ci.yaml -b main -s success -L 1 \ - --json databaseId -q '.[0].databaseId' -R "$REPO" 2>/dev/null) || RUN_ID="" -if [ -z "$RUN_ID" ]; then - echo "No CI run with index cache found, skipping." +RUN_IDS=$(gh run list -w ci.yaml -b main -s success -L 10 \ + --json databaseId -q '.[].databaseId' -R "$REPO" 2>/dev/null) || RUN_IDS="" +if [ -z "$RUN_IDS" ]; then + echo "No successful CI runs on main found, skipping." exit 1 fi -# Download the artifact -echo "Downloading index cache from CI run $RUN_ID..." -if ! gh run download "$RUN_ID" -n "$ARTIFACT_NAME" -D "$DOWNLOAD_DIR" -R "$REPO" 2>/dev/null; then - echo "Failed to download index cache artifact, skipping." - exit 1 -fi +# Phase timings: this script is on the critical path of every CI shard that +# uses it, and "the import took a while" is not actionable — the download and +# the replay have different fixes (a smaller artifact vs. a cheaper load). +# Report them separately, with the size that explains both. +DOWNLOAD_START=$(date +%s) +CACHE_FILE="" +for ARTIFACT_NAME in $ARTIFACT_NAMES; do + CANDIDATE="$DOWNLOAD_DIR/${ARTIFACT_NAME}.sql.gz" + for RUN_ID in $RUN_IDS; do + echo "Trying artifact ${ARTIFACT_NAME} from CI run $RUN_ID..." + if gh run download "$RUN_ID" -n "$ARTIFACT_NAME" -D "$DOWNLOAD_DIR" -R "$REPO" 2>/dev/null \ + && [ -f "$CANDIDATE" ]; then + echo "Downloaded ${ARTIFACT_NAME} from CI run $RUN_ID." + CACHE_FILE="$CANDIDATE" + break + fi + rm -rf "$DOWNLOAD_DIR" + done + if [ -n "$CACHE_FILE" ]; then + break + fi +done -CACHE_FILE="$DOWNLOAD_DIR/boxel-index-cache.sql.gz" -if [ ! -f "$CACHE_FILE" ]; then - echo "Cache file not found in artifact, skipping." +if [ -z "$CACHE_FILE" ] || [ ! -f "$CACHE_FILE" ]; then + echo "No index cache artifact found in recent CI runs, skipping." exit 1 fi +echo "Index cache download took $(( $(date +%s) - DOWNLOAD_START ))s ($(du -h "$CACHE_FILE" | cut -f1) compressed)." + +REPLAY_START=$(date +%s) # Clear any partial data before importing. echo "Truncating index tables..." @@ -87,4 +120,5 @@ else | docker exec -i boxel-pg psql $PSQL_OPTS fi +echo "Index cache replay took $(( $(date +%s) - REPLAY_START ))s." echo "Index cache imported successfully."