ci: Update index cache for host use

.github/workflows/ci.yaml

@@ -1071,3 +1071,20 @@ jobs:
           name: boxel-index-cache
           path: /tmp/boxel-index-cache.sql.gz
           retention-days: 30
+      # The host-scoped snapshot covers only the realms a host test shard
+      # serves, so a shard downloads and replays a fraction of the full one.
+      # Verified with its own expected-tables sidecar (the script's default
+      # would resolve to the full snapshot's, which happens to list the same
+      # tables — passing it explicitly keeps that a fact rather than a
+      # coincidence).
+      - name: Verify host-scoped index cache is complete
+        run: |
+          scripts/verify-index-cache.sh \
+            /tmp/boxel-index-cache-host.sql.gz \
+            /tmp/boxel-index-cache-host.tables
+      - name: Upload host-scoped index cache
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: boxel-index-cache-host
+          path: /tmp/boxel-index-cache-host.sql.gz
+          retention-days: 30

mise-tasks/ci/cache-index

@@ -5,6 +5,7 @@
 set -euo pipefail
 
 export PATH="./node_modules/.bin:$PATH"
+REPO_ROOT="$(cd "../.." && pwd)"
 
 READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson"
 # Strip whichever scheme is in front; realm-server speaks HTTPS+HTTP/2 in
@@ -24,6 +25,30 @@ for realm in base catalog skills submissions experiments openrouter software-fac
 done
 READINESS_URLS="${READINESS_URLS}|${MATRIX_URL_VAL}|http://localhost:5001|${ICONS_URL}|${HOST_URL}"
 
+# Give the realms whose rows this snapshot is imported from a content-derived
+# mtime before anything indexes them, so the mtimes baked into the dump
+# describe the content rather than when this runner happened to clone it. A
+# consumer that normalizes the same way sees matching mtimes for every file
+# it hasn't changed, and its boot index skips them. Without this the dump
+# records checkout times that exist nowhere else and every consumer
+# re-indexes the whole realm. See scripts/normalize-realm-mtimes.mjs.
+#
+# Scoped to base, skills and openrouter — the realms a test stack boots
+# without one of the SKIP_* flags switching it off, so the ones a consumer
+# reconciles against its own checkout. A consumer has to normalize exactly
+# this set: a realm normalized on only one side matches nothing and
+# re-indexes wholesale. The other realms indexed below still land in the
+# dump; they just carry this runner's checkout mtimes.
+#
+# skills:setup first because the skills realm is a separate clone that isn't
+# on disk yet in this job; it no-ops if the content is already present, and
+# start:development below would otherwise be the one to fetch it — after
+# this point.
+echo "Normalizing realm file mtimes..."
+pnpm --dir=../skills-realm skills:setup
+node "$REPO_ROOT/scripts/normalize-realm-mtimes.mjs" \
+  ../base ../skills-realm/contents ../openrouter-realm
+
 echo "Starting services..."
 SKIP_BOXEL_HOMEPAGE=true \
   NODE_NO_WARNINGS=1 \
@@ -92,3 +117,99 @@ gzip /tmp/boxel-index-cache.sql
 
 echo "Index cache created at /tmp/boxel-index-cache.sql.gz"
 ls -lh /tmp/boxel-index-cache.sql.gz
+
+# A second, smaller snapshot carrying only the realms a host test shard
+# actually serves. Every shard pays the download and the replay of whatever it
+# imports, and the realms left out here are the instance-heavy ones (catalog,
+# experiments, software-factory, submissions) — the shard neither mounts nor
+# queries them, so their rows are pure cost. Same table set and same COPY
+# format as the full snapshot above, so a consumer replays either one
+# unchanged.
+#
+# pg_dump takes no row filter, so the blocks are assembled here: a
+# `COPY public.<table> (<cols>) FROM stdin;` header, the matching
+# `COPY (SELECT <cols> …) TO STDOUT` body, and a `\.` terminator. Header and
+# SELECT share one generated column list so their order cannot drift, and
+# `-q` keeps psql from printing a command tag into the middle of the data.
+HOST_CACHE_REALMS=(base skills openrouter)
+HOST_DUMP=/tmp/boxel-index-cache-host.sql
+
+echo "Dumping host-scoped index cache for realms: ${HOST_CACHE_REALMS[*]}"
+
+# A realm's indexed `realm_url` is not reliably its serving URL: the base
+# realm registers under its canonical `https://cardstack.com/base/`, while
+# skills and openrouter carry the serving origin. Reconstructing either form
+# from REALM_BASE_URL gets base wrong. Read the realm_urls the index actually
+# holds and keep the ones whose path is a realm we want, so the filter follows
+# whatever the indexer wrote.
+REALM_URLS_RAW=$(docker exec boxel-pg psql -U postgres -d boxel -tAc \
+  "SELECT DISTINCT realm_url FROM boxel_index")
+
+# Each scoped realm must contribute at least one realm_url. Matching nothing
+# would still produce a structurally valid dump — empty COPY blocks, which the
+# verify step accepts because it checks for the block rather than its contents
+# — and would send every consumer back to indexing from scratch with nothing
+# to show it had happened. Fail here instead, while the cause is still legible.
+HOST_REALM_URLS=""
+for realm in "${HOST_CACHE_REALMS[@]}"; do
+  matched=0
+  while IFS= read -r realm_url; do
+    [ -z "$realm_url" ] && continue
+    case "$realm_url" in
+      */"${realm}"/) ;;
+      *) continue ;;
+    esac
+    # These go into SQL as literals. A quote in a realm URL is not a thing
+    # that happens, and if it ever did, stopping beats emitting broken SQL.
+    case "$realm_url" in
+      *"'"*)
+        echo "::error title=Host index cache::realm_url ${realm_url} contains a quote and cannot be used as a SQL literal."
+        exit 1
+        ;;
+    esac
+    HOST_REALM_URLS+="${HOST_REALM_URLS:+, }'${realm_url}'"
+    matched=$((matched + 1))
+    echo "  ${realm}: ${realm_url}"
+  done <<EOF
+${REALM_URLS_RAW}
+EOF
+  if [ "$matched" -eq 0 ]; then
+    echo "::error title=Host index cache empty::No indexed realm_url ends in '/${realm}/'. Either the realm did not index in this job, or the realm_url form changed — compare against the mount paths in mise-tasks/services/realm-server."
+    exit 1
+  fi
+done
+
+{
+  # The client settings pg_dump puts at the head of a data-only dump, so this
+  # file replays under the same ones as the full snapshot.
+  echo "SET statement_timeout = 0;"
+  echo "SET client_encoding = 'UTF8';"
+  echo "SET standard_conforming_strings = on;"
+  echo "SET check_function_bodies = false;"
+  echo "SET client_min_messages = warning;"
+  echo
+  for t in "${CACHE_TABLES[@]}"; do
+    cols=$(docker exec boxel-pg psql -U postgres -d boxel -tAc \
+      "SELECT string_agg(quote_ident(column_name), ', ' ORDER BY ordinal_position)
+         FROM information_schema.columns
+        WHERE table_schema = 'public' AND table_name = '${t}'")
+    if [ -z "$cols" ]; then
+      echo "::error title=Host index cache incomplete::Table '${t}' has no columns in information_schema — it was likely renamed or removed by a migration. Update CACHE_TABLES here and the TRUNCATE list in scripts/import-cached-index.sh." >&2
+      exit 1
+    fi
+    echo "COPY public.${t} (${cols}) FROM stdin;"
+    docker exec boxel-pg psql -U postgres -d boxel -q --no-psqlrc \
+      -c "COPY (SELECT ${cols} FROM ${t} WHERE realm_url IN (${HOST_REALM_URLS})) TO STDOUT"
+    echo '\.'
+    echo
+  done
+} > "$HOST_DUMP"
+
+# Same table list as the full snapshot, written alongside the host dump so the
+# verify step can be pointed at this one explicitly.
+printf '%s\n' "${CACHE_TABLES[@]}" > /tmp/boxel-index-cache-host.tables
+
+gzip "$HOST_DUMP"
+
+echo "Host-scoped index cache created at ${HOST_DUMP}.gz"
+ls -lh "${HOST_DUMP}.gz"

scripts/normalize-realm-mtimes.mjs

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+// Rewrites every file's mtime under the given realm directories to a value
+// derived from the file's own content, so that two checkouts of the same
+// content agree on mtime even though nothing else about them does.
+//
+// Why: the indexer decides what a from-scratch pass has to revisit by
+// comparing each file's filesystem mtime against the mtime recorded on its
+// `boxel_index` row (see `discoverInvalidations`), and it skips the files
+// where the two are equal. That comparison is what lets a realm boot on an
+// imported index snapshot and re-render only what actually changed. But
+// `git clone` stamps every file with the checkout time, so a snapshot taken
+// on one runner and imported on another has no matching mtime anywhere and
+// the pass re-renders the whole realm — the cache buys nothing.
+//
+// Content-derived mtimes give the comparison the signal it actually wants:
+// same bytes → same mtime → skipped; different bytes → different mtime →
+// revisited, along with everything the invalidation fan-out reaches from it.
+// A file's history doesn't enter into it, so this works on the shallow
+// clones CI uses and on the separately-cloned skills realm, neither of
+// which carries the history a commit-time scheme would need.
+//
+// Run this identically on the exporting side (before indexing, so the
+// snapshot records these mtimes) and on the importing side (before the
+// realm server boots). It is idempotent: a second run over unchanged
+// content recomputes the same timestamps.
+//
+// The timestamps are stable but not meaningful as dates — a file's mtime is
+// a fingerprint of its content, not when anyone touched it. Keep the two
+// sides in agreement about which directories get normalized, and expect
+// `last-modified` on these realms' source files (and any UI derived from
+// it) to read as a fixed date rather than "just now".
+//
+// Usage: normalize-realm-mtimes.mjs <dir> [<dir> ...]
+
+import { createHash } from 'node:crypto';
+import { readdirSync, readFileSync, lstatSync, utimesSync } from 'node:fs';
+import { join } from 'node:path';
+
+// Directories that are never part of a realm's indexed surface. `.git`
+// matters most: the skills realm is a clone, so its `.git` holds far more
+// bytes than the realm itself and hashing it would dominate the run.
+const SKIP_DIRS = new Set(['.git', 'node_modules']);
+
+// The window derived timestamps land in: epoch seconds 1e9 (2001-09-09)
+// through 2e9 (2033-05-18). Comfortably inside what every filesystem and
+// Postgres `bigint` column round-trips, and far enough from now that a
+// normalized mtime is recognizable as synthetic when someone is staring at
+// one wondering why a base card claims to have been saved in 2014.
+const WINDOW_START = 1_000_000_000;
+const WINDOW_SIZE = 1_000_000_000;
+
+// Whole seconds, because that is all the realm's reader preserves:
+// `NodeAdapter` reports mtimes through `unixTime()`, which floors to
+// seconds. Sub-second precision here would be truncated on the way into
+// the index and every comparison would miss by the remainder.
+function deriveMtime(contents) {
+  let digest = createHash('sha256').update(contents).digest();
+  return WINDOW_START + (digest.readUInt32BE(0) % WINDOW_SIZE);
+}
+
+function* walk(dir) {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch (err) {
+    if (err?.code === 'ENOENT') {
+      return;
+    }
+    throw err;
+  }
+  for (let entry of entries) {
+    if (entry.isDirectory()) {
+      if (!SKIP_DIRS.has(entry.name)) {
+        yield* walk(join(dir, entry.name));
+      }
+    } else if (entry.isFile()) {
+      yield join(dir, entry.name);
+    }
+    // Symlinks and other non-regular entries are skipped: there is no
+    // content of their own to hash, and following them risks wandering
+    // outside the realm directory.
+  }
+}
+
+let dirs = process.argv.slice(2);
+if (dirs.length === 0) {
+  console.error('usage: normalize-realm-mtimes.mjs <dir> [<dir> ...]');
+  process.exit(1);
+}
+
+let exitCode = 0;
+for (let dir of dirs) {
+  let stat;
+  try {
+    stat = lstatSync(dir);
+  } catch (err) {
+    if (err?.code !== 'ENOENT') {
+      throw err;
+    }
+  }
+  if (!stat?.isDirectory()) {
+    // A missing directory is not fatal, but it does mean this realm's files
+    // keep their checkout mtimes and will be re-indexed wholesale on the
+    // importing side. That is a silent loss of the entire point, so say so
+    // loudly rather than exiting clean.
+    console.error(
+      `::warning::normalize-realm-mtimes: ${dir} is not a directory — its files keep their checkout mtimes and will not match a cached index`,
+    );
+    exitCode = 1;
+    continue;
+  }
+
+  let count = 0;
+  for (let file of walk(dir)) {
+    let mtime = deriveMtime(readFileSync(file));
+    // atime is set to the same value only because utimesSync requires it;
+    // nothing in the indexer reads it.
+    utimesSync(file, mtime, mtime);
+    count++;
+  }
+  console.log(`normalized mtimes for ${count} files under ${dir}`);
+}
+
+process.exit(exitCode);