diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a4e0d0e2743..c20e7818405 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1071,3 +1071,20 @@ jobs: name: boxel-index-cache path: /tmp/boxel-index-cache.sql.gz retention-days: 30 + # The host-scoped snapshot covers only the realms a host test shard + # serves, so a shard downloads and replays a fraction of the full one. + # Verified with its own expected-tables sidecar (the script's default + # would resolve to the full snapshot's, which happens to list the same + # tables — passing it explicitly keeps that a fact rather than a + # coincidence). + - name: Verify host-scoped index cache is complete + run: | + scripts/verify-index-cache.sh \ + /tmp/boxel-index-cache-host.sql.gz \ + /tmp/boxel-index-cache-host.tables + - name: Upload host-scoped index cache + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: boxel-index-cache-host + path: /tmp/boxel-index-cache-host.sql.gz + retention-days: 30 diff --git a/mise-tasks/ci/cache-index b/mise-tasks/ci/cache-index index 7a00fdafe71..ce0b7388317 100755 --- a/mise-tasks/ci/cache-index +++ b/mise-tasks/ci/cache-index @@ -5,6 +5,7 @@ set -euo pipefail export PATH="./node_modules/.bin:$PATH" +REPO_ROOT="$(cd "../.." && pwd)" READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" # Strip whichever scheme is in front; realm-server speaks HTTPS+HTTP/2 in @@ -24,6 +25,30 @@ for realm in base catalog skills submissions experiments openrouter software-fac done READINESS_URLS="${READINESS_URLS}|${MATRIX_URL_VAL}|http://localhost:5001|${ICONS_URL}|${HOST_URL}" +# Give the realms whose rows this snapshot is imported from a content-derived +# mtime before anything indexes them, so the mtimes baked into the dump +# describe the content rather than when this runner happened to clone it. A +# consumer that normalizes the same way sees matching mtimes for every file +# it hasn't changed, and its boot index skips them. Without this the dump +# records checkout times that exist nowhere else and every consumer +# re-indexes the whole realm. See scripts/normalize-realm-mtimes.mjs. +# +# Scoped to base, skills and openrouter — the realms a test stack boots +# without one of the SKIP_* flags switching it off, so the ones a consumer +# reconciles against its own checkout. A consumer has to normalize exactly +# this set: a realm normalized on only one side matches nothing and +# re-indexes wholesale. The other realms indexed below still land in the +# dump; they just carry this runner's checkout mtimes. +# +# skills:setup first because the skills realm is a separate clone that isn't +# on disk yet in this job; it no-ops if the content is already present, and +# start:development below would otherwise be the one to fetch it — after +# this point. +echo "Normalizing realm file mtimes..." +pnpm --dir=../skills-realm skills:setup +node "$REPO_ROOT/scripts/normalize-realm-mtimes.mjs" \ + ../base ../skills-realm/contents ../openrouter-realm + echo "Starting services..." SKIP_BOXEL_HOMEPAGE=true \ NODE_NO_WARNINGS=1 \ @@ -92,3 +117,99 @@ gzip /tmp/boxel-index-cache.sql echo "Index cache created at /tmp/boxel-index-cache.sql.gz" ls -lh /tmp/boxel-index-cache.sql.gz + +# A second, smaller snapshot carrying only the realms a host test shard +# actually serves. Every shard pays the download and the replay of whatever it +# imports, and the realms left out here are the instance-heavy ones (catalog, +# experiments, software-factory, submissions) — the shard neither mounts nor +# queries them, so their rows are pure cost. Same table set and same COPY +# format as the full snapshot above, so a consumer replays either one +# unchanged. +# +# pg_dump takes no row filter, so the blocks are assembled here: a +# `COPY public. () FROM stdin;` header, the matching +# `COPY (SELECT …) TO STDOUT` body, and a `\.` terminator. Header and +# SELECT share one generated column list so their order cannot drift, and +# `-q` keeps psql from printing a command tag into the middle of the data. +HOST_CACHE_REALMS=(base skills openrouter) +HOST_DUMP=/tmp/boxel-index-cache-host.sql + +echo "Dumping host-scoped index cache for realms: ${HOST_CACHE_REALMS[*]}" + +# A realm's indexed `realm_url` is not reliably its serving URL: the base +# realm registers under its canonical `https://cardstack.com/base/`, while +# skills and openrouter carry the serving origin. Reconstructing either form +# from REALM_BASE_URL gets base wrong. Read the realm_urls the index actually +# holds and keep the ones whose path is a realm we want, so the filter follows +# whatever the indexer wrote. +REALM_URLS_RAW=$(docker exec boxel-pg psql -U postgres -d boxel -tAc \ + "SELECT DISTINCT realm_url FROM boxel_index") + +# Each scoped realm must contribute at least one realm_url. Matching nothing +# would still produce a structurally valid dump — empty COPY blocks, which the +# verify step accepts because it checks for the block rather than its contents +# — and would send every consumer back to indexing from scratch with nothing +# to show it had happened. Fail here instead, while the cause is still legible. +HOST_REALM_URLS="" +for realm in "${HOST_CACHE_REALMS[@]}"; do + matched=0 + while IFS= read -r realm_url; do + [ -z "$realm_url" ] && continue + case "$realm_url" in + */"${realm}"/) ;; + *) continue ;; + esac + # These go into SQL as literals. A quote in a realm URL is not a thing + # that happens, and if it ever did, stopping beats emitting broken SQL. + case "$realm_url" in + *"'"*) + echo "::error title=Host index cache::realm_url ${realm_url} contains a quote and cannot be used as a SQL literal." + exit 1 + ;; + esac + HOST_REALM_URLS+="${HOST_REALM_URLS:+, }'${realm_url}'" + matched=$((matched + 1)) + echo " ${realm}: ${realm_url}" + done <&2 + exit 1 + fi + echo "COPY public.${t} (${cols}) FROM stdin;" + docker exec boxel-pg psql -U postgres -d boxel -q --no-psqlrc \ + -c "COPY (SELECT ${cols} FROM ${t} WHERE realm_url IN (${HOST_REALM_URLS})) TO STDOUT" + echo '\.' + echo + done +} > "$HOST_DUMP" + +# Same table list as the full snapshot, written alongside the host dump so the +# verify step can be pointed at this one explicitly. +printf '%s\n' "${CACHE_TABLES[@]}" > /tmp/boxel-index-cache-host.tables + +gzip "$HOST_DUMP" + +echo "Host-scoped index cache created at ${HOST_DUMP}.gz" +ls -lh "${HOST_DUMP}.gz" diff --git a/scripts/normalize-realm-mtimes.mjs b/scripts/normalize-realm-mtimes.mjs new file mode 100755 index 00000000000..aa75f989144 --- /dev/null +++ b/scripts/normalize-realm-mtimes.mjs @@ -0,0 +1,124 @@ +#!/usr/bin/env node +// Rewrites every file's mtime under the given realm directories to a value +// derived from the file's own content, so that two checkouts of the same +// content agree on mtime even though nothing else about them does. +// +// Why: the indexer decides what a from-scratch pass has to revisit by +// comparing each file's filesystem mtime against the mtime recorded on its +// `boxel_index` row (see `discoverInvalidations`), and it skips the files +// where the two are equal. That comparison is what lets a realm boot on an +// imported index snapshot and re-render only what actually changed. But +// `git clone` stamps every file with the checkout time, so a snapshot taken +// on one runner and imported on another has no matching mtime anywhere and +// the pass re-renders the whole realm — the cache buys nothing. +// +// Content-derived mtimes give the comparison the signal it actually wants: +// same bytes → same mtime → skipped; different bytes → different mtime → +// revisited, along with everything the invalidation fan-out reaches from it. +// A file's history doesn't enter into it, so this works on the shallow +// clones CI uses and on the separately-cloned skills realm, neither of +// which carries the history a commit-time scheme would need. +// +// Run this identically on the exporting side (before indexing, so the +// snapshot records these mtimes) and on the importing side (before the +// realm server boots). It is idempotent: a second run over unchanged +// content recomputes the same timestamps. +// +// The timestamps are stable but not meaningful as dates — a file's mtime is +// a fingerprint of its content, not when anyone touched it. Keep the two +// sides in agreement about which directories get normalized, and expect +// `last-modified` on these realms' source files (and any UI derived from +// it) to read as a fixed date rather than "just now". +// +// Usage: normalize-realm-mtimes.mjs [ ...] + +import { createHash } from 'node:crypto'; +import { readdirSync, readFileSync, lstatSync, utimesSync } from 'node:fs'; +import { join } from 'node:path'; + +// Directories that are never part of a realm's indexed surface. `.git` +// matters most: the skills realm is a clone, so its `.git` holds far more +// bytes than the realm itself and hashing it would dominate the run. +const SKIP_DIRS = new Set(['.git', 'node_modules']); + +// The window derived timestamps land in: epoch seconds 1e9 (2001-09-09) +// through 2e9 (2033-05-18). Comfortably inside what every filesystem and +// Postgres `bigint` column round-trips, and far enough from now that a +// normalized mtime is recognizable as synthetic when someone is staring at +// one wondering why a base card claims to have been saved in 2014. +const WINDOW_START = 1_000_000_000; +const WINDOW_SIZE = 1_000_000_000; + +// Whole seconds, because that is all the realm's reader preserves: +// `NodeAdapter` reports mtimes through `unixTime()`, which floors to +// seconds. Sub-second precision here would be truncated on the way into +// the index and every comparison would miss by the remainder. +function deriveMtime(contents) { + let digest = createHash('sha256').update(contents).digest(); + return WINDOW_START + (digest.readUInt32BE(0) % WINDOW_SIZE); +} + +function* walk(dir) { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch (err) { + if (err?.code === 'ENOENT') { + return; + } + throw err; + } + for (let entry of entries) { + if (entry.isDirectory()) { + if (!SKIP_DIRS.has(entry.name)) { + yield* walk(join(dir, entry.name)); + } + } else if (entry.isFile()) { + yield join(dir, entry.name); + } + // Symlinks and other non-regular entries are skipped: there is no + // content of their own to hash, and following them risks wandering + // outside the realm directory. + } +} + +let dirs = process.argv.slice(2); +if (dirs.length === 0) { + console.error('usage: normalize-realm-mtimes.mjs [ ...]'); + process.exit(1); +} + +let exitCode = 0; +for (let dir of dirs) { + let stat; + try { + stat = lstatSync(dir); + } catch (err) { + if (err?.code !== 'ENOENT') { + throw err; + } + } + if (!stat?.isDirectory()) { + // A missing directory is not fatal, but it does mean this realm's files + // keep their checkout mtimes and will be re-indexed wholesale on the + // importing side. That is a silent loss of the entire point, so say so + // loudly rather than exiting clean. + console.error( + `::warning::normalize-realm-mtimes: ${dir} is not a directory — its files keep their checkout mtimes and will not match a cached index`, + ); + exitCode = 1; + continue; + } + + let count = 0; + for (let file of walk(dir)) { + let mtime = deriveMtime(readFileSync(file)); + // atime is set to the same value only because utimesSync requires it; + // nothing in the indexer reads it. + utimesSync(file, mtime, mtime); + count++; + } + console.log(`normalized mtimes for ${count} files under ${dir}`); +} + +process.exit(exitCode);