From 785604172e5f1f6d10fbc46376d2e5d13efe63e0 Mon Sep 17 00:00:00 2001 From: LiveOak Date: Thu, 6 Aug 2026 16:48:09 -0500 Subject: [PATCH] fix(anatomy): never delete entries a capped scan did not reach `walkDir` returns the moment the collected set reaches `anatomy.max_files`, and it walks each directory in alphabetical order. So a capped run does not sample the tree - it stops partway through it. `scanProject` then did `existing.files = fresh.files`, which reads "I never got there" as "this file is gone" and deletes the entire alphabetical tail. On one real project - 4,969 eligible files, 530 indexed, cap 500 - a scan kept 245 files from an early bulk directory and would have removed every entry under `Strategies/` (1,546 files), `05 Decisions/`, `07 Dashboards/` and `09 Coordinator/`. The entry count only fell 530 -> 500, so nothing looked wrong afterwards. It is also reachable with nobody typing `openwolf scan`: the daemon's cron engine runs `scan_project` on a schedule. A truncated walk has no way to distinguish the two cases, so it is no longer allowed to delete: it merges and prints what it kept. Pruning stays available on any complete walk, which is every project under its cap - so the full-replace behaviour is unchanged for anyone the bug never affected. `buildAnatomy` now returns `truncated`. It is `fileCount >= maxFiles`, not `>`, because a walk that ended exactly at the cap is indistinguishable from one that would have continued, and the safe reading of an ambiguous signal is the pessimistic one. The lock-contention fallback is skipped when truncated, for the same reason: it writes `renderStore(fresh)` straight to anatomy.md, and `fresh` holds only the part of the tree the walk reached. Losing the lock is not a reason to publish a partial render over the file humans read and `importFromMarkdown` reads back. tests/anatomy-scan-never-shrink.test.ts covers it, including that a complete walk still prunes a genuinely deleted file and that curated `md-import` descriptions survive. Reverting only the scanner change and rebuilding makes 5 of its 6 cases fail; the one that still passes is the under-cap case, which was never broken. The test imports the built module and builds on demand if `dist/` is absent: `npm test` runs off the TypeScript with type-stripping, which does not rewrite relative import specifiers, and `anatomy-scanner.ts` imports `./description-extractor.js`. Every existing test happens to target a module with no relative imports, so this has not come up before - happy to restructure if you would prefer the suite built first, or the scanner split so the decision is unit-testable off the source. node --test tests/anatomy-scan-never-shrink.test.ts 6/6 npm test 32/32 npx tsc --noEmit clean except the pre-existing src/daemon/cron-engine.ts TS2503, present on an unmodified checkout too Co-Authored-By: Claude Opus 5 (1M context) --- src/scanner/anatomy-scanner.ts | 56 +++++++-- tests/anatomy-scan-never-shrink.test.ts | 158 ++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 13 deletions(-) create mode 100644 tests/anatomy-scan-never-shrink.test.ts diff --git a/src/scanner/anatomy-scanner.ts b/src/scanner/anatomy-scanner.ts index 929ac497..ad017800 100644 --- a/src/scanner/anatomy-scanner.ts +++ b/src/scanner/anatomy-scanner.ts @@ -171,7 +171,10 @@ function walkDir( /** * Scan the project and return the anatomy content and file count WITHOUT writing to disk. */ -export function buildAnatomy(wolfDir: string, projectRoot: string): { content: string; fileCount: number; store: AnatomyStoreData } { +export function buildAnatomy( + wolfDir: string, + projectRoot: string +): { content: string; fileCount: number; store: AnatomyStoreData; truncated: boolean } { const configPath = path.join(wolfDir, "config.json"); const config = readJSON(configPath, { version: 1, @@ -186,19 +189,41 @@ export function buildAnatomy(wolfDir: string, projectRoot: string): { content: s }); const store = newStore(); - walkDir( - projectRoot, - projectRoot, - config.openwolf.anatomy.exclude_patterns, - config.openwolf.anatomy.max_files, - store.files - ); - - return { content: renderStore(store), fileCount: Object.keys(store.files).length, store }; + const maxFiles = config.openwolf.anatomy.max_files; + walkDir(projectRoot, projectRoot, config.openwolf.anatomy.exclude_patterns, maxFiles, store.files); + + const fileCount = Object.keys(store.files).length; + // `walkDir` returns as soon as the collected set reaches max_files, so a run that ends at + // the cap did not survey the tree — it stopped partway through it. `>=` rather than `>`: + // a walk that finished exactly at the cap is indistinguishable from one that would have + // continued, and the safe reading of an ambiguous signal is the pessimistic one. + return { content: renderStore(store), fileCount, store, truncated: fileCount >= maxFiles }; } export function scanProject(wolfDir: string, projectRoot: string): number { - const { fileCount, store: fresh } = buildAnatomy(wolfDir, projectRoot); + const { fileCount, store: fresh, truncated } = buildAnatomy(wolfDir, projectRoot); + + // A capped walk stops partway through the tree in alphabetical order, so it cannot tell + // "this file is gone" from "I never got there" — and the full-replace below reads the + // second as the first. On a project whose index has outgrown max_files that deletes the + // entire alphabetical tail: on one real vault (530 entries indexed, cap 500, 4,969 files + // eligible) a scan kept 245 files from an early bulk folder and would have removed every + // entry under Strategies/ (1,546 files), 05 Decisions/, 07 Dashboards/ and 09 Coordinator/. + // The count only fell 530 -> 500, so nothing looked wrong afterwards. + // + // This is reachable without anyone typing `openwolf scan`: the daemon's cron engine runs + // scan_project on a schedule. + // + // So a truncated walk is no longer permitted to delete. It merges, and says so. Pruning + // stays available on any complete walk, which is every project under its cap. + if (truncated) { + process.stderr.write( + `\n⚠ openwolf: the scan reached anatomy.max_files (${fileCount}) and stopped partway through ` + + `the tree. Entries it never reached have been KEPT rather than deleted, so stale entries ` + + `cannot be pruned until a scan completes. Raise openwolf.anatomy.max_files in ` + + `.wolf/config.json, or add bulk directories to exclude_patterns.\n\n` + ); + } const result = withAnatomyLock(wolfDir, CLI_LOCK_BUDGET_MS, () => { // Absorb md-side edits, then full-replace: the fresh disk walk defines @@ -212,15 +237,20 @@ export function scanProject(wolfDir: string, projectRoot: string): number { if (prev.hash === entry.hash && prev.symbols) entry.symbols = prev.symbols; } } - existing.files = fresh.files; + existing.files = truncated ? { ...existing.files, ...fresh.files } : fresh.files; existing.meta.lastScanned = new Date().toISOString(); renderToFile(wolfDir, existing); saveStore(wolfDir, existing); return true; }); - if (result === null) { + if (result === null && !truncated) { // Lock contention: fall back to writing the render directly (rare; the // next locked writer reconciles via the md import path). + // + // Skipped when the walk was truncated: `fresh` holds only the part of the tree the walk + // reached, so writing it here would gut anatomy.md — the file humans read, and the file + // `importFromMarkdown` reads back — even though the store itself was left intact. Losing + // the lock is not a reason to publish a partial render; the next complete scan writes it. writeText(path.join(wolfDir, "anatomy.md"), renderStore(fresh)); } diff --git a/tests/anatomy-scan-never-shrink.test.ts b/tests/anatomy-scan-never-shrink.test.ts new file mode 100644 index 00000000..0f3b85f4 --- /dev/null +++ b/tests/anatomy-scan-never-shrink.test.ts @@ -0,0 +1,158 @@ +import { test, describe } from "node:test"; +import * as assert from "node:assert"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +/** + * Imported from `dist/`, not from `src/`, and built on demand if it is not there. + * + * `npm test` runs the suite straight off the TypeScript with type-stripping, which does not + * rewrite relative import specifiers — and `anatomy-scanner.ts` imports + * `./description-extractor.js`, a path that only exists after a build. Every existing test + * happens to target a module with no relative imports, so this constraint has not come up + * before. Building here costs ~7s once and keeps `npm test` working on a clean checkout. + * + * Happy to restructure if you would rather the suite built first, or the scanner were + * split so the decision is unit-testable off the source. + */ +const repoRoot = path.resolve(import.meta.dirname, ".."); +const distScanner = path.join(repoRoot, "dist", "src", "scanner", "anatomy-scanner.js"); +if (!fs.existsSync(distScanner)) { + // tsc exits non-zero on the pre-existing src/daemon/cron-engine.ts TS2503, but still + // emits, so the emitted file — not the exit code — is what decides success. + try { + execFileSync("npx", ["tsc", "-p", "tsconfig.json"], { cwd: repoRoot, stdio: "ignore", shell: true }); + } catch {} + if (!fs.existsSync(distScanner)) throw new Error(`build produced no ${distScanner}`); +} +const { scanProject, buildAnatomy } = await import(pathToFileURL(distScanner).href); + +const tmpDir = () => fs.mkdtempSync(path.join(os.tmpdir(), "wolf-shrink-")); + +/** + * A project whose index already knows about every file, laid out so a capped walk stops + * inside the alphabetically-early directory and never reaches the late one. + * + * This is the real-world shape: a bulk evidence folder sorts before the curated content and + * consumes the whole budget. + */ +function makeProject(opts: { cap: number; early: number; late: number }) { + const root = tmpDir(); + const wolfDir = path.join(root, ".wolf"); + fs.mkdirSync(wolfDir, { recursive: true }); + fs.mkdirSync(path.join(root, "01-bulk"), { recursive: true }); + fs.mkdirSync(path.join(root, "zz-curated"), { recursive: true }); + + const files: Record = {}; + const add = (rel: string) => { + fs.writeFileSync(path.join(root, rel), `# ${rel}\n\nbody\n`, "utf-8"); + files[rel] = { + description: `curated ${rel}`, + tokens: 5, + hash: "stale-on-purpose", + size: 10, + mtimeMs: 1, + updatedAt: "2026-01-01T00:00:00.000Z", + source: "md-import", + }; + }; + for (let i = 0; i < opts.early; i++) add(`01-bulk/a${String(i).padStart(4, "0")}.md`); + for (let i = 0; i < opts.late; i++) add(`zz-curated/c${String(i).padStart(4, "0")}.md`); + + fs.writeFileSync( + path.join(wolfDir, "config.json"), + JSON.stringify({ + version: 1, + openwolf: { + anatomy: { + max_description_length: 100, + max_files: opts.cap, + exclude_patterns: [".git", ".wolf"], + }, + token_audit: { chars_per_token_code: 3.5, chars_per_token_prose: 4.0 }, + }, + }), + "utf-8" + ); + fs.writeFileSync( + path.join(wolfDir, "anatomy-index.json"), + JSON.stringify({ + version: 1, + meta: { lastScanned: "2026-01-01T00:00:00.000Z", fileCount: Object.keys(files).length }, + files, + }), + "utf-8" + ); + fs.writeFileSync(path.join(wolfDir, "anatomy.md"), "# anatomy\n", "utf-8"); + return { root, wolfDir, indexed: Object.keys(files).length }; +} + +const readIndex = (wolfDir: string) => + JSON.parse(fs.readFileSync(path.join(wolfDir, "anatomy-index.json"), "utf-8")) as { + files: Record; + }; +const keysUnder = (wolfDir: string, prefix: string) => + Object.keys(readIndex(wolfDir).files).filter((k) => k.startsWith(prefix)); + +describe("scanProject: a capped walk must not delete", () => { + test("entries the walk never reached are kept, not pruned", () => { + // 60 early + 40 late = 100 indexed; a cap of 50 dies inside 01-bulk. + const p = makeProject({ cap: 50, early: 60, late: 40 }); + scanProject(p.wolfDir, p.root); + + const after = readIndex(p.wolfDir).files; + assert.equal( + Object.keys(after).length, + p.indexed, + "a truncated scan deleted entries it never looked at" + ); + assert.equal(keysUnder(p.wolfDir, "zz-curated/").length, 40); + fs.rmSync(p.root, { recursive: true, force: true }); + }); + + test("buildAnatomy reports the truncation", () => { + const p = makeProject({ cap: 50, early: 60, late: 40 }); + assert.equal(buildAnatomy(p.wolfDir, p.root).truncated, true); + fs.rmSync(p.root, { recursive: true, force: true }); + }); + + test("a walk that ends exactly at the cap counts as truncated", () => { + // Indistinguishable from one that would have continued, so it must be read as unsafe. + const p = makeProject({ cap: 10, early: 5, late: 5 }); + assert.equal(buildAnatomy(p.wolfDir, p.root).truncated, true); + fs.rmSync(p.root, { recursive: true, force: true }); + }); + + test("a complete walk still prunes a file that is genuinely gone", () => { + // The guard must not cost the feature: pruning is why the full-replace exists. + const p = makeProject({ cap: 5000, early: 3, late: 3 }); + fs.rmSync(path.join(p.root, "zz-curated", "c0000.md")); + scanProject(p.wolfDir, p.root); + + assert.equal(buildAnatomy(p.wolfDir, p.root).truncated, false); + assert.equal(keysUnder(p.wolfDir, "zz-curated/").length, 2, "deleted file was not pruned"); + assert.equal(Object.keys(readIndex(p.wolfDir).files).length, p.indexed - 1); + fs.rmSync(p.root, { recursive: true, force: true }); + }); + + test("a complete walk under the cap keeps everything", () => { + const p = makeProject({ cap: 5000, early: 3, late: 3 }); + scanProject(p.wolfDir, p.root); + assert.equal(Object.keys(readIndex(p.wolfDir).files).length, p.indexed); + fs.rmSync(p.root, { recursive: true, force: true }); + }); + + test("curated descriptions survive a truncated scan", () => { + // The entries the walk DID reach are refreshed; md-import descriptions are preserved, + // which is the behaviour the full-replace path already had and must not regress. + const p = makeProject({ cap: 50, early: 60, late: 40 }); + scanProject(p.wolfDir, p.root); + const files = readIndex(p.wolfDir).files as Record; + assert.equal(files["01-bulk/a0000.md"]?.description, "curated 01-bulk/a0000.md"); + assert.equal(files["zz-curated/c0000.md"]?.description, "curated zz-curated/c0000.md"); + fs.rmSync(p.root, { recursive: true, force: true }); + }); +});