From cc4c8002ca837be2f6a5e91e2fd0abb1c8811b31 Mon Sep 17 00:00:00 2001 From: Deepan Wadhwa Date: Tue, 22 Sep 2026 15:19:49 -0400 Subject: [PATCH 01/30] Add bounded Chutni inventory activation --- Makefile | 2 +- assets/app.html | 41 +- docs/CHUTNI_JOBS_RECOVERY_PLAN.md | 539 ++++++++++++++++++ src/samosa_fs.c | 173 +++++- src/samosa_gateway.c | 240 +++++++- .../contracts/chutni_preflight_response.json | 21 +- .../chutni_scope_create_request.json | 1 + tests/test_chutni_controls.sh | 5 +- tests/test_chutni_gateway.sh | 30 +- tests/test_chutni_inventory.sh | 42 ++ tests/test_ui_chutni_contracts.py | 4 +- vendor/chutni | 2 +- 12 files changed, 1050 insertions(+), 50 deletions(-) create mode 100644 docs/CHUTNI_JOBS_RECOVERY_PLAN.md diff --git a/Makefile b/Makefile index bb68175..c8aacf5 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,7 @@ test-native-summarizer-real: samosa-summarizer tests/test_native_summarizer.c $(BUILD_DIR)/test_native_summarizer $(BUILD_DIR)/samosa-summarizer "$(SAMOSA_SUMMARIZER_TEST_MODEL)" # The HTTP controller invokes the same generic service that MCP hosts use. -chutni-gateway-test: samosa-gateway chutni-service test_fake_openai_backend tests/test_chutni_gateway.sh tests/test_chutni_controls.sh +chutni-gateway-test: samosa-gateway samosa-fs chutni-service test_fake_openai_backend tests/test_chutni_gateway.sh tests/test_chutni_controls.sh sh tests/test_chutni_gateway.sh sh tests/test_chutni_controls.sh diff --git a/assets/app.html b/assets/app.html index de37f1c..d59d98b 100644 --- a/assets/app.html +++ b/assets/app.html @@ -7150,6 +7150,7 @@

Delete conversation?

let chutniScopes = []; // full summaries, server truth let chutniPollTimer = null; let chutniPreflightId = null; // set once a preflight has been accepted + let chutniPolicyFingerprint = null; let chutniForgetId = null; function activeChutniScope() { @@ -7247,7 +7248,7 @@

Delete conversation?

if (!jobEls.memoryAccess) return; const folder = (jobEls.folder.value || "").trim(); const scope = folder && Array.isArray(chutniScopes) - ? chutniScopes.find(item => item.canonical_root === folder && item.state === "ready") + ? chutniScopes.find(item => item.canonical_root === folder && ["ready", "ready_partial"].includes(item.state)) : null; jobEls.memoryAccess.innerHTML = ""; const strong = document.createElement("strong"); @@ -7365,7 +7366,7 @@

Delete conversation?

} catch (_) { return { ...entry, state: entry.state || "unknown", unreadable: true }; } })); chutniScopes = full; - readyChutniMemoryCount = full.filter(scope => scope.state === "ready").length; + readyChutniMemoryCount = full.filter(scope => ["ready", "ready_partial"].includes(scope.state)).length; chutniEls.error.hidden = true; } catch (err) { // A library that cannot be read is its own state: say so and offer @@ -7438,6 +7439,10 @@

Delete conversation?

return { chip: improving ? "ready · improving summaries" : "ready", tone: "ok", why: warn.length ? null : null, actions: ["build", "forget"] }; } + case "ready_partial": + return { chip: "partial index", tone: "warn", + why: `The safety limit “${s.limiting_reason || "unspecified"}” stopped this build early. Searchable evidence is available, but this folder is not fully indexed. Refresh after adjusting the scope or limits.`, + actions: ["ask", "build", "forget"] }; default: return { chip: s.state || "unknown", tone: null, why: null, actions: ["build", "forget"] }; } @@ -7637,16 +7642,18 @@

Delete conversation?

const chat = activeChat(); const selectedHere = !!(chat && chat.directory_context && chat.directory_context.scope_id === s.id); - if (s.state === "ready") { + if (s.state === "ready" || s.state === "ready_partial") { const ready = document.createElement("p"); ready.className = "chutni-why"; - ready.textContent = selectedHere + ready.textContent = s.state === "ready_partial" + ? "This partial memory is active in the current conversation. Search results may omit files beyond the scan limit." + : selectedHere ? "This memory is active in the current conversation. Ask questions in Chat." : "Memory is ready. Ask questions about these files in Chat."; li.appendChild(ready); } const actions = p.actions.slice(); - if (s.state === "ready") { + if (s.state === "ready" || s.state === "ready_partial") { actions.unshift("ask"); if (selectedHere) actions.splice(1, 0, "unuse"); } @@ -7753,7 +7760,7 @@

Delete conversation?

} function closeChutniPanels() { chutniEls.addPanel.hidden = true; chutniEls.forgetPanel.hidden = true; - chutniPreflightId = null; chutniForgetId = null; + chutniPreflightId = null; chutniPolicyFingerprint = null; chutniForgetId = null; chutniEls.preflight.hidden = true; chutniEls.preflight.innerHTML = ""; chutniEls.addNext.textContent = "Check folder"; renderChutni(); @@ -7764,7 +7771,7 @@

Delete conversation?

chutniEls.summaryBudget.value = "3000"; chutniEls.summaryBudget.setCustomValidity(""); chutniEls.preflight.hidden = true; chutniEls.preflight.innerHTML = ""; - chutniPreflightId = null; + chutniPreflightId = null; chutniPolicyFingerprint = null; chutniEls.addNext.textContent = "Check folder"; chutniEls.pane.hidden = true; chutniEls.empty.hidden = true; chutniEls.addPanel.hidden = false; @@ -7795,6 +7802,7 @@

Delete conversation?

if (!r.ok) throw new Error((body && body.error && body.error.message) || "That folder could not be checked."); renderPreflight(body); chutniPreflightId = body.preflight_id || "accepted"; + chutniPolicyFingerprint = body.policy_fingerprint || null; chutniEls.addNext.textContent = "Build memory"; } catch (err) { chutniEls.preflight.hidden = false; @@ -7812,6 +7820,7 @@

Delete conversation?

method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ preflight_id: chutniPreflightId, + policy_fingerprint: chutniPolicyFingerprint, display_name: baseName(path), summary_token_budget: summaryTokenBudget, client_request_id: `scope-${Date.now()}` @@ -7830,6 +7839,7 @@

Delete conversation?

function renderPreflight(pf) { const est = pf.estimate || {}, pol = pf.effective_policy || {}; const chutni = pf.chutni || {}; + const inventory = pf.inventory || {}; const dl = document.createElement("dl"); const row = (k, v) => { if (v === null || v === undefined) return; @@ -7840,7 +7850,11 @@

Delete conversation?

// "About" is honest for a sampled estimate; an exact-looking number // from a sample would be fake precision. const approx = est.sampled ? "About " : ""; - if (typeof est.files === "number") row("Files", `${approx}${est.files}`); + if (typeof inventory.regular_files === "number") row("Files found", inventory.regular_files); + else if (typeof est.files === "number") row("Files", `${approx}${est.files}`); + if (typeof inventory.regular_file_bytes === "number") row("File size found", bytesLabel(inventory.regular_file_bytes)); + if (typeof inventory.skipped === "number") row("Skipped", inventory.skipped); + if (inventory.partial) row("Inventory", `Partial: stopped at ${inventory.limiting_reason || "a safety limit"}`); row("Source size", bytesLabel(est.source_bytes)); const need = bytesLabel(est.peak_additional_disk_bytes), free = bytesLabel(est.available_disk_bytes); if (need) row("Extra disk needed", free ? `${need} (${free} free)` : need); @@ -7867,7 +7881,12 @@

Delete conversation?

p.textContent = typeof w === "string" ? w : (w.message || JSON.stringify(w)); chutniEls.preflight.appendChild(p); } - if (bad.length) { chutniEls.addNext.textContent = "Check folder"; chutniPreflightId = null; } + if (inventory.partial) { + const p = document.createElement("p"); p.className = "chutni-why"; + p.textContent = "This preview reached a safety limit. Review the partial counts before building memory."; + chutniEls.preflight.appendChild(p); + } + if (bad.length) { chutniEls.addNext.textContent = "Check folder"; chutniPreflightId = null; chutniPolicyFingerprint = null; } chutniEls.preflight.hidden = false; } @@ -8610,7 +8629,7 @@

Delete conversation?

initialPath: chutniEls.path.value.trim() || undefined, onSelect: path => { chutniEls.path.value = path; - chutniPreflightId = null; + chutniPreflightId = null; chutniPolicyFingerprint = null; chutniEls.addNext.textContent = "Check folder"; chutniEls.preflight.hidden = true; chutniEls.addNext.focus(); @@ -8624,7 +8643,7 @@

Delete conversation?

// the user never confirmed. chutniEls.path.oninput = () => { if (!chutniPreflightId) return; - chutniPreflightId = null; + chutniPreflightId = null; chutniPolicyFingerprint = null; chutniEls.addNext.textContent = "Check folder"; chutniEls.preflight.hidden = true; }; diff --git a/docs/CHUTNI_JOBS_RECOVERY_PLAN.md b/docs/CHUTNI_JOBS_RECOVERY_PLAN.md new file mode 100644 index 0000000..c073f42 --- /dev/null +++ b/docs/CHUTNI_JOBS_RECOVERY_PLAN.md @@ -0,0 +1,539 @@ +# Chutni and Jobs recovery plan + +**Status:** implementation plan; no product changes from this document alone + +**Updated:** 2026-09-22 + +**Primary target:** 16 GB Apple-silicon MacBook Air + +**Scope:** safe folder inventory, useful Chutni retrieval, practical Jobs, +durable scheduling, and a small local decision-model runtime + +This plan replaces optimistic completion claims in older Jobs documents for the +current app surface. Those documents remain useful implementation history, but +the acceptance gates below define whether Chutni and Jobs are useful now. + +## 1. Outcome + +Samosa should let a user select an ordinary mixed folder—even one containing a +software project—and safely do two things: + +1. Build a searchable Chutni without entering virtual environments, dependency + trees, caches, build output, repositories, or symlink escapes. +2. Run small, repeatable Jobs such as reporting, finding, classifying, and + organizing files, with a visible plan, durable progress, review gates, and + undo. + +The filesystem remains the source of truth. Models may make bounded semantic +decisions; they may not construct paths, choose arbitrary tools, overwrite +files, delete files, or bypass the approval policy. + +## 2. Verified baseline and why the current behavior fails + +### Chutni + +- `vendor/chutni/src/chutni.c` records configurable exclusion policy but does + not enforce `exclude_globs` during traversal. +- `vendor/chutni/src/scan.c` has unbounded depth when a maximum is absent. +- The scanner traverses broadly and creates source/hash work before deciding + whether a file has useful text content. +- `src/samosa_gateway.c` sends folder activation without an enforceable + exclusion policy or traversal budget. The UI can display an effective policy + that is not the policy the scanner actually executes. +- The fixed ignored-directory list catches a few exact names such as `.venv` + and `node_modules`, but it is not enough for generated trees, nested package + managers, case variants, custom environments, or user rules. + +### Jobs + +- The simple Jobs UI sends `goal`, `folder`, and a requested mode, but the + current backend ignores the mode. +- Intent routing collapses the model's `organize` answer into the report path. +- The ordinary Jobs path can therefore enumerate or summarize a folder but + does not reliably create and execute a useful action plan. +- More capable compiled-definition machinery exists, but the normal app does + not expose it as a small set of understandable, end-to-end workflows. +- Earlier regression evidence validates older paths and isolated endpoints; it + is not proof that the current browser flow completes useful Jobs. + +The repair starts below these UI symptoms: one shared, policy-enforcing folder +inventory must be used by Chutni preflight, Chutni ingestion, Jobs preview, and +Jobs execution. + +## 3. Updated decision-model finding + +### What OpenDecision contributes + +The OpenDecision repository was reviewed at commit +[`20b2a78f9822130322125c0345976a12a87c76a4`](https://github.com/deepanwadhwa/OpenDecision/tree/20b2a78f9822130322125c0345976a12a87c76a4). +It is not a generative model. It is an Apache-2.0 Python decision engine over a +zero-shot classifier and exposes four useful result shapes: + +- `Choice`: select one declared option; +- `Noul`: score whether a statement is supported; +- `Score`: choose a position on an ordered rubric; +- `Relation`: distinguish support, contradiction, and missing evidence. + +Its default model is +[`MoritzLaurer/ModernBERT-large-zeroshot-v2.0`](https://huggingface.co/MoritzLaurer/ModernBERT-large-zeroshot-v2.0), +an approximately 0.4B-parameter ModernBERT classifier. The model configuration +has `max_position_embeddings: 8192`, and OpenDecision's direct entailment paths +explicitly tokenize up to 8,192 tokens. + +The 8K claim needs three qualifications: + +1. OpenDecision's document API defaults to 384-token chunks and restricts an + individual chunk to at most 4,096 tokens. It retrieves passages before the + final decision rather than routinely sending an entire 8K document. +2. `Choice` delegates tokenization to the Transformers zero-shot pipeline, + while `Noul` and the evidence backend set their own limits. Samosa must use + explicit token accounting and must not assume every upstream route behaves + identically at 8K. +3. The model card says a later model was being prepared to make fuller use of + the 8K window. Capacity is not proof that relevant evidence is used equally + well at the beginning, middle, and end of a long input. + +OpenDecision's long-document pattern is still valuable: split evidence, +retrieve question-relevant passages, pack them under a known token budget, make +a typed decision, and return the passages that caused it. + +### OpenDecision/ModernBERT versus Laya + +Laya and OpenDecision should be evaluated through one Samosa decision contract, +not integrated as two unrelated features. + +| Candidate | Best expected role | Context actually used by current package | Important limitation | +| --- | --- | --- | --- | +| OpenDecision + ModernBERT zero-shot | intent, evidence relation, longer file/document classification | architecture supports 8,192; direct entailment uses 8,192; document chunks default to 384 and cap at 4,096 | upstream Python/PyTorch service is not an appropriate shipped Samosa dependency; scores remain uncalibrated for Samosa Jobs | +| Laya root | very short bounded routing/triage | Laya config uses `max_len: 512`, despite its ModernBERT encoder being 8K-capable | base checkpoint is not the strongest typed-decision checkpoint | +| Laya typed-decisions | short `Choice`/`Score`/`Noul` decisions | Laya config uses `max_len: 1024` | benchmark specialization may not transfer; higher-cardinality choices and domain calibration need testing | + +The initial hypothesis is: + +- prefer ModernBERT zero-shot for decisions that benefit from retrieved file + evidence or explicit support/contradiction/unknown; +- prefer Laya typed-decisions only if it is materially faster or more accurate + on short Samosa routing and classification cases; +- ship one default decision model, not both resident at once, on a 16 GB host; +- retain deterministic rules for cases that do not need a model. + +This is a measured selection gate, not a pre-decided model addition. + +## 4. Target architecture + +```text +User-selected root + | + v +Policy-enforcing inventory (one shared implementation) + |-- pruned directories and skipped-file reasons + |-- bounded metadata for eligible files + |-- stable file identity and content hash when required + | + +--> Chutni manifest -> extraction -> chunks -> FTS -> cited retrieval + | + +--> Jobs inventory -> deterministic recipe -> optional typed decisions + | + v + reviewable action plan + | + journaled executor + | + undo +``` + +Chutni and Jobs may reuse content-addressed extraction results. Jobs must not +require a Chutni to exist, and building a Chutni must never authorize Jobs to +modify the source folder. + +## 5. Phase A — build one safe folder inventory + +### A1. Policy layers + +Apply policies before opening a directory or file: + +1. **Non-overridable safety rules** + - do not follow symbolic links; + - do not cross out of the selected root; + - optionally remain on the selected filesystem for large scopes; + - skip sockets, devices, FIFOs, and other non-regular files; + - exclude Samosa's own state and job journals if they appear under a scope. +2. **Generated-tree defaults** + - VCS: `.git`, `.hg`, `.svn`; + - Python: `.venv`, `venv`, `env`, `__pycache__`, `.mypy_cache`, + `.pytest_cache`, `.ruff_cache`, `site-packages`; + - JavaScript: `node_modules`, `.next`, `.nuxt`, `.yarn`, `.pnpm-store`; + - build/cache: `build`, `dist`, `target`, `DerivedData`, `.gradle`, + `.cache`, coverage output, and common IDE indexes. +3. **Marker-based pruning** + - recognize environments by markers such as `pyvenv.cfg`, not only by the + directory name; + - recognize package/build trees by a small, documented marker set. +4. **User policy** + - add/remove exclusions in preflight; + - persist the exact accepted policy with the Chutni or Job; + - match path components with the selected filesystem's case behavior. + +Defaults should be conservative and explainable. An expert may opt a generated +tree back in, but may not disable symlink/root-boundary protections. + +### A2. Traversal and work budgets + +Every inventory request has explicit limits: + +- maximum depth; +- maximum directories and files visited; +- maximum eligible bytes and per-file bytes; +- deadline and cancellation token; +- maximum errors retained in the response; +- no materialization of the whole tree merely to sort it; +- backpressure between traversal, extraction, and indexing. + +Reaching a limit produces `partial` plus the exact limiting reason. It never +silently claims completion. + +### A3. Eligibility before expensive work + +- Decide from file type, extension, and a bounded magic-byte probe whether a + file can contribute searchable content. +- Do not hash or extract unsupported files during the initial walk. +- Hash eligible content only when needed for ingestion, change detection, or a + duplicate-finding recipe. +- Re-stat before and after reading; if identity, size, or modification time + changes, defer the file instead of publishing inconsistent content. +- Reuse `read_cache` by content hash and parser fingerprint. + +### A4. One preflight contract + +Preflight and execution consume the same inventory implementation and policy. +The preflight response includes: + +- directories visited and pruned; +- eligible, skipped, unreadable, and unsupported file counts; +- eligible bytes and estimated extraction work; +- exclusions grouped by reason with representative paths; +- limits and whether the result is complete; +- a policy fingerprint that must accompany confirmation. + +If the root or policy changes between preview and confirmation, execution +rejects the stale fingerprint and asks for a new preview. + +### A5. Inventory acceptance gates + +- A synthetic `.venv` and `node_modules`, each containing at least 10,000 + decoy files, are pruned at their roots. No descendant is opened or hashed. +- A differently named folder containing `pyvenv.cfg` is also pruned. +- A symlink to an outside secret and a symlink cycle are never followed. +- Preflight and execution report the same considered/skipped counts for an + unchanged tree. +- Cancellation interrupts a large walk promptly and leaves a truthful partial + checkpoint. +- Warm reruns inspect metadata and changed files only; unchanged content is not + re-extracted. + +## 6. Phase B — make Chutni demonstrably useful + +### B1. Activation flow + +1. Select folder. +2. Run the shared preflight. +3. Show what will be indexed and what will be ignored. +4. Let the user adjust non-safety exclusions. +5. Confirm the policy fingerprint. +6. Build a durable, reconnectable index with pause and cancel. + +The build status reports real units: inventory, extraction, chunks indexed, +files deferred, and model-summary work if any. It does not invent a percentage +when total work is unknown. + +### B2. Search and retrieval proof + +- Keep raw, provenance-bearing chunks in SQLite FTS alongside path and parser + metadata. +- Make incremental add/change/rename/delete reconciliation the normal refresh. +- Add **Test this Chutni** to the completion screen. It runs a real query and + displays retrieved passages and paths before the user relies on chat. +- Chat injects only useful retrieved evidence and renders citations back to the + file and passage. +- A retrieval miss stays a miss; the app must not imply the answer came from + the selected folder. + +### B3. Chutni acceptance gates + +- Planted facts in Markdown, text, HTML, and supported PDFs are found with the + correct file/path citation. +- A fact inside an excluded dependency tree is not indexed. +- Removing or changing a source file updates results after refresh. +- Restarting the app during a build resumes from a durable checkpoint. +- Existing indexes created under the unenforced policy are marked for rebuild; + they are not silently labeled safe and current. + +## 7. Phase C — replace the Jobs blank box with useful recipes + +The main Jobs screen begins with a small set of productized recipes. A freeform +goal may map into one of them, but pasted internal JSON is not required. + +### C1. First recipes + +1. **Folder report** + - counts by type, age, size, unreadable/unsupported reason, and duplicate + candidates; + - read-only and deterministic. +2. **Find files about…** + - filename/metadata pass, bounded cached skim, semantic shortlist, and cited + verification; + - returns evidence rather than merely filenames. +3. **Sort by file type** + - deterministic destination names; + - preview, confirm, execute, and undo. +4. **Classify an inbox** + - choose among user-visible categories such as billing, medical, work, + personal, and review; + - semantic decisions may suggest categories; ambiguous items go to review. +5. **Find duplicates** + - size grouping followed by content hashes; + - read-only by default; no automatic deletion. +6. **Watch a folder** + - run a chosen recipe only on added/changed files; + - scheduled write actions always produce a review batch unless the user has + explicitly enabled a narrowly defined safe rule. + +### C2. Durable `JobSpec` + +Compile every UI submission into a versioned structure containing: + +- job and recipe version; +- selected root identity; +- accepted inventory policy fingerprint; +- inputs, categories, schedule, and action mode; +- decision-model identifier/revision and calibrated policy version; +- planned operations and their preconditions; +- approval state, event sequence, checkpoint, result, and undo journal. + +The event log is append-only. Summary state is written atomically. Restarting +the gateway reconstructs an accurate state and resumes only resumable work. + +### C3. Safety boundary + +- The model sees normalized evidence and a closed list of options. +- The model returns a typed label/distribution/abstention, never a path or shell + command. +- The deterministic compiler validates every source and destination beneath + the authorized root. +- The executor uses no-follow operations and revalidates file identity + immediately before mutation. +- No overwrite, deletion, or implicit conflict suffixing. +- Every move is journaled with enough identity information to undo safely. +- If the source changed since preview, that item returns to review. + +### C4. Jobs acceptance gates + +- Each recipe is launchable from the current browser app and reaches a durable + terminal result. +- Report and find recipes cause no filesystem writes outside the job journal. +- Sort preview performs no moves; confirmed execution matches the preview; + undo restores the original tree and hashes. +- Ambiguous classification is visibly held for review. +- Schedules operate on deltas, not a full rescan/reclassification of unchanged + content. +- The UI history survives browser and gateway restarts and explains failures + at item level. + +## 8. Phase D — a native, catalogue-managed decision runtime + +### D1. Stable Samosa decision contract + +Create one internal interface independent of either candidate model: + +```json +{ + "operation": "choice", + "evidence": [{"id": "file-17", "text": "..."}], + "question": "Which inbox category fits this file?", + "options": [ + {"id": "billing", "description": "Invoices, receipts, and payments"}, + {"id": "medical", "description": "Care, insurance, and health records"}, + {"id": "review", "description": "Insufficient or conflicting evidence"} + ], + "max_input_tokens": 4096, + "policy_id": "inbox-v1" +} +``` + +The response contains model revision, selected option, all scores, calibrated +confidence, abstention/review reason, evidence IDs, token count, latency, and +runtime memory observations. `relation` and ordered `score` use similarly +closed schemas. + +### D2. Development evaluation before catalogue exposure + +Use OpenDecision at its pinned commit as a development oracle/harness, not as +the shipped Python server. Build a Samosa-specific gold set covering: + +- find/report/organize intent; +- inbox categories and deliberately ambiguous files; +- support/contradiction/unknown evidence; +- file relevance for search; +- adversarial filenames and instructions embedded inside documents; +- class counts of 2, 3–5, 6–10, and more than 10; +- English plus any languages Samosa intends to claim. + +Run both candidates at 512 and 1,024 tokens. Run the ModernBERT zero-shot +candidate additionally at 2,048, 4,096, and 8,192 tokens with the decisive +evidence placed at the beginning, middle, and end. Record accuracy, macro-F1, +abstention precision/recall, expected calibration error, p50/p95 latency, peak +RSS, memory pressure, swap growth, and thermal behavior. + +An 8K candidate passes only if it improves Samosa decisions enough to justify +its latency and memory cost. Retrieval plus a smaller packed input remains the +default when it is equally accurate. + +### D3. Runtime qualification on the 16 GB M3 Air + +Prototype the reference PyTorch/MPS path only to establish correctness. Do not +ship Python 3.13, PyTorch, Transformers, or a FastAPI server with Samosa. + +Run a native-runtime bakeoff using pinned official weights: + +- official ModernBERT safetensors as the parity reference; +- official ModernBERT ONNX FP16/int8/Q4 variants where the chosen runtime + supports all ModernBERT operators correctly; +- a reproducible Core ML or MLX conversion only after logits and labels match + the reference within declared tolerances; +- official Laya weights and a reproducible native conversion if Laya wins a + decision class. + +The helper process should be `samosa-decision`, sandboxable and loaded on +demand. It unloads after a bounded idle period and yields to interactive chat. +On 16 GB systems it must not coexist with a memory-heavy chat backend unless a +measured resource gate says that combination is safe. + +### D4. Model catalogue work + +Decision models are auxiliary components, not selectable chat personalities. +Extend the catalogue and UI with: + +- `family: "decision"`, `category: "decision"`, `role: "auxiliary"`; +- `backend_kind: "decision_native"`; +- `routing: "automatic"` and `load_policy: "on_demand_per_job_batch"`; +- exact upstream revision, artifact byte counts, SHA-256 hashes, tokenizer, + precision, maximum qualified input, languages, and qualification report; +- a **Jobs intelligence** section in Settings with download, verification, + repair, remove, and active/idle status; +- an in-context Jobs prompt when a recipe needs an uninstalled decision model. + +The current catalogue's artifact resolver and dependency allowlist are partly +hardcoded by model ID/backend. Add a validated generic auxiliary-model install +resolution under Samosa home, plus an allowlisted `samosa-decision` dependency, +instead of adding one more scattered special case. + +Add a candidate to `assets/models.json` only after the artifact/runtime parity, +accuracy, memory, and licensing gates pass. If neither candidate passes, ship +the deterministic recipes without semantic classification rather than +pretending the model is reliable. + +## 9. Phase E — made-up-folder dogfood + +Create the fixture outside the repository, under a new temporary directory: + +```text +Samosa-Dogfood/ + Documents/ + project-plan.md + anonymous-scan.pdf # contains a planted Titli vaccination fact + invoice-september.txt + Inbox/ + clear-billing.txt + clear-medical.txt + clear-work.txt + mixed-ambiguous.txt + Software/ + README.md + src/main.c + custom-python-env/pyvenv.cfg + custom-python-env/... # 10,000+ decoys + node_modules/... # 10,000+ decoys + build/... + .git/... + outside-secret.txt + escape-link -> ../outside-secret.txt +``` + +Automate these end-to-end scenarios through the same HTTP/UI paths used by the +app: + +1. **Chutni preflight:** excluded roots and counts are shown; no excluded + descendant or outside target is opened. +2. **Chutni build:** the eligible files are indexed and the policy fingerprint + in the manifest matches the preview. +3. **Retrieval:** planted facts are found with exact file/passages; unrelated + questions do not fabricate a match. +4. **Folder report:** produces deterministic counts that agree with the shared + inventory. +5. **Sort by type:** preview, confirm, execute, and undo; the final tree and + file hashes equal the initial snapshot. +6. **Find Titli:** a meaningless PDF filename is found through bounded content + evidence, with no dependency-tree scan. +7. **Inbox triage:** clear cases are suggested correctly and the mixed case is + held for review. +8. **Scheduled delta:** adding one file processes one file; unchanged files are + not reclassified. +9. **Crash/reload:** interrupt inventory, indexing, classification, and a move + batch at controlled points; after restart each job is either resumable or in + an honest review/terminal state. + +Keep fixture construction deterministic. Record a machine-readable result and +a short evidence document for every live-model qualification run. + +## 10. Delivery order + +1. Freeze current failing browser/API scenarios as regression tests. +2. Implement and test the shared safe inventory. +3. Route Chutni preflight and ingestion through it; rebuild unsafe old indexes. +4. Prove Chutni retrieval and chat citations on the dogfood folder. +5. Ship durable deterministic Jobs recipes: report, sort, duplicates, and + watch-folder delta handling. +6. Add the decision contract and gold evaluation harness. +7. Compare ModernBERT/OpenDecision behavior with Laya; select runtime/model per + measured gate. +8. Add the qualified auxiliary model to the catalogue and app download flow. +9. Enable semantic find and inbox classification with calibrated review gates. +10. Run the full dogfood, restart, memory, and UI acceptance matrix on the + 16 GB M3 MacBook Air. + +Do not block the safe scanner or deterministic Jobs on the decision-model +research. The product becomes useful in increments, and model intelligence is +added only where it beats deterministic behavior. + +## 11. Definition of done + +The recovery is complete only when all of the following are evidenced: + +- Chutni prunes dependency/generated trees before descendant access. +- The visible preflight policy is the executed policy. +- Chutni search and chat retrieve planted evidence with citations. +- The normal Jobs UI completes useful recipes without pasted internal JSON. +- Job state/history is durable and scheduled work is delta-based. +- Filesystem changes are previewed, revalidated, journaled, non-overwriting, + and undoable. +- Semantic ambiguity is held for review rather than forced into a move. +- The selected decision model has measured Samosa accuracy and calibration, + native-runtime parity, pinned artifacts, and a safe 16 GB M3 profile. +- Model download/verify/repair/remove works through the app. +- Documentation reports the exact tested scope and does not convert a single + fixture or upstream benchmark into a broad reliability claim. + +## 12. Source notes + +- OpenDecision repository and API shapes: + +- OpenDecision decision engine at the reviewed revision: + +- OpenDecision document retrieval at the reviewed revision: + +- ModernBERT zero-shot model: + +- ModernBERT architecture documentation: + +- Laya model and published evaluation artifacts: + diff --git a/src/samosa_fs.c b/src/samosa_fs.c index 9ecdbbc..85e273b 100644 --- a/src/samosa_fs.c +++ b/src/samosa_fs.c @@ -24,9 +24,11 @@ #include #include #include +#include #include #include #include +#include #include #ifndef PATH_MAX @@ -751,12 +753,20 @@ typedef struct { int cross_filesystems; char **exclude_names; size_t exclude_count; + unsigned int max_depth; + unsigned long long max_files; + unsigned long long max_directories; + unsigned long long max_seconds; + unsigned long long deadline_ms; } ChutniPolicy; typedef struct { unsigned long long files; unsigned long long skipped; + unsigned long long directories_seen; unsigned long long directories_entered; + const char *limiting_reason; + int stop; } ChutniCounters; static volatile sig_atomic_t g_chutni_canceled = 0; @@ -766,6 +776,13 @@ static void chutni_on_signal(int signum) { g_chutni_canceled = 1; } +static unsigned long long chutni_monotonic_ms(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) return 0; + return (unsigned long long)ts.tv_sec * 1000ull + + (unsigned long long)ts.tv_nsec / 1000000ull; +} + /* Test-only determinism seam: SAMOSA_CHUTNI_TEST_DELAY_US inserts a per-file * delay so cancellation tests can exercise the two-second stop bound against * a small fixture instead of needing hundreds of thousands of real files. @@ -780,14 +797,34 @@ static long chutni_test_delay_us(void) { return delay; } -static int chutni_name_excluded(const ChutniPolicy *policy, const char *name) { - size_t i; - for (i = 0; i < policy->exclude_count; ++i) - if (strcmp(policy->exclude_names[i], name) == 0) +static int chutni_generated_name_excluded(const char *name) { + static const char *generated[] = { + ".git", ".svn", ".hg", "node_modules", ".cache", "__pycache__", + ".venv", "venv", "env", "target", "build", "dist", "DerivedData", + ".Trash", ".mypy_cache", ".pytest_cache", ".ruff_cache", + "site-packages", ".next", ".nuxt", ".yarn", ".pnpm-store", + ".gradle", "coverage", ".idea", NULL + }; + for (const char **p = generated; *p; p++) + if (!strcasecmp(*p, name)) return 1; + return 0; +} + +static int chutni_user_name_excluded(const ChutniPolicy *policy, const char *name) { + for (size_t i = 0; i < policy->exclude_count; ++i) + if (strcasecmp(policy->exclude_names[i], name) == 0) return 1; return 0; } +static int chutni_python_environment(const char *parent, const char *name) { + char marker[PATH_MAX]; + struct stat st; + int n = snprintf(marker, sizeof marker, "%s/%s/pyvenv.cfg", parent, name); + return n >= 0 && (size_t)n < sizeof marker && + lstat(marker, &st) == 0 && S_ISREG(st.st_mode); +} + static int chutni_emit_file(const char *rel_path, const struct stat *st) { Buffer out = {0}; int ok = buf_put(&out, "{\"type\":\"file\",\"rel_path\":") && @@ -814,14 +851,41 @@ static int chutni_emit_skip(const char *rel_path, const char *reason) { return 1; } +static void chutni_inventory_policy_fingerprint(const ChutniPolicy *policy, + const struct stat *root_st, + char out[65]) { + Sha256 sha; + unsigned char digest[32]; + char identity[256]; + int n = snprintf(identity, sizeof identity, + "samosa-folder-policy-v1\n%llu:%llu\n%d:%d:%u:%llu:%llu:%llu\n", + (unsigned long long)root_st->st_dev, (unsigned long long)root_st->st_ino, + policy->include_hidden, policy->cross_filesystems, policy->max_depth, + policy->max_files, policy->max_directories, policy->max_seconds); + sha256_init(&sha); + if (n > 0) sha256_update(&sha, identity, (size_t)n); + for (size_t i = 0; i < policy->exclude_count; i++) { + sha256_update(&sha, policy->exclude_names[i], strlen(policy->exclude_names[i])); + sha256_update(&sha, "\n", 1); + } + sha256_final(&sha, digest); + sha256_hex(digest, out); +} + /* Returns 0 only on an unrecoverable output failure (caller reports * output_too_large); an unreadable directory is a recorded skip, not a * failure, so the scan can continue past it. */ static int chutni_walk(dev_t root_dev, const char *dir_abs, const char *rel_prefix, + unsigned int depth, const ChutniPolicy *policy, ChutniCounters *counters) { DIR *dir; struct dirent *de; if (g_chutni_canceled) return 1; + if (policy->deadline_ms && chutni_monotonic_ms() >= policy->deadline_ms) { + counters->limiting_reason = "deadline"; + counters->stop = 1; + return 1; + } dir = opendir(dir_abs); if (!dir) { if (!chutni_emit_skip(rel_prefix[0] ? rel_prefix : ".", @@ -831,11 +895,16 @@ static int chutni_walk(dev_t root_dev, const char *dir_abs, const char *rel_pref return 1; } counters->directories_entered++; - while (!g_chutni_canceled && (de = readdir(dir)) != NULL) { + while (!g_chutni_canceled && !counters->stop && (de = readdir(dir)) != NULL) { char child_abs[PATH_MAX]; char child_rel[PATH_MAX]; struct stat st; int n; + if (policy->deadline_ms && chutni_monotonic_ms() >= policy->deadline_ms) { + counters->limiting_reason = "deadline"; + counters->stop = 1; + break; + } if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) continue; { @@ -874,18 +943,42 @@ static int chutni_walk(dev_t root_dev, const char *dir_abs, const char *rel_pref counters->skipped++; continue; } - if (chutni_name_excluded(policy, de->d_name)) { + if (chutni_generated_name_excluded(de->d_name)) { + if (!chutni_emit_skip(child_rel, "generated_tree")) { closedir(dir); return 0; } + counters->skipped++; + continue; + } + if (chutni_user_name_excluded(policy, de->d_name)) { if (!chutni_emit_skip(child_rel, "user_exclusion")) { closedir(dir); return 0; } counters->skipped++; continue; } if (S_ISDIR(st.st_mode)) { + if (chutni_python_environment(dir_abs, de->d_name)) { + if (!chutni_emit_skip(child_rel, "python_environment_marker")) { closedir(dir); return 0; } + counters->skipped++; + continue; + } + if (counters->directories_seen >= policy->max_directories) { + if (!chutni_emit_skip(child_rel, "directory_limit")) { closedir(dir); return 0; } + counters->skipped++; + counters->limiting_reason = "maximum_directories"; + counters->stop = 1; + continue; + } + counters->directories_seen++; + if (depth >= policy->max_depth) { + if (!chutni_emit_skip(child_rel, "depth_limit")) { closedir(dir); return 0; } + counters->skipped++; + counters->limiting_reason = "maximum_depth"; + continue; + } if (!policy->cross_filesystems && st.st_dev != root_dev) { if (!chutni_emit_skip(child_rel, "cross_filesystem")) { closedir(dir); return 0; } counters->skipped++; continue; } - if (!chutni_walk(root_dev, child_abs, child_rel, policy, counters)) { + if (!chutni_walk(root_dev, child_abs, child_rel, depth + 1, policy, counters)) { closedir(dir); return 0; } @@ -896,6 +989,13 @@ static int chutni_walk(dev_t root_dev, const char *dir_abs, const char *rel_pref counters->skipped++; continue; } + if (counters->files >= policy->max_files) { + if (!chutni_emit_skip(child_rel, "file_limit")) { closedir(dir); return 0; } + counters->skipped++; + counters->limiting_reason = "maximum_files"; + counters->stop = 1; + break; + } if (!chutni_emit_file(child_rel, &st)) { closedir(dir); return 0; } counters->files++; } @@ -911,21 +1011,40 @@ static int command_chutni_inventory(const char *root, const ChutniPolicy *policy put_error("folder_unavailable"); return 65; } + if (!policy->max_directories || !policy->max_files) { + put_error("invalid_budget"); + return 64; + } + if (policy->deadline_ms == 0) { + /* A zero deadline is reserved for an expired monotonic clock; the CLI + * always supplies a positive duration. */ + put_error("invalid_budget"); + return 64; + } if (lstat(root_abs, &root_st) != 0 || !S_ISDIR(root_st.st_mode)) { put_error("folder_unavailable"); return 65; } - if (!chutni_walk(root_st.st_dev, root_abs, "", policy, &counters)) { + counters.directories_seen = 1; + char policy_fingerprint[65]; + chutni_inventory_policy_fingerprint(policy, &root_st, policy_fingerprint); + if (!chutni_walk(root_st.st_dev, root_abs, "", 0, policy, &counters)) { put_error("output_too_large"); return 65; } { Buffer out = {0}; int ok = buf_printf(&out, - "{\"type\":\"done\",\"canceled\":%s,\"files\":%llu,\"skipped\":%llu," - "\"directories_entered\":%llu}\n", + "{\"type\":\"done\",\"canceled\":%s,\"partial\":%s,\"files\":%llu,\"skipped\":%llu," + "\"directories_seen\":%llu,\"directories_entered\":%llu,\"limiting_reason\":\"%s\"," + "\"policy_fingerprint\":\"%s\"}\n", g_chutni_canceled ? "true" : "false", - counters.files, counters.skipped, counters.directories_entered); + (g_chutni_canceled || counters.limiting_reason) ? "true" : "false", + counters.files, counters.skipped, counters.directories_seen, + counters.directories_entered, + g_chutni_canceled ? "canceled" : + (counters.limiting_reason ? counters.limiting_reason : "none"), + policy_fingerprint); if (!ok) { free(out.data); put_error("output_too_large"); return 65; } fputs(out.data, stdout); free(out.data); @@ -1364,6 +1483,10 @@ static void usage(void) { " samosa-fs undo --root ROOT SRC DST\n" " samosa-fs chutni-inventory --root ROOT [--include-hidden]\n" " [--cross-filesystems] [--exclude NAME]...\n" + " [--max-depth N] [--max-files N]\n" + " [--max-directories N] [--max-seconds N]\n" + " [--max-depth N] [--max-files N]\n" + " [--max-directories N] [--max-seconds N]\n" " samosa-fs chutni-hash --root ROOT PATH\n" " samosa-fs --version\n", stderr); } @@ -1379,6 +1502,16 @@ static int parse_size_arg(const char *text, size_t *out) { return 1; } +static int parse_ull_arg(const char *text, unsigned long long *out) { + char *end = NULL; + unsigned long long parsed; + errno = 0; + parsed = strtoull(text, &end, 10); + if (errno || !end || *end) return 0; + *out = parsed; + return 1; +} + int main(int argc, char **argv) { const char *cmd; const char *path = NULL; @@ -1387,6 +1520,10 @@ int main(int argc, char **argv) { const char *dst = NULL; const char *expected_hash = NULL; size_t max_bytes = DEFAULT_MAX_FILE_BYTES; + unsigned long long max_files = 100000; + unsigned long long max_directories = 20000; + unsigned long long max_seconds = 60; + unsigned long long max_depth = 64; off_t expected_size = 0; double expected_mtime = 0.0; int have_size = 0; @@ -1447,6 +1584,14 @@ int main(int argc, char **argv) { include_hidden = 1; } else if (strcmp(argv[i], "--cross-filesystems") == 0) { cross_filesystems = 1; + } else if (strcmp(argv[i], "--max-files") == 0 && i + 1 < argc) { + if (!parse_ull_arg(argv[++i], &max_files) || !max_files || max_files > 1000000) { usage(); return 64; } + } else if (strcmp(argv[i], "--max-directories") == 0 && i + 1 < argc) { + if (!parse_ull_arg(argv[++i], &max_directories) || !max_directories || max_directories > 100000) { usage(); return 64; } + } else if (strcmp(argv[i], "--max-seconds") == 0 && i + 1 < argc) { + if (!parse_ull_arg(argv[++i], &max_seconds) || !max_seconds || max_seconds > 3600) { usage(); return 64; } + } else if (strcmp(argv[i], "--max-depth") == 0 && i + 1 < argc) { + if (!parse_ull_arg(argv[++i], &max_depth) || max_depth > 64) { usage(); return 64; } } else if (strcmp(argv[i], "--exclude") == 0 && i + 1 < argc) { char **next; if (exclude_count == exclude_cap) { @@ -1478,6 +1623,12 @@ int main(int argc, char **argv) { ChutniPolicy policy = {0}; policy.include_hidden = include_hidden; policy.cross_filesystems = cross_filesystems; + policy.max_depth = (unsigned int)max_depth; + policy.max_files = max_files; + policy.max_directories = max_directories; + policy.max_seconds = max_seconds; + unsigned long long now = chutni_monotonic_ms(); + policy.deadline_ms = now ? now + max_seconds * 1000ull : 0; policy.exclude_names = exclude_names; policy.exclude_count = exclude_count; signal(SIGINT, chutni_on_signal); diff --git a/src/samosa_gateway.c b/src/samosa_gateway.c index 9c84f73..8fca55f 100644 --- a/src/samosa_gateway.c +++ b/src/samosa_gateway.c @@ -21346,6 +21346,7 @@ static int chutni_scope_root_exists(Gateway *g, const char *canonical_root) { static int chutni_scope_metadata_create(Gateway *g, const char *scope_id, const char *display_name, const char *canonical_root, + const char *policy_fingerprint, int summary_token_budget) { if (chutni_scope_root_exists(g, canonical_root)) return 0; char scopes_path[PATH_MAX], scope_path[PATH_MAX], metadata_path[PATH_MAX]; @@ -21379,8 +21380,9 @@ static int chutni_scope_metadata_create(Gateway *g, const char *scope_id, text_json_string(&json, identity) && text_add(&json, ",\"summary_token_budget\":") && text_add(&json, budget) && - text_add(&json, ",\"policy_fingerprint\":\"chutni-reference-scan-v1\"," - "\"state\":\"unbuilt\",\"freshness_state\":\"complete\"," + text_add(&json, ",\"policy_fingerprint\":") && + text_json_string(&json, policy_fingerprint) && + text_add(&json, ",\"state\":\"unbuilt\",\"freshness_state\":\"complete\"," "\"active_job_id\":\"\",\"phase\":\"idle\"," "\"evidence_generation\":0,\"enhancement_revision\":0," "\"regular_files_seen\":0,\"files_indexed\":0," @@ -21401,10 +21403,19 @@ static int chutni_scope_metadata_create(Gateway *g, const char *scope_id, text_json_string(&json, now) && text_add(&json, ",\"active_database\":\"\"," "\"effective_policy\":{\"include_hidden\":false," - "\"cross_filesystems\":false,\"maximum_file_bytes\":67108864," + "\"cross_filesystems\":false,\"follow_symlinks\":false," + "\"maximum_depth\":32,\"maximum_files\":10000," + "\"maximum_directories\":5000,\"maximum_seconds\":20," + "\"maximum_file_bytes\":67108864," "\"mandatory_exclusions\":[\".git\",\".svn\",\".hg\"," "\"node_modules\",\".cache\",\"__pycache__\",\".venv\"," - "\"venv\",\"target\",\".Trash\"],\"user_exclusions\":[]}," + "\"venv\",\"env\",\"target\",\"build\",\"dist\"," + "\"DerivedData\",\".Trash\",\".mypy_cache\"," + "\".pytest_cache\",\".ruff_cache\",\"site-packages\"," + "\".next\",\".nuxt\",\".yarn\",\".pnpm-store\"," + "\".gradle\",\"coverage\",\".idea\"]," + "\"marker_exclusions\":[\"directory containing pyvenv.cfg\"]," + "\"user_exclusions\":[]}," "\"warnings\":[]}\n") && write_small_file(metadata_path, json.data) && chutni_scope_registry_write(g); @@ -21463,6 +21474,29 @@ static void chutni_json_set_number(jval *object, const char *key, double value) field->num = value; } +static void chutni_json_set_bool(jval *object, const char *key, int value) { + jval *field = object && object->t == J_OBJ ? json_get(object, key) : NULL; + if (!field && object && object->t == J_OBJ) { + char **keys = realloc(object->keys, (size_t)(object->len + 1) * sizeof(*keys)); + if (!keys) return; + object->keys = keys; + jval **kids = realloc(object->kids, (size_t)(object->len + 1) * sizeof(*kids)); + if (!kids) return; + object->kids = kids; + field = calloc(1, sizeof(*field)); + char *key_copy = strdup(key); + if (!field || !key_copy) { free(field); free(key_copy); return; } + field->t = J_NULL; + object->keys[object->len] = key_copy; + object->kids[object->len] = field; + object->len++; + } + if (!field) return; + if (field->t == J_STR) { free(field->str); field->str = NULL; } + field->t = J_BOOL; + field->boolean = !!value; +} + static int chutni_scope_summary_token_budget_write(Gateway *g, const char *scope_id, int budget) { @@ -21567,11 +21601,17 @@ static int chutni_scope_publish(Gateway *g, const char *scope_id, content_artifacts && content_artifacts->t == J_NUM ? content_artifacts->num : (text && text->t == J_NUM ? text->num : 0); char now[32] = {0}; rfc3339_now_to(now, sizeof(now)); - chutni_json_set_string(scope, "state", "ready"); + jval *partial_value = json_get(scan, "partial"); + int partial = partial_value && partial_value->t == J_BOOL && partial_value->boolean; + jval *limiting_reason = json_get(scan, "limiting_reason"); + chutni_json_set_string(scope, "state", partial ? "ready_partial" : "ready"); chutni_json_set_string(scope, "active_job_id", ""); chutni_json_set_string(scope, "phase", "complete"); chutni_json_set_string(scope, "current_file", ""); - chutni_json_set_string(scope, "freshness_state", "complete"); + chutni_json_set_string(scope, "freshness_state", partial ? "partial" : "complete"); + chutni_json_set_bool(scope, "build_partial", partial); + if (limiting_reason && limiting_reason->t == J_STR) + chutni_json_set_string(scope, "limiting_reason", limiting_reason->str); chutni_json_set_number(scope, "evidence_generation", (double)generation); chutni_json_set_number(scope, "regular_files_seen", seen->num); chutni_json_set_number(scope, "files_indexed", indexed->num); @@ -21596,7 +21636,7 @@ static int chutni_scope_publish(Gateway *g, const char *scope_id, (progress_raw = read_file_limit(progress_path, 1 << 20))) progress = json_parse(progress_raw, &progress_arena); chutni_scope_overlay_progress(scope, progress); - chutni_json_set_string(scope, "phase", "complete"); + chutni_json_set_string(scope, "phase", partial ? "partial" : "complete"); chutni_json_set_string(scope, "current_file", ""); json_free(progress); free(progress_arena); free(progress_raw); TextBuffer output = {0}; @@ -21681,8 +21721,10 @@ static void *chutni_worker(void *opaque) { chutni_scope_summary_token_budget(g, args->scope_id); int prepared = chutni_scope_metadata(g, args->scope_id, root_path, display_name) && - text_add(&request_json, "{\"path\":") && + text_add(&request_json, "{") && + text_add(&request_json, "\"path\":") && text_json_string(&request_json, root_path) && + text_add(&request_json, ",\"max_depth\":32,\"max_files\":10000,\"max_directories\":5000,\"max_seconds\":20,\"max_file_size_bytes\":67108864") && text_add(&request_json, ",\"confirmed\":true,\"register\":true,\"label\":") && text_json_string(&request_json, display_name) && text_add(&request_json, ",\"app_name\":\"Samosa\",\"app_version\":") && @@ -21804,16 +21846,22 @@ static void *chutni_worker(void *opaque) { write_small_file(protocol_path, service_output) && chutni_scope_publish(g, args->scope_id, service_output, args->generation)) { - final_state = "completed"; - message = enrichment.failed + char *result_arena = NULL; + jval *result = json_parse(service_output, &result_arena); + jval *result_scan = result && result->t == J_OBJ ? json_get(result, "scan") : NULL; + jval *result_partial = result_scan && result_scan->t == J_OBJ ? json_get(result_scan, "partial") : NULL; + int partial = result_partial && result_partial->t == J_BOOL && result_partial->boolean; + final_state = partial ? "completed_partial" : "completed"; + message = partial ? "Chutni saved a partial index because a scan safety limit was reached." : enrichment.failed ? "Portable Chutni memory is ready; some optional enrichment was unavailable." : "Portable Chutni memory is ready with reusable content artifacts."; + json_free(result); free(result_arena); } else { message = "Chutni completed but Samosa could not publish its status."; } } path_copy(progress.phase, sizeof(progress.phase), - !strcmp(final_state, "completed") ? "complete" : + (!strcmp(final_state, "completed") || !strcmp(final_state, "completed_partial")) ? "complete" : !strcmp(final_state, "paused_user") ? "paused" : !strcmp(final_state, "canceled") ? "canceled" : "failed"); progress.current_file[0] = 0; @@ -21822,6 +21870,7 @@ static void *chutni_worker(void *opaque) { chutni_job_write(g, args->scope_id, args->job_id, final_state, phase, args->generation, message); chutni_job_event(g, args->scope_id, args->job_id, + !strcmp(final_state, "completed_partial") ? "completed_partial" : !strcmp(final_state, "completed") ? "completed" : !strcmp(final_state, "paused_user") ? "paused_user" : !strcmp(final_state, "canceled") ? "canceled" : "failed", @@ -21911,6 +21960,113 @@ static void chutni_wait_worker_idle(Gateway *g, const char *scope_id, const char } } +typedef struct { + char reason[64]; + char representative[128]; + unsigned long long count; +} ChutniPreflightSkip; + +static int chutni_preflight_inventory(Gateway *g, const char *canonical, + TextBuffer *out, + char fingerprint_out[65]) { + char *argv[] = {g->samosa_fs, (char *)"chutni-inventory", (char *)"--root", + (char *)canonical, (char *)"--max-depth", (char *)"32", + (char *)"--max-files", (char *)"10000", + (char *)"--max-directories", (char *)"5000", + (char *)"--max-seconds", (char *)"20", NULL}; + int status = 0, saw_done = 0, partial = 0; + char *raw = run_capture(g, g->samosa_fs, argv, 8 << 20, &status); + if (!raw || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + free(raw); + return 0; + } + + unsigned long long files = 0, bytes = 0, skipped = 0; + unsigned long long directories_seen = 0, directories_entered = 0; + char limiting_reason[64] = "none", fingerprint[65] = ""; + ChutniPreflightSkip reasons[8] = {0}; + size_t reason_count = 0; + const char *line = raw; + while (*line) { + const char *end = strchr(line, '\n'); + size_t length = end ? (size_t)(end - line) : strlen(line); + char *record = malloc(length + 1), *arena = NULL; + if (!record) { free(raw); return 0; } + memcpy(record, line, length); record[length] = 0; + jval *item = json_parse(record, &arena); + free(record); + if (!item || item->t != J_OBJ) { + json_free(item); free(arena); free(raw); return 0; + } + jval *type = json_get(item, "type"); + if (type && type->t == J_STR && !strcmp(type->str, "file")) { + files++; + jval *size = json_get(item, "size"); + if (size && size->t == J_NUM && size->num > 0) + bytes += (unsigned long long)size->num; + } else if (type && type->t == J_STR && !strcmp(type->str, "skip")) { + skipped++; + jval *reason = json_get(item, "reason"); + jval *path = json_get(item, "rel_path"); + if (reason && reason->t == J_STR) { + size_t slot = 0; + while (slot < reason_count && strcmp(reasons[slot].reason, reason->str)) slot++; + if (slot == reason_count && reason_count < sizeof(reasons) / sizeof(reasons[0])) { + snprintf(reasons[slot].reason, sizeof(reasons[slot].reason), "%s", reason->str); + if (path && path->t == J_STR) + snprintf(reasons[slot].representative, sizeof(reasons[slot].representative), "%s", path->str); + reason_count++; + } + if (slot < reason_count) reasons[slot].count++; + } + } else if (type && type->t == J_STR && !strcmp(type->str, "done")) { + saw_done = 1; + jval *value = json_get(item, "partial"); + partial = value && value->t == J_BOOL && value->boolean; + value = json_get(item, "directories_seen"); + if (value && value->t == J_NUM && value->num >= 0) directories_seen = (unsigned long long)value->num; + value = json_get(item, "directories_entered"); + if (value && value->t == J_NUM && value->num >= 0) directories_entered = (unsigned long long)value->num; + value = json_get(item, "limiting_reason"); + if (value && value->t == J_STR) snprintf(limiting_reason, sizeof(limiting_reason), "%s", value->str); + value = json_get(item, "policy_fingerprint"); + if (value && value->t == J_STR) snprintf(fingerprint, sizeof(fingerprint), "%s", value->str); + } + json_free(item); free(arena); + if (!end) break; + line = end + 1; + } + free(raw); + if (!saw_done || !fingerprint[0]) return 0; + path_copy(fingerprint_out, 65, fingerprint); + + char n_files[32], n_bytes[32], n_skipped[32], n_seen[32], n_entered[32]; + snprintf(n_files, sizeof(n_files), "%llu", files); + snprintf(n_bytes, sizeof(n_bytes), "%llu", bytes); + snprintf(n_skipped, sizeof(n_skipped), "%llu", skipped); + snprintf(n_seen, sizeof(n_seen), "%llu", directories_seen); + snprintf(n_entered, sizeof(n_entered), "%llu", directories_entered); + if (!text_add(out, "{\"regular_files\":") || !text_add(out, n_files) || + !text_add(out, ",\"regular_file_bytes\":") || !text_add(out, n_bytes) || + !text_add(out, ",\"skipped\":") || !text_add(out, n_skipped) || + !text_add(out, ",\"directories_seen\":") || !text_add(out, n_seen) || + !text_add(out, ",\"directories_entered\":") || !text_add(out, n_entered) || + !text_add(out, partial ? ",\"complete\":false,\"partial\":true,\"limiting_reason\":" : + ",\"complete\":true,\"partial\":false,\"limiting_reason\":" ) || + !text_json_string(out, limiting_reason) || + !text_add(out, ",\"policy_fingerprint\":") || !text_json_string(out, fingerprint) || + !text_add(out, ",\"skip_reasons\":[")) return 0; + for (size_t i = 0; i < reason_count; i++) { + char count[32]; snprintf(count, sizeof(count), "%llu", reasons[i].count); + if ((i && !text_add(out, ",")) || + !text_add(out, "{\"reason\":") || !text_json_string(out, reasons[i].reason) || + !text_add(out, ",\"count\":") || !text_add(out, count) || + !text_add(out, ",\"representative_path\":") || + !text_json_string(out, reasons[i].representative) || !text_add(out, "}")) return 0; + } + return text_add(out, "]}"); +} + static int chutni_preflight(Gateway *g, int fd, const SamosaHttpRequest *request) { char *arena = NULL; jval *root = json_parse(request->body, &arena); jval *kind = root && root->t == J_OBJ ? json_get(root, "kind") : NULL; @@ -21924,7 +22080,14 @@ static int chutni_preflight(Gateway *g, int fd, const SamosaHttpRequest *request json_free(root); free(arena); return samosa_http_json_error(fd, 400, "invalid_preflight", "A readable folder root is required."); } - char id[40]; if (!durable_job_id_generate(id)) { json_free(root); free(arena); return samosa_http_json_error(fd, 500, "id_generation_failed", "A preflight could not be created."); } + TextBuffer inventory = {0}; + char policy_fingerprint[65] = {0}; + if (!chutni_preflight_inventory(g, canonical, &inventory, policy_fingerprint)) { + json_free(root); free(arena); free(inventory.data); + return samosa_http_json_error(fd, 503, "inventory_unavailable", + "The selected folder could not be inventoried within its safety budget."); + } + char id[40]; if (!durable_job_id_generate(id)) { json_free(root); free(arena); free(inventory.data); return samosa_http_json_error(fd, 500, "id_generation_failed", "A preflight could not be created."); } char dir[PATH_MAX], file[PATH_MAX]; int ok = path_join(dir, sizeof(dir), g->chutni_root, "preflights") && mkdirs(dir) && path_join(file, sizeof(file), dir, id); TextBuffer service_args = {0}; @@ -21948,7 +22111,7 @@ static int chutni_preflight(Gateway *g, int fd, const SamosaHttpRequest *request action && action->t == J_STR; if (!service_ok) { json_free(status_json); free(status_arena); free(folder_status); - json_free(root); free(arena); + json_free(root); free(arena); free(inventory.data); return samosa_http_json_error(fd, 503, "chutni_unavailable", "The bundled Chutni service could not inspect that folder."); } @@ -21956,7 +22119,7 @@ static int chutni_preflight(Gateway *g, int fd, const SamosaHttpRequest *request !strcmp(action->str, "unsupported_store") || !strcmp(action->str, "invalid_store")) { json_free(status_json); free(status_arena); free(folder_status); - json_free(root); free(arena); + json_free(root); free(arena); free(inventory.data); return samosa_http_json_error(fd, 409, action->str, "The adjacent memory path exists but cannot be opened safely."); } @@ -21968,18 +22131,29 @@ static int chutni_preflight(Gateway *g, int fd, const SamosaHttpRequest *request text_add(&b, ",\"kind\":\"folder\",\"canonical_root\":") && text_json_string(&b, canonical) && text_add(&b, ",\"volume_identity\":") && text_json_string(&b, volume) && text_add(&b, ",\"root_file_identity\":") && text_json_string(&b, identity) && + text_add(&b, ",\"policy_fingerprint\":") && text_json_string(&b, policy_fingerprint) && text_add(&b, ",\"effective_policy\":{\"include_hidden\":false," - "\"cross_filesystems\":false,\"maximum_file_bytes\":67108864," + "\"cross_filesystems\":false,\"maximum_depth\":32," + "\"maximum_files\":10000,\"maximum_directories\":5000," + "\"maximum_seconds\":20,\"maximum_file_bytes\":67108864," + "\"follow_symlinks\":false," "\"mandatory_exclusions\":[\".git\",\".svn\",\".hg\"," "\"node_modules\",\".cache\",\"__pycache__\",\".venv\"," - "\"venv\",\"target\",\".Trash\"],\"user_exclusions\":[]}," + "\"venv\",\"env\",\"target\",\"build\",\"dist\"," + "\"DerivedData\",\".Trash\",\".mypy_cache\"," + "\".pytest_cache\",\".ruff_cache\",\"site-packages\"," + "\".next\",\".nuxt\",\".yarn\",\".pnpm-store\"," + "\".gradle\",\"coverage\",\".idea\"]," + "\"marker_exclusions\":[\"directory containing pyvenv.cfg\"]," + "\"user_exclusions\":[]}," "\"chutni\":") && text_add(&b, folder_status) && + text_add(&b, ",\"inventory\":") && text_add(&b, inventory.data) && text_add(&b, ",\"warnings\":[]}\n") && write_small_file(file, b.data); free(b.data); json_free(status_json); free(status_arena); free(folder_status); - json_free(root); free(arena); + json_free(root); free(arena); free(inventory.data); if (!ok) return samosa_http_json_error(fd, 500, "preflight_failed", "The preflight could not be saved."); char *saved = read_file_limit(file, 8192); int sent = saved && samosa_http_response(fd, 200, "application/json", saved, NULL); free(saved); return sent; @@ -22012,10 +22186,17 @@ static int chutni_scope_create(Gateway *g, int fd, const SamosaHttpRequest *requ return samosa_http_json_error(fd, 400, "invalid_preflight", "That preflight is unavailable."); } jval *kind = json_get(p, "kind"), *canonical = json_get(p, "canonical_root"); + jval *saved_fingerprint = json_get(p, "policy_fingerprint"); + jval *accepted_fingerprint = root && root->t == J_OBJ + ? json_get(root, "policy_fingerprint") : NULL; if (!kind || kind->t != J_STR || !canonical || canonical->t != J_STR || + !saved_fingerprint || saved_fingerprint->t != J_STR || + !accepted_fingerprint || accepted_fingerprint->t != J_STR || + strcmp(saved_fingerprint->str, accepted_fingerprint->str) || !name || name->t != J_STR || !*name->str) { json_free(root); free(arena); json_free(p); free(pf_arena); free(preflight); - return samosa_http_json_error(fd, 400, "invalid_scope", "display_name and a valid preflight are required."); + return samosa_http_json_error(fd, 409, "stale_policy", + "The accepted inventory policy changed. Check the folder again."); } char canonical_copy[PATH_MAX], name_copy[256]; if (!path_copy(canonical_copy, sizeof(canonical_copy), canonical->str) || @@ -22024,9 +22205,28 @@ static int chutni_scope_create(Gateway *g, int fd, const SamosaHttpRequest *requ return samosa_http_json_error(fd, 400, "invalid_scope", "The folder path or display name is too long."); } + jval *saved_identity = json_get(p, "root_file_identity"); + char resolved_root[PATH_MAX], current_identity[96]; + struct stat current_root; + if (!saved_identity || saved_identity->t != J_STR || + !realpath(canonical_copy, resolved_root) || strcmp(resolved_root, canonical_copy) || + stat(canonical_copy, ¤t_root) != 0 || !S_ISDIR(current_root.st_mode)) { + json_free(root); free(arena); json_free(p); free(pf_arena); free(preflight); + return samosa_http_json_error(fd, 409, "stale_root", + "The selected folder changed after preview. Check it again."); + } + snprintf(current_identity, sizeof(current_identity), "%llu:%llu", + (unsigned long long)current_root.st_dev, + (unsigned long long)current_root.st_ino); + if (strcmp(saved_identity->str, current_identity)) { + json_free(root); free(arena); json_free(p); free(pf_arena); free(preflight); + return samosa_http_json_error(fd, 409, "stale_root", + "The selected folder changed after preview. Check it again."); + } char scope_id[40]; if (!durable_job_id_generate(scope_id)) { json_free(root); free(arena); json_free(p); free(pf_arena); free(preflight); return samosa_http_json_error(fd, 500, "id_generation_failed", "A scope could not be created."); } int created_ok = chutni_scope_metadata_create( - g, scope_id, name_copy, canonical_copy, summary_token_budget); + g, scope_id, name_copy, canonical_copy, saved_fingerprint->str, + summary_token_budget); json_free(root); free(arena); json_free(p); free(pf_arena); free(preflight); if (!created_ok) return samosa_http_json_error(fd, 409, "scope_exists", "That folder already has a Chutni scope or cannot be registered."); char job_id[40]; if (!durable_job_id_generate(job_id) || !chutni_start_worker(g, scope_id, job_id, 1, "queued")) @@ -22051,7 +22251,7 @@ static int chutni_scope_show(Gateway *g, int fd, const char *scope_id) { lifecycle state without claiming that unfinished evidence is ready. */ char job_id[96] = {0}, job_state[32] = {0}; unsigned long long generation = 0; if (chutni_job_load(g, scope_id, job_id, job_state, &generation) && - strcmp(job_state, "completed")) { + strcmp(job_state, "completed") && strcmp(job_state, "completed_partial")) { char *arena = NULL; jval *root = json_parse(raw, &arena); jval *state = root && root->t == J_OBJ ? json_get(root, "state") : NULL; const char *visible = !strcmp(job_state, "paused_user") ? "paused_user" : diff --git a/tests/fixtures/ui_chutni/contracts/chutni_preflight_response.json b/tests/fixtures/ui_chutni/contracts/chutni_preflight_response.json index d3565bb..befaef7 100644 --- a/tests/fixtures/ui_chutni/contracts/chutni_preflight_response.json +++ b/tests/fixtures/ui_chutni/contracts/chutni_preflight_response.json @@ -2,6 +2,7 @@ "preflight_id": "random-id", "expires_at": "ISO-8601", "kind": "folder", + "policy_fingerprint": "sha256:policy-digest", "roots": [ { "root_id": "random-id", @@ -13,13 +14,29 @@ } ], "effective_policy": { - "policy_fingerprint": "sha256", "cross_filesystems": false, "include_hidden": false, - "maximum_file_bytes": 268435456, + "follow_symlinks": false, + "maximum_depth": 32, + "maximum_files": 10000, + "maximum_directories": 5000, + "maximum_seconds": 20, + "maximum_file_bytes": 67108864, "mandatory_exclusions": [], "user_exclusions": [] }, + "inventory": { + "regular_files": 100, + "regular_file_bytes": 1234, + "skipped": 0, + "directories_seen": 4, + "directories_entered": 4, + "complete": true, + "partial": false, + "limiting_reason": "none", + "policy_fingerprint": "sha256:policy-digest", + "skip_reasons": [] + }, "estimate": { "sampled": true, "files": 100, diff --git a/tests/fixtures/ui_chutni/contracts/chutni_scope_create_request.json b/tests/fixtures/ui_chutni/contracts/chutni_scope_create_request.json index 9ca6169..c815284 100644 --- a/tests/fixtures/ui_chutni/contracts/chutni_scope_create_request.json +++ b/tests/fixtures/ui_chutni/contracts/chutni_scope_create_request.json @@ -1,5 +1,6 @@ { "client_request_id": "same-caller-random-id", "preflight_id": "random-id", + "policy_fingerprint": "sha256:policy-digest", "display_name": "Research" } diff --git a/tests/test_chutni_controls.sh b/tests/test_chutni_controls.sh index 9865f16..851fc6e 100755 --- a/tests/test_chutni_controls.sh +++ b/tests/test_chutni_controls.sh @@ -33,6 +33,7 @@ REAL_CHUTNI_MCP="$ROOT/$BUILD_DIR/chutni-mcp" \ SAMOSA_HOME="$TMP/home" SAMOSA_PORT="$PORT" \ SAMOSA_BACKEND_URL="http://127.0.0.1:$((PORT + 1))" \ SAMOSA_CHUTNI_SERVICE="$ROOT/tests/fake_chutni_slow.sh" \ +SAMOSA_FS="$ROOT/$BUILD_DIR/samosa-fs" \ SAMOSA_APP_HTML="$ROOT/assets/app.html" \ "$ROOT/$BUILD_DIR/samosa-gateway" >"$TMP/gateway.log" 2>&1 & GW_PID=$! @@ -53,9 +54,11 @@ PREFLIGHT=$(curl -fsS -H "X-Samosa-Token: $TOKEN" -H 'Content-Type: application/ --data-binary "{\"kind\":\"folder\",\"roots\":[{\"path\":\"$TMP/source\"}]}") PREFLIGHT_ID=$(printf '%s' "$PREFLIGHT" | sed -n 's/.*"preflight_id":"\([^"]*\)".*/\1/p') [ -n "$PREFLIGHT_ID" ] || fail "failed to get preflight id" +POLICY=$(printf '%s' "$PREFLIGHT" | tr '\n' ' ' | sed -n 's/.*"policy_fingerprint":"\([^"]*\)".*/\1/p') +[ -n "$POLICY" ] || fail "preflight did not include a policy fingerprint" CREATED=$(curl -fsS -H "X-Samosa-Token: $TOKEN" -H 'Content-Type: application/json' \ -X POST "http://127.0.0.1:$PORT/v1/chutni/scopes" \ - --data-binary "{\"preflight_id\":\"$PREFLIGHT_ID\",\"display_name\":\"Control fixture\"}") + --data-binary "{\"preflight_id\":\"$PREFLIGHT_ID\",\"policy_fingerprint\":\"$POLICY\",\"display_name\":\"Control fixture\"}") SCOPE=$(printf '%s' "$CREATED" | sed -n 's/.*"scope_id":"\([^"]*\)".*/\1/p') JOB=$(printf '%s' "$CREATED" | sed -n 's/.*"job_id":"\([^"]*\)".*/\1/p') [ -n "$SCOPE" ] && [ -n "$JOB" ] || fail "scope ID missing from create response" diff --git a/tests/test_chutni_gateway.sh b/tests/test_chutni_gateway.sh index 11ccea2..c1cb689 100755 --- a/tests/test_chutni_gateway.sh +++ b/tests/test_chutni_gateway.sh @@ -67,6 +67,7 @@ SAMOSA_QWEN_ENGINE="$ROOT/$BUILD_DIR/test_fake_openai_backend" \ SAMOSA_QWEN_MODEL="$TMP/home/qwen-model" \ SAMOSA_TOKENIZER="$TMP/tokenizer.json" \ SAMOSA_CHUTNI_SERVICE="$ROOT/$BUILD_DIR/chutni-mcp" \ +SAMOSA_FS="$ROOT/$BUILD_DIR/samosa-fs" \ SAMOSA_EXTRACT="$SAMOSA_EXTRACT" \ SAMOSA_OCR="$ROOT/tests/fake_ocr_sidecar.sh" \ SAMOSA_APP_VERSION="test-enrichment-1" \ @@ -93,19 +94,46 @@ PF=$(curl -fsS -H "X-Samosa-Token: $TOKEN" -H 'Content-Type: application/json' - --data-binary "{\"kind\":\"folder\",\"roots\":[{\"path\":\"$TMP/source\"}]}") PREFLIGHT=$(printf '%s' "$PF" | sed -n 's/.*"preflight_id":"\([^"]*\)".*/\1/p') [ -n "$PREFLIGHT" ] || fail "missing preflight_id" +POLICY=$(printf '%s' "$PF" | tr '\n' ' ' | sed -n 's/.*"policy_fingerprint":"\([^"]*\)".*/\1/p') +[ -n "$POLICY" ] || fail "missing policy fingerprint" +printf '%s' "$PF" | grep -q '"inventory":{"regular_files":' +printf '%s' "$PF" | grep -q '"directories_entered":' printf '%s' "$PF" | grep -q '"action":"create_store"' printf '%s' "$PF" | grep -q '"store_path":' printf '%s' "$PF" | grep -q '\.chutni' STORE=$(printf '%s' "$PF" | sed -n 's/.*"store_path":"\([^"]*\)".*/\1/p') [ -n "$STORE" ] || fail "missing store_path" +STALE_POLICY_CODE=$(curl -sS -o "$TMP/stale-policy.json" -w '%{http_code}' \ + -H "X-Samosa-Token: $TOKEN" -H 'Content-Type: application/json' -X POST \ + "http://127.0.0.1:$PORT/v1/chutni/scopes" \ + --data-binary "{\"preflight_id\":\"$PREFLIGHT\",\"policy_fingerprint\":\"wrong\",\"display_name\":\"Research\"}") +[ "$STALE_POLICY_CODE" = 409 ] || fail "scope confirmation accepted a stale policy fingerprint" + CREATED=$(curl -fsS -H "X-Samosa-Token: $TOKEN" -H 'Content-Type: application/json' -X POST \ "http://127.0.0.1:$PORT/v1/chutni/scopes" \ - --data-binary "{\"preflight_id\":\"$PREFLIGHT\",\"display_name\":\"Research\",\"summary_token_budget\":128}") + --data-binary "{\"preflight_id\":\"$PREFLIGHT\",\"policy_fingerprint\":\"$POLICY\",\"display_name\":\"Research\",\"summary_token_budget\":128}") SCOPE=$(printf '%s' "$CREATED" | sed -n 's/.*"scope_id":"\([^"]*\)".*/\1/p') JOB=$(printf '%s' "$CREATED" | sed -n 's/.*"job_id":"\([^"]*\)".*/\1/p') [ -n "$SCOPE" ] && [ -n "$JOB" ] || fail "scope ID missing from create response" +# A matching policy token is insufficient if the selected directory itself was +# replaced after preview. +mkdir -p "$TMP/stale-source" +printf 'stale root fixture\n' >"$TMP/stale-source/file.txt" +STALE_PF=$(curl -fsS -H "X-Samosa-Token: $TOKEN" -H 'Content-Type: application/json' -X POST \ + "http://127.0.0.1:$PORT/v1/chutni/preflight" \ + --data-binary "{\"kind\":\"folder\",\"roots\":[{\"path\":\"$TMP/stale-source\"}]}") +STALE_ID=$(printf '%s' "$STALE_PF" | sed -n 's/.*"preflight_id":"\([^"]*\)".*/\1/p') +STALE_POLICY=$(printf '%s' "$STALE_PF" | tr '\n' ' ' | sed -n 's/.*"policy_fingerprint":"\([^"]*\)".*/\1/p') +mv "$TMP/stale-source" "$TMP/stale-source-old" +mkdir "$TMP/stale-source" +STALE_ROOT_CODE=$(curl -sS -o "$TMP/stale-root.json" -w '%{http_code}' \ + -H "X-Samosa-Token: $TOKEN" -H 'Content-Type: application/json' -X POST \ + "http://127.0.0.1:$PORT/v1/chutni/scopes" \ + --data-binary "{\"preflight_id\":\"$STALE_ID\",\"policy_fingerprint\":\"$STALE_POLICY\",\"display_name\":\"Stale fixture\"}") +[ "$STALE_ROOT_CODE" = 409 ] || fail "scope confirmation accepted a replaced folder root" + i=0 while [ "$i" -lt 300 ]; do STATUS=$(curl -fsS -H "X-Samosa-Token: $TOKEN" "http://127.0.0.1:$PORT/v1/chutni/scopes/$SCOPE") diff --git a/tests/test_chutni_inventory.sh b/tests/test_chutni_inventory.sh index 1b26e35..8b74ec4 100755 --- a/tests/test_chutni_inventory.sh +++ b/tests/test_chutni_inventory.sh @@ -93,6 +93,48 @@ grep -q '"type":"done","canceled":false' "$OUT" || { echo "FAIL: missing clean d sh "$GEN" fix-perms "$ROOT" rm -rf "$ROOT" +# --- explicit traversal budgets stop work and report a truthful partial result --- +mkdir -p "$TMP/budget/child" +i=0 +while [ "$i" -lt 8 ]; do + printf 'budget fixture %s\n' "$i" >"$TMP/budget/file-$i.txt" + i=$((i + 1)) +done +printf 'nested fixture\n' >"$TMP/budget/child/nested.txt" +"$FS" chutni-inventory --root "$TMP/budget" --max-files 2 >"$TMP/budget-files.ndjson" +FILE_COUNT=$(grep -c '"type":"file"' "$TMP/budget-files.ndjson") +[ "$FILE_COUNT" = 2 ] || { echo "FAIL: --max-files 2 emitted $FILE_COUNT file records"; cat "$TMP/budget-files.ndjson"; exit 1; } +grep -q '"type":"done".*"partial":true.*"limiting_reason":"maximum_files"' "$TMP/budget-files.ndjson" \ + || { echo "FAIL: file limit did not produce a truthful partial summary"; cat "$TMP/budget-files.ndjson"; exit 1; } + +"$FS" chutni-inventory --root "$TMP/budget" --max-depth 0 >"$TMP/budget-depth.ndjson" +grep -q '"rel_path":"child","reason":"depth_limit"' "$TMP/budget-depth.ndjson" \ + || { echo "FAIL: depth limit did not prune the child directory"; cat "$TMP/budget-depth.ndjson"; exit 1; } +grep -q '"type":"done".*"partial":true.*"limiting_reason":"maximum_depth"' "$TMP/budget-depth.ndjson" \ + || { echo "FAIL: depth limit did not produce a truthful partial summary"; cat "$TMP/budget-depth.ndjson"; exit 1; } + +# --- generated trees and marker-detected environments are pruned in inventory --- +mkdir -p "$TMP/policy/.venv" "$TMP/policy/NODE_MODULES" "$TMP/policy/build" \ + "$TMP/policy/custom-python-env" +printf 'decoy\n' >"$TMP/policy/.venv/decoy.txt" +printf 'decoy\n' >"$TMP/policy/NODE_MODULES/decoy.txt" +printf 'decoy\n' >"$TMP/policy/build/decoy.txt" +printf 'home = /python\n' >"$TMP/policy/custom-python-env/pyvenv.cfg" +printf 'decoy\n' >"$TMP/policy/custom-python-env/decoy.txt" +"$FS" chutni-inventory --root "$TMP/policy" --include-hidden >"$TMP/policy.ndjson" +for name in .venv NODE_MODULES build; do + grep -q "\"rel_path\":\"$name\",\"reason\":\"generated_tree\"" "$TMP/policy.ndjson" \ + || { echo "FAIL: generated directory $name was not pruned"; cat "$TMP/policy.ndjson"; exit 1; } + if grep -q "\"rel_path\":\"$name/decoy.txt\"" "$TMP/policy.ndjson"; then + echo "FAIL: inventory entered generated directory $name"; exit 1 + fi +done +grep -q '"rel_path":"custom-python-env","reason":"python_environment_marker"' "$TMP/policy.ndjson" \ + || { echo "FAIL: custom Python environment was not recognized by marker"; cat "$TMP/policy.ndjson"; exit 1; } +if grep -q '"rel_path":"custom-python-env/decoy.txt"' "$TMP/policy.ndjson"; then + echo "FAIL: inventory entered custom Python environment"; exit 1 +fi + # --- chutni-hash: a complete SHA-256, never a prefix, with identity checks --- sh "$GEN" build "$ROOT" EXPECTED=$(sha256_file "$ROOT/plain/report.md") || exit 1 diff --git a/tests/test_ui_chutni_contracts.py b/tests/test_ui_chutni_contracts.py index d1df908..f0110bc 100644 --- a/tests/test_ui_chutni_contracts.py +++ b/tests/test_ui_chutni_contracts.py @@ -55,7 +55,7 @@ # API-version change, not a silent edit. CHUTNI_SCOPE_KINDS = {"folder", "drive", "computer"} CHUTNI_SCOPE_STATES = { - "unbuilt", "building", "ready", "checking", "updating", "rebuilding", + "unbuilt", "building", "ready", "ready_partial", "checking", "updating", "rebuilding", "paused_user", "paused_chat", "needs_permission", "disconnected", "needs_attention", "failed_initial", "canceled_initial", "forgetting", } @@ -89,7 +89,7 @@ SELECTION_STATES = {"queued", "loading", "ready", "failed"} DURABLE_JOB_STATES = { "queued", "running", "paused_user", "paused_chat", "canceling", - "canceled", "failed", "completed", + "canceled", "failed", "completed", "completed_partial", } SETUP_NEXT_STEPS = {"name", "welcome", "model", "download", "chat"} diff --git a/vendor/chutni b/vendor/chutni index 1fe03ea..9753788 160000 --- a/vendor/chutni +++ b/vendor/chutni @@ -1 +1 @@ -Subproject commit 1fe03eaca2bcab170173bbd6d0933cc3414b4e29 +Subproject commit 975378854dd9e237fb24bb54c418307abb30452d From a41d52bacffcd1e27abd50018d55b4bbe401cff0 Mon Sep 17 00:00:00 2001 From: Deepan Wadhwa Date: Tue, 22 Sep 2026 16:16:52 -0400 Subject: [PATCH 02/30] Add bounded Jobs duplicate and sort recipes --- assets/app.html | 53 +++- docs/CHUTNI_JOBS_RECOVERY_PLAN.md | 10 +- src/samosa_gateway.c | 405 +++++++++++++++++++++++++++++- tests/test_compiled_gateway.sh | 93 +++++++ tests/test_jobs_ui.mjs | 26 ++ 5 files changed, 572 insertions(+), 15 deletions(-) diff --git a/assets/app.html b/assets/app.html index d59d98b..5053329 100644 --- a/assets/app.html +++ b/assets/app.html @@ -1754,7 +1754,11 @@

Jobs

Jobs reads the folder directly. It does not use Chutni memory; it checks the current files on disk for this run.
- + + +
+
+
@@ -6580,7 +6584,7 @@

Delete conversation?

} // --- Jobs (Models -> Tools -> Jobs) -------------------------------- const jobEls = { - view: $("jobsView"), form: $("jobForm"), folder: $("jobFolder"), goal: $("jobGoal"), + view: $("jobsView"), form: $("jobForm"), folder: $("jobFolder"), goal: $("jobGoal"), recipe: $("jobRecipe"), browse: $("jobBrowse"), memoryAccess: $("jobMemoryAccess"), run: $("jobRun"), auto: $("jobAuto"), activity: $("jobActivity"), bar: $("jobBar"), progress: $("jobProgress"), result: $("jobResult"), resultLabel: $("jobResultLabel"), resultText: $("jobResultText"), @@ -8088,6 +8092,34 @@

Delete conversation?

const parts = Object.entries(grouped).map(([kind, count]) => `${count} ${kind}`).join(", "); const l = jobLine("▦", `${evt.total} files found`, "step-done"); if (parts) addSub(l, parts); break; } + case "report": { + const ageLabels = ["under 30 days", "30–365 days", "over a year", "unknown/future"]; + const sizeLabels = ["up to 1 MB", "1–10 MB", "over 10 MB"]; + const types = Object.entries(evt.by_type || {}).map(([ext, n]) => `${n} ${ext}`).join(", ") || "No files found"; + let md = `### Folder report\n\n- **${evt.total} files** · ${bytesLabel(evt.bytes)}\n- **File types:** ${types}\n`; + md += `- **Age:** ${(evt.age || []).map((n, i) => `${n} ${ageLabels[i]}`).join(", ")}\n`; + md += `- **Size:** ${(evt.size_bands || []).map((n, i) => `${n} ${sizeLabels[i]}`).join(", ")}\n`; + md += `- **Duplicate candidates:** ${evt.duplicate_candidates || 0} files in ${evt.duplicate_size_groups || 0} same-size groups (size only; contents not compared)\n`; + const skips = Object.entries(evt.skip_reasons || {}); + if (skips.length) md += `- **Skipped:** ${skips.map(([reason, n]) => `${n} ${reason}`).join(", ")}\n`; + if (evt.partial) md += `\n**Partial report:** stopped at ${evt.limiting_reason || "a safety limit"}.\n`; + showJobResult(md); break; + } + case "duplicates": { + let md = `### Duplicate files\n\n${evt.duplicate_groups || 0} verified duplicate group${evt.duplicate_groups === 1 ? "" : "s"}. ` + + `${evt.candidate_files || 0} same-size candidates checked; ${evt.hashed_files || 0} files hashed.`; + if (evt.groups && evt.groups.length) { + md += "\n\n"; + evt.groups.forEach(group => { + md += `- **${bytesLabel(group.size)}** · SHA-256 \`${group.sha256}\`\n`; + group.files.forEach(path => { md += ` - \`${path}\`\n`; }); + }); + } + if (evt.skipped) md += `\n${evt.skipped} entries skipped by inventory policy.\n`; + if (evt.deferred_files) md += `\n${evt.deferred_files} candidates deferred by a hashing safety limit.\n`; + if (evt.partial) md += `\n**Partial duplicate search:** stopped at ${evt.limiting_reason || "a safety limit"}.\n`; + showJobResult(md); break; + } case "indexing": ctx.indexLine = jobLine("·", `Checking ${evt.total} filenames`); break; case "index_complete": { @@ -8112,6 +8144,7 @@

Delete conversation?

} case "await_apply": ctx.jobId = evt.job_id; lastJobId = evt.job_id; + ctx.awaitingApply = true; showBar(`${evt.moves} file${evt.moves === 1 ? "" : "s"} ready to move.`, [["Apply moves", "apply", () => applyJob(evt.job_id, ctx)]]); break; case "await_user": { @@ -8256,7 +8289,7 @@

Delete conversation?

if (jobEls.result.hidden) showJobResult(summary); loadReviewQueue(ctx.jobId || lastJobId, ctx); if (ctx.moved && !ctx.undone) showBar("Done — files were moved.", [["Undo", "undo", () => undoJob(ctx.jobId || lastJobId, ctx)]]); - else jobEls.bar.hidden = true; + else if (!ctx.awaitingApply) jobEls.bar.hidden = true; break; } case "error": showJobResult(evt.message, true); break; @@ -8319,11 +8352,11 @@

Delete conversation?

} async function runJob() { if (jobRunning) return; - const folder = jobEls.folder.value.trim(), goal = jobEls.goal.value.trim(); - if (!folder || !goal) { jobEls.folder.focus(); return; } + const folder = jobEls.folder.value.trim(), goal = jobEls.goal.value.trim(), recipe = jobEls.recipe.value; + if (!folder || (recipe === "find" && !goal)) { jobEls.folder.focus(); return; } jobRunning = true; jobEls.run.disabled = true; jobEls.run.textContent = "Running"; clearJobOutput(); const ctx = { folder }; - try { await streamJob("/v1/jobs/run", { goal, folder, mode: jobEls.auto.checked ? "execute" : "confirm" }, ctx); } + try { await streamJob("/v1/jobs/run", recipe !== "find" ? { recipe, folder } : { goal, folder, recipe, mode: jobEls.auto.checked ? "execute" : "confirm" }, ctx); } catch (e) { renderJobEvent({ type: "error", message: String(e) }, ctx); } finally { jobRunning = false; jobEls.run.disabled = false; jobEls.run.textContent = "Run job"; } } @@ -8685,6 +8718,14 @@

Delete conversation?

await refreshChutni(); }; jobEls.form.onsubmit=e=>{e.preventDefault();runJob();}; + const syncJobRecipe = () => { + const deterministic = jobEls.recipe.value !== "find"; + jobEls.goal.disabled = deterministic; + jobEls.goal.placeholder = deterministic ? "No description needed for this recipe" : "Find my cat's medical records"; + $("jobGoalLabel").textContent = deterministic ? "Task description (optional)" : "What should I find?"; + }; + jobEls.recipe.onchange = syncJobRecipe; + syncJobRecipe(); jobEls.goal.onkeydown=e=>{if(e.key==="Enter"&&(e.metaKey||e.ctrlKey)){e.preventDefault();runJob();}}; jobEls.previewDef.onclick=previewDefinitionJob; jobEls.runDef.onclick=runDefinitionJob; diff --git a/docs/CHUTNI_JOBS_RECOVERY_PLAN.md b/docs/CHUTNI_JOBS_RECOVERY_PLAN.md index c073f42..dc6ad24 100644 --- a/docs/CHUTNI_JOBS_RECOVERY_PLAN.md +++ b/docs/CHUTNI_JOBS_RECOVERY_PLAN.md @@ -1,6 +1,6 @@ # Chutni and Jobs recovery plan -**Status:** implementation plan; no product changes from this document alone +**Status:** in progress; Phase C recipes are being implemented incrementally **Updated:** 2026-09-22 @@ -298,6 +298,14 @@ goal may map into one of them, but pasted internal JSON is not required. - scheduled write actions always produce a review batch unless the user has explicitly enabled a narrowly defined safe rule. +**Implementation checkpoint (2026-09-22):** Folder report, Find files about…, +and Find duplicates are exposed in Jobs. Find duplicates hashes only bounded +same-size candidates and reports a partial result when work is capped or a +candidate cannot be read. Sort by file type is now being added through the +existing persisted move-plan, explicit Apply, and Undo path. Inbox +classification and watch-folder deltas remain unimplemented Phase C work; this +checkpoint does not claim the Phase C acceptance gates are complete. + ### C2. Durable `JobSpec` Compile every UI submission into a versioned structure containing: diff --git a/src/samosa_gateway.c b/src/samosa_gateway.c index 8fca55f..09d3d1f 100644 --- a/src/samosa_gateway.c +++ b/src/samosa_gateway.c @@ -5744,6 +5744,373 @@ static int model_find_intent(Gateway *g, const char *goal) { return is_find; } +typedef struct { + char key[32]; + unsigned long long count; + unsigned long long bytes; +} JobsReportGroup; + +static int jobs_u64_cmp(const void *a, const void *b) { + unsigned long long x = *(const unsigned long long *)a; + unsigned long long y = *(const unsigned long long *)b; + return x < y ? -1 : x > y ? 1 : 0; +} + +static void jobs_report_type(const char *path, char out[32]) { + const char *base = strrchr(path, '/'); + const char *dot; + size_t n; + base = base ? base + 1 : path; + dot = strrchr(base, '.'); + if (!dot || dot == base || !dot[1]) { path_copy(out, 32, "[no extension]"); return; } + n = strlen(dot); + if (n > 31) n = 31; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)dot[i]; + out[i] = (char)(c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c); + } + out[n] = 0; +} + +/* Productized read-only report recipe. It consumes the same bounded, + * metadata-only walk as Chutni preflight; no file contents or hashes are + * read. Size-equal files are called duplicate candidates, never duplicates. */ +static int jobs_folder_report(Gateway *g, int fd, const char *folder, + const char *existing_job_id) { + char *argv[] = {g->samosa_fs, (char *)"chutni-inventory", (char *)"--root", + (char *)folder, (char *)"--max-depth", (char *)"32", + (char *)"--max-files", (char *)"10000", + (char *)"--max-directories", (char *)"5000", + (char *)"--max-seconds", (char *)"20", NULL}; + int status = 0; + char *raw = run_capture(g, g->samosa_fs, argv, 8 << 20, &status); + if (!raw || !WIFEXITED(status) || WEXITSTATUS(status)) { + free(raw); + return samosa_http_json_error(fd, 400, "folder_scan_failed", "The folder could not be inspected within the report safety budget."); + } + JobsReportGroup types[64] = {0}, skips[32] = {0}; + unsigned long long sizes[10000], file_count = 0, total_bytes = 0; + unsigned long long age[4] = {0}, size_band[3] = {0}; + size_t type_count = 0, skip_count = 0; + int partial = 0, saw_done = 0; + char limiting_reason[64] = "none", job_id[64]; + time_t now = time(NULL); + const char *line = raw; + while (*line) { + const char *end = strchr(line, '\n'); + size_t len = end ? (size_t)(end - line) : strlen(line); + char *record = malloc(len + 1), *arena = NULL; + if (!record) { free(raw); return 0; } + memcpy(record, line, len); record[len] = 0; + jval *obj = json_parse(record, &arena); free(record); + if (!obj || obj->t != J_OBJ) { json_free(obj); free(arena); free(raw); return samosa_http_json_error(fd, 500, "invalid_inventory", "The folder inventory returned invalid data."); } + jval *type = json_get(obj, "type"); + if (type && type->t == J_STR && !strcmp(type->str, "file")) { + jval *path = json_get(obj, "rel_path"), *size = json_get(obj, "size"), *mtime = json_get(obj, "mtime"); + unsigned long long bytes = size && size->t == J_NUM && size->num > 0 ? (unsigned long long)size->num : 0; + if (!path || path->t != J_STR || file_count >= 10000) { json_free(obj); free(arena); free(raw); return samosa_http_json_error(fd, 500, "invalid_inventory", "The folder inventory exceeded its declared budget."); } + sizes[file_count++] = bytes; total_bytes += bytes; + char ext[32]; jobs_report_type(path->str, ext); + size_t k = 0; while (k < type_count && strcmp(types[k].key, ext)) k++; + if (k == type_count && type_count < 64) { path_copy(types[k].key, sizeof(types[k].key), ext); type_count++; } + if (k < type_count) { types[k].count++; types[k].bytes += bytes; } + if (bytes <= 1024ull*1024ull) size_band[0]++; else if (bytes <= 10ull*1024ull*1024ull) size_band[1]++; else size_band[2]++; + if (!mtime || mtime->t != J_NUM || mtime->num > (double)now + 86400.0) age[3]++; + else { double days = ((double)now - mtime->num) / 86400.0; age[days < 30 ? 0 : days < 365 ? 1 : 2]++; } + } else if (type && type->t == J_STR && !strcmp(type->str, "skip")) { + jval *reason = json_get(obj, "reason"); + if (reason && reason->t == J_STR) { + size_t k = 0; while (k < skip_count && strcmp(skips[k].key, reason->str)) k++; + if (k == skip_count && skip_count < 32) { path_copy(skips[k].key, sizeof(skips[k].key), reason->str); skip_count++; } + if (k < skip_count) skips[k].count++; + } + } else if (type && type->t == J_STR && !strcmp(type->str, "done")) { + saw_done = 1; jval *v = json_get(obj, "partial"); partial = v && v->t == J_BOOL && v->boolean; + v = json_get(obj, "limiting_reason"); if (v && v->t == J_STR) path_copy(limiting_reason, sizeof(limiting_reason), v->str); + } + json_free(obj); free(arena); + if (!end) break; + line = end + 1; + } + free(raw); + if (!saw_done) return samosa_http_json_error(fd, 500, "invalid_inventory", "The folder inventory did not finish."); + qsort(sizes, (size_t)file_count, sizeof(sizes[0]), jobs_u64_cmp); + unsigned long long duplicate_groups = 0, duplicate_candidates = 0; + for (size_t i = 0; i < file_count;) { size_t j = i + 1; while (j < file_count && sizes[j] == sizes[i]) j++; if (j - i > 1) { duplicate_groups++; duplicate_candidates += j - i; } i = j; } + if (!samosa_http_stream_headers(fd)) return 0; + if (existing_job_id) path_copy(job_id, sizeof(job_id), existing_job_id); + else snprintf(job_id, sizeof(job_id), "job-%ld-%ld-%lld", (long)time(NULL), (long)getpid(), (long long)monotonic_millis()); + if (!save_job_state(g, job_id, "Folder report", folder)) return 0; + TextBuffer event = {0}; char num[64]; + int ok = text_add(&event, "{\"seq\":1,\"type\":\"decode_intent\",\"job_id\":") && text_json_string(&event, job_id) && + text_add(&event, ",\"goal\":\"Folder report\",\"folder\":") && text_json_string(&event, folder) && text_add(&event, "}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + free(event.data); event = (TextBuffer){0}; + if (!ok) goto report_done; + snprintf(num, sizeof(num), "%llu", file_count); + ok = text_add(&event, "{\"seq\":2,\"type\":\"report\",\"recipe\":\"folder_report\",\"total\":") && text_add(&event, num) && text_add(&event, ",\"bytes\":"); + snprintf(num, sizeof(num), "%llu", total_bytes); ok = ok && text_add(&event, num) && text_add(&event, ",\"partial\":") && text_add(&event, partial ? "true" : "false") && text_add(&event, ",\"limiting_reason\":") && text_json_string(&event, limiting_reason) && text_add(&event, ",\"by_type\":{"); + for (size_t i = 0; ok && i < type_count; i++) { if (i) ok = text_add(&event, ","); snprintf(num, sizeof(num), "%llu", types[i].count); ok = ok && text_json_string(&event, types[i].key) && text_add(&event, ":") && text_add(&event, num); } + ok = ok && text_add(&event, "},\"age\":["); + for (int i = 0; ok && i < 4; i++) { if (i) ok = text_add(&event, ","); snprintf(num, sizeof(num), "%llu", age[i]); ok = ok && text_add(&event, num); } + ok = ok && text_add(&event, "],\"size_bands\":["); + for (int i = 0; ok && i < 3; i++) { if (i) ok = text_add(&event, ","); snprintf(num, sizeof(num), "%llu", size_band[i]); ok = ok && text_add(&event, num); } + snprintf(num, sizeof(num), "%llu", duplicate_candidates); ok = ok && text_add(&event, "],\"duplicate_candidates\":") && text_add(&event, num); + snprintf(num, sizeof(num), "%llu", duplicate_groups); ok = ok && text_add(&event, ",\"duplicate_size_groups\":") && text_add(&event, num) && text_add(&event, ",\"skip_reasons\":{"); + for (size_t i = 0; ok && i < skip_count; i++) { if (i) ok = text_add(&event, ","); snprintf(num, sizeof(num), "%llu", skips[i].count); ok = ok && text_json_string(&event, skips[i].key) && text_add(&event, ":") && text_add(&event, num); } + ok = ok && text_add(&event, "}}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + free(event.data); event.data = NULL; + if (ok) { + snprintf(num, sizeof(num), "{\"seq\":3,\"type\":\"done\",\"summary\":\"%llu files inspected%s.\"}", file_count, partial ? "; report is partial" : ""); + ok = job_sse_json(g, fd, job_id, num) && samosa_send_all(fd, "data: [DONE]\n\n", 14); + } +report_done: + free(event.data); + return ok; +} + +typedef struct { unsigned long long size; char *path; char hash[65]; } JobsDuplicateFile; + +static int jobs_duplicate_file_cmp(const void *a, const void *b) { + const JobsDuplicateFile *x = a, *y = b; + if (x->size != y->size) return x->size < y->size ? -1 : 1; + return strcmp(x->path, y->path); +} + +/* Exact duplicate recipe. Size groups are only candidates; only matching + * complete SHA-256 values are reported as duplicates. Hashing is bounded to + * 128 candidate files / 128 MiB, with the inventory's own limits in front. */ +static int jobs_find_duplicates(Gateway *g, int fd, const char *folder) { + enum { MAX_FILES = 10000, MAX_HASH_FILES = 128 }; + const unsigned long long max_hash_bytes = 128ull * 1024ull * 1024ull; + char *argv[] = {g->samosa_fs, (char *)"chutni-inventory", (char *)"--root", + (char *)folder, (char *)"--max-depth", (char *)"32", + (char *)"--max-files", (char *)"10000", + (char *)"--max-directories", (char *)"5000", + (char *)"--max-seconds", (char *)"20", NULL}; + unsigned long long start = monotonic_millis(), total = 0, skipped = 0, candidate_files = 0, + hashed_files = 0, deferred = 0, hashed_bytes = 0; + JobsDuplicateFile *files = calloc(MAX_FILES, sizeof(*files)); + char *raw = NULL, job_id[64], reason[64] = "none"; + int status = 0, partial = 0, saw_done = 0, ok = 0; + if (!files) return 0; + raw = run_capture(g, g->samosa_fs, argv, 8 << 20, &status); + if (!raw || !WIFEXITED(status) || WEXITSTATUS(status)) { + free(raw); free(files); + return samosa_http_json_error(fd, 400, "folder_scan_failed", "The folder could not be inspected within the duplicate search safety budget."); + } + const char *line = raw; + while (*line) { + const char *end = strchr(line, '\n'); size_t len = end ? (size_t)(end - line) : strlen(line); + char *record = malloc(len + 1), *record_arena = NULL; + if (!record) goto cleanup; + memcpy(record, line, len); record[len] = 0; + jval *obj = json_parse(record, &record_arena); free(record); + if (!obj || obj->t != J_OBJ) { json_free(obj); free(record_arena); goto cleanup; } + jval *type = json_get(obj, "type"); + if (type && type->t == J_STR && !strcmp(type->str, "file")) { + jval *path = json_get(obj, "rel_path"), *size = json_get(obj, "size"); + if (!path || path->t != J_STR || !size || size->t != J_NUM || size->num < 0 || total >= MAX_FILES) { + json_free(obj); free(record_arena); goto cleanup; + } + files[total].size = (unsigned long long)size->num; + files[total].path = strdup(path->str); + if (!files[total].path) { json_free(obj); free(record_arena); goto cleanup; } + total++; + } else if (type && type->t == J_STR && !strcmp(type->str, "skip")) skipped++; + else if (type && type->t == J_STR && !strcmp(type->str, "done")) { + saw_done = 1; jval *v = json_get(obj, "partial"); partial = v && v->t == J_BOOL && v->boolean; + v = json_get(obj, "limiting_reason"); if (v && v->t == J_STR) path_copy(reason, sizeof(reason), v->str); + } + json_free(obj); free(record_arena); + if (!end) break; line = end + 1; + } + free(raw); raw = NULL; + if (!saw_done) goto cleanup; + qsort(files, (size_t)total, sizeof(*files), jobs_duplicate_file_cmp); + for (size_t i = 0; i < total;) { + size_t end = i + 1; while (end < total && files[end].size == files[i].size) end++; + if (end - i > 1) { + size_t group_count = end - i; + candidate_files += group_count; + unsigned long long group_bytes = files[i].size * (unsigned long long)group_count; + if (files[i].size > 64ull * 1024ull * 1024ull || hashed_files + group_count > MAX_HASH_FILES) { + deferred += group_count; partial = 1; path_copy(reason, sizeof(reason), "hash_file_limit"); i = end; continue; + } + if (hashed_bytes > max_hash_bytes || group_bytes > max_hash_bytes - hashed_bytes) { + deferred += group_count; partial = 1; path_copy(reason, sizeof(reason), "hash_byte_limit"); i = end; continue; + } + if (monotonic_millis() - start >= 20000ull) { + deferred += group_count; partial = 1; path_copy(reason, sizeof(reason), "hash_time_limit"); i = end; continue; + } + for (size_t k = i; k < end; k++) { + char *hash_argv[] = {g->samosa_fs, (char *)"chutni-hash", (char *)"--root", + (char *)folder, files[k].path, NULL}; + int hash_status = 0; char *hash_raw = run_capture(g, g->samosa_fs, hash_argv, 4096, &hash_status); + char *hash_arena = NULL; jval *hash_obj = hash_raw ? json_parse(hash_raw, &hash_arena) : NULL; + jval *hash = hash_obj && hash_obj->t == J_OBJ ? json_get(hash_obj, "sha256") : NULL; + if (!hash_raw || !WIFEXITED(hash_status) || WEXITSTATUS(hash_status) || !hash || hash->t != J_STR || strlen(hash->str) != 64) { + deferred++; partial = 1; path_copy(reason, sizeof(reason), "hash_error"); + } else { path_copy(files[k].hash, sizeof(files[k].hash), hash->str); hashed_files++; hashed_bytes += files[k].size; } + json_free(hash_obj); free(hash_arena); free(hash_raw); + } + } + i = end; + } + if (!samosa_http_stream_headers(fd)) goto cleanup; + snprintf(job_id, sizeof(job_id), "job-%ld-%ld-%lld", (long)time(NULL), (long)getpid(), (long long)monotonic_millis()); + if (!save_job_state(g, job_id, "Find duplicates", folder)) goto cleanup; + TextBuffer event = {0}; char num[64]; + ok = text_add(&event, "{\"seq\":1,\"type\":\"decode_intent\",\"job_id\":") && text_json_string(&event, job_id) && text_add(&event, ",\"goal\":\"Find duplicates\",\"folder\":") && text_json_string(&event, folder) && text_add(&event, "}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + free(event.data); event = (TextBuffer){0}; + unsigned long long duplicate_groups = 0; + if (ok) { + ok = text_add(&event, "{\"seq\":2,\"type\":\"duplicates\",\"recipe\":\"find_duplicates\",\"candidate_files\":"); + snprintf(num, sizeof(num), "%llu", candidate_files); ok = ok && text_add(&event, num) && text_add(&event, ",\"hashed_files\":"); + snprintf(num, sizeof(num), "%llu", hashed_files); ok = ok && text_add(&event, num) && text_add(&event, ",\"deferred_files\":"); + snprintf(num, sizeof(num), "%llu", deferred); ok = ok && text_add(&event, num) && text_add(&event, ",\"skipped\":"); + snprintf(num, sizeof(num), "%llu", skipped); ok = ok && text_add(&event, num) && text_add(&event, ",\"partial\":") && text_add(&event, partial ? "true" : "false") && text_add(&event, ",\"limiting_reason\":") && text_json_string(&event, reason) && text_add(&event, ",\"groups\":["); + int first_group = 1; + for (size_t i = 0; ok && i < total;) { + size_t end = i + 1; while (end < total && files[end].size == files[i].size) end++; + for (size_t k = i; k < end; k++) { + if (!files[k].hash[0]) continue; + size_t matches = 0; for (size_t x = i; x < end; x++) if (!strcmp(files[k].hash, files[x].hash)) matches++; + if (matches < 2) continue; + int earlier = 0; for (size_t x = i; x < k; x++) if (!strcmp(files[k].hash, files[x].hash)) earlier = 1; + if (earlier) continue; + if (!first_group) ok = text_add(&event, ","); first_group = 0; duplicate_groups++; + snprintf(num, sizeof(num), "%llu", files[k].size); + ok = ok && text_add(&event, "{\"size\":") && text_add(&event, num) && text_add(&event, ",\"sha256\":") && text_json_string(&event, files[k].hash) && text_add(&event, ",\"files\":["); + int first_path = 1; + for (size_t x = i; ok && x < end; x++) if (!strcmp(files[k].hash, files[x].hash)) { + if (!first_path) ok = text_add(&event, ","); first_path = 0; ok = ok && text_json_string(&event, files[x].path); + } + ok = ok && text_add(&event, "]}"); + } + i = end; + } + ok = ok && text_add(&event, "],\"duplicate_groups\":"); snprintf(num, sizeof(num), "%llu", duplicate_groups); ok = ok && text_add(&event, num) && text_add(&event, "}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + } + free(event.data); + if (ok) { + snprintf(num, sizeof(num), "{\"seq\":3,\"type\":\"done\",\"summary\":\"%llu verified duplicate group%s%s.\"}", duplicate_groups, duplicate_groups == 1 ? "" : "s", partial ? "; some candidates need another pass" : ""); + ok = job_sse_json(g, fd, job_id, num) && samosa_send_all(fd, "data: [DONE]\n\n", 14); + } +cleanup: + free(raw); + for (size_t i = 0; i < total; i++) free(files[i].path); + free(files); + return ok; +} + +/* Sort-by-type is a deterministic preview recipe. It stages only moves within + * the selected folder; the ordinary apply endpoint revalidates each operation + * and the existing undo journal records successful moves. */ +static int jobs_sort_by_type(Gateway *g, int fd, const char *folder) { + enum { MAX_FILES = 10000 }; + char *argv[] = {g->samosa_fs, (char *)"chutni-inventory", (char *)"--root", + (char *)folder, (char *)"--max-depth", (char *)"32", + (char *)"--max-files", (char *)"10000", + (char *)"--max-directories", (char *)"5000", + (char *)"--max-seconds", (char *)"20", NULL}; + typedef struct { char *src; char *dst; unsigned long long size; double mtime; } Move; + Move *moves = calloc(MAX_FILES, sizeof(*moves)); + char *raw = NULL, job_id[64], plan_path[PATH_MAX]; + int status = 0, partial = 0, saw_done = 0, ok = 0; + size_t count = 0; + if (!moves) return 0; + raw = run_capture(g, g->samosa_fs, argv, 8 << 20, &status); + if (!raw || !WIFEXITED(status) || WEXITSTATUS(status)) { + free(raw); free(moves); + return samosa_http_json_error(fd, 400, "folder_scan_failed", "The folder could not be inspected within the sort safety budget."); + } + const char *line = raw; + while (*line) { + const char *end = strchr(line, '\n'); size_t len = end ? (size_t)(end - line) : strlen(line); + char *record = malloc(len + 1), *record_arena = NULL; + if (!record) goto sort_cleanup; + memcpy(record, line, len); record[len] = 0; + jval *obj = json_parse(record, &record_arena); free(record); + if (!obj || obj->t != J_OBJ) { json_free(obj); free(record_arena); goto sort_cleanup; } + jval *type = json_get(obj, "type"); + if (type && type->t == J_STR && !strcmp(type->str, "file")) { + jval *path = json_get(obj, "rel_path"), *size = json_get(obj, "size"), *mtime = json_get(obj, "mtime"); + if (!path || path->t != J_STR || !size || size->t != J_NUM || !mtime || mtime->t != J_NUM || count >= MAX_FILES) { json_free(obj); free(record_arena); goto sort_cleanup; } + if (!strncmp(path->str, "Sorted by type/", 15)) { json_free(obj); free(record_arena); if (!end) break; line = end + 1; continue; } + char ext[32], src[PATH_MAX], dst[PATH_MAX]; + jobs_report_type(path->str, ext); + const char *label = !strcmp(ext, "[no extension]") ? "no-extension" : ext + 1; + if (snprintf(src, sizeof(src), "%s/%s", folder, path->str) >= (int)sizeof(src) || + snprintf(dst, sizeof(dst), "%s/Sorted by type/%s/%s", folder, label, path->str) >= (int)sizeof(dst)) { + json_free(obj); free(record_arena); goto sort_cleanup; + } + if (strcmp(src, dst)) { + moves[count].src = strdup(src); moves[count].dst = strdup(dst); + if (!moves[count].src || !moves[count].dst) { json_free(obj); free(record_arena); goto sort_cleanup; } + moves[count].size = (unsigned long long)size->num; moves[count].mtime = mtime->num; + count++; + } + } else if (type && type->t == J_STR && !strcmp(type->str, "done")) { + saw_done = 1; jval *v = json_get(obj, "partial"); partial = v && v->t == J_BOOL && v->boolean; + } + json_free(obj); free(record_arena); + if (!end) break; line = end + 1; + } + free(raw); raw = NULL; + if (!saw_done) goto sort_cleanup; + snprintf(job_id, sizeof(job_id), "job-%ld-%ld-%lld", (long)time(NULL), (long)getpid(), (long long)monotonic_millis()); + if (!save_job_state(g, job_id, "Sort by file type", folder) || + !job_state_path(g, job_id, "plan.jsonl", plan_path, 1)) goto sort_cleanup; + TextBuffer plan_file = {0}, event = {0}; + for (size_t i = 0; i < count; i++) { + TextBuffer rec = {0}; + char size[32], mtime[64]; snprintf(size, sizeof(size), "%llu", moves[i].size); + snprintf(mtime, sizeof(mtime), "%.9f", moves[i].mtime); + int added = text_add(&rec, "{\"src\":") && text_json_string(&rec, moves[i].src) && + text_add(&rec, ",\"dst\":") && text_json_string(&rec, moves[i].dst) && + text_add(&rec, ",\"size\":") && text_add(&rec, size) && + text_add(&rec, ",\"mtime\":") && text_add(&rec, mtime) && text_add(&rec, "}\n"); + if (!added || !text_add(&plan_file, rec.data)) { free(rec.data); free(plan_file.data); goto sort_cleanup; } + free(rec.data); + } + if (!write_small_file(plan_path, plan_file.data ? plan_file.data : "")) { free(plan_file.data); goto sort_cleanup; } + free(plan_file.data); + if (!samosa_http_stream_headers(fd)) goto sort_cleanup; + ok = text_add(&event, "{\"seq\":1,\"type\":\"decode_intent\",\"job_id\":") && text_json_string(&event, job_id) && text_add(&event, ",\"goal\":\"Sort by file type\",\"folder\":") && text_json_string(&event, folder) && text_add(&event, "}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + free(event.data); event = (TextBuffer){0}; + ok = ok && text_add(&event, "{\"seq\":2,\"type\":\"plan\",\"moves\":["); + for (size_t i = 0; ok && i < count; i++) { + if (i) ok = text_add(&event, ","); + ok = ok && text_add(&event, "{\"src\":") && text_json_string(&event, moves[i].src) && text_add(&event, ",\"dst\":") && text_json_string(&event, moves[i].dst) && text_add(&event, "}"); + } + ok = ok && text_add(&event, "],\"skips\":[]}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + free(event.data); event = (TextBuffer){0}; + if (ok && !partial && count) { + char number[32]; snprintf(number, sizeof(number), "%zu", count); + ok = text_add(&event, "{\"seq\":3,\"type\":\"await_apply\",\"job_id\":") && text_json_string(&event, job_id) && text_add(&event, ",\"moves\":") && text_add(&event, number) && text_add(&event, "}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + } + free(event.data); event = (TextBuffer){0}; + if (ok) { + char summary[160]; snprintf(summary, sizeof(summary), partial + ? "Inventory was partial; no moves can be applied. Raise the folder limit or retry." + : "Prepared %zu type-based move%s.", count, count == 1 ? "" : "s"); + ok = text_add(&event, "{\"seq\":4,\"type\":\"done\",\"job_id\":") && text_json_string(&event, job_id) && text_add(&event, ",\"summary\":") && text_json_string(&event, summary) && text_add(&event, "}") && job_sse_json(g, fd, job_id, event.data) && samosa_send_all(fd, "data: [DONE]\n\n", 14); + } + free(event.data); +sort_cleanup: + free(raw); + for (size_t i = 0; i < count; i++) { free(moves[i].src); free(moves[i].dst); } + free(moves); + return ok; +} + static int jobs_report(Gateway *g, int fd, const char *goal, const char *folder, const char *existing_job_id) { char *argv[] = {g->samosa_fs, "survey", "--max-file-bytes", "104857600", @@ -5824,14 +6191,23 @@ static int jobs_run(Gateway *g, int fd, const SamosaHttpRequest *request) { char *arena = NULL; jval *root = json_parse(request->body, &arena); jval *goal = root && root->t == J_OBJ ? json_get(root, "goal") : NULL; jval *folder = root && root->t == J_OBJ ? json_get(root, "folder") : NULL; - if (!goal || goal->t != J_STR || !folder || folder->t != J_STR) { + jval *recipe = root && root->t == J_OBJ ? json_get(root, "recipe") : NULL; + int folder_report = recipe && recipe->t == J_STR && !strcmp(recipe->str, "folder_report"); + int find_duplicates = recipe && recipe->t == J_STR && !strcmp(recipe->str, "find_duplicates"); + int sort_by_type = recipe && recipe->t == J_STR && !strcmp(recipe->str, "sort_by_type"); + if ((!folder_report && !find_duplicates && !sort_by_type && (!goal || goal->t != J_STR || !goal->str[0])) || + (recipe && (recipe->t != J_STR || (!folder_report && !find_duplicates && !sort_by_type))) || + !folder || folder->t != J_STR || !folder->str[0]) { json_free(root); free(arena); - return samosa_http_json_error(fd, 400, "invalid_job", "goal and folder are required."); + return samosa_http_json_error(fd, 400, "invalid_job", "A supported recipe or goal and a folder are required."); } - char *goal_copy = strdup(goal->str), *folder_copy = strdup(folder->str); + char *goal_copy = strdup(folder_report ? "Folder report" : find_duplicates ? "Find duplicates" : sort_by_type ? "Sort by file type" : goal->str), *folder_copy = strdup(folder->str); json_free(root); free(arena); if (!goal_copy || !folder_copy) { free(goal_copy); free(folder_copy); return 0; } - int result = jobs_report(g, fd, goal_copy, folder_copy, NULL); + int result = sort_by_type ? jobs_sort_by_type(g, fd, folder_copy) + : find_duplicates ? jobs_find_duplicates(g, fd, folder_copy) + : folder_report ? jobs_folder_report(g, fd, folder_copy, NULL) + : jobs_report(g, fd, goal_copy, folder_copy, NULL); free(goal_copy); free(folder_copy); return result; } @@ -6296,13 +6672,26 @@ static int jobs_apply_or_undo(Gateway *g, int fd, const SamosaHttpRequest *reque if (!job_state_path(g, job_id, undo ? "applied.jsonl" : "plan.jsonl", plan_path, 0) || !job_state_path(g, job_id, "applied.jsonl", applied_path, 1)) { free(folder); return 0; } char *raw = read_file_limit(plan_path, 1 << 20); if (!raw) { free(folder); return samosa_http_json_error(fd, 404, "plan_not_found", "There is no pending move plan."); } - if (!samosa_http_stream_headers(fd)) { free(raw); free(folder); return 0; } + char *iteration = strdup(raw); + if (!iteration) { free(raw); free(folder); return 0; } + if (!samosa_http_stream_headers(fd)) { free(iteration); free(raw); free(folder); return 0; } int moved = 0, total = 0; char *save = NULL; - for (char *line = strtok_r(raw, "\n", &save); line; line = strtok_r(NULL, "\n", &save)) { + for (char *line = strtok_r(iteration, "\n", &save); line; line = strtok_r(NULL, "\n", &save)) { char *line_arena = NULL; jval *move = json_parse(line, &line_arena); jval *src = json_get(move, "src"), *dst = json_get(move, "dst"); if (!src || src->t != J_STR || !dst || dst->t != J_STR) { json_free(move); free(line_arena); continue; } - char *argv[] = {g->samosa_fs, undo ? "undo" : "move", "--root", folder, src->str, dst->str, NULL}; + jval *size = json_get(move, "size"), *mtime = json_get(move, "mtime"); + char size_text[32], mtime_text[64]; + char *argv[12]; int argc = 0; + argv[argc++] = g->samosa_fs; argv[argc++] = undo ? "undo" : "move"; + argv[argc++] = "--root"; argv[argc++] = folder; + if (!undo && size && size->t == J_NUM && size->num >= 0 && mtime && mtime->t == J_NUM) { + snprintf(size_text, sizeof(size_text), "%.0f", size->num); + snprintf(mtime_text, sizeof(mtime_text), "%.9f", mtime->num); + argv[argc++] = "--size"; argv[argc++] = size_text; + argv[argc++] = "--mtime"; argv[argc++] = mtime_text; + } + argv[argc++] = src->str; argv[argc++] = dst->str; argv[argc] = NULL; int status = 0; char *result = run_capture(g, g->samosa_fs, argv, 65536, &status); ++total; int ok_move = result && WIFEXITED(status) && !WEXITSTATUS(status) && strstr(result, "\"moved\":true"); if (ok_move) ++moved; @@ -6321,7 +6710,7 @@ static int jobs_apply_or_undo(Gateway *g, int fd, const SamosaHttpRequest *reque snprintf(event, sizeof(event), "{\"type\":\"done\",\"job_id\":\"%s\",\"summary\":\"%s %d file%s.\"}", job_id, undo ? "Restored" : "Moved", moved, moved == 1 ? "" : "s"); ok = ok && sse_json(fd, event) && samosa_send_all(fd, "data: [DONE]\n\n", 14); - free(raw); free(folder); return ok; + free(iteration); free(raw); free(folder); return ok; } static int backend_available(Gateway *g, const char *name) { diff --git a/tests/test_compiled_gateway.sh b/tests/test_compiled_gateway.sh index 43df136..25300eb 100644 --- a/tests/test_compiled_gateway.sh +++ b/tests/test_compiled_gateway.sh @@ -37,6 +37,21 @@ printf 'png\n' >"$TMP/logo.png" printf "Titli vaccination record, rabies booster 2026.\n" >"$TMP/files/cat-medical-note.txt" printf "Miso vaccination record.\n" >"$TMP/files/miso-record.txt" printf "Cafe total 4.50\n" >"$TMP/files/receipt-b.txt" +/bin/mkdir "$TMP/report-files" +printf 'same size A\n' >"$TMP/report-files/first.txt" +printf 'same size B\n' >"$TMP/report-files/second.txt" +/bin/cp "$TMP/report-files/first.txt" "$TMP/report-files/copy.txt" +printf 'a tiny csv value\n' >"$TMP/report-files/third.csv" +printf 'outside secret\n' >"$TMP/outside-report-secret.txt" +/bin/ln -s "$TMP/outside-report-secret.txt" "$TMP/report-files/escape-link" +/bin/mkdir "$TMP/duplicate-limit-files" +i=1 +while [ "$i" -le 129 ]; do + printf 'same-size candidate\n' >"$TMP/duplicate-limit-files/item-$i.txt" + i=$((i + 1)) +done +/bin/mkdir "$TMP/sort-stale-files" +printf 'before preview\n' >"$TMP/sort-stale-files/item.txt" /bin/mkdir "$TMP/interlock-files" printf "First interlock receipt.\n" >"$TMP/interlock-files/a.txt" printf "Second interlock receipt.\n" >"$TMP/interlock-files/b.txt" @@ -182,6 +197,84 @@ report=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ printf '%s' "$report" | /usr/bin/grep -q '"type":"report"' printf '%s' "$report" | /usr/bin/grep -q '"type":"done"' +# The productized Folder report recipe takes the bounded metadata inventory, +# reports size-only duplicate candidates honestly, and leaves the source tree +# untouched. It does not invoke model intent classification. +report_recipe=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ + -H 'Content-Type: application/json' \ + --data-binary "{\"recipe\":\"folder_report\",\"folder\":\"$TMP/report-files\"}") +printf '%s' "$report_recipe" | /usr/bin/grep -q '"recipe":"folder_report"' +printf '%s' "$report_recipe" | /usr/bin/grep -q '"total":4' +printf '%s' "$report_recipe" | /usr/bin/grep -q '"duplicate_candidates":3' +printf '%s' "$report_recipe" | /usr/bin/grep -q '"duplicate_size_groups":1' +printf '%s' "$report_recipe" | /usr/bin/grep -q '"symlink":1' +printf '%s' "$report_recipe" | /usr/bin/grep -q '"size_bands":\[' +printf '%s' "$report_recipe" | /usr/bin/grep -q '"age":\[' +[ "$(/usr/bin/find "$TMP/report-files" -type f | /usr/bin/wc -l | /usr/bin/tr -d ' ')" = 4 ] +report_again=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ + -H 'Content-Type: application/json' \ + --data-binary "{\"recipe\":\"folder_report\",\"folder\":\"$TMP/report-files\"}") +REPORT_SNAPSHOT=$(printf '%s' "$report_recipe" | /usr/bin/grep '"type":"report"' | /usr/bin/sed 's/.*"type":"report"/"type":"report"/') +REPORT_AGAIN_SNAPSHOT=$(printf '%s' "$report_again" | /usr/bin/grep '"type":"report"' | /usr/bin/sed 's/.*"type":"report"/"type":"report"/') +[ "$REPORT_SNAPSHOT" = "$REPORT_AGAIN_SNAPSHOT" ] || { echo "Folder report was not deterministic" >&2; exit 1; } +DUPLICATES=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ + -H 'Content-Type: application/json' \ + --data-binary "{\"recipe\":\"find_duplicates\",\"folder\":\"$TMP/report-files\"}") +printf '%s' "$DUPLICATES" | /usr/bin/grep -q '"type":"duplicates"' +printf '%s' "$DUPLICATES" | /usr/bin/grep -q '"candidate_files":3' +printf '%s' "$DUPLICATES" | /usr/bin/grep -q '"hashed_files":3' +printf '%s' "$DUPLICATES" | /usr/bin/grep -q '"duplicate_groups":1' +printf '%s' "$DUPLICATES" | /usr/bin/grep -q 'first.txt' +printf '%s' "$DUPLICATES" | /usr/bin/grep -q 'copy.txt' +! printf '%s' "$DUPLICATES" | /usr/bin/grep -q 'second.txt' +printf '%s' "$DUPLICATES" | /usr/bin/grep -q '"partial":false' +[ "$(/usr/bin/find "$TMP/report-files" -type f | /usr/bin/wc -l | /usr/bin/tr -d ' ')" = 4 ] +LIMITED_DUPLICATES=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ + -H 'Content-Type: application/json' \ + --data-binary "{\"recipe\":\"find_duplicates\",\"folder\":\"$TMP/duplicate-limit-files\"}") +printf '%s' "$LIMITED_DUPLICATES" | /usr/bin/grep -q '"candidate_files":129' +printf '%s' "$LIMITED_DUPLICATES" | /usr/bin/grep -q '"hashed_files":0' +printf '%s' "$LIMITED_DUPLICATES" | /usr/bin/grep -q '"deferred_files":129' +printf '%s' "$LIMITED_DUPLICATES" | /usr/bin/grep -q '"partial":true' +printf '%s' "$LIMITED_DUPLICATES" | /usr/bin/grep -q '"limiting_reason":"hash_file_limit"' +SORT_PREVIEW=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ + -H 'Content-Type: application/json' \ + --data-binary "{\"recipe\":\"sort_by_type\",\"folder\":\"$TMP/report-files\"}") +printf '%s' "$SORT_PREVIEW" | /usr/bin/grep -q '"type":"plan"' +printf '%s' "$SORT_PREVIEW" | /usr/bin/grep -q '"moves":4' +printf '%s' "$SORT_PREVIEW" | /usr/bin/grep -q 'Sorted by type/txt/first.txt' +[ -f "$TMP/report-files/first.txt" ] +[ ! -e "$TMP/report-files/Sorted by type" ] +SORT_JOB=$(printf '%s' "$SORT_PREVIEW" | /usr/bin/sed -n 's/.*"type":"await_apply","job_id":"\([^"]*\)".*/\1/p' | /usr/bin/head -1) +[ -n "$SORT_JOB" ] +printf '%s' "$SORT_PREVIEW" | /usr/bin/grep -q '"type":"done"' +sorted=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/apply" \ + -H 'Content-Type: application/json' --data-binary "{\"job_id\":\"$SORT_JOB\"}") +printf '%s' "$sorted" | /usr/bin/grep -q '"applied":4' +[ -f "$TMP/report-files/Sorted by type/txt/first.txt" ] +[ -f "$TMP/report-files/Sorted by type/csv/third.csv" ] +unsorted=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/undo" \ + -H 'Content-Type: application/json' --data-binary "{\"job_id\":\"$SORT_JOB\"}") +printf '%s' "$unsorted" | /usr/bin/grep -q '"undone":4' +[ -f "$TMP/report-files/first.txt" ] +[ -f "$TMP/report-files/third.csv" ] +[ ! -e "$TMP/report-files/Sorted by type/txt/first.txt" ] +[ ! -e "$TMP/report-files/Sorted by type/csv/third.csv" ] +STALE_SORT=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ + -H 'Content-Type: application/json' \ + --data-binary "{\"recipe\":\"sort_by_type\",\"folder\":\"$TMP/sort-stale-files\"}") +STALE_SORT_JOB=$(printf '%s' "$STALE_SORT" | /usr/bin/sed -n 's/.*"type":"await_apply","job_id":"\([^"]*\)".*/\1/p' | /usr/bin/head -1) +[ -n "$STALE_SORT_JOB" ] +printf 'changed after preview with a different size\n' >"$TMP/sort-stale-files/item.txt" +STALE_APPLY=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/apply" \ + -H 'Content-Type: application/json' --data-binary "{\"job_id\":\"$STALE_SORT_JOB\"}") +printf '%s' "$STALE_APPLY" | /usr/bin/grep -q '"applied":0,"skipped":1' +/usr/bin/grep -q 'changed after preview' "$TMP/sort-stale-files/item.txt" +[ ! -e "$TMP/sort-stale-files/Sorted by type" ] +REPORT_JOB=$(printf '%s' "$report_recipe" | /usr/bin/sed -n 's/.*"job_id":"\([^"]*\)".*/\1/p' | /usr/bin/head -1) +[ -f "$HOME_DIR/jobs/$REPORT_JOB/events.jsonl" ] +/usr/bin/grep -q '"recipe":"folder_report"' "$HOME_DIR/jobs/$REPORT_JOB/events.jsonl" + # Phase JI find: model triages every filename (Phase A), the verify loop reads # content and ends with a structured finish() result card (JI.2/JI.4/JI.5). No # C keyword scoring, no canned question, no prose "answer" as the ending. diff --git a/tests/test_jobs_ui.mjs b/tests/test_jobs_ui.mjs index 94277ac..ac3ec64 100644 --- a/tests/test_jobs_ui.mjs +++ b/tests/test_jobs_ui.mjs @@ -50,8 +50,15 @@ globalThis.jobEls = { result: new Element(), resultLabel: new Element(), resultText: new Element(), review: new Element(), reviewList: new Element(), reviewMeta: new Element(), }; globalThis.lastJobId = null; +globalThis.bytesLabel = n => `${n} bytes`; +globalThis.showJobResult = text => { jobEls.resultText.innerHTML = text; jobEls.result.hidden = false; }; const app = readFileSync(new URL("../assets/app.html", import.meta.url), "utf8"); +assert.match(app, /id="jobRecipe"[\s\S]*value="folder_report"/); +assert.match(app, /value="find_duplicates">Find duplicates/); +assert.match(app, /value="sort_by_type">Sort by file type/); +assert.match(app, /recipe !== "find" \? \{ recipe, folder \}/); +assert.match(app, /jobEls\.goal\.disabled = deterministic/); const begin = app.indexOf(" const baseName ="); const end = app.indexOf(" async function streamJob"); assert.ok(begin >= 0 && end > begin, "Jobs renderer block must remain extractable"); @@ -95,4 +102,23 @@ assert.match(jobEls.resultText.innerHTML, /certificate\.pdf/); assert.match(jobEls.resultText.innerHTML, /Rabies vaccination/); assert.match(jobEls.resultText.innerHTML, /scan\.png/); +renderJobEvent({ type: "report", total: 3, bytes: 42, by_type: { ".txt": 2, ".csv": 1 }, age: [1, 1, 1, 0], size_bands: [3, 0, 0], duplicate_candidates: 2, duplicate_size_groups: 1, skip_reasons: {}, partial: true, limiting_reason: "maximum_files" }, ctx); +assert.match(jobEls.resultText.innerHTML, /Folder report/); +assert.match(jobEls.resultText.innerHTML, /2 \.txt/); +assert.match(jobEls.resultText.innerHTML, /size only; contents not compared/); +assert.match(jobEls.resultText.innerHTML, /Partial report.*maximum_files/); + +renderJobEvent({ type: "duplicates", duplicate_groups: 1, candidate_files: 3, hashed_files: 3, deferred_files: 0, skipped: 1, partial: false, groups: [{ size: 12, sha256: "a".repeat(64), files: ["first.txt", "copy.txt"] }] }, ctx); +assert.match(jobEls.resultText.innerHTML, /1 verified duplicate group/); +assert.match(jobEls.resultText.innerHTML, /first\.txt/); +assert.match(jobEls.resultText.innerHTML, /SHA-256/); +renderJobEvent({ type: "duplicates", duplicate_groups: 0, candidate_files: 129, hashed_files: 0, deferred_files: 129, skipped: 0, partial: true, limiting_reason: "hash_file_limit", groups: [] }, ctx); +assert.match(jobEls.resultText.innerHTML, /129 candidates deferred/); +assert.match(jobEls.resultText.innerHTML, /Partial duplicate search.*hash_file_limit/); + +renderJobEvent({ type: "await_apply", job_id: "sort-job", moves: 4 }, ctx); +assert.equal(ctx.awaitingApply, true); +assert.equal(jobEls.barText.textContent, "4 files ready to move."); +assert.equal(jobEls.barActions.children[0].textContent, "Apply moves"); + process.stdout.write("jobs UI DOM fixtures: PASS\n"); From 25e3cf72d41e12f43a97144a6abcb853d3b9b9ce Mon Sep 17 00:00:00 2001 From: Deepan Wadhwa Date: Tue, 22 Sep 2026 16:21:53 -0400 Subject: [PATCH 03/30] Add conservative inbox filename triage --- assets/app.html | 11 +++- docs/CHUTNI_JOBS_RECOVERY_PLAN.md | 10 ++-- src/samosa_gateway.c | 96 +++++++++++++++++++++++++++++-- tests/test_compiled_gateway.sh | 18 ++++++ tests/test_jobs_ui.mjs | 8 +++ 5 files changed, 134 insertions(+), 9 deletions(-) diff --git a/assets/app.html b/assets/app.html index 5053329..9a98c40 100644 --- a/assets/app.html +++ b/assets/app.html @@ -1755,7 +1755,7 @@

Jobs

- +
@@ -8120,6 +8120,15 @@

Delete conversation?

if (evt.partial) md += `\n**Partial duplicate search:** stopped at ${evt.limiting_reason || "a safety limit"}.\n`; showJobResult(md); break; } + case "inbox_classification": { + const names = ["billing", "medical", "work", "personal", "review"]; + let md = "### Inbox classification (filename clues only)\n\n"; + (evt.items || []).forEach(item => { md += `- **${item.category}** · \`${item.path}\` — ${item.signal}\n`; }); + md += "\n" + names.map((name, i) => `${evt.counts?.[i] || 0} ${name}`).join(" · "); + if (evt.skipped) md += `\n\n${evt.skipped} entries skipped by inventory policy.`; + if (evt.partial) md += `\n\n**Partial classification:** stopped at ${evt.limiting_reason || "a safety limit"}.`; + showJobResult(md); break; + } case "indexing": ctx.indexLine = jobLine("·", `Checking ${evt.total} filenames`); break; case "index_complete": { diff --git a/docs/CHUTNI_JOBS_RECOVERY_PLAN.md b/docs/CHUTNI_JOBS_RECOVERY_PLAN.md index dc6ad24..0eacdd7 100644 --- a/docs/CHUTNI_JOBS_RECOVERY_PLAN.md +++ b/docs/CHUTNI_JOBS_RECOVERY_PLAN.md @@ -301,10 +301,12 @@ goal may map into one of them, but pasted internal JSON is not required. **Implementation checkpoint (2026-09-22):** Folder report, Find files about…, and Find duplicates are exposed in Jobs. Find duplicates hashes only bounded same-size candidates and reports a partial result when work is capped or a -candidate cannot be read. Sort by file type is now being added through the -existing persisted move-plan, explicit Apply, and Undo path. Inbox -classification and watch-folder deltas remain unimplemented Phase C work; this -checkpoint does not claim the Phase C acceptance gates are complete. +candidate cannot be read. Sort by file type uses the persisted move plan, +explicit Apply, source size/mtime revalidation, and Undo path. Inbox +classification is a bounded filename-only pass: a single category clue yields +a suggestion, while unknown or cross-category clues go to review. It does not +read content or move files. Watch-folder deltas remain unimplemented Phase C +work; this checkpoint does not claim the Phase C acceptance gates are complete. ### C2. Durable `JobSpec` diff --git a/src/samosa_gateway.c b/src/samosa_gateway.c index 09d3d1f..f74fa3c 100644 --- a/src/samosa_gateway.c +++ b/src/samosa_gateway.c @@ -6111,6 +6111,92 @@ static int jobs_sort_by_type(Gateway *g, int fd, const char *folder) { return ok; } +typedef struct { char *path; const char *category; const char *signal; } JobsInboxItem; + +static const char *jobs_inbox_category(const char *path, const char **signal) { + const char *base = strrchr(path, '/'); base = base ? base + 1 : path; + char lower[PATH_MAX]; size_t n = strlen(base); if (n >= sizeof(lower)) n = sizeof(lower) - 1; + for (size_t i = 0; i < n; i++) { unsigned char c = (unsigned char)base[i]; lower[i] = (char)(c >= 'A' && c <= 'Z' ? c + 32 : c); } + lower[n] = 0; + static const char *billing[] = {"invoice", "receipt", "payment", "billing", "statement", "tax", "bill", NULL}; + static const char *medical[] = {"medical", "health", "doctor", "clinic", "vaccine", "vaccination", "prescription", "lab-result", NULL}; + static const char *work[] = {"work", "meeting", "project", "resume", "cv", "proposal", "roadmap", "sprint", NULL}; + static const char *personal[] = {"personal", "family", "travel", "diary", "journal", "photo", NULL}; + const char *const *sets[] = {billing, medical, work, personal}; + const char *names[] = {"billing", "medical", "work", "personal"}; + int found = -1, categories = 0; int matched[4] = {0}; *signal = NULL; + for (int k = 0; k < 4; k++) for (const char *const *word = sets[k]; *word; word++) { + if (strstr(lower, *word) && !matched[k]) { matched[k] = 1; found = k; categories++; if (categories == 1) *signal = *word; } + } + if (categories != 1) { *signal = categories ? "conflicting filename clues" : "no clear filename clue"; return "review"; } + return names[found]; +} + +/* Bounded filename-only inbox triage. It exposes its matching filename clue, + * never reads file contents, and sends ambiguous/conflicting names to review. */ +static int jobs_classify_inbox(Gateway *g, int fd, const char *folder) { + enum { MAX_FILES = 10000 }; + char *argv[] = {g->samosa_fs, (char *)"chutni-inventory", (char *)"--root", + (char *)folder, (char *)"--max-depth", (char *)"32", + (char *)"--max-files", (char *)"10000", + (char *)"--max-directories", (char *)"5000", + (char *)"--max-seconds", (char *)"20", NULL}; + JobsInboxItem *items = calloc(MAX_FILES, sizeof(*items)); + char *raw = NULL, job_id[64], reason[64] = "none"; + int status = 0, partial = 0, saw_done = 0, ok = 0; size_t count = 0, skipped = 0; + if (!items) return 0; + raw = run_capture(g, g->samosa_fs, argv, 8 << 20, &status); + if (!raw || !WIFEXITED(status) || WEXITSTATUS(status)) { free(raw); free(items); return samosa_http_json_error(fd, 400, "folder_scan_failed", "The inbox could not be inspected within the classification safety budget."); } + const char *line = raw; + while (*line) { + const char *end = strchr(line, '\n'); size_t len = end ? (size_t)(end - line) : strlen(line); + char *record = malloc(len + 1), *record_arena = NULL; + if (!record) goto inbox_cleanup; + memcpy(record, line, len); record[len] = 0; + jval *obj = json_parse(record, &record_arena); free(record); + if (!obj || obj->t != J_OBJ) { json_free(obj); free(record_arena); goto inbox_cleanup; } + jval *type = json_get(obj, "type"); + if (type && type->t == J_STR && !strcmp(type->str, "file")) { + jval *path = json_get(obj, "rel_path"); + if (!path || path->t != J_STR || count >= MAX_FILES) { json_free(obj); free(record_arena); goto inbox_cleanup; } + items[count].path = strdup(path->str); + if (!items[count].path) { json_free(obj); free(record_arena); goto inbox_cleanup; } + items[count].category = jobs_inbox_category(path->str, &items[count].signal); count++; + } else if (type && type->t == J_STR && !strcmp(type->str, "skip")) skipped++; + else if (type && type->t == J_STR && !strcmp(type->str, "done")) { + saw_done = 1; jval *v = json_get(obj, "partial"); partial = v && v->t == J_BOOL && v->boolean; + v = json_get(obj, "limiting_reason"); if (v && v->t == J_STR) path_copy(reason, sizeof(reason), v->str); + } + json_free(obj); free(record_arena); + if (!end) break; line = end + 1; + } + free(raw); raw = NULL; if (!saw_done) goto inbox_cleanup; + snprintf(job_id, sizeof(job_id), "job-%ld-%ld-%lld", (long)time(NULL), (long)getpid(), (long long)monotonic_millis()); + if (!save_job_state(g, job_id, "Classify an inbox", folder) || !samosa_http_stream_headers(fd)) goto inbox_cleanup; + TextBuffer event = {0}; char num[32]; + ok = text_add(&event, "{\"seq\":1,\"type\":\"decode_intent\",\"job_id\":") && text_json_string(&event, job_id) && text_add(&event, ",\"goal\":\"Classify an inbox\",\"folder\":") && text_json_string(&event, folder) && text_add(&event, "}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + free(event.data); event = (TextBuffer){0}; + unsigned long long totals[5] = {0}; + ok = ok && text_add(&event, "{\"seq\":2,\"type\":\"inbox_classification\",\"items\":["); + for (size_t i = 0; ok && i < count; i++) { + int c = !strcmp(items[i].category, "billing") ? 0 : !strcmp(items[i].category, "medical") ? 1 : !strcmp(items[i].category, "work") ? 2 : !strcmp(items[i].category, "personal") ? 3 : 4; + totals[c]++; if (i) ok = text_add(&event, ","); + ok = ok && text_add(&event, "{\"path\":") && text_json_string(&event, items[i].path) && text_add(&event, ",\"category\":") && text_json_string(&event, items[i].category) && text_add(&event, ",\"signal\":") && text_json_string(&event, items[i].signal) && text_add(&event, "}"); + } + ok = ok && text_add(&event, "],\"counts\":["); + for (int i = 0; ok && i < 5; i++) { if (i) ok = text_add(&event, ","); snprintf(num, sizeof(num), "%llu", totals[i]); ok = ok && text_add(&event, num); } + ok = ok && text_add(&event, "],\"skipped\":"); snprintf(num, sizeof(num), "%zu", skipped); ok = ok && text_add(&event, num) && text_add(&event, ",\"partial\":") && text_add(&event, partial ? "true" : "false") && text_add(&event, ",\"limiting_reason\":") && text_json_string(&event, reason) && text_add(&event, "}"); + if (ok) ok = job_sse_json(g, fd, job_id, event.data); + free(event.data); + if (ok) { + char summary[160]; snprintf(summary, sizeof(summary), "%zu files classified by filename; %llu held for review%s.", count, totals[4], partial ? " (partial inventory)" : ""); + TextBuffer done = {0}; ok = text_add(&done, "{\"seq\":3,\"type\":\"done\",\"job_id\":") && text_json_string(&done, job_id) && text_add(&done, ",\"summary\":") && text_json_string(&done, summary) && text_add(&done, "}") && job_sse_json(g, fd, job_id, done.data) && samosa_send_all(fd, "data: [DONE]\n\n", 14); free(done.data); + } +inbox_cleanup: + free(raw); for (size_t i = 0; i < count; i++) free(items[i].path); free(items); return ok; +} + static int jobs_report(Gateway *g, int fd, const char *goal, const char *folder, const char *existing_job_id) { char *argv[] = {g->samosa_fs, "survey", "--max-file-bytes", "104857600", @@ -6195,16 +6281,18 @@ static int jobs_run(Gateway *g, int fd, const SamosaHttpRequest *request) { int folder_report = recipe && recipe->t == J_STR && !strcmp(recipe->str, "folder_report"); int find_duplicates = recipe && recipe->t == J_STR && !strcmp(recipe->str, "find_duplicates"); int sort_by_type = recipe && recipe->t == J_STR && !strcmp(recipe->str, "sort_by_type"); - if ((!folder_report && !find_duplicates && !sort_by_type && (!goal || goal->t != J_STR || !goal->str[0])) || - (recipe && (recipe->t != J_STR || (!folder_report && !find_duplicates && !sort_by_type))) || + int classify_inbox = recipe && recipe->t == J_STR && !strcmp(recipe->str, "classify_inbox"); + if ((!folder_report && !find_duplicates && !sort_by_type && !classify_inbox && (!goal || goal->t != J_STR || !goal->str[0])) || + (recipe && (recipe->t != J_STR || (!folder_report && !find_duplicates && !sort_by_type && !classify_inbox))) || !folder || folder->t != J_STR || !folder->str[0]) { json_free(root); free(arena); return samosa_http_json_error(fd, 400, "invalid_job", "A supported recipe or goal and a folder are required."); } - char *goal_copy = strdup(folder_report ? "Folder report" : find_duplicates ? "Find duplicates" : sort_by_type ? "Sort by file type" : goal->str), *folder_copy = strdup(folder->str); + char *goal_copy = strdup(folder_report ? "Folder report" : find_duplicates ? "Find duplicates" : sort_by_type ? "Sort by file type" : classify_inbox ? "Classify an inbox" : goal->str), *folder_copy = strdup(folder->str); json_free(root); free(arena); if (!goal_copy || !folder_copy) { free(goal_copy); free(folder_copy); return 0; } - int result = sort_by_type ? jobs_sort_by_type(g, fd, folder_copy) + int result = classify_inbox ? jobs_classify_inbox(g, fd, folder_copy) + : sort_by_type ? jobs_sort_by_type(g, fd, folder_copy) : find_duplicates ? jobs_find_duplicates(g, fd, folder_copy) : folder_report ? jobs_folder_report(g, fd, folder_copy, NULL) : jobs_report(g, fd, goal_copy, folder_copy, NULL); diff --git a/tests/test_compiled_gateway.sh b/tests/test_compiled_gateway.sh index 25300eb..6c27fd8 100644 --- a/tests/test_compiled_gateway.sh +++ b/tests/test_compiled_gateway.sh @@ -52,6 +52,13 @@ while [ "$i" -le 129 ]; do done /bin/mkdir "$TMP/sort-stale-files" printf 'before preview\n' >"$TMP/sort-stale-files/item.txt" +/bin/mkdir "$TMP/inbox-files" +printf 'September invoice\n' >"$TMP/inbox-files/invoice-september.txt" +printf 'Vaccination history\n' >"$TMP/inbox-files/vaccination-history.txt" +printf 'Project sprint plan\n' >"$TMP/inbox-files/sprint-plan.txt" +printf 'Family travel notes\n' >"$TMP/inbox-files/family-travel.txt" +printf 'Conflicting evidence\n' >"$TMP/inbox-files/medical-invoice.pdf" +printf 'Unclear document\n' >"$TMP/inbox-files/misc.pdf" /bin/mkdir "$TMP/interlock-files" printf "First interlock receipt.\n" >"$TMP/interlock-files/a.txt" printf "Second interlock receipt.\n" >"$TMP/interlock-files/b.txt" @@ -260,6 +267,17 @@ printf '%s' "$unsorted" | /usr/bin/grep -q '"undone":4' [ -f "$TMP/report-files/third.csv" ] [ ! -e "$TMP/report-files/Sorted by type/txt/first.txt" ] [ ! -e "$TMP/report-files/Sorted by type/csv/third.csv" ] +INBOX=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ + -H 'Content-Type: application/json' \ + --data-binary "{\"recipe\":\"classify_inbox\",\"folder\":\"$TMP/inbox-files\"}") +printf '%s' "$INBOX" | /usr/bin/grep -q '"type":"inbox_classification"' +printf '%s' "$INBOX" | /usr/bin/grep -q '"category":"billing","signal":"invoice"' +printf '%s' "$INBOX" | /usr/bin/grep -q '"category":"medical","signal":"vaccination"' +printf '%s' "$INBOX" | /usr/bin/grep -q '"category":"work","signal":"sprint"' +printf '%s' "$INBOX" | /usr/bin/grep -q '"category":"personal","signal":"family"' +printf '%s' "$INBOX" | /usr/bin/grep -q '"category":"review","signal":"conflicting filename clues"' +printf '%s' "$INBOX" | /usr/bin/grep -q '"category":"review","signal":"no clear filename clue"' +[ "$(/usr/bin/find "$TMP/inbox-files" -type f | /usr/bin/wc -l | /usr/bin/tr -d ' ')" = 6 ] STALE_SORT=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/run" \ -H 'Content-Type: application/json' \ --data-binary "{\"recipe\":\"sort_by_type\",\"folder\":\"$TMP/sort-stale-files\"}") diff --git a/tests/test_jobs_ui.mjs b/tests/test_jobs_ui.mjs index ac3ec64..be612c6 100644 --- a/tests/test_jobs_ui.mjs +++ b/tests/test_jobs_ui.mjs @@ -57,6 +57,7 @@ const app = readFileSync(new URL("../assets/app.html", import.meta.url), "utf8") assert.match(app, /id="jobRecipe"[\s\S]*value="folder_report"/); assert.match(app, /value="find_duplicates">Find duplicates/); assert.match(app, /value="sort_by_type">Sort by file type/); +assert.match(app, /value="classify_inbox">Classify an inbox/); assert.match(app, /recipe !== "find" \? \{ recipe, folder \}/); assert.match(app, /jobEls\.goal\.disabled = deterministic/); const begin = app.indexOf(" const baseName ="); @@ -120,5 +121,12 @@ renderJobEvent({ type: "await_apply", job_id: "sort-job", moves: 4 }, ctx); assert.equal(ctx.awaitingApply, true); assert.equal(jobEls.barText.textContent, "4 files ready to move."); assert.equal(jobEls.barActions.children[0].textContent, "Apply moves"); +renderJobEvent({ type: "inbox_classification", counts: [1, 1, 0, 0, 1], items: [ + { path: "invoice.pdf", category: "billing", signal: "invoice" }, + { path: "medical-invoice.pdf", category: "review", signal: "conflicting filename clues" }, +] }, ctx); +assert.match(jobEls.resultText.innerHTML, /filename clues only/); +assert.match(jobEls.resultText.innerHTML, /invoice\.pdf/); +assert.match(jobEls.resultText.innerHTML, /conflicting filename clues/); process.stdout.write("jobs UI DOM fixtures: PASS\n"); From 66a092a386bcac4b9476f6f2bb909c03ddd70690 Mon Sep 17 00:00:00 2001 From: Deepan Wadhwa Date: Tue, 22 Sep 2026 16:33:44 -0400 Subject: [PATCH 04/30] Add scheduled Jobs folder watch deltas --- assets/app.html | 39 +++++- docs/CHUTNI_JOBS_RECOVERY_PLAN.md | 9 +- src/samosa_gateway.c | 213 ++++++++++++++++++++++++++++++ tests/test_compiled_gateway.sh | 26 ++++ tests/test_jobs_ui.mjs | 3 + 5 files changed, 286 insertions(+), 4 deletions(-) diff --git a/assets/app.html b/assets/app.html index 9a98c40..b5aeffe 100644 --- a/assets/app.html +++ b/assets/app.html @@ -1755,7 +1755,11 @@

Jobs

- + +
+
@@ -6585,6 +6589,7 @@

Delete conversation?

// --- Jobs (Models -> Tools -> Jobs) -------------------------------- const jobEls = { view: $("jobsView"), form: $("jobForm"), folder: $("jobFolder"), goal: $("jobGoal"), recipe: $("jobRecipe"), + watchRecipe: $("jobWatchRecipe"), watchRecipeField: $("jobWatchRecipeField"), browse: $("jobBrowse"), memoryAccess: $("jobMemoryAccess"), run: $("jobRun"), auto: $("jobAuto"), activity: $("jobActivity"), bar: $("jobBar"), progress: $("jobProgress"), result: $("jobResult"), resultLabel: $("jobResultLabel"), resultText: $("jobResultText"), @@ -8365,7 +8370,36 @@

Delete conversation?

if (!folder || (recipe === "find" && !goal)) { jobEls.folder.focus(); return; } jobRunning = true; jobEls.run.disabled = true; jobEls.run.textContent = "Running"; clearJobOutput(); const ctx = { folder }; - try { await streamJob("/v1/jobs/run", recipe !== "find" ? { recipe, folder } : { goal, folder, recipe, mode: jobEls.auto.checked ? "execute" : "confirm" }, ctx); } + try { + if (recipe === "watch_folder") { + const storageKey = `samosa.watch.${folder}`; + const existingWatch = localStorage.getItem(storageKey); + if (existingWatch) { + showBar("Folder watch is active.", [["Stop watching", "undo", async () => { + const stopped = await authFetch("/v1/jobs/schedule/stop", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ job_id: existingWatch }) }); + const result = await stopped.json(); if (!stopped.ok || !result.ok) throw new Error(result.error?.message || "Could not stop folder watch"); + localStorage.removeItem(storageKey); jobEls.bar.hidden = true; showJobResult("Folder watch stopped."); + }]]); + showJobResult("This folder already has an active watch. Stop it here before starting another."); + return; + } + const jobId = `watch-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const job = { job_id: jobId, input: { folder }, watch_recipe: jobEls.watchRecipe.value, resources: { run_on_battery: true } }; + const response = await authFetch("/v1/jobs/schedule/arm", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ job, window_start: "00:00", window_end: "00:00", missed_policy: "run_next_start", keep_awake: false }) }); + const data = await response.json(); + if (!response.ok || !data.ok) throw new Error(data.error?.message || "Could not start folder watch"); + localStorage.setItem(storageKey, jobId); + jobLine("●", `Watching every five minutes · ${jobEls.watchRecipe.value === "classify_inbox" ? "classify inbox filenames" : "folder report"}`, "step-done"); + showJobResult("The current folder state is saved as the baseline. Only added or changed files will run through the selected read-only recipe."); + showBar("Folder watch is active.", [["Stop watching", "undo", async () => { + const stopped = await authFetch("/v1/jobs/schedule/stop", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ job_id: jobId }) }); + const result = await stopped.json(); if (!stopped.ok || !result.ok) throw new Error(result.error?.message || "Could not stop folder watch"); + localStorage.removeItem(storageKey); jobEls.bar.hidden = true; showJobResult("Folder watch stopped."); + }]]); + } else { + await streamJob("/v1/jobs/run", recipe !== "find" ? { recipe, folder } : { goal, folder, recipe, mode: jobEls.auto.checked ? "execute" : "confirm" }, ctx); + } + } catch (e) { renderJobEvent({ type: "error", message: String(e) }, ctx); } finally { jobRunning = false; jobEls.run.disabled = false; jobEls.run.textContent = "Run job"; } } @@ -8729,6 +8763,7 @@

Delete conversation?

jobEls.form.onsubmit=e=>{e.preventDefault();runJob();}; const syncJobRecipe = () => { const deterministic = jobEls.recipe.value !== "find"; + jobEls.watchRecipeField.hidden = jobEls.recipe.value !== "watch_folder"; jobEls.goal.disabled = deterministic; jobEls.goal.placeholder = deterministic ? "No description needed for this recipe" : "Find my cat's medical records"; $("jobGoalLabel").textContent = deterministic ? "Task description (optional)" : "What should I find?"; diff --git a/docs/CHUTNI_JOBS_RECOVERY_PLAN.md b/docs/CHUTNI_JOBS_RECOVERY_PLAN.md index 0eacdd7..4eac2a7 100644 --- a/docs/CHUTNI_JOBS_RECOVERY_PLAN.md +++ b/docs/CHUTNI_JOBS_RECOVERY_PLAN.md @@ -305,8 +305,13 @@ candidate cannot be read. Sort by file type uses the persisted move plan, explicit Apply, source size/mtime revalidation, and Undo path. Inbox classification is a bounded filename-only pass: a single category clue yields a suggestion, while unknown or cross-category clues go to review. It does not -read content or move files. Watch-folder deltas remain unimplemented Phase C -work; this checkpoint does not claim the Phase C acceptance gates are complete. +read content or move files. Watch folder saves a complete metadata baseline and +checks every five minutes; the supported read-only inbox or report recipe runs +only on added or changed paths. An incomplete inventory does not advance its +checkpoint. Watch runs are stoppable and remain scheduled until stopped. These +recipes complete the Phase C productized recipe list, while the durability, +restart-history, and exact-tree acceptance gates still need their full dogfood +evidence; this checkpoint does not claim the overall recovery plan is complete. ### C2. Durable `JobSpec` diff --git a/src/samosa_gateway.c b/src/samosa_gateway.c index f74fa3c..a84cb58 100644 --- a/src/samosa_gateway.c +++ b/src/samosa_gateway.c @@ -1561,6 +1561,9 @@ static int write_schedule_with_status(const char *path, jval *schedule, const ch free(out.data); return 0; } +static int jobs_watch_delta(Gateway *g, const char *job_id, const char *folder, + const char *recipe, int initialize, TextBuffer *summary); + static int jobs_schedule_arm(Gateway *g, int fd, const SamosaHttpRequest *request) { char *arena = NULL; jval *body = json_parse(request->body, &arena); jval *job = body && body->t == J_OBJ ? json_get(body, "job") : NULL; @@ -1576,6 +1579,15 @@ static int jobs_schedule_arm(Gateway *g, int fd, const SamosaHttpRequest *reques return samosa_http_json_error(fd, 400, "invalid_schedule", "A job object or job_path is required."); } jval *id = json_get(job, "job_id"); + jval *watch_recipe = json_get(job, "watch_recipe"); + jval *watch_input = json_get(job, "input"); + jval *watch_folder = watch_input && watch_input->t == J_OBJ ? json_get(watch_input, "folder") : NULL; + if (watch_recipe && (watch_recipe->t != J_STR || + (strcmp(watch_recipe->str, "classify_inbox") && strcmp(watch_recipe->str, "folder_report")) || + !watch_folder || watch_folder->t != J_STR || !watch_folder->str[0])) { + json_free(loaded_job); free(job_arena); free(job_raw); json_free(body); free(arena); + return samosa_http_json_error(fd, 400, "invalid_watch", "Watch jobs require a folder and a read-only supported recipe."); + } char job_id[128]; if (id && id->t == J_STR) path_copy(job_id, sizeof(job_id), id->str); else slugify_to(job_id, sizeof(job_id), job_path_value && job_path_value->t == J_STR ? path_basename_const(job_path_value->str) : "scheduled-job"); @@ -1637,6 +1649,18 @@ static int jobs_schedule_arm(Gateway *g, int fd, const SamosaHttpRequest *reques text_add(&schedule, ",\"run_on_battery\":") && text_add(&schedule, (run_batt && run_batt->t == J_BOOL && run_batt->boolean) ? "true" : "false") && text_add(&schedule, ",\"review_required_policy\":\"queue\",\"armed_at\":") && text_json_string(&schedule, now) && text_add(&schedule, "}\n") && write_small_file(schedule_path, schedule.data); + if (ok && watch_recipe) { + TextBuffer baseline = {0}; + char watch_path[PATH_MAX]; + int has_baseline = job_state_path(g, job_id, "watch-state.json", watch_path, 0) && access(watch_path, F_OK) == 0; + if (!jobs_watch_delta(g, job_id, watch_folder->str, watch_recipe->str, !has_baseline, &baseline)) ok = 0; + free(baseline.data); + if (ok) { + char *schedule_arena = NULL; jval *schedule_obj = json_parse(schedule.data, &schedule_arena); + ok = schedule_obj && write_schedule_with_status(schedule_path, schedule_obj, "watching", 1, "baseline_ready"); + json_free(schedule_obj); free(schedule_arena); + } + } TextBuffer response = {0}; if (ok) ok = text_add(&response, "{\"ok\":true,\"job_id\":") && text_json_string(&response, job_id) && text_add(&response, ",\"schedule_path\":") && text_json_string(&response, schedule_path) && @@ -1647,6 +1671,26 @@ static int jobs_schedule_arm(Gateway *g, int fd, const SamosaHttpRequest *reques json_free(loaded_job); free(job_arena); free(job_raw); json_free(body); free(arena); return sent; } +static int jobs_schedule_stop(Gateway *g, int fd, const SamosaHttpRequest *request) { + char *arena = NULL; jval *body = json_parse(request->body, &arena); + jval *id = body && body->t == J_OBJ ? json_get(body, "job_id") : NULL; + if (!id || id->t != J_STR || !valid_job_id(id->str)) { json_free(body); free(arena); return samosa_http_json_error(fd, 400, "invalid_job_id", "A valid watch job_id is required."); } + char path[PATH_MAX]; char job_id[128]; path_copy(job_id, sizeof(job_id), id->str); + if (!job_state_path(g, job_id, "schedule.json", path, 0)) { json_free(body); free(arena); return samosa_http_json_error(fd, 404, "schedule_not_found", "That watch schedule is unavailable."); } + char job_path[PATH_MAX]; job_state_path(g, job_id, "job.json", job_path, 0); + char *raw = read_file_limit(path, 1 << 20), *schedule_arena = NULL; jval *schedule = raw ? json_parse(raw, &schedule_arena) : NULL; + char *job_raw = read_file_limit(job_path, 1 << 20), *job_arena = NULL; jval *job = job_raw ? json_parse(job_raw, &job_arena) : NULL; + if (!schedule || schedule->t != J_OBJ || !job || job->t != J_OBJ || !json_get(job, "watch_recipe")) { + json_free(schedule); free(schedule_arena); free(raw); json_free(job); free(job_arena); free(job_raw); json_free(body); free(arena); + return samosa_http_json_error(fd, 404, "watch_not_found", "That watch schedule is unavailable."); + } + int ok = write_schedule_with_status(path, schedule, "stopped", 0, "user_stopped"); + TextBuffer response = {0}; + if (ok) ok = text_add(&response, "{\"ok\":true,\"job_id\":") && text_json_string(&response, job_id) && text_add(&response, ",\"stopped\":true}"); + int sent = ok && samosa_http_response(fd, 200, "application/json", response.data, NULL); + free(response.data); json_free(schedule); free(schedule_arena); free(raw); json_free(job); free(job_arena); free(job_raw); json_free(body); free(arena); return sent; +} + static int append_job_event_file(const char *path, int *seq, const char *type, const char *json_fields) { char *old = read_file_limit(path, 16 << 20); @@ -3029,11 +3073,28 @@ static int run_scheduled_job_native(Gateway *g, const char *schedule_path, jval json_free(job); free(job_arena); free(job_raw); return 0; } int seq = 1; + char *prior_events = read_file_limit(events_path, 16 << 20); + for (const char *p = prior_events; p && *p; p++) if (*p == '\n') seq++; + free(prior_events); TextBuffer fields = {0}; text_add(&fields, "\"job_id\":"); text_json_string(&fields, job_id_value->str); text_add(&fields, ",\"job_path\":"); text_json_string(&fields, job_path_value->str); append_job_event_file(events_path, &seq, "scheduled_job_start", fields.data); free(fields.data); + jval *watch_recipe = json_get(job, "watch_recipe"); + if (watch_recipe && watch_recipe->t == J_STR && folder && folder->t == J_STR) { + TextBuffer delta = {0}; + int watch_ok = jobs_watch_delta(g, job_id_value->str, folder->str, watch_recipe->str, 0, &delta); + char *existing_events = read_file_limit(events_path, 16 << 20); seq = 1; + for (const char *p = existing_events; p && *p; p++) if (*p == '\n') seq++; + append_job_event_file(events_path, &seq, watch_ok ? "watch_delta" : "error", + watch_ok ? delta.data : "\"message\":\"watch scan failed or was partial\""); + free(existing_events); free(delta.data); + json_free(job); free(job_arena); free(job_raw); + write_schedule_with_status(schedule_path, schedule, watch_ok ? "watching" : "watching_with_error", 1, + watch_ok ? "delta_checked" : "delta_incomplete"); + return watch_ok; + } /* Comparison workflow: fetch the user's public URLs, persist only new/changed pages. The local folder (if any) and the changed items are both left on disk for a later model comparison step; the runner does the deterministic @@ -6197,6 +6258,156 @@ static int jobs_classify_inbox(Gateway *g, int fd, const char *folder) { free(raw); for (size_t i = 0; i < count; i++) free(items[i].path); free(items); return ok; } +typedef struct { char *path; unsigned long long size, dev, ino; double mtime; } JobsWatchEntry; +static int jobs_watch_entry_cmp(const void *a, const void *b) { + return strcmp(((const JobsWatchEntry *)a)->path, ((const JobsWatchEntry *)b)->path); +} +static ssize_t jobs_watch_find(const JobsWatchEntry *entries, size_t count, const char *path) { + size_t lo = 0, hi = count; + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; int cmp = strcmp(entries[mid].path, path); + if (!cmp) return (ssize_t)mid; + if (cmp < 0) lo = mid + 1; else hi = mid; + } + return -1; +} + +/* Persist a metadata snapshot and execute the selected read-only recipe only + * for added or changed paths. A partial inventory never replaces the last + * complete snapshot, so the next poll retries the same unprocessed delta. */ +static int jobs_watch_delta(Gateway *g, const char *job_id, const char *folder, + const char *recipe, int initialize, TextBuffer *summary) { + enum { MAX_FILES = 10000 }; + char *argv[] = {g->samosa_fs, (char *)"chutni-inventory", (char *)"--root", + (char *)folder, (char *)"--max-depth", (char *)"32", + (char *)"--max-files", (char *)"10000", + (char *)"--max-directories", (char *)"5000", + (char *)"--max-seconds", (char *)"20", NULL}; + JobsWatchEntry *current = calloc(MAX_FILES, sizeof(*current)), *previous = calloc(MAX_FILES, sizeof(*previous)); + char state_path[PATH_MAX], *raw = NULL, *old_state = NULL, *old_arena = NULL; + TextBuffer state = {0}; + size_t ncurrent = 0, nprevious = 0, skipped = 0, added = 0, changed = 0, removed = 0; + int status = 0, partial = 0, done = 0, ok = 0; + if (!current || !previous || !job_state_path(g, job_id, "watch-state.json", state_path, 1)) goto watch_done; + old_state = read_file_limit(state_path, 4 << 20); + if (old_state) { + jval *obj = json_parse(old_state, &old_arena), *files = obj && obj->t == J_OBJ ? json_get(obj, "files") : NULL; + if (!files || files->t != J_ARR || files->len > MAX_FILES) { json_free(obj); goto watch_done; } + for (int i = 0; i < files->len; i++) { + jval *row = files->kids[i], *path = json_get(row, "path"), *size = json_get(row, "size"), + *mtime = json_get(row, "mtime"), *dev = json_get(row, "dev"), *ino = json_get(row, "ino"); + if (!path || path->t != J_STR || !size || size->t != J_NUM || !mtime || mtime->t != J_NUM || + !dev || dev->t != J_NUM || !ino || ino->t != J_NUM) { json_free(obj); goto watch_done; } + previous[nprevious].path = strdup(path->str); if (!previous[nprevious].path) { json_free(obj); goto watch_done; } + previous[nprevious].size = (unsigned long long)size->num; previous[nprevious].mtime = mtime->num; + previous[nprevious].dev = (unsigned long long)dev->num; previous[nprevious].ino = (unsigned long long)ino->num; nprevious++; + } + json_free(obj); + } + raw = run_capture(g, g->samosa_fs, argv, 8 << 20, &status); + if (!raw || !WIFEXITED(status) || WEXITSTATUS(status)) goto watch_done; + const char *line = raw; + while (*line) { + const char *end = strchr(line, '\n'); size_t len = end ? (size_t)(end - line) : strlen(line); + char *record = malloc(len + 1), *record_arena = NULL; if (!record) goto watch_done; + memcpy(record, line, len); record[len] = 0; jval *obj = json_parse(record, &record_arena); free(record); + if (!obj || obj->t != J_OBJ) { json_free(obj); free(record_arena); goto watch_done; } + jval *type = json_get(obj, "type"); + if (type && type->t == J_STR && !strcmp(type->str, "file")) { + jval *path = json_get(obj, "rel_path"), *size = json_get(obj, "size"), *mtime = json_get(obj, "mtime"), + *dev = json_get(obj, "dev"), *ino = json_get(obj, "ino"); + if (!path || path->t != J_STR || !size || size->t != J_NUM || !mtime || mtime->t != J_NUM || + !dev || dev->t != J_NUM || !ino || ino->t != J_NUM || ncurrent >= MAX_FILES) { + json_free(obj); free(record_arena); goto watch_done; + } + current[ncurrent].path = strdup(path->str); if (!current[ncurrent].path) { json_free(obj); free(record_arena); goto watch_done; } + current[ncurrent].size = (unsigned long long)size->num; current[ncurrent].mtime = mtime->num; + current[ncurrent].dev = (unsigned long long)dev->num; current[ncurrent].ino = (unsigned long long)ino->num; ncurrent++; + } else if (type && type->t == J_STR && !strcmp(type->str, "skip")) skipped++; + else if (type && type->t == J_STR && !strcmp(type->str, "done")) { + done = 1; jval *v = json_get(obj, "partial"); partial = v && v->t == J_BOOL && v->boolean; + } + json_free(obj); free(record_arena); if (!end) break; line = end + 1; + } + if (!done || partial) goto watch_done; + qsort(current, ncurrent, sizeof(*current), jobs_watch_entry_cmp); + qsort(previous, nprevious, sizeof(*previous), jobs_watch_entry_cmp); + size_t i = 0, j = 0; + while (i < ncurrent) { + while (j < nprevious && strcmp(previous[j].path, current[i].path) < 0) j++; + if (j >= nprevious || strcmp(previous[j].path, current[i].path)) added++; + else if (current[i].size != previous[j].size || current[i].mtime != previous[j].mtime || + current[i].dev != previous[j].dev || current[i].ino != previous[j].ino) changed++; + i++; + } + i = j = 0; + while (j < nprevious) { + while (i < ncurrent && strcmp(current[i].path, previous[j].path) < 0) i++; + if (i >= ncurrent || strcmp(current[i].path, previous[j].path)) removed++; + j++; + } + int built = text_add(&state, "{\"schema_version\":1,\"files\":["); + for (i = 0; built && i < ncurrent; i++) { + if (i) built = text_add(&state, ","); + char size[32], dev[32], ino[32], mtime[64]; + snprintf(size, sizeof(size), "%llu", current[i].size); snprintf(dev, sizeof(dev), "%llu", current[i].dev); + snprintf(ino, sizeof(ino), "%llu", current[i].ino); snprintf(mtime, sizeof(mtime), "%.9f", current[i].mtime); + built = built && text_add(&state, "{\"path\":") && text_json_string(&state, current[i].path) && + text_add(&state, ",\"size\":") && text_add(&state, size) && text_add(&state, ",\"mtime\":") && text_add(&state, mtime) && + text_add(&state, ",\"dev\":") && text_add(&state, dev) && text_add(&state, ",\"ino\":") && text_add(&state, ino) && text_add(&state, "}"); + } + built = built && text_add(&state, "]}\n"); if (!built) goto watch_done; + if (initialize) added = changed = removed = 0; + char count_text[64], add_text[64], change_text[64], remove_text[64], skip_text[64], processed_text[64]; + snprintf(count_text, sizeof(count_text), "%zu", ncurrent); snprintf(add_text, sizeof(add_text), "%zu", added); + snprintf(change_text, sizeof(change_text), "%zu", changed); snprintf(remove_text, sizeof(remove_text), "%zu", removed); + snprintf(skip_text, sizeof(skip_text), "%zu", skipped); snprintf(processed_text, sizeof(processed_text), "%zu", added + changed); + ok = text_add(summary, "\"recipe\":") && text_json_string(summary, recipe) && text_add(summary, ",\"checked\":") && text_add(summary, count_text) && + text_add(summary, ",\"added\":") && text_add(summary, add_text) && text_add(summary, ",\"changed\":") && text_add(summary, change_text) && + text_add(summary, ",\"removed\":") && text_add(summary, remove_text) && text_add(summary, ",\"skipped\":") && text_add(summary, skip_text) && + text_add(summary, ",\"processed\":") && text_add(summary, initialize ? "0" : processed_text); + if (ok && !initialize && !strcmp(recipe, "classify_inbox")) { + ok = text_add(summary, ",\"items\":["); int first = 1; + for (i = 0; ok && i < ncurrent; i++) { + ssize_t old_index = jobs_watch_find(previous, nprevious, current[i].path); + int is_new = old_index < 0 || current[i].size != previous[old_index].size || current[i].mtime != previous[old_index].mtime || + current[i].dev != previous[old_index].dev || current[i].ino != previous[old_index].ino; + if (!is_new) continue; + const char *signal = NULL, *category = jobs_inbox_category(current[i].path, &signal); + if (!first) ok = text_add(summary, ","); first = 0; + ok = ok && text_add(summary, "{\"path\":") && text_json_string(summary, current[i].path) && text_add(summary, ",\"category\":") && text_json_string(summary, category) && text_add(summary, ",\"signal\":") && text_json_string(summary, signal) && text_add(summary, "}"); + } + ok = ok && text_add(summary, "]"); + } + if (ok && !initialize && !strcmp(recipe, "folder_report")) { + struct { char key[32]; unsigned long long count; } groups[64] = {0}; size_t group_count = 0; + ok = text_add(summary, ",\"by_type\":{"); int first = 1; + for (i = 0; ok && i < ncurrent; i++) { + ssize_t old_index = jobs_watch_find(previous, nprevious, current[i].path); + int is_new = old_index < 0 || current[i].size != previous[old_index].size || current[i].mtime != previous[old_index].mtime || + current[i].dev != previous[old_index].dev || current[i].ino != previous[old_index].ino; + if (!is_new) continue; + char ext[32]; jobs_report_type(current[i].path, ext); + size_t k = 0; while (k < group_count && strcmp(groups[k].key, ext)) k++; + if (k == group_count && group_count < 64) { path_copy(groups[k].key, sizeof(groups[k].key), ext); group_count++; } + if (k < group_count) groups[k].count++; + } + for (size_t k = 0; ok && k < group_count; k++) { + char number[32]; snprintf(number, sizeof(number), "%llu", groups[k].count); + if (!first) ok = text_add(summary, ","); first = 0; + ok = ok && text_json_string(summary, groups[k].key) && text_add(summary, ":") && text_add(summary, number); + } + ok = ok && text_add(summary, "}"); + } + ok = ok && text_add(summary, ",\"initialized\":") && text_add(summary, initialize ? "true" : "false"); + if (ok) ok = write_small_file(state_path, state.data); +watch_done: + free(state.data); free(raw); free(old_state); free(old_arena); + for (i = 0; i < ncurrent; i++) free(current[i].path); + for (i = 0; i < nprevious; i++) free(previous[i].path); + free(current); free(previous); return ok; +} + static int jobs_report(Gateway *g, int fd, const char *goal, const char *folder, const char *existing_job_id) { char *argv[] = {g->samosa_fs, "survey", "--max-file-bytes", "104857600", @@ -23582,6 +23793,8 @@ static int gateway_handler(SamosaHttpServer *server, int fd, return jobs_apply_or_undo(g, fd, request, 1); if (!strcmp(request->method, "POST") && !strcmp(request->path, "/v1/jobs/schedule/arm")) return jobs_schedule_arm(g, fd, request); + if (!strcmp(request->method, "POST") && !strcmp(request->path, "/v1/jobs/schedule/stop")) + return jobs_schedule_stop(g, fd, request); if (!strcmp(request->method, "POST") && !strcmp(request->path, "/v1/jobsd/once")) return jobsd_once_native(g, fd, request); if (!strcmp(request->method, "GET") && !strcmp(request->path, "/v1/jobs/launchd-plist")) diff --git a/tests/test_compiled_gateway.sh b/tests/test_compiled_gateway.sh index 6c27fd8..993b91a 100644 --- a/tests/test_compiled_gateway.sh +++ b/tests/test_compiled_gateway.sh @@ -59,6 +59,9 @@ printf 'Project sprint plan\n' >"$TMP/inbox-files/sprint-plan.txt" printf 'Family travel notes\n' >"$TMP/inbox-files/family-travel.txt" printf 'Conflicting evidence\n' >"$TMP/inbox-files/medical-invoice.pdf" printf 'Unclear document\n' >"$TMP/inbox-files/misc.pdf" +/bin/mkdir "$TMP/watch-folder" +printf 'Initial invoice\n' >"$TMP/watch-folder/invoice-initial.txt" +printf 'Steady file\n' >"$TMP/watch-folder/steady.txt" /bin/mkdir "$TMP/interlock-files" printf "First interlock receipt.\n" >"$TMP/interlock-files/a.txt" printf "Second interlock receipt.\n" >"$TMP/interlock-files/b.txt" @@ -787,6 +790,29 @@ printf '%s' "$missed" | /usr/bin/grep -q '"job_id":"missed-skip","action":"defer printf '%s' "$missed" | /usr/bin/grep -q '"job_id":"missed-run","action":"run","reason":"missed_window"' /usr/bin/grep -q '"type":"scheduled_job_complete"' "$HOME_DIR/jobs/missed-run/events.jsonl" +# Folder watches snapshot a baseline, process only additions/changes, repeat +# on the daemon's five-minute poll, and can be stopped from the UI session. +WATCH_DEF="{\"job\":{\"job_id\":\"watch-delta\",\"input\":{\"folder\":\"$TMP/watch-folder\"},\"watch_recipe\":\"classify_inbox\",\"resources\":{\"run_on_battery\":true}},\"window_start\":\"00:00\",\"window_end\":\"00:00\",\"missed_policy\":\"run_next_start\",\"keep_awake\":false}" +/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/schedule/arm" \ + -H 'Content-Type: application/json' --data-binary "$WATCH_DEF" | /usr/bin/grep -q '"ok":true' +[ -f "$HOME_DIR/jobs/watch-delta/watch-state.json" ] +printf 'Changed invoice with a longer body\n' >"$TMP/watch-folder/invoice-initial.txt" +printf 'New family trip\n' >"$TMP/watch-folder/family-trip.txt" +watch_run=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobsd/once" \ + -H 'Content-Type: application/json' --data-binary '{"now_minutes":720,"on_battery":false}') +printf '%s' "$watch_run" | /usr/bin/grep -q '"job_id":"watch-delta","action":"run"' +/usr/bin/grep -q '"type":"watch_delta".*"added":1,"changed":1.*"processed":2' "$HOME_DIR/jobs/watch-delta/events.jsonl" +/usr/bin/grep -q '"category":"billing"' "$HOME_DIR/jobs/watch-delta/events.jsonl" +/usr/bin/grep -q '"category":"personal"' "$HOME_DIR/jobs/watch-delta/events.jsonl" +watch_again=$(/usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobsd/once" \ + -H 'Content-Type: application/json' --data-binary '{"now_minutes":720,"on_battery":false}') +printf '%s' "$watch_again" | /usr/bin/grep -q '"job_id":"watch-delta","action":"run"' +/usr/bin/grep -q '"type":"watch_delta".*"added":0,"changed":0.*"processed":0' "$HOME_DIR/jobs/watch-delta/events.jsonl" +stopped=$(/usr/bin/curl -fsS -H "X-Samosa-Token: $MAIN_TOKEN" -X POST \ + "http://127.0.0.1:$PORT/v1/jobs/schedule/stop" -H 'Content-Type: application/json' \ + --data-binary '{"job_id":"watch-delta"}') +printf '%s' "$stopped" | /usr/bin/grep -q '"stopped":true' + # --- launchd lifecycle (dry-run, temp LaunchAgents dir) --- /usr/bin/curl -fsS "http://127.0.0.1:$PORT/v1/jobs/launchd/status" | /usr/bin/grep -q '"installed":false' /usr/bin/curl -fsS -X POST "http://127.0.0.1:$PORT/v1/jobs/launchd/install" | /usr/bin/grep -q '"ok":true' diff --git a/tests/test_jobs_ui.mjs b/tests/test_jobs_ui.mjs index be612c6..89ee757 100644 --- a/tests/test_jobs_ui.mjs +++ b/tests/test_jobs_ui.mjs @@ -58,6 +58,9 @@ assert.match(app, /id="jobRecipe"[\s\S]*value="folder_report"/); assert.match(app, /value="find_duplicates">Find duplicates/); assert.match(app, /value="sort_by_type">Sort by file type/); assert.match(app, /value="classify_inbox">Classify an inbox/); +assert.match(app, /value="watch_folder">Watch a folder/); +assert.match(app, /Recipe to run on added or changed files/); +assert.match(app, /v1\/jobs\/schedule\/stop/); assert.match(app, /recipe !== "find" \? \{ recipe, folder \}/); assert.match(app, /jobEls\.goal\.disabled = deterministic/); const begin = app.indexOf(" const baseName ="); From 20175987e142c7363dbebbc87488e03b5740ae03 Mon Sep 17 00:00:00 2001 From: Deepan Wadhwa Date: Tue, 22 Sep 2026 17:55:31 -0400 Subject: [PATCH 05/30] Fix bundled Jobs helper startup regressions --- Makefile | 2 +- assets/app.html | 1 + dist/samosa | 5 +++++ src/samosa_gateway.c | 7 +++++-- tests/test_samosa_detached_service.sh | 15 ++++++++++++--- 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c8aacf5..b0ce775 100644 --- a/Makefile +++ b/Makefile @@ -282,7 +282,7 @@ chutni-gateway-test: samosa-gateway samosa-fs chutni-service test_fake_openai_ba sh tests/test_chutni_gateway.sh sh tests/test_chutni_controls.sh -detached-service-test: samosa-gateway chutni-service test_fake_openai_backend tests/test_samosa_detached_service.sh +detached-service-test: samosa-gateway samosa-fs chutni-service test_fake_openai_backend tests/test_samosa_detached_service.sh sh tests/test_samosa_detached_service.sh app-lifecycle-test: samosa-gateway chutni-service test_fake_openai_backend tests/test_samosa_app_lifecycle.sh tests/test_samosa_llama_lifecycle.sh diff --git a/assets/app.html b/assets/app.html index b5aeffe..dad86df 100644 --- a/assets/app.html +++ b/assets/app.html @@ -740,6 +740,7 @@ .job-form { display: grid; gap: 11px; margin-bottom: 18px; } .job-form label { font-size: 12px; font-weight: 600; color: var(--ink-2); } .job-field { display: grid; gap: 6px; } + .job-field[hidden] { display: none; } .job-form input:not([type="checkbox"]), .job-form textarea { width: 100%; margin: 0; border: 1px solid var(--line); border-radius: var(--r-sm); background: var(--surface); color: var(--ink); padding: 10px 12px; outline: none; font: inherit; resize: vertical; } .job-form #jobGoal { min-height: 86px; max-height: 180px; padding: 13px; font-size: 16px; line-height: 1.45; } .job-form input:not([type="checkbox"]):focus, .job-form textarea:focus { border-color: var(--line-strong); } diff --git a/dist/samosa b/dist/samosa index 6ca3c59..51fa624 100644 --- a/dist/samosa +++ b/dist/samosa @@ -439,6 +439,7 @@ serve_foreground() { SAMOSA_VOICE_TRACE_AUTO="$VOICE_TRACE_AUTO" \ SAMOSA_APP_VERSION="$SAMOSA_VERSION" \ SAMOSA_CHUTNI_SERVICE="$RELEASE_DIR/bin/chutni-mcp" \ + SAMOSA_FS="$RELEASE_DIR/bin/samosa-fs" \ SAMOSA_MODELS_CATALOG="$MODELS_CATALOG" \ exec "$GATEWAY" } @@ -472,6 +473,7 @@ write_launchd_plist() { x_voice_trace_auto=$(xml_escape "$VOICE_TRACE_AUTO") x_version=$(xml_escape "$SAMOSA_VERSION") x_chutni=$(xml_escape "$RELEASE_DIR/bin/chutni-mcp") + x_samosa_fs=$(xml_escape "$RELEASE_DIR/bin/samosa-fs") x_models_catalog=$(xml_escape "$MODELS_CATALOG") x_log=$(xml_escape "$SERVER_LOG") cat >"$LAUNCHD_PLIST" <SAMOSA_VOICE_TRACE_AUTO$x_voice_trace_auto SAMOSA_APP_VERSION$x_version SAMOSA_CHUTNI_SERVICE$x_chutni + SAMOSA_FS$x_samosa_fs SAMOSA_MODELS_CATALOG$x_models_catalog RunAtLoad @@ -540,6 +543,7 @@ start_server_detached() { SAMOSA_VOICE_TRACE_AUTO="$VOICE_TRACE_AUTO" \ SAMOSA_APP_VERSION="$SAMOSA_VERSION" SAMOSA_APP_LIFECYCLE="$APP_LIFECYCLE" \ SAMOSA_CHUTNI_SERVICE="$RELEASE_DIR/bin/chutni-mcp" \ + SAMOSA_FS="$RELEASE_DIR/bin/samosa-fs" \ SAMOSA_MODELS_CATALOG="$MODELS_CATALOG" \ "$GATEWAY" >"$SERVER_LOG" 2>&1 "$SERVER_LOG" 2>&1 mu); + pthread_mutex_lock(&g->job_pid_mu); if (add) { for (size_t i = 0; i < sizeof(g->job_pids) / sizeof(g->job_pids[0]); ++i) if (!g->job_pids[i]) { g->job_pids[i] = pid; break; } @@ -435,7 +436,7 @@ static void track_job_pid(Gateway *g, pid_t pid, int add) { for (size_t i = 0; i < sizeof(g->job_pids) / sizeof(g->job_pids[0]); ++i) if (g->job_pids[i] == pid) { g->job_pids[i] = 0; break; } } - pthread_mutex_unlock(&g->mu); + pthread_mutex_unlock(&g->job_pid_mu); } static void document_child_set(Gateway *g, pid_t pid) { @@ -23878,6 +23879,7 @@ static int load_config(Gateway *g) { g->summarizer_write_fd = -1; g->summarizer_read_fd = -1; pthread_mutex_init(&g->mu, NULL); + pthread_mutex_init(&g->job_pid_mu, NULL); pthread_mutex_init(&g->summarizer_mu, NULL); pthread_mutex_init(&g->install_mu, NULL); pthread_mutex_init(&g->selection_mu, NULL); @@ -24174,6 +24176,7 @@ int main(int argc, char **argv) { backend_stop(&gateway); samosa_http_server_destroy(&server); samosa_mm_supervisor_destroy(&gateway.multimodal_supervisor); + pthread_mutex_destroy(&gateway.job_pid_mu); pthread_mutex_destroy(&gateway.mu); pthread_mutex_destroy(&gateway.summarizer_mu); pthread_mutex_destroy(&gateway.voice_mu); diff --git a/tests/test_samosa_detached_service.sh b/tests/test_samosa_detached_service.sh index e2b4b19..9ed2388 100644 --- a/tests/test_samosa_detached_service.sh +++ b/tests/test_samosa_detached_service.sh @@ -22,6 +22,7 @@ trap cleanup EXIT HUP INT TERM mkdir -p "$TMP/home" "$TMP/release/bin" "$TMP/release/model" cp "$ROOT/$BUILD_DIR/samosa-gateway" "$TMP/release/bin/samosa-gateway" +cp "$ROOT/$BUILD_DIR/samosa-fs" "$TMP/release/bin/samosa-fs" cp "$ROOT/$BUILD_DIR/test_fake_openai_backend" "$TMP/release/bin/qwen36b" cp "$ROOT/$BUILD_DIR/chutni-mcp" "$TMP/release/bin/chutni-mcp" cp "$ROOT/assets/app.html" "$TMP/release/app.html" @@ -74,7 +75,11 @@ if [ "$(uname -s)" = "Darwin" ]; then } fi -HEALTH=$(curl -fsS --max-time 5 "http://127.0.0.1:$PORT/healthz") +HEALTH=$(curl -fsS --max-time 5 "http://127.0.0.1:$PORT/healthz") || { + echo "FAIL: detached service health request failed" >&2 + cat "$TMP/home/server.log" >&2 2>/dev/null || true + exit 1 +} printf '%s' "$HEALTH" | grep -q '"gateway":true' printf '%s' "$HEALTH" | grep -q '"chutni":{"available":true' printf '%s' "$HEALTH" | grep -q '"managed_by":"samosa"' @@ -83,10 +88,14 @@ printf '%s' "$HEALTH" | grep -q '"can_create_memory":true' # Prove this surviving Samosa process can invoke its bundled Chutni runtime. # Preflight reads only a repository fixture and does not create a memory. TOKEN=$(tr -d '\n' <"$TMP/home/run/ui-token") -PREFLIGHT=$(curl -fsS --max-time 5 \ +PREFLIGHT=$(curl -fsS --max-time 30 \ -H "X-Samosa-Token: $TOKEN" -H 'Content-Type: application/json' -X POST \ "http://127.0.0.1:$PORT/v1/chutni/preflight" \ - --data-binary "{\"kind\":\"folder\",\"roots\":[{\"path\":\"$ROOT/tests/fixtures/chutni_browser_e2e\"}]}") + --data-binary "{\"kind\":\"folder\",\"roots\":[{\"path\":\"$ROOT/tests/fixtures/chutni_browser_e2e\"}]}") || { + echo "FAIL: detached service Chutni preflight request failed" >&2 + cat "$TMP/home/server.log" >&2 2>/dev/null || true + exit 1 +} printf '%s' "$PREFLIGHT" | grep -q '"preflight_id":' printf '%s' "$PREFLIGHT" | grep -q '"action":"create_store"' From 8adcad3d9167c7defb161f8570a60f252f697b1c Mon Sep 17 00:00:00 2001 From: Deepan Wadhwa Date: Sat, 3 Oct 2026 15:34:08 -0400 Subject: [PATCH 06/30] Add persistent file workflows and decision-routed folder chat --- Makefile | 32 +- assets/app.html | 1188 ++++++-- dist/install.sh | 4 +- docs/CHUTNI_JOBS_RECOVERY_PLAN.md | 249 +- .../FILE_WORKFLOW_IMPLEMENTATION_INVENTORY.md | 27 + docs/FOLDER_FILE_DECISION_HARNESS.md | 114 + docs/ISSUE_TASKS.md | 7 + docs/PDF_READING_PLAN.md | 11 +- docs/TASKS_FILE_WORKFLOWS.md | 412 +++ docs/TASKS_JOBS.md | 2 + docs/TASKS_JOBS_INTELLIGENCE.md | 2 + docs/evidence/decision/README.md | 108 + .../decision/laya-typed-1024-report.json | 107 + docs/evidence/decision/laya-typed-1024.jsonl | 36 + .../decision/laya-typed-512-report.json | 107 + docs/evidence/decision/laya-typed-512.jsonl | 36 + .../decision/modernbert-1024-report.json | 107 + docs/evidence/decision/modernbert-1024.jsonl | 36 + .../decision/modernbert-2048-report.json | 107 + docs/evidence/decision/modernbert-2048.jsonl | 36 + .../decision/modernbert-4096-report.json | 107 + docs/evidence/decision/modernbert-4096.jsonl | 36 + .../decision/modernbert-512-report.json | 107 + docs/evidence/decision/modernbert-512.jsonl | 36 + .../decision/modernbert-8192-report.json | 107 + docs/evidence/decision/modernbert-8192.jsonl | 36 + docs/evidence/decision/run-manifest.json | 43 + .../evidence/decision/screening-decision.json | 34 + docs/evidence/dogfood/README.md | 18 + .../dogfood/mixed-folder-2026-09-22.json | 63 + .../2026-10-01-connected/README.md | 367 +++ .../automatic-memory-full-suite-passed.log | 923 ++++++ .../automatic-memory-initial-full-failure.log | 834 ++++++ .../automatic-memory-install.log | 53 + ...atic-memory-later-jobs-fixture-failure.log | 5 + .../automatic-memory-later-jobs-passed.log | 2 + ...tomatic-memory-live-connection-failure.log | 60 + .../automatic-memory-native-passed.log | 5 + .../automatic-memory-ornith-passed.log | 6 + .../automatic-memory-qwen-passed.log | 5 + .../automatic-memory-restored.json | 1 + .../2026-10-01-connected/automatic-memory.md | 119 + .../bounded-selection-harness.log | 14 + .../citation-repeat-moved-plan-failed.log | 34 + .../compact-selector-workflow-passed.log | 9 + .../connected-gateway.log | 63 + .../copy-staging-permissions-passed.log | 3 + .../coverage-exhaustive-retry.log | 2 + .../coverage-first-run.log | 29 + .../2026-10-01-connected/document-harness.log | 14 + .../document-timeout-gateway.log | 63 + .../2026-10-01-connected/full-suite.log | 925 ++++++ .../index-cache-gates.log | 69 + .../model-fork-organize-passed.log | 8 + .../named-exhaustive-coverage.log | 2 + .../2026-10-01-connected/organize-gateway.log | 63 + .../ornith-auto-failed.log | 15 + .../2026-10-01-connected/ornith-passed.log | 4 + .../qwen-current-selector-passed.log | 6 + .../2026-10-01-connected/qwen-passed.log | 4 + .../2026-10-01-connected/qwen-timeout.log | 42 + .../safety-budget-index-restored.log | 1 + .../safety-budget-partial-index.log | 3 + .../safety-budget-partial-workflow-passed.log | 9 + .../selected-reader-gateway.log | 63 + .../shared-cache-harness.log | 14 + .../shared-index-cache-events.json | 15 + .../shared-index-forgotten-passed.log | 3 + .../shared-index-incomplete-build.log | 9 + ...hared-index-incomplete-organize-passed.log | 9 + .../shared-index-incomplete-source-passed.log | 2 + .../shared-index-moved-passed.log | 2 + .../shared-index-organize-passed.log | 9 + .../shared-index-page-failed.log | 7 + .../shared-index-page-passed.log | 2 + .../shared-index-stale-passed.log | 4 + .../shared-index-status.json | 91 + .../shared-v2-discovery-failed.log | 6 + .../shared-v2-ornith-passed.log | 10 + .../shared-v2-qwen-bounded-passed.log | 6 + .../shared-v2-qwen-timeout.log | 41 + .../stop-recovery-gateway.log | 63 + .../ui-cleanup-full-suite-passed.log | 927 ++++++ .../ui-cleanup-sandbox-failure.log | 36 + .../2026-10-01-connected/ui-cleanup.md | 52 + .../2026-10-01-selection/README.md | 60 + .../2026-10-01-selection/build.log | 5 + .../2026-10-01-selection/compiled-gateway.log | 63 + .../2026-10-01-selection/focused-tests.log | 8 + .../2026-10-01-selection/full-test.log | 915 ++++++ .../2026-10-02-document-coverage/README.md | 99 + .../attachments.log | 8 + .../conversation-prefix.log | 1 + .../document-harness.log | 1 + .../failed-review.log | 1 + .../final-gap-instruction-live.json | 15 + .../installed-runtime.json | 12 + .../mixed-40-full.log | 3 + .../mixed-40-ornith-live.log | 9 + .../mixed-40-small-context.log | 3 + .../model-review-concise.json | 1 + .../model-review-truncated.json | 1 + .../ocr-routing.log | 1 + .../ocr-runtime.log | 2 + .../reader-contract.log | 5 + .../ui-fixtures.log | 16 + .../2026-10-03-decision-harness/README.md | 97 + .../absolute-proof-score-failure.log | 13 + .../final-build.log | 11 + .../full-local-suite.log | 936 ++++++ .../2026-10-03-decision-harness/install.log | 53 + .../installed-qwen-answers.json | 43 + .../installed-qwen-answers.log | 7 + .../installed-ui/composer-focus.png | Bin 0 -> 55966 bytes .../installed-ui/folder-memory.png | Bin 0 -> 117798 bytes .../installed-ui/results.json | 11 + .../laya-routing.json | 372 +++ .../laya-source-probe.jsonl | 8 + .../laya-source-probe.stderr | 2 + .../opendecision-routing.json | 376 +++ .../ornith-absence-coverage-failure.json | 30 + .../ornith-absence-coverage-failure.log | 5 + .../ornith-browser-answers.json | 75 + .../ornith-browser-earlier-failures.json | 56 + ...rnith-filename-interpretation-failure.json | 51 + ...ornith-filename-interpretation-failure.log | 8 + .../ornith-general-answers.json | 44 + .../ornith-general-answers.log | 23 + .../ornith-missing-file-final.json | 30 + .../ornith-missing-file-final.log | 5 + .../ornith-missing-subject-failure.json | 32 + .../ornith-missing-subject-failure.log | 19 + .../ornith-overview-overstrict-test.json | 26 + .../ornith-overview-overstrict-test.log | 20 + .../post-review-checks.log | 69 + .../qwen-general-answers.json | 63 + .../qwen-general-answers.log | 11 + .../qwen-overview-overstrict-test.json | 28 + .../2026-10-03-decision-harness/run.json | 20 + .../topic-proof-probe.log | 11 + .../2026-10-03-fw6-continuation/README.md | 75 + .../focused-regressions.log | 60 + .../full-suite-failure.log | 784 +++++ .../2026-10-03-fw6-continuation/install.log | 53 + .../ornith-final-retry-failure.log | 15 + .../ornith-review-failure.log | 15 + .../2026-10-03-walkthrough-feedback/README.md | 13 + .../fixed-composer.png | Bin 0 -> 55465 bytes .../fixed-memory-card.png | Bin 0 -> 97816 bytes src/samosa_fs.c | 312 +- src/samosa_gateway.c | 2547 ++++++++++++++--- src/samosa_http.h | 1 + src/samosa_memory.h | 572 ++++ tests/assert_chutni_inventory_parity.py | 26 + tests/assert_folder_memory.py | 100 + tests/assert_folder_memory_live.py | 32 + tests/assert_job_selection.py | 368 +++ tests/assert_jobs_report_inventory.py | 19 + tests/chutni_real_slow_wrapper.sh | 4 + tests/decision/README.md | 51 + tests/decision/evaluate.py | 247 ++ tests/decision/evaluate_memory.py | 63 + tests/decision/gold.jsonl | 36 + tests/decision/memory_cases.jsonl | 11 + tests/decision/requirements-macos-arm64.txt | 26 + tests/decision/run_model.py | 203 ++ tests/fake_chutni_slow.sh | 3 + tests/fake_openai_backend.c | 61 +- tests/make_test_pdf.py | 41 + tests/test_chutni_controls.sh | 72 +- tests/test_chutni_crash_resume.sh | 113 + tests/test_chutni_gateway.sh | 334 ++- tests/test_chutni_inventory.sh | 47 +- tests/test_chutni_ui.mjs | 63 +- tests/test_compiled_gateway.sh | 327 ++- tests/test_decision_eval.py | 88 + tests/test_document_detail_ui.mjs | 29 + tests/test_document_harness.py | 49 +- tests/test_document_mixed_live.py | 82 + tests/test_document_mixed_pdf.py | 149 + tests/test_document_reader_contract.c | 111 + tests/test_file_workflow_live.py | 185 ++ tests/test_jobs_ui.mjs | 308 +- tests/test_memory_harness.c | 113 + tests/test_memory_harness_browser.mjs | 39 + tests/test_memory_harness_live.py | 182 ++ tests/test_runtime_only_release.sh | 2 +- tests/test_runtime_settings.c | 29 + tests/test_samosa_decision.py | 341 +++ tests/test_samosa_detached_service.sh | 7 +- tests/test_samosa_serve.c | 3 +- tests/test_samosa_wrapper.sh | 28 +- tests/test_selected_copy.py | 132 + tests/test_sidebar_ui.mjs | 55 + tests/tree_manifest.py | 35 + tools/gen_file_workflow_fixture.py | 45 + tools/gen_mixed_document_fixture.py | 58 + tools/install_local_dev.sh | 22 +- tools/package_hf.py | 1 + tools/samosa_decision.py | 820 ++++++ vendor/chutni | 2 +- 201 files changed, 21040 insertions(+), 801 deletions(-) create mode 100644 docs/FILE_WORKFLOW_IMPLEMENTATION_INVENTORY.md create mode 100644 docs/FOLDER_FILE_DECISION_HARNESS.md create mode 100644 docs/TASKS_FILE_WORKFLOWS.md create mode 100644 docs/evidence/decision/README.md create mode 100644 docs/evidence/decision/laya-typed-1024-report.json create mode 100644 docs/evidence/decision/laya-typed-1024.jsonl create mode 100644 docs/evidence/decision/laya-typed-512-report.json create mode 100644 docs/evidence/decision/laya-typed-512.jsonl create mode 100644 docs/evidence/decision/modernbert-1024-report.json create mode 100644 docs/evidence/decision/modernbert-1024.jsonl create mode 100644 docs/evidence/decision/modernbert-2048-report.json create mode 100644 docs/evidence/decision/modernbert-2048.jsonl create mode 100644 docs/evidence/decision/modernbert-4096-report.json create mode 100644 docs/evidence/decision/modernbert-4096.jsonl create mode 100644 docs/evidence/decision/modernbert-512-report.json create mode 100644 docs/evidence/decision/modernbert-512.jsonl create mode 100644 docs/evidence/decision/modernbert-8192-report.json create mode 100644 docs/evidence/decision/modernbert-8192.jsonl create mode 100644 docs/evidence/decision/run-manifest.json create mode 100644 docs/evidence/decision/screening-decision.json create mode 100644 docs/evidence/dogfood/README.md create mode 100644 docs/evidence/dogfood/mixed-folder-2026-09-22.json create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/README.md create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-full-suite-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-initial-full-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-install.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-later-jobs-fixture-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-later-jobs-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-live-connection-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-native-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-ornith-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-qwen-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory-restored.json create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/automatic-memory.md create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/bounded-selection-harness.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/citation-repeat-moved-plan-failed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/compact-selector-workflow-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/connected-gateway.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/copy-staging-permissions-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/coverage-exhaustive-retry.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/coverage-first-run.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/document-harness.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/document-timeout-gateway.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/full-suite.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/index-cache-gates.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/model-fork-organize-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/named-exhaustive-coverage.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/organize-gateway.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/ornith-auto-failed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/ornith-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/qwen-current-selector-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/qwen-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/qwen-timeout.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/safety-budget-index-restored.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/safety-budget-partial-index.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/safety-budget-partial-workflow-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/selected-reader-gateway.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-cache-harness.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-cache-events.json create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-forgotten-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-incomplete-build.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-incomplete-organize-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-incomplete-source-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-moved-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-organize-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-page-failed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-page-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-stale-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-index-status.json create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-v2-discovery-failed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-v2-ornith-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-v2-qwen-bounded-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/shared-v2-qwen-timeout.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/stop-recovery-gateway.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/ui-cleanup-full-suite-passed.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/ui-cleanup-sandbox-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-01-connected/ui-cleanup.md create mode 100644 docs/regressions/file-workflows/2026-10-01-selection/README.md create mode 100644 docs/regressions/file-workflows/2026-10-01-selection/build.log create mode 100644 docs/regressions/file-workflows/2026-10-01-selection/compiled-gateway.log create mode 100644 docs/regressions/file-workflows/2026-10-01-selection/focused-tests.log create mode 100644 docs/regressions/file-workflows/2026-10-01-selection/full-test.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/README.md create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/attachments.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/conversation-prefix.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/document-harness.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/failed-review.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/final-gap-instruction-live.json create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/installed-runtime.json create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/mixed-40-full.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/mixed-40-ornith-live.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/mixed-40-small-context.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/model-review-concise.json create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/model-review-truncated.json create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/ocr-routing.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/ocr-runtime.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/reader-contract.log create mode 100644 docs/regressions/file-workflows/2026-10-02-document-coverage/ui-fixtures.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/README.md create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/absolute-proof-score-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/final-build.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/full-local-suite.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/install.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/installed-qwen-answers.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/installed-qwen-answers.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/installed-ui/composer-focus.png create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/installed-ui/folder-memory.png create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/installed-ui/results.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/laya-routing.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/laya-source-probe.jsonl create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/laya-source-probe.stderr create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/opendecision-routing.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-absence-coverage-failure.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-absence-coverage-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-browser-answers.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-browser-earlier-failures.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-filename-interpretation-failure.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-filename-interpretation-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-general-answers.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-general-answers.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-missing-file-final.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-missing-file-final.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-missing-subject-failure.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-missing-subject-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-overview-overstrict-test.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/ornith-overview-overstrict-test.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/post-review-checks.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/qwen-general-answers.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/qwen-general-answers.log create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/qwen-overview-overstrict-test.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/run.json create mode 100644 docs/regressions/file-workflows/2026-10-03-decision-harness/topic-proof-probe.log create mode 100644 docs/regressions/file-workflows/2026-10-03-fw6-continuation/README.md create mode 100644 docs/regressions/file-workflows/2026-10-03-fw6-continuation/focused-regressions.log create mode 100644 docs/regressions/file-workflows/2026-10-03-fw6-continuation/full-suite-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-03-fw6-continuation/install.log create mode 100644 docs/regressions/file-workflows/2026-10-03-fw6-continuation/ornith-final-retry-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-03-fw6-continuation/ornith-review-failure.log create mode 100644 docs/regressions/file-workflows/2026-10-03-walkthrough-feedback/README.md create mode 100644 docs/regressions/file-workflows/2026-10-03-walkthrough-feedback/fixed-composer.png create mode 100644 docs/regressions/file-workflows/2026-10-03-walkthrough-feedback/fixed-memory-card.png create mode 100644 src/samosa_memory.h create mode 100644 tests/assert_chutni_inventory_parity.py create mode 100644 tests/assert_folder_memory.py create mode 100644 tests/assert_folder_memory_live.py create mode 100644 tests/assert_job_selection.py create mode 100644 tests/assert_jobs_report_inventory.py create mode 100755 tests/chutni_real_slow_wrapper.sh create mode 100644 tests/decision/README.md create mode 100644 tests/decision/evaluate.py create mode 100644 tests/decision/evaluate_memory.py create mode 100644 tests/decision/gold.jsonl create mode 100644 tests/decision/memory_cases.jsonl create mode 100644 tests/decision/requirements-macos-arm64.txt create mode 100644 tests/decision/run_model.py create mode 100644 tests/make_test_pdf.py create mode 100755 tests/test_chutni_crash_resume.sh create mode 100644 tests/test_decision_eval.py create mode 100644 tests/test_document_detail_ui.mjs create mode 100644 tests/test_document_mixed_live.py create mode 100644 tests/test_document_mixed_pdf.py create mode 100644 tests/test_file_workflow_live.py create mode 100644 tests/test_memory_harness.c create mode 100644 tests/test_memory_harness_browser.mjs create mode 100644 tests/test_memory_harness_live.py create mode 100644 tests/test_samosa_decision.py create mode 100644 tests/test_selected_copy.py create mode 100644 tests/tree_manifest.py create mode 100644 tools/gen_file_workflow_fixture.py create mode 100644 tools/gen_mixed_document_fixture.py create mode 100644 tools/samosa_decision.py diff --git a/Makefile b/Makefile index b0ce775..d7f23ee 100644 --- a/Makefile +++ b/Makefile @@ -234,7 +234,7 @@ motto-test: samosa-gateway samosa-ocr test_fake_openai_backend tests/test_motto_ tier2-test: samosa-gateway samosa-ocr test_fake_openai_backend tests/test_tier2_escalation.sh sh tests/test_tier2_escalation.sh -samosa-gateway: src/samosa_gateway.c src/samosa_multimodal.c src/samosa_multimodal.h src/samosa_evidence.c src/samosa_evidence.h src/samosa_html.c src/samosa_html.h src/samosa_http.h src/json.h chutni-service +samosa-gateway: src/samosa_gateway.c src/samosa_memory.h src/samosa_multimodal.c src/samosa_multimodal.h src/samosa_evidence.c src/samosa_evidence.h src/samosa_html.c src/samosa_html.h src/samosa_http.h src/json.h chutni-service @mkdir -p $(BUILD_DIR) $(CC) -O2 $(CWARN) -Wno-unused-function -std=c11 -pthread -Isrc \ src/samosa_gateway.c $(GATEWAY_SUPPORT_SRCS) -o $(BUILD_DIR)/samosa-gateway $(DL_LDFLAGS) @@ -278,9 +278,10 @@ test-native-summarizer-real: samosa-summarizer tests/test_native_summarizer.c $(BUILD_DIR)/test_native_summarizer $(BUILD_DIR)/samosa-summarizer "$(SAMOSA_SUMMARIZER_TEST_MODEL)" # The HTTP controller invokes the same generic service that MCP hosts use. -chutni-gateway-test: samosa-gateway samosa-fs chutni-service test_fake_openai_backend tests/test_chutni_gateway.sh tests/test_chutni_controls.sh +chutni-gateway-test: samosa-gateway samosa-fs chutni-service test_fake_openai_backend tests/test_chutni_gateway.sh tests/test_chutni_controls.sh tests/test_chutni_crash_resume.sh sh tests/test_chutni_gateway.sh sh tests/test_chutni_controls.sh + sh tests/test_chutni_crash_resume.sh detached-service-test: samosa-gateway samosa-fs chutni-service test_fake_openai_backend tests/test_samosa_detached_service.sh sh tests/test_samosa_detached_service.sh @@ -295,7 +296,7 @@ test-lan-access: samosa-gateway test_fake_openai_backend tests/test_lan_access.s # samosa-jobsd is the same source under a launchd-friendly name. Invoked as # `samosa-jobsd jobsd-once` it polls armed schedules and exits — no listener, # no backend — which is exactly what the installed launchd plist fires. -samosa-jobsd: src/samosa_gateway.c src/samosa_multimodal.c src/samosa_multimodal.h src/samosa_evidence.c src/samosa_evidence.h src/samosa_html.c src/samosa_html.h src/samosa_http.h src/json.h +samosa-jobsd: src/samosa_gateway.c src/samosa_memory.h src/samosa_multimodal.c src/samosa_multimodal.h src/samosa_evidence.c src/samosa_evidence.h src/samosa_html.c src/samosa_html.h src/samosa_http.h src/json.h @mkdir -p $(BUILD_DIR) $(CC) -O2 $(CWARN) -Wno-unused-function -std=c11 -pthread -Isrc \ src/samosa_gateway.c $(GATEWAY_SUPPORT_SRCS) -o $(BUILD_DIR)/samosa-jobsd $(DL_LDFLAGS) @@ -310,18 +311,24 @@ test_fake_native_summarizer: tests/fake_native_summarizer.c $(CC) -O2 $(CWARN) -std=c11 tests/fake_native_summarizer.c \ -o $(BUILD_DIR)/test_fake_native_summarizer -test-native-summarizer-supervisor: test_fake_native_summarizer tests/test_native_summarizer_supervisor.c src/samosa_gateway.c src/samosa_multimodal.c src/samosa_evidence.c src/samosa_html.c +test-native-summarizer-supervisor: test_fake_native_summarizer tests/test_native_summarizer_supervisor.c src/samosa_gateway.c src/samosa_memory.h src/samosa_multimodal.c src/samosa_evidence.c src/samosa_html.c $(CC) -O1 $(CWARN) -Wno-unused-function -std=c11 -pthread -Isrc \ tests/test_native_summarizer_supervisor.c $(GATEWAY_SUPPORT_SRCS) -o $(BUILD_DIR)/test_native_summarizer_supervisor $(DL_LDFLAGS) $(BUILD_DIR)/test_native_summarizer_supervisor \ $(BUILD_DIR)/test_fake_native_summarizer tests/fixtures/native-summarizer/model.gguf -test-runtime-settings: tests/test_runtime_settings.c src/samosa_gateway.c src/samosa_multimodal.c src/samosa_evidence.c src/samosa_html.c +test-runtime-settings: tests/test_runtime_settings.c src/samosa_gateway.c src/samosa_memory.h src/samosa_multimodal.c src/samosa_evidence.c src/samosa_html.c @mkdir -p $(BUILD_DIR) $(CC) -O1 $(CWARN) -Wno-unused-function -std=c11 -pthread -Isrc \ tests/test_runtime_settings.c $(GATEWAY_SUPPORT_SRCS) -o $(BUILD_DIR)/test_runtime_settings $(DL_LDFLAGS) $(BUILD_DIR)/test_runtime_settings +test-memory-harness: tests/test_memory_harness.c src/samosa_gateway.c src/samosa_memory.h src/samosa_multimodal.c src/samosa_evidence.c src/samosa_html.c + @mkdir -p $(BUILD_DIR) + $(CC) -O1 $(CWARN) -Wno-unused-function -std=c11 -pthread -Isrc \ + tests/test_memory_harness.c $(GATEWAY_SUPPORT_SRCS) -o $(BUILD_DIR)/test_memory_harness $(DL_LDFLAGS) + $(BUILD_DIR)/test_memory_harness + # fake_model_download_server: deterministic stand-in for the trusted model # catalog's artifact host (docs/TASKS_UI_CHUTNI.md T0.1/T2.2). Ordinary tests # must never fetch a real multi-gigabyte model artifact. @@ -385,7 +392,7 @@ compiled-gateway-test: samosa-gateway samosa-extract samosa-jobsd samosa-fs test # regression for the PDF page-batch-cap fix. The required routing gate below # must fail when PDFium is absent; it must never turn a portable build into a # false passing/skip result. -test-document-reader-contract: tests/test_document_reader_contract.c src/samosa_gateway.c src/samosa_multimodal.c src/samosa_evidence.c src/samosa_html.c +test-document-reader-contract: tests/test_document_reader_contract.c src/samosa_gateway.c src/samosa_memory.h src/samosa_multimodal.c src/samosa_evidence.c src/samosa_html.c @mkdir -p $(BUILD_DIR) $(CC) -O1 $(CWARN) -Wno-unused-function -std=c11 -pthread -Isrc \ tests/test_document_reader_contract.c $(GATEWAY_SUPPORT_SRCS) \ @@ -394,6 +401,7 @@ test-document-reader-contract: tests/test_document_reader_contract.c src/samosa_ test-document-harness: samosa-gateway test_fake_openai_backend test-document-reader-contract tests/test_document_harness.py tests/document_reader_spy.py BUILD_DIR="$(BUILD_DIR)" python3 tests/test_document_harness.py + node tests/test_document_detail_ui.mjs test-pdf-ocr-runtime: samosa-ocr test-document-reader-contract SAMOSA_OCR="$$PWD/$(BUILD_DIR)/samosa-ocr" \ @@ -401,6 +409,11 @@ test-pdf-ocr-runtime: samosa-ocr test-document-reader-contract SAMOSA_EXTRACT="$${SAMOSA_EXTRACT:-$$PWD/$(BUILD_DIR)/samosa-extract}" \ $(OCR_TEST_PYTHON) tests/test_pdf_ocr_runtime.py +test-document-mixed-pdf: samosa-gateway samosa-ocr test_fake_openai_backend + BUILD_DIR="$(BUILD_DIR)" $(OCR_TEST_PYTHON) tests/test_document_mixed_pdf.py + BUILD_DIR="$(BUILD_DIR)" SAMOSA_MIXED_TEST_CONTEXT=8192 $(OCR_TEST_PYTHON) tests/test_document_mixed_pdf.py + BUILD_DIR="$(BUILD_DIR)" SAMOSA_MIXED_TEST_CONTEXT=8192 SAMOSA_FAKE_DOCUMENT_REVIEW_FAIL=1 $(OCR_TEST_PYTHON) tests/test_document_mixed_pdf.py + test-pdf-ocr-routing: tests/test_pdf_ocr_routing.py @if [ -n "$${SAMOSA_EXTRACT:-}" ]; then \ extractor="$${SAMOSA_EXTRACT}"; \ @@ -503,7 +516,7 @@ test-docx-extractor: tests/test_samosa_docx.c tests/test_samosa_docx.sh src/samo $(MINIZ_READ_SRCS) -o $(BUILD_DIR)/test-samosa-docx SAMOSA_DOCX_TEST_RUNNER=./$(BUILD_DIR)/test-samosa-docx sh tests/test_samosa_docx.sh -test: pagecache-residency-test test-evidence-contract test-html-extractor test-docx-extractor tests/test_expert_cache.c tests/test_kv_cache.c tests/test_repetition_guard.c tests/test_thinking_budget.c tests/test_groupwise_q4.c tests/test_samosa_serve.c tests/test_samosa_wrapper.sh tests/test_atomic_install.sh tests/test_install_path.sh tests/test_gateway_installer.sh tests/test_runtime_only_release.sh tests/test_local_document_runtime_guard.sh tests/test_thinking_output.py tests/test_regression_gate.py tests/test_openrouter_control.py tests/test_route_analysis.py tests/test_spec_accept.py tests/test_converter_quant.py tests/test_package_pdfium.py +test: pagecache-residency-test test-evidence-contract test-html-extractor test-docx-extractor tests/test_expert_cache.c tests/test_kv_cache.c tests/test_repetition_guard.c tests/test_thinking_budget.c tests/test_groupwise_q4.c tests/test_samosa_serve.c tests/test_samosa_wrapper.sh tests/test_atomic_install.sh tests/test_install_path.sh tests/test_gateway_installer.sh tests/test_runtime_only_release.sh tests/test_local_document_runtime_guard.sh tests/test_thinking_output.py tests/test_regression_gate.py tests/test_openrouter_control.py tests/test_route_analysis.py tests/test_spec_accept.py tests/test_converter_quant.py tests/test_package_pdfium.py tests/test_decision_eval.py @mkdir -p $(BUILD_DIR) $(CC) -O1 -Isrc tests/test_expert_cache.c src/expert_cache.c -o $(BUILD_DIR)/test_expert_cache && ./$(BUILD_DIR)/test_expert_cache $(CC) -O1 -Itests tests/test_kv_cache.c tests/kv_cache.c -o $(BUILD_DIR)/test_kv_cache -lm && ./$(BUILD_DIR)/test_kv_cache @@ -523,12 +536,17 @@ test: pagecache-residency-test test-evidence-contract test-html-extractor test-d python3 tests/test_route_analysis.py python3 tests/test_spec_accept.py python3 tests/test_package_pdfium.py + python3 tests/test_decision_eval.py + python3 tests/test_samosa_decision.py + $(MAKE) test-memory-harness @if [ -n "$(NUMPY_PYTHON)" ]; then $(NUMPY_PYTHON) tests/test_converter_quant.py; \ else echo "converter quant tests: SKIP (NumPy environment unavailable)"; fi # The compiled gateway, Chutni, and the Kimi preflight are part of the default # gate. They were previously reachable only by name, so a regression in any of # them left `make test` green. Run as sub-makes, not prerequisites: several # bind fixed ports and must not overlap under `make -j`. + $(MAKE) samosa-fs + python3 tests/test_selected_copy.py $(MAKE) compiled-gateway-test $(MAKE) test-chutni $(MAKE) chutni-gateway-test diff --git a/assets/app.html b/assets/app.html index dad86df..319107a 100644 --- a/assets/app.html +++ b/assets/app.html @@ -5,31 +5,31 @@ - Samosa Chat + Samosa @@ -1555,15 +1544,15 @@

Choose a model