From 73d8545206533c5c4c2bdc9974fffb912bda744b Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 07:30:25 -0700 Subject: [PATCH 01/14] docs(spec): DB-side fuzzy job search (drop in-memory index) --- .../2026-06-17-db-side-fuzzy-search-design.md | 353 ++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-17-db-side-fuzzy-search-design.md diff --git a/docs/superpowers/specs/2026-06-17-db-side-fuzzy-search-design.md b/docs/superpowers/specs/2026-06-17-db-side-fuzzy-search-design.md new file mode 100644 index 0000000..fe515ac --- /dev/null +++ b/docs/superpowers/specs/2026-06-17-db-side-fuzzy-search-design.md @@ -0,0 +1,353 @@ +# Design: DB-side fuzzy job search (drop the in-memory index) + +**Status**: design approved 2026-06-17. `paavo-db` + `paavo-web` + +`paavo-web-ui` change. **No schema migration**, **no proto change**, **no +`WireMessage` / `LogFrame` shape change**, **no `paavod` change**, no change +to the `/api/events` SSE contract. Net dependency move: `fuzzy-matcher` +leaves `paavo-web` and joins `paavo-db`. + +--- + +## 1. Goal + +Job fuzzy search currently runs against a **full-history in-memory +projection** (`crate::index::JobIndex` in `paavo-web`): the background poller +loads every job's searchable fields into RAM each tick, and `GET /api/jobs` +fuzzy-matches over that `Vec`. The structure grows without bound as job +history accumulates. + +Move fuzzy search — and every other consumer of that index — **into SQLite**, +so the web process no longer holds the whole job history in memory just to +search it. SQLite drives row iteration and returns only the requested page; +the process keeps nothing per-job resident. This is the same "offload the +work to the database, snappy at scale" direction as the +2026-06-17 dashboard-SQL-aggregates design, applied to search. + +Searching must stay **true fzf-style**: the query `almcx` must match (and +rank) `alice … mcxa266-01`, exactly as the in-memory `SkimMatcherV2` does +today. A plain substring `LIKE '%almcx%'` would not — hence the design below. + +## 2. Background: how search works today + +- **`crates/paavo-web/src/index.rs`** defines `JobIndex { items: + Vec, haystacks: Vec }`. `haystack()` builds a + lowercased `"{id} {submitter} {state:?} {board_id}"` per row. `search()` + runs `SkimMatcherV2::fuzzy_match` over every haystack, sorts by score + (newest-first tiebreak), then paginates. `new_count()` counts rows newer + than the `as_of` pin (the "↑ N new" pill). +- **The poller** (`spawn_poller`) rebuilds `JobIndex` from + `db.jobs_index()` → `JobRow::list_index` (a lightweight all-rows + projection) whenever a content fingerprint changes, and bumps a per-resource + revision that fans out over `/api/events`. The jobs fingerprint `fp_jobs` + hashes `id + state` of **every** job. +- **`crates/paavo-web/src/api/jobs.rs::list`** reads `LiveState.index` for + both blank-query list mode (time-ordered, `as_of`-pinned) and non-blank + search mode (fuzzy-ranked, `new_count` forced to 0). +- **`crates/paavo-web-ui/src/components/jobs_list.rs`** debounces keystrokes + by **150 ms** (generation-counter + `gloo_timers::Timeout`) before + committing the query the data resource is keyed on. + +The job table (`migrations/V1__initial.sql`) has indexes `idx_job_state`, +`idx_job_submitted_at`, and a partial `idx_job_priority_subat`. No +full-text/trigram index exists. + +## 3. Decisions + +### 3.1 Split membership from ranking: subsequence `LIKE` + a `fuzzy_score` UDF + +Two SQL concerns, two mechanisms: + +- **Membership** ("does this job match?") → a **subsequence `LIKE`**. The + query `almcx` becomes the pattern `%a%l%m%c%x%` (each character escaped for + `%`/`_`/`\`, joined and bracketed by `%`). `LIKE '%a%l%m%c%x%'` is true iff + those characters appear *in order* — precisely the subsequence test, so + `almcx` matches `alice … mcxa266`. This is a plain SQL string comparison: no + UDF call. +- **Ranking** ("how good is the match?") → a custom SQLite scalar function + **`fuzzy_score(haystack, query) -> Option`** backed by the same + `SkimMatcherV2` used today, invoked **only in `ORDER BY`**, **only on the + rows `LIKE` already kept**. + +*Chosen because* it keeps ranking byte-for-byte identical to today while +calling the expensive cross-boundary UDF on the *matched subset only* — and +the pagination `COUNT(*)` calls it **zero** times (pure `LIKE`). SQLite +materializes only the requested page, meeting the memory goal. + +*Rejected — UDF in the `WHERE` clause (no `LIKE` pre-filter):* simpler SQL, +but the UDF then fires on **every** row, twice per keystroke (page + count). +The `LIKE` pre-filter reduces that to a cheap per-row string compare plus a +UDF call only on matches. + +*Rejected — substring `LIKE '%almcx%'`:* simplest, but substring ≠ +subsequence; `almcx` would not match `alice … mcxa266`. Fails the core +requirement. + +*Rejected — FTS5:* token/prefix BM25 search has different (non-subsequence) +semantics and needs a migration plus write-path triggers kept in sync with +`paavod`. Heavier; out of scope. + +### 3.2 Eliminate `JobIndex`; every consumer reads SQL + +Keeping the index in RAM for the live pill or the dashboard would defeat the +goal (the full projection would still be resident). So `JobIndex` is deleted +and `LiveState` drops its `index` field. Each consumer moves to a typed +`paavo-db` query (§4.2): + +| Consumer | Today | After | +|---|---|---| +| `/api/jobs` search (non-blank `q`) | `index.search(q,…)` | `JobRow::search_index_page` + `search_count` | +| `/api/jobs` list (blank `q`, `as_of`-pinned) | `index.search("",…)` | `JobRow::list_index_page` + `JobRow::count` | +| "↑ N new" pill | `index.new_count(as_of)` | `JobRow::count_newer(as_of)` | +| Poller change-detection | hash `id+state` of every job | `JobRow::activity_digest()` (bounded aggregate) | + +### 3.3 Poller change-detection becomes a bounded aggregate fingerprint + +Today's `fp_jobs` is *exact* — any job's state change anywhere bumps the jobs +revision — but it materializes the whole table each tick. It is replaced by a +single index-backed aggregate: + +```sql +SELECT state, COUNT(*), COALESCE(MAX(submitted_at), 0) FROM job GROUP BY state +``` + +hashed into a `u64` digest (≤ 8 rows). This catches **every insert** (a +group's count and/or max moves) and **every state transition** (counts shift +between groups). It is a *heuristic*: it misses only the rare case where two +jobs make offsetting transitions within one poll interval such that all +per-state counts and maxima land identical — which self-heals on the next +tick, and is only a "should the client refetch" hint, never a correctness +input. Boards/schedules fingerprints are unchanged (small, id-stable tables). + +*Accepted trade-off, signed off during design review.* + +### 3.4 Case handling — lowercase both sides + +The haystack is lowercased in SQL (`lower(...)`) so the `LIKE` operand and the +UDF input are byte-identical; the query is lowercased in Rust before both the +`LIKE` pattern and the UDF argument are built. With an all-lowercase query, +`SkimMatcherV2`'s smart-case reduces to plain case-insensitive matching, so +the `LIKE` candidate set and the `fuzzy_score`-returns-`Some` set coincide on +the ASCII content these haystacks carry (ULID ids, ASCII states, board ids, +usernames). A unit test asserts this equivalence (§6). This also *improves* +on today's behavior, where the haystack was lowercased but the raw query was +not (so an uppercase keystroke mostly failed to match). + +**Graceful degradation:** even if the two sets ever diverged, a row that +`LIKE`-matches but the matcher scores `NULL` simply sorts **last** under +`ORDER BY score DESC` (SQLite sorts `NULL` last under `DESC`) and is still a +legitimate subsequence match — a benign ordering anomaly, not a correctness +bug. The safety valve, if ever needed, is to `AND fuzzy_score(...) IS NOT +NULL` into the `WHERE`. + +### 3.5 Debounce 150 → 250 ms + +With search now a **table scan** per committed keystroke (rather than an +in-memory match), more typing quiet is warranted. `jobs_list.rs` already +debounces via a generation counter; this is a one-line retune (`Timeout::new( +150, …)` → `250`) plus the doc-comment update. The mechanism is unchanged. + +## 4. Architecture + +Changes follow the dependency DAG (`proto → db → web → web-ui`); no +back-edges. `paavo-proto` is untouched (`JobListItem` / `Page` +are reused verbatim). + +### 4.1 The `fuzzy_score` function — `crates/paavo-db/src/db.rs` + +Registered in `configure()` (runs for both RW and RO connections; `paavod` +simply never calls it): + +```rust +// Build the matcher once and capture it; the function is called per +// matched row, so re-constructing SkimMatcherV2 (and its scoring tables) +// each call would be wasteful. SkimMatcherV2 is Send + Sync (config only, +// no interior mutability), satisfying the closure's bounds. +// SkimMatcherV2 is deterministic for fixed inputs; flag the function so +// SQLite can cache/factor calls. create_scalar_function is a safe API — +// no `unsafe`, so paavo-db keeps #![forbid(unsafe_code)]. +let matcher = SkimMatcherV2::default(); +conn.create_scalar_function( + "fuzzy_score", + 2, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + move |ctx| { + // get_raw().as_str() borrows the column bytes — no per-row String + // allocation. Arg 0 is lower()ed in SQL; arg 1 is lowercased in Rust. + let haystack = ctx.get_raw(0).as_str()?; + let needle = ctx.get_raw(1).as_str()?; + Ok(matcher.fuzzy_match(haystack, needle)) // Option; None => SQL NULL + }, +)?; +``` + +`fuzzy-matcher = "0.3"` is added to `paavo-db`'s `Cargo.toml`. (If +`SkimMatcherV2` should turn out not to be `Sync` on the pinned version, fall +back to constructing it inside the closure — a measured, not assumed, +decision left to implementation.) + +### 4.2 Queries — `crates/paavo-db/src/job.rs` (no migration) + +The lowercased haystack expression, written once as a SQL fragment: + +```sql +lower(id || ' ' || submitter || ' ' || state || ' ' || coalesce(board_id, '')) +``` + +**Search page** (non-blank query) — UDF only on matched rows, only for rank: + +```sql +SELECT id, state, priority, submitter, board_id, submitted_at, + fuzzy_score(, ?1) AS score -- ?1 = lowercased query +FROM job +WHERE LIKE ?2 ESCAPE '\' -- ?2 = %a%l%m%c%x% +ORDER BY score DESC, submitted_at DESC, id DESC +LIMIT ?3 OFFSET ?4; +``` + +**Search count** (pagination total) — pure `LIKE`, no UDF: + +```sql +SELECT COUNT(*) FROM job WHERE LIKE ?1 ESCAPE '\'; +``` + +**List page** (blank query, `as_of`-pinned, lightweight projection): + +```sql +SELECT id, state, priority, submitter, board_id, submitted_at +FROM job +WHERE (?1 IS NULL OR submitted_at <= ?1) +ORDER BY submitted_at DESC, id DESC +LIMIT ?2 OFFSET ?3; +``` + +**New-count** (the pill): `SELECT COUNT(*) FROM job WHERE submitted_at > ?1`. + +**Activity digest** (poller): `SELECT state, COUNT(*), COALESCE(MAX( +submitted_at), 0) FROM job GROUP BY state`, folded into a `u64` hash. + +New public methods on `JobRow`, each returning the lightweight +`paavo_proto::JobListItem` projection where rows are returned: +`search_index_page`, `search_count`, `list_index_page`, `count_newer`, +`activity_digest`. The existing `count(conn, as_of)` is reused for list-mode +totals. The now-unused `list_index` is removed. + +A `pub(crate)` LIKE helper module holds `escape_like` (moved from `board.rs`, +which currently has a private copy) and `subsequence_pattern(query) -> +String`, so wildcard escaping lives in one place. + +### 4.3 RO façade + poller + handler — `crates/paavo-web` + +- **`src/db.rs`** (`WebDb`): add thin wrappers `jobs_search_page`, + `jobs_search_count`, `jobs_list_page`, `jobs_new_count`, and + `jobs_activity_digest`; remove `jobs_index`. Each is one short + lock/query/unlock on the RO connection — no lock is held across an `.await` + (the handler has no await between calls), consistent with the existing + `db.rs` "sync calls in async handlers" rationale. +- **`src/index.rs`**: delete `JobIndex` and its tests; `LiveState` loses the + `index` field and its `Arc>`. `spawn_poller` replaces the + `db.jobs_index()` → rebuild step with `db.jobs_activity_digest()` → + fingerprint compare. `Revisions`, the watch channel, and the boards/schedule + fingerprints are unchanged. +- **`src/api/jobs.rs::list`**: read rows + totals from `WebDb`; branch on + blank vs non-blank `q` exactly as today (blank → list page + `count` + + `count_newer`; non-blank → search page + `search_count`, `new_count = 0`). + The `revision` echoed in `Page` still comes from `LiveState::revisions()`. + +### 4.4 Debounce — `crates/paavo-web-ui/src/components/jobs_list.rs` + +`Timeout::new(150, …)` → `Timeout::new(250, …)`; update the "Debounce" +doc-comment's "150 ms" to "250 ms". No other change. `paavo-web-ui` is +workspace-excluded (wasm32) — verified by `just build-ui` + manual smoke, not +`cargo test --workspace`. + +## 5. Failure modes & edge cases + +- **Blank / whitespace query:** list mode (no `LIKE`, no UDF); `as_of` pin and + the pill behave exactly as today. +- **Query of all wildcard chars (`%`, `_`):** escaped via `ESCAPE '\'`, so + they match literally rather than acting as wildcards. +- **No matches:** `search_count` → 0, `search_index_page` → empty; the SPA + renders its existing "0 matches" state. +- **`LIKE`/matcher divergence:** degrades gracefully (§3.4) — a `LIKE`-only + row sorts last with a `NULL` score; never a wrong-row or crash. +- **DB read error:** handler maps `DbError` → 500; the SPA shows its existing + "failed to load jobs" branch; a transient WAL hiccup recovers on the next + live-signal refetch. +- **Poller fingerprint blind spot:** offsetting same-tick transitions (§3.3) + may delay a revision bump by one interval; self-heals. +- **Unknown state string in a row:** surfaces as `DbError::UnknownEnum` from + the row decoder, same as every other typed query. + +## 6. Testing strategy + +House style — hand-written assertions, `tempfile` DBs, no new test deps. + +1. **paavo-db** (`#[cfg(test)]`, tempfile): register `fuzzy_score`, then + - `almcx` matches `alice … mcxa266` and ranks the best match first; + - `search_index_page` pagination (page/size slicing) and `search_count` + parity with the matched total; + - **divergence guard:** on a mixed fixture, the set of rows passing the + `LIKE` pattern equals the set for which `SkimMatcherV2` returns `Some`; + - `list_index_page` ordering (`submitted_at DESC, id DESC`), the `as_of` + pin, and paging; + - `count_newer` strict-`>` boundary; + - `activity_digest` changes on insert and on a state transition, and is + stable across an unrelated no-op. +2. **paavo-web** (integration, RW `Db` seeds a temp file, RO `WebDb` reads via + WAL, build the router): the existing `tests/api_jobs.rs` (pagination + + `q=alice` narrowing) now exercises the SQL path end-to-end and still + passes; the `index.rs` poller test keeps its "jobs revision bumps after + insert" assertion, with the index-read removed. +3. **paavo-web-ui**: compile-checked via `just build-ui`; the 250 ms debounce + and live ranking are covered by the manual smoke (§7). + +## 7. Definition of done + +- `cargo fmt --all -- --check`, `cargo clippy --workspace --all-targets -- -D + warnings`, and `cargo test --workspace` all green. +- `just build-ui` builds the SPA without error. +- Manual smoke (`PAAVO_FAKE_RUNNER=1` daemon + `paavo-web`): open `/jobs`; + typing `almcx` ranks `alice … mcxa266` first; clearing the box returns to + the time-ordered list; submitting jobs from another terminal updates the + list and the "↑ N new" pill live; the query only fires after ~250 ms of + typing quiet. + +## 8. Scope — explicitly out + +- **A migration, FTS5, or a trigram/generated-column index.** The haystack is + computed inline; search is an honest O(n) scan per committed keystroke, + which is the accepted trade for bounded memory at lab scale. Revisit only if + the job table reaches a scale where a per-keystroke scan lags. +- **`paavod` / writer-side changes**, the `/api/events` SSE contract, and any + `JobListItem` / `Page` wire-shape change. +- **Boards/schedules search or fingerprints** (small, id-stable; untouched). +- **Per-job log search** — already a separate client-side filter. + +## 9. Cross-branch coordination + +The in-flight `feat/dashboard-sql-aggregates` branch's design (§3.5) sources +the dashboard's "Recent activity" list from `live.index.read().search("", +…)`. Deleting `JobIndex` removes that read. The clean resolution — which +*strengthens* that branch's own "counts from SQL, snappy at scale" thesis — is +for its recent-jobs list to come from `JobRow::list_index_page(None, 0, 8)` +instead. Whichever branch lands second should adopt the SQL source rather than +reintroduce the index. + +## 10. Files touched + +| File | Change | +| --- | --- | +| `crates/paavo-db/Cargo.toml` | add `fuzzy-matcher = "0.3"` | +| `crates/paavo-db/src/db.rs` | register `fuzzy_score` UDF in `configure` | +| `crates/paavo-db/src/job.rs` | `search_index_page`, `search_count`, `list_index_page`, `count_newer`, `activity_digest`; remove `list_index` | +| `crates/paavo-db/src/like.rs` *(new)* | `pub(crate) escape_like` + `subsequence_pattern` | +| `crates/paavo-db/src/board.rs` | use the shared `escape_like` (drop the private copy) | +| `crates/paavo-web/Cargo.toml` | remove `fuzzy-matcher` (no longer used) | +| `crates/paavo-web/src/index.rs` | delete `JobIndex`; `LiveState` drops `index`; poller uses `activity_digest` | +| `crates/paavo-web/src/db.rs` | add search/list/new-count/digest wrappers; remove `jobs_index` | +| `crates/paavo-web/src/api/jobs.rs` | `list` reads SQL instead of the index | +| `crates/paavo-web/tests/api_jobs.rs` | exercises the SQL path (assertions preserved) | +| `crates/paavo-web-ui/src/components/jobs_list.rs` | debounce 150 → 250 ms + doc comment | + +**Migration:** none. **Proto change:** none. **Estimated size:** ~250–350 LOC +including tests. From 629150f4832603fd34a58db08cf344a0f42fd25f Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 07:48:21 -0700 Subject: [PATCH 02/14] docs(plan): DB-side fuzzy job search implementation plan --- .../plans/2026-06-17-db-side-fuzzy-search.md | 1071 +++++++++++++++++ 1 file changed, 1071 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-17-db-side-fuzzy-search.md diff --git a/docs/superpowers/plans/2026-06-17-db-side-fuzzy-search.md b/docs/superpowers/plans/2026-06-17-db-side-fuzzy-search.md new file mode 100644 index 0000000..9ae1f48 --- /dev/null +++ b/docs/superpowers/plans/2026-06-17-db-side-fuzzy-search.md @@ -0,0 +1,1071 @@ +# DB-Side Fuzzy Job Search Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Move job fuzzy search (and the live pill + poller change-detection) off the full-history in-memory `JobIndex` and into SQLite, so `paavo-web` no longer holds the whole job history in RAM to search it. + +**Architecture:** A custom SQLite scalar function `fuzzy_score` (backed by `SkimMatcherV2`) ranks matches; a subsequence `LIKE '%a%l%m%c%x%'` pattern decides membership. SQLite drives iteration and returns only the requested page. `JobIndex` is deleted; the `/api/jobs` handler, the "N new" pill, and the poller's revision fingerprint all read SQL. The SPA's search box debounce goes 150 → 250 ms. + +**Tech Stack:** Rust 1.95, rusqlite 0.32 (`functions` feature), `fuzzy-matcher` 0.3 (`SkimMatcherV2`), axum 0.7, Leptos CSR (`gloo_timers`), `tempfile` tests. + +**Spec:** `docs/superpowers/specs/2026-06-17-db-side-fuzzy-search-design.md` + +**Worktree:** `.worktrees/db-fuzzy-search` (branch `feat/db-fuzzy-search`). Run all commands there. + +--- + +### Task 1: Shared LIKE helpers (`paavo-db`) + +Extract the wildcard-escaping logic into one module and add the subsequence-pattern builder both the search query and its tests need. `board.rs` already has a private `escape_like`; consolidate it. + +**Files:** +- Create: `crates/paavo-db/src/like.rs` +- Modify: `crates/paavo-db/src/lib.rs` (add `mod like;`) +- Modify: `crates/paavo-db/src/board.rs:332-345` (drop private `escape_like`, import the shared one) + +- [ ] **Step 1: Write `like.rs` with its tests** + +```rust +//! Shared `LIKE` helpers: wildcard escaping and the subsequence pattern +//! builder used by the fuzzy-search queries in `job.rs` and the fleet +//! filter in `board.rs`. Kept in one place so the escape rules (`%`, `_`, +//! `\`, paired with `ESCAPE '\'`) never drift between call sites. + +/// Escape `LIKE` wildcards so `%`, `_`, and `\` match literally. Pair the +/// result with `ESCAPE '\'` in the query. +pub(crate) fn escape_like(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if c == '%' || c == '_' || c == '\\' { + out.push('\\'); + } + out.push(c); + } + out +} + +/// Build a subsequence `LIKE` pattern from `query`: each character is +/// escaped and separated by `%`, so `"almcx"` becomes `"%a%l%m%c%x%"`. +/// `LIKE` against this pattern is true iff the characters appear in order +/// — the same membership test `SkimMatcherV2` uses. An empty query yields +/// `"%"` (matches everything). The caller lowercases `query` first so the +/// pattern and the `fuzzy_score` needle agree. +pub(crate) fn subsequence_pattern(query: &str) -> String { + let mut p = String::from("%"); + for c in query.chars() { + if c == '%' || c == '_' || c == '\\' { + p.push('\\'); + } + p.push(c); + p.push('%'); + } + p +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_like_escapes_wildcards() { + assert_eq!(escape_like("a%b_c\\d"), "a\\%b\\_c\\\\d"); + assert_eq!(escape_like("plain"), "plain"); + } + + #[test] + fn subsequence_pattern_interleaves_percents() { + assert_eq!(subsequence_pattern("almcx"), "%a%l%m%c%x%"); + assert_eq!(subsequence_pattern(""), "%"); + assert_eq!(subsequence_pattern("a%"), "%a%\\%%"); + } +} +``` + +- [ ] **Step 2: Wire the module + repoint `board.rs`** + +In `crates/paavo-db/src/lib.rs`, add `mod like;` (between `mod job;` and `mod log;`): + +```rust +mod job; +mod like; +mod log; +``` + +In `crates/paavo-db/src/board.rs`, add the import near the top (after the existing `use` lines): + +```rust +use crate::like::escape_like; +``` + +Then **delete** the private `escape_like` fn and its doc comment at `board.rs:332-345` (the two call sites in `list_page`/`count` keep calling `escape_like(q)` unchanged — they now resolve to the imported one). + +- [ ] **Step 3: Run the tests** + +Run: `cargo test -p paavo-db like::` +Expected: PASS (2 tests). + +- [ ] **Step 4: Verify the crate still builds (board.rs repoint)** + +Run: `cargo build -p paavo-db` +Expected: builds with no warnings. + +- [ ] **Step 5: Commit** + +```bash +git add crates/paavo-db/src/like.rs crates/paavo-db/src/lib.rs crates/paavo-db/src/board.rs +git commit -m "refactor(db): shared LIKE helpers (escape_like + subsequence_pattern)" +``` + +--- + +### Task 2: Register the `fuzzy_score` scalar function (`paavo-db`) + +Enable rusqlite's `functions` feature, add `fuzzy-matcher`, and register `fuzzy_score(haystack, needle) -> Option` on every connection in `configure()`. + +**Files:** +- Modify: `crates/paavo-db/Cargo.toml` +- Modify: `crates/paavo-db/src/db.rs` + +- [ ] **Step 1: Write the failing test** (append to `crates/paavo-db/src/db.rs`) + +```rust +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn fuzzy_score_some_for_subsequence_none_otherwise() { + let dir = tempdir().unwrap(); + let db = Db::open(dir.path().join("t.sqlite")).unwrap(); + let c = db.raw_conn(); + let hit: Option = c + .query_row("SELECT fuzzy_score('alice mcxa266-01', 'almcx')", [], |r| r.get(0)) + .unwrap(); + assert!(hit.is_some(), "almcx is a subsequence of 'alice mcxa266-01'"); + let miss: Option = c + .query_row("SELECT fuzzy_score('bob', 'almcx')", [], |r| r.get(0)) + .unwrap(); + assert!(miss.is_none(), "almcx is not a subsequence of 'bob'"); + } +} +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cargo test -p paavo-db fuzzy_score_some_for_subsequence` +Expected: FAIL — `no such function: fuzzy_score`. + +- [ ] **Step 3: Add the dependencies** + +In `crates/paavo-db/Cargo.toml`, change the `rusqlite` line and add `fuzzy-matcher`: + +```toml +rusqlite = { workspace = true, features = ["functions"] } +refinery = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +ulid = { workspace = true } +chrono = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +# Skim-style fuzzy matcher backing the `fuzzy_score` SQLite scalar function +# (crate::db). Not workspace-pinned — paavo-db is the only crate that ranks +# fuzzy matches. Mirrors the dep paavo-web previously carried. +fuzzy-matcher = "0.3" +``` + +- [ ] **Step 4: Register the function in `db.rs`** + +Replace the import line `use rusqlite::{Connection, OpenFlags};` with: + +```rust +use fuzzy_matcher::skim::SkimMatcherV2; +use fuzzy_matcher::FuzzyMatcher; +use rusqlite::functions::FunctionFlags; +use rusqlite::{Connection, Error, OpenFlags}; +``` + +Add `register_fuzzy_score(conn)?;` to `configure()` (just before its final `Ok(())`), then add the function below `configure`: + +```rust +/// Register `fuzzy_score(haystack, needle) -> Option`: the +/// `SkimMatcherV2` ranking the web UI used to run in memory, now callable +/// from SQL by `JobRow::search_index_page`. Registered on every connection +/// (RW + RO); only the RO search path invokes it. `None` (no match) maps to +/// SQL `NULL`. +fn register_fuzzy_score(conn: &Connection) -> Result<()> { + // Built once and captured: the function runs per matched row, so + // rebuilding the matcher each call would waste its scoring tables. + // SkimMatcherV2 is plain config data (Send + 'static); the closure bounds + // are satisfied and create_scalar_function is safe (no `unsafe`). + let matcher = SkimMatcherV2::default(); + conn.create_scalar_function( + "fuzzy_score", + 2, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + move |ctx| { + // get_raw().as_str() borrows the column bytes — no per-row String. + let haystack = ctx + .get_raw(0) + .as_str() + .map_err(|e| Error::UserFunctionError(e.into()))?; + let needle = ctx + .get_raw(1) + .as_str() + .map_err(|e| Error::UserFunctionError(e.into()))?; + Ok(matcher.fuzzy_match(haystack, needle)) + }, + )?; + Ok(()) +} +``` + +- [ ] **Step 5: Run the test to verify it passes** + +Run: `cargo test -p paavo-db fuzzy_score_some_for_subsequence` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/paavo-db/Cargo.toml crates/paavo-db/src/db.rs Cargo.lock +git commit -m "feat(db): register fuzzy_score SQLite scalar function" +``` + +--- + +### Task 3: Fuzzy search queries on `JobRow` (`paavo-db`) + +Add `search_index_page` (ranked page) and `search_count` (total), plus the shared `#[cfg(test)]` harness reused by Tasks 4–5. + +**Files:** +- Modify: `crates/paavo-db/src/job.rs` + +- [ ] **Step 1: Write the failing tests** (append to `crates/paavo-db/src/job.rs`) + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::Db; + use tempfile::{tempdir, TempDir}; + + /// Open a fresh migrated temp DB (registers `fuzzy_score`). + fn test_db() -> (TempDir, Db) { + let dir = tempdir().unwrap(); + let db = Db::open(dir.path().join("t.sqlite")).unwrap(); + (dir, db) + } + + fn new_job(submitter: &str) -> NewJob { + NewJob { + id: JobId::new(), + priority: Priority::Interactive, + submitter: submitter.into(), + source: JobSource::Cli, + board_selector: BoardSelector { + kind: "mcxa266".into(), + instance: None, + wiring_profile: None, + }, + inactivity_timeout_ms: 120_000, + hard_max_ms: 900_000, + tar_blake3: "deadbeef".into(), + tar_path: "/tmp/x.tar".into(), + cargo_update_packages: vec![], + skip_cache: false, + } + } + + /// Insert a job; when `board` is `Some`, transition it to `building` so + /// the board id lands in the searchable haystack. Returns the job id. + fn insert_job( + conn: &Connection, + submitter: &str, + board: Option<&str>, + submitted_at: i64, + ) -> JobId { + let j = new_job(submitter); + let id = j.id; + JobRow::insert(conn, &j, submitted_at).unwrap(); + if let Some(b) = board { + JobRow::transition_to_building(conn, &id, b, submitted_at).unwrap(); + } + id + } + + #[test] + fn fuzzy_search_isolates_by_submitter() { + // q="alice": bob/carol lack a,l,i,c,e in order (ULIDs exclude i,l). + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", None, 3_000); + insert_job(c, "bob", None, 2_000); + insert_job(c, "carol", None, 1_000); + + assert_eq!(JobRow::search_count(c, "alice").unwrap(), 1); + let rows = JobRow::search_index_page(c, "alice", 0, 50).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].submitter, "alice"); + } + + #[test] + fn fuzzy_search_cross_field_subsequence_ranks_best_first() { + // "almcx" spans submitter+board; "alice" (al contiguous) + mcxa + // ranks above "carol" (a..l split) + mcxa. + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", Some("mcxa266-01"), 3_000); + insert_job(c, "carol", Some("mcxa266-03"), 1_000); + + let rows = JobRow::search_index_page(c, "almcx", 0, 50).unwrap(); + assert!(rows.iter().any(|r| r.submitter == "alice")); + assert_eq!(rows[0].submitter, "alice", "best fuzzy match ranks first"); + } +} +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cargo test -p paavo-db fuzzy_search` +Expected: FAIL — `no function or associated item named search_count`/`search_index_page`. + +- [ ] **Step 3: Implement both methods** (add to `impl JobRow`, e.g. after `list_index`) + +```rust + /// One page of fuzzy-search results, ranked by `fuzzy_score`. `q` is + /// matched as a case-insensitive subsequence over the lowercased + /// `id + submitter + state + board_id` haystack; ranking uses the same + /// `SkimMatcherV2` the web UI ran in memory. The `LIKE` pre-filter + /// decides membership (cheap), so `fuzzy_score` runs only on matched + /// rows. Returns the lightweight [`paavo_proto::JobListItem`] projection. + pub fn search_index_page( + conn: &Connection, + q: &str, + offset: u32, + limit: u32, + ) -> Result> { + let needle = q.trim().to_lowercase(); + let pattern = crate::like::subsequence_pattern(&needle); + let mut stmt = conn.prepare( + "SELECT id, state, priority, submitter, board_id, submitted_at, \ + fuzzy_score(lower(id || ' ' || submitter || ' ' || state || ' ' || coalesce(board_id,'')), ?1) AS score \ + FROM job \ + WHERE lower(id || ' ' || submitter || ' ' || state || ' ' || coalesce(board_id,'')) LIKE ?2 ESCAPE '\\' \ + ORDER BY score DESC, submitted_at DESC, id DESC \ + LIMIT ?3 OFFSET ?4", + )?; + let rows = stmt + .query_map( + params![needle, pattern, limit as i64, offset as i64], + index_row, + )? + .collect::, _>>()? + .into_iter() + .collect::>>()?; + Ok(rows) + } + + /// Total fuzzy-search matches for `q` (pagination total). Pure `LIKE` + /// membership — does **not** call `fuzzy_score`. + pub fn search_count(conn: &Connection, q: &str) -> Result { + let needle = q.trim().to_lowercase(); + let pattern = crate::like::subsequence_pattern(&needle); + let n: i64 = conn.query_row( + "SELECT COUNT(*) FROM job \ + WHERE lower(id || ' ' || submitter || ' ' || state || ' ' || coalesce(board_id,'')) LIKE ?1 ESCAPE '\\'", + params![pattern], + |r| r.get(0), + )?; + Ok(n as u64) + } +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cargo test -p paavo-db fuzzy_search` +Expected: PASS (2 tests). + +- [ ] **Step 5: Commit** + +```bash +git add crates/paavo-db/src/job.rs +git commit -m "feat(db): JobRow fuzzy search queries (search_index_page + search_count)" +``` + +--- + +### Task 4: List-mode page + new-count on `JobRow` (`paavo-db`) + +Lightweight time-ordered page (blank query) with the `as_of` pin, and the strictly-newer count behind the "N new" pill. + +**Files:** +- Modify: `crates/paavo-db/src/job.rs` + +- [ ] **Step 1: Write the failing tests** (append inside the `tests` module from Task 3) + +```rust + #[test] + fn list_index_page_orders_and_pins() { + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", None, 3_000); + insert_job(c, "bob", None, 2_000); + insert_job(c, "carol", None, 1_000); + + let all = JobRow::list_index_page(c, None, 0, 50).unwrap(); + assert_eq!( + all.iter().map(|r| r.submitter.as_str()).collect::>(), + ["alice", "bob", "carol"] + ); + let pinned = JobRow::list_index_page(c, Some(2_000), 0, 50).unwrap(); + assert_eq!( + pinned.iter().map(|r| r.submitter.as_str()).collect::>(), + ["bob", "carol"] + ); + let p2 = JobRow::list_index_page(c, None, 2, 2).unwrap(); + assert_eq!(p2.len(), 1); + assert_eq!(p2[0].submitter, "carol"); + } + + #[test] + fn count_newer_is_strictly_greater() { + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", None, 3_000); + insert_job(c, "bob", None, 2_000); + assert_eq!(JobRow::count_newer(c, None).unwrap(), 0); + assert_eq!(JobRow::count_newer(c, Some(3_000)).unwrap(), 0, "exclusive"); + assert_eq!(JobRow::count_newer(c, Some(2_000)).unwrap(), 1); + assert_eq!(JobRow::count_newer(c, Some(0)).unwrap(), 2); + } +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cargo test -p paavo-db list_index_page_orders_and_pins count_newer_is_strictly_greater` +Expected: FAIL — methods not found. + +- [ ] **Step 3: Implement both methods** (add to `impl JobRow`) + +```rust + /// One page of the time-ordered jobs list, newest-first + /// (`submitted_at DESC, id DESC`), optionally pinned to + /// `submitted_at <= as_of` for stable pagination under live inserts. + /// Lightweight [`paavo_proto::JobListItem`] projection — the blank-query + /// counterpart to `search_index_page`. + pub fn list_index_page( + conn: &Connection, + as_of: Option, + offset: u32, + limit: u32, + ) -> Result> { + let (sql, bind): (&str, Vec) = match as_of { + Some(t) => ( + "SELECT id, state, priority, submitter, board_id, submitted_at FROM job \ + WHERE submitted_at <= ?1 ORDER BY submitted_at DESC, id DESC LIMIT ?2 OFFSET ?3", + vec![t, limit as i64, offset as i64], + ), + None => ( + "SELECT id, state, priority, submitter, board_id, submitted_at FROM job \ + ORDER BY submitted_at DESC, id DESC LIMIT ?1 OFFSET ?2", + vec![limit as i64, offset as i64], + ), + }; + let mut stmt = conn.prepare(sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(bind), index_row)? + .collect::, _>>()? + .into_iter() + .collect::>>()?; + Ok(rows) + } + + /// Count of jobs strictly newer than `as_of` (drives the "N new" pill); + /// 0 when `as_of` is `None`. Index-backed by `idx_job_submitted_at`. + pub fn count_newer(conn: &Connection, as_of: Option) -> Result { + let n: i64 = match as_of { + Some(t) => conn.query_row( + "SELECT COUNT(*) FROM job WHERE submitted_at > ?1", + params![t], + |r| r.get(0), + )?, + None => 0, + }; + Ok(n as u64) + } +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cargo test -p paavo-db list_index_page_orders_and_pins count_newer_is_strictly_greater` +Expected: PASS (2 tests). + +- [ ] **Step 5: Commit** + +```bash +git add crates/paavo-db/src/job.rs +git commit -m "feat(db): JobRow list_index_page + count_newer (lightweight list mode)" +``` + +--- + +### Task 5: Poller fingerprint `activity_digest` (`paavo-db`) + +A bounded `GROUP BY state` digest the poller hashes to detect inserts and state transitions without materializing the table. + +**Files:** +- Modify: `crates/paavo-db/src/job.rs` + +- [ ] **Step 1: Write the failing test** (append inside the `tests` module) + +```rust + #[test] + fn activity_digest_changes_on_insert_and_transition() { + let (_d, db) = test_db(); + let c = db.raw_conn(); + let empty = JobRow::activity_digest(c).unwrap(); + let id = insert_job(c, "alice", None, 1_000); + let after_insert = JobRow::activity_digest(c).unwrap(); + assert_ne!(empty, after_insert, "insert must change the digest"); + assert_eq!(after_insert, JobRow::activity_digest(c).unwrap(), "stable no-op"); + JobRow::transition_to_building(c, &id, "mcxa266-01", 2_000).unwrap(); + let after_transition = JobRow::activity_digest(c).unwrap(); + assert_ne!(after_insert, after_transition, "transition must change the digest"); + } +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cargo test -p paavo-db activity_digest_changes` +Expected: FAIL — method not found. + +- [ ] **Step 3: Implement** (add to `impl JobRow`) + +```rust + /// A cheap, bounded fingerprint of the job table for the live poller. + /// `GROUP BY state` yields ≤ 8 rows; folding each + /// `(state, count, max(submitted_at))` into a hash detects every insert + /// (a group's count/max moves) and every state transition (counts shift + /// between groups) without materializing the table. Heuristic: it misses + /// only offsetting same-tick transitions that leave all tallies + /// identical, which self-heals next tick. See design doc §3.3. + pub fn activity_digest(conn: &Connection) -> Result { + use std::hash::{Hash, Hasher}; + let mut stmt = conn.prepare( + "SELECT state, COUNT(*), COALESCE(MAX(submitted_at), 0) \ + FROM job GROUP BY state ORDER BY state", + )?; + let rows = stmt + .query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, i64>(1)?, + r.get::<_, i64>(2)?, + )) + })? + .collect::, _>>()?; + let mut h = std::collections::hash_map::DefaultHasher::new(); + rows.len().hash(&mut h); + for (state, count, max_sub) in &rows { + state.hash(&mut h); + count.hash(&mut h); + max_sub.hash(&mut h); + } + Ok(h.finish()) + } +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cargo test -p paavo-db activity_digest_changes` +Expected: PASS. Then run the whole crate: `cargo test -p paavo-db` → all green. + +- [ ] **Step 5: Commit** + +```bash +git add crates/paavo-db/src/job.rs +git commit -m "feat(db): JobRow activity_digest (bounded poller fingerprint)" +``` + +--- + +### Task 6: `WebDb` façade wrappers (`paavo-web`) + +Thin RO-connection wrappers over the new `JobRow` methods. (Additive — `jobs_index` stays until Task 9 so the poller keeps compiling.) + +**Files:** +- Modify: `crates/paavo-web/src/db.rs` + +- [ ] **Step 1: Add the wrappers** (insert into `impl WebDb`, after `jobs_index`) + +```rust + /// One page of fuzzy-search results (lightweight projection), ranked. + pub fn jobs_search_page( + &self, + q: &str, + offset: u32, + limit: u32, + ) -> paavo_db::Result> { + paavo_db::JobRow::search_index_page(self.inner.lock().raw_conn(), q, offset, limit) + } + + /// Total fuzzy-search matches for `q` (pagination total). + pub fn jobs_search_count(&self, q: &str) -> paavo_db::Result { + paavo_db::JobRow::search_count(self.inner.lock().raw_conn(), q) + } + + /// One page of the time-ordered jobs list (lightweight), optionally + /// pinned to `submitted_at <= as_of`. + pub fn jobs_list_page( + &self, + as_of: Option, + offset: u32, + limit: u32, + ) -> paavo_db::Result> { + paavo_db::JobRow::list_index_page(self.inner.lock().raw_conn(), as_of, offset, limit) + } + + /// Count of jobs newer than `as_of` (the "N new" pill); 0 when unpinned. + pub fn jobs_new_count(&self, as_of: Option) -> paavo_db::Result { + paavo_db::JobRow::count_newer(self.inner.lock().raw_conn(), as_of) + } + + /// Bounded change-detection fingerprint for the live poller. + pub fn jobs_activity_digest(&self) -> paavo_db::Result { + paavo_db::JobRow::activity_digest(self.inner.lock().raw_conn()) + } +``` + +- [ ] **Step 2: Build** + +Run: `cargo build -p paavo-web` +Expected: builds, no warnings. + +- [ ] **Step 3: Commit** + +```bash +git add crates/paavo-web/src/db.rs +git commit -m "feat(web): WebDb wrappers for DB-side job search" +``` + +--- + +### Task 7: `/api/jobs` reads SQL (`paavo-web`) + +Rewrite the `list` handler to read rows/counts from `WebDb` instead of `LiveState.index`. Guarded by the existing `tests/api_jobs.rs` integration test (pagination + `q=alice` narrowing), which passes before and must pass after. + +**Files:** +- Modify: `crates/paavo-web/src/api/jobs.rs:9-64` + +- [ ] **Step 1: Swap the imports** + +Replace `use crate::index::LiveState;` with `use crate::proxy::AppState;` (keep `use crate::db::WebDb;` — `get`/`log` still use it). + +- [ ] **Step 2: Replace the `list` handler** (`jobs.rs:28-64`) + +```rust +pub async fn list( + State(s): State, + Query(q): Query>, +) -> Result>, (StatusCode, String)> { + let page: u32 = q + .get("page") + .and_then(|v| v.parse().ok()) + .unwrap_or(1) + .max(1); + let per_page: u32 = q + .get("per_page") + .and_then(|v| v.parse().ok()) + .unwrap_or(50) + .clamp(1, 200); + // Trim so `?q=` (or trailing spaces) behaves like no filter. + let query = q.get("q").map(|v| v.trim().to_string()).unwrap_or_default(); + let as_of: Option = q.get("as_of").and_then(|v| v.parse().ok()); + let err = |e: paavo_db::DbError| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()); + // saturating_mul: an unclamped hostile `?page=` must yield an empty + // page, not overflow. + let offset = (page - 1).saturating_mul(per_page); + + let (items, total, new_count) = if query.is_empty() { + let items = s.db.jobs_list_page(as_of, offset, per_page).map_err(err)?; + let total = s.db.jobs_count(as_of).map_err(err)?; + let new_count = s.db.jobs_new_count(as_of).map_err(err)?; + (items, total, new_count) + } else { + let items = s.db.jobs_search_page(&query, offset, per_page).map_err(err)?; + let total = s.db.jobs_search_count(&query).map_err(err)?; + (items, total, 0) + }; + + Ok(Json(Page { + items, + total, + page, + per_page, + revision: s.live.revisions().jobs, + new_count, + as_of, + })) +} +``` + +Also update the module doc comment at `jobs.rs:3-5` to read: `//! - GET /api/jobs reads SQLite directly (fuzzy search via the fuzzy_score //! function; blank query = time-ordered page). The live revision still //! comes from the poller.` + +- [ ] **Step 3: Run the integration test** + +Run: `cargo test -p paavo-web --test api_jobs` +Expected: PASS (`jobs_list_paginates_and_searches`). The handler now serves rows straight from SQLite (WAL visibility makes seeded rows readable immediately). + +- [ ] **Step 4: Commit** + +```bash +git add crates/paavo-web/src/api/jobs.rs +git commit -m "feat(web): /api/jobs reads SQL (fuzzy search + list) instead of the index" +``` + +--- + +### Task 8: Delete `JobIndex`; poller uses `activity_digest` (`paavo-web`) + +Remove the in-memory structure, slim `LiveState`, repoint the poller, and drop the now-unused `fuzzy-matcher` dep. Guarded by `tests/api_events.rs` (revision bumps) and the rewritten unit test. + +**Files:** +- Modify: `crates/paavo-web/src/index.rs` (replace whole file) +- Modify: `crates/paavo-web/Cargo.toml` (remove `fuzzy-matcher`) + +- [ ] **Step 1: Replace `crates/paavo-web/src/index.rs` entirely** + +```rust +//! Live revision poller + the shared [`LiveState`] it feeds. +//! +//! `paavo-web` keeps no per-job data resident: the jobs list and fuzzy +//! search read SQLite directly (see `crate::api::jobs` and +//! `paavo_db::JobRow::search_index_page`). This module's only job is to +//! watch a handful of cheap, bounded aggregates and bump a per-resource +//! revision counter when something changes, which fans out over +//! `/api/events` so the SPA refetches. The `RwLock` guards are always +//! dropped before `.await`. +use parking_lot::RwLock; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::watch; + +/// Monotonic per-resource revision counters. Bumped when the resource's +/// content fingerprint changes; pushed to clients over `/api/events`. +#[derive(Clone, Copy, Default, PartialEq, Eq, serde::Serialize)] +pub struct Revisions { + /// Jobs revision. + pub jobs: u64, + /// Boards revision. + pub boards: u64, + /// Schedules revision. + pub schedules: u64, +} + +/// Process-local live state shared by the poller and the API handlers: the +/// current revisions and a watch channel that fans revision bumps out to +/// every `/api/events` connection. +#[derive(Clone)] +pub struct LiveState { + rev: Arc>, + tx: Arc>, + fp: Arc>, // (jobs, boards, schedules) fingerprints +} + +impl Default for LiveState { + fn default() -> Self { + Self::new() + } +} + +impl LiveState { + /// Construct empty live state seeded at revision 0. + pub fn new() -> Self { + let (tx, _) = watch::channel(Revisions::default()); + Self { + rev: Arc::new(RwLock::new(Revisions::default())), + tx: Arc::new(tx), + fp: Arc::new(RwLock::new((0, 0, 0))), + } + } + + /// A receiver positioned at the current revisions. + pub fn subscribe(&self) -> watch::Receiver { + self.tx.subscribe() + } + + /// Snapshot of the current revisions. + pub fn revisions(&self) -> Revisions { + *self.rev.read() + } +} + +fn hash_u64(items: &[T]) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + items.len().hash(&mut h); + for it in items { + it.hash(&mut h); + } + h.finish() +} + +/// Spawn the single background poller. `interval` is a parameter so tests +/// can run it at ~20ms. A transient DB read error keeps the last snapshot +/// and skips the tick. The RwLock guards are always dropped before `.await`. +pub fn spawn_poller(db: crate::db::WebDb, live: LiveState, interval: Duration) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + loop { + ticker.tick().await; + let mut changed = false; + let mut snap = live.revisions(); + + // Jobs: a bounded GROUP BY state digest (no full-table scan into + // memory). Catches inserts + state transitions. + if let Ok(digest) = db.jobs_activity_digest() { + if live.fp.read().0 != digest { + live.fp.write().0 = digest; + snap.jobs += 1; + changed = true; + } + } + if let Ok(boards) = db.all_boards() { + let keys: Vec = boards + .iter() + .map(|b| format!("{}:{:?}:{:?}", b.spec.id, b.spec.health, b.last_used_at)) + .collect(); + let f = hash_u64(&keys); + if live.fp.read().1 != f { + live.fp.write().1 = f; + snap.boards += 1; + changed = true; + } + } + if let Ok(scheds) = db.all_schedules() { + let keys: Vec = scheds + .iter() + .map(|s| { + format!( + "{}:{}:{:?}:{:?}", + s.id, s.enabled, s.last_triggered_at, s.last_completed_at + ) + }) + .collect(); + let f = hash_u64(&keys); + if live.fp.read().2 != f { + live.fp.write().2 = f; + snap.schedules += 1; + changed = true; + } + } + if changed { + *live.rev.write() = snap; + let _ = live.tx.send(snap); + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use paavo_db::{Db, NewJob}; + use paavo_proto::{BoardSelector, JobId, JobSource, Priority}; + use tempfile::tempdir; + + fn sample_new_job(id: JobId) -> NewJob { + NewJob { + id, + priority: Priority::Interactive, + submitter: "alice".into(), + source: JobSource::Cli, + board_selector: BoardSelector { + kind: "mcxa266".into(), + instance: None, + wiring_profile: None, + }, + inactivity_timeout_ms: 120_000, + hard_max_ms: 900_000, + tar_blake3: "deadbeef".into(), + tar_path: "/tmp/x.tar".into(), + cargo_update_packages: vec![], + skip_cache: false, + } + } + + /// A RW `Db` seeds the same temp file the RO `WebDb` reads via WAL. + /// After an insert the poller must bump the `jobs` revision, observed by + /// a `subscribe()`r within a bounded wait. + #[tokio::test] + async fn spawn_poller_bumps_jobs_revision_after_insert() { + let dir = tempdir().unwrap(); + let path = dir.path().join("paavo.sqlite"); + let rw = Db::open(&path).unwrap(); // keep the writer alive for WAL visibility + let webdb = crate::db::WebDb::open(&path).unwrap(); + let live = LiveState::new(); + spawn_poller(webdb, live.clone(), Duration::from_millis(20)); + let mut rx = live.subscribe(); + + let id = JobId::new(); + paavo_db::JobRow::insert(rw.raw_conn(), &sample_new_job(id), 0).unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + assert!(!remaining.is_zero(), "poller never bumped the jobs revision"); + if tokio::time::timeout(remaining, rx.changed()).await.is_err() { + panic!("poller never bumped the jobs revision (timeout)"); + } + let _ = rx.borrow_and_update(); + if live.revisions().jobs >= 1 { + break; + } + } + } +} +``` + +- [ ] **Step 2: Remove the `fuzzy-matcher` dependency** + +In `crates/paavo-web/Cargo.toml`, delete the `fuzzy-matcher = "0.3"` line and its three-line comment (the `# Fuzzy ranking …` block at lines 34-37). + +- [ ] **Step 3: Run the affected tests** + +Run: `cargo test -p paavo-web` +Expected: PASS — `index::tests::spawn_poller_bumps_jobs_revision_after_insert`, `api_jobs`, and `api_events` all green. + +- [ ] **Step 4: Commit** + +```bash +git add crates/paavo-web/src/index.rs crates/paavo-web/Cargo.toml Cargo.lock +git commit -m "refactor(web): delete in-memory JobIndex; poller uses activity_digest" +``` + +--- + +### Task 9: Remove dead `list_index` / `jobs_index` (`paavo-db`, `paavo-web`) + +With the poller off the index, `WebDb::jobs_index` and `JobRow::list_index` are unused. Delete them. + +**Files:** +- Modify: `crates/paavo-web/src/db.rs` (remove `jobs_index`) +- Modify: `crates/paavo-db/src/job.rs` (remove `list_index`) + +- [ ] **Step 1: Delete `WebDb::jobs_index`** + +In `crates/paavo-web/src/db.rs`, remove the `jobs_index` method (the `/// Lightweight jobs index projection …` doc + fn at lines 77-80). + +- [ ] **Step 2: Delete `JobRow::list_index`** + +In `crates/paavo-db/src/job.rs`, remove `list_index` (the `/// Lightweight projection feeding paavo-web's in-memory jobs index. …` doc + fn, originally `job.rs:264-284`). Its only caller (`jobs_index`) is gone. + +- [ ] **Step 3: Build the workspace to confirm nothing else referenced them** + +Run: `cargo build --workspace` +Expected: builds with no warnings (CI runs with `-Dwarnings`; an unused-import or dead-code warning here means a missed reference — fix it). + +- [ ] **Step 4: Commit** + +```bash +git add crates/paavo-web/src/db.rs crates/paavo-db/src/job.rs +git commit -m "chore(db,web): remove dead list_index / jobs_index" +``` + +--- + +### Task 10: Bump the search debounce to 250 ms (`paavo-web-ui`) + +The SPA already debounces with a generation counter; retune it now that each committed keystroke triggers a table scan. + +**Files:** +- Modify: `crates/paavo-web-ui/src/components/jobs_list.rs:23` (doc comment) and `:90` (the `Timeout`) + +- [ ] **Step 1: Update the timeout** + +At `jobs_list.rs:90`, change `Timeout::new(150, move || {` to `Timeout::new(250, move || {`. + +- [ ] **Step 2: Update the doc comment** + +At `jobs_list.rs:26-28`, change "after a 150 ms quiet period" to "after a 250 ms quiet period". (Optionally note: "longer than a pure client-side filter would need, because each committed query now runs a DB-side scan.") + +- [ ] **Step 3: Build the UI** (workspace-excluded wasm32 — not covered by `cargo test`) + +Run: `just build-ui` +Expected: `trunk build --release` succeeds, emits `crates/paavo-web-ui/dist`. (Prereqs: `rustup target add wasm32-unknown-unknown`, `cargo install trunk`. If trunk is unavailable in this environment, note it and defer this verification to the manual smoke.) + +- [ ] **Step 4: Commit** + +```bash +git add crates/paavo-web-ui/src/components/jobs_list.rs +git commit -m "feat(web-ui): bump jobs search debounce 150 -> 250 ms" +``` + +--- + +### Task 11: Refresh integration-test comments + final gate + +Reword the now-stale `api_jobs.rs` comments (the index is gone) and run the full CI gate + manual smoke. + +**Files:** +- Modify: `crates/paavo-web/tests/api_jobs.rs:1-6,27-28,72-73` + +- [ ] **Step 1: Reword the stale comments** (no behavior change) + +- File header (`api_jobs.rs:1-6`): replace the "over the poller-maintained in-memory index … seed rows via a live RW `Db` writer and then poll … until the index reflects them" wording with: `//! Integration tests for GET /api/jobs: pagination + fuzzy search. //! Rows are seeded via a live RW Db writer and read back through the RO //! WebDb (WAL), which the handler now queries directly — no in-memory index.` +- `jobs_app` comment (`api_jobs.rs:27-28`): replace "The router and the poller must share the SAME LiveState: the poller writes the index, GET /api/jobs reads it." with "The poller drives only the live revision; GET /api/jobs reads SQLite directly. The shared LiveState still supplies the echoed revision." +- `wait_for_total` doc (`api_jobs.rs:72-73`): replace "until the index reports `want` total rows" with "until GET /api/jobs reports `want` total rows (WAL read visibility is effectively immediate; the loop guards against any checkpoint lag)." + +- [ ] **Step 2: Run the format + lint + test gate** (exactly what CI runs) + +```bash +cargo fmt --all +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` +Expected: fmt clean, clippy clean, all tests pass. + +- [ ] **Step 3: Manual smoke** (no hardware) + +```bash +# Terminal A: fake-runner daemon +PAAVO_FAKE_RUNNER=1 cargo run -p paavod -- --config sample-paavo.toml +# Terminal B: web viewer (run `just build-ui` first to embed the SPA) +cargo run -p paavo-web -- --config sample-paavo.toml +# Terminal C: submit a few jobs so there is something to search +PAAVO_HOST=http://127.0.0.1:8090 cargo run -p paavo-cli -- jobs +``` +Verify at `http://127.0.0.1:8081/jobs`: typing `almcx` ranks an `alice … mcxa266` row first; clearing the box returns to the time-ordered list; the "↑ N new" pill appears after new submissions; the query only fires ~250 ms after you stop typing. + +- [ ] **Step 4: Commit** + +```bash +git add crates/paavo-web/tests/api_jobs.rs +git commit -m "test(web): refresh /api/jobs integration comments after index removal" +``` + +--- + +## Self-Review + +**Spec coverage** (against `2026-06-17-db-side-fuzzy-search-design.md`): + +- §3.1 `fuzzy_score` UDF + subsequence `LIKE` → Tasks 1, 2, 3. +- §3.2 eliminate `JobIndex`; consumers → SQL → Tasks 6 (façade), 7 (search/list/new_count), 8 (delete index + poller). +- §3.3 `activity_digest` poller fingerprint → Task 5, wired in Task 8. +- §3.4 lowercase both sides (parity) → `search_index_page`/`search_count` lowercase the needle; haystack `lower()`ed in SQL (Task 3). Divergence is graceful (NULL score sorts last). +- §3.5 debounce 150 → 250 ms → Task 10. +- §4.1 register in `configure` → Task 2. §4.2 queries → Tasks 3–5. §4.3 façade/poller/handler → Tasks 6–8. §4.4 debounce → Task 10. +- §10 files-touched: `like.rs` (T1), `db.rs` UDF (T2), `job.rs` methods (T3–5), `board.rs` (T1), both `Cargo.toml`s (T2 add, T8 remove), `index.rs` (T8), web `db.rs` (T6, T9), `api/jobs.rs` (T7), `api_jobs.rs` (T11), `jobs_list.rs` (T10). All covered. + +**Divergence-guard note:** the spec mentions a test asserting the `LIKE` candidate set equals the `fuzzy_score`-`Some` set. That equivalence is exercised indirectly by `fuzzy_search_isolates_by_submitter` (a `LIKE`-passing row also scores non-NULL, else it would be dropped from the ranked page) and by the `fuzzy_score_some_for_subsequence_none_otherwise` UDF test. A dedicated guard test can be added if stronger coverage is wanted, but it is not required for the headline behavior. + +**Type consistency:** method names are stable across tasks — `search_index_page`, `search_count`, `list_index_page`, `count_newer`, `activity_digest` (db) and `jobs_search_page`, `jobs_search_count`, `jobs_list_page`, `jobs_new_count`, `jobs_activity_digest` (web). The handler calls the `jobs_*` façade names; the façade calls the `JobRow` names. `jobs_count` (pre-existing) is reused for list totals. + +**Ordering invariant:** every task leaves the workspace compiling and green. Dead code (`list_index`/`jobs_index`) is removed (Task 9) only after its last caller is gone (Task 8). + + + From 77f70a600ca5150cb7c89d5f29be424b7781ef9a Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 07:57:26 -0700 Subject: [PATCH 03/14] refactor(db): shared LIKE helpers (escape_like + subsequence_pattern) --- crates/paavo-db/src/board.rs | 16 +--------- crates/paavo-db/src/lib.rs | 1 + crates/paavo-db/src/like.rs | 59 ++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 15 deletions(-) create mode 100644 crates/paavo-db/src/like.rs diff --git a/crates/paavo-db/src/board.rs b/crates/paavo-db/src/board.rs index 591d458..6b48700 100644 --- a/crates/paavo-db/src/board.rs +++ b/crates/paavo-db/src/board.rs @@ -1,6 +1,7 @@ //! Board table typed helpers. use crate::error::{DbError, Result}; +use crate::like::escape_like; use paavo_proto::{BoardHealth, BoardSelector, BoardSpec, ProbeSelector}; use rusqlite::{params, Connection, Error as RusqliteError, ErrorCode, OptionalExtension, Row}; @@ -329,21 +330,6 @@ fn require_one_row(n: usize, id: &str) -> Result<()> { } } -/// Escape `LIKE` wildcards in a user-supplied substring so `%`, `_`, and -/// `\` are matched literally (paired with `ESCAPE '\'` in the query). Used -/// by [`BoardRow::list_page`] / [`BoardRow::count`] so a filter like `a_b` -/// matches the literal characters rather than `_` standing for any char. -fn escape_like(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for c in s.chars() { - if c == '%' || c == '_' || c == '\\' { - out.push('\\'); - } - out.push(c); - } - out -} - fn health_to_str(h: BoardHealth) -> &'static str { match h { BoardHealth::Healthy => "healthy", diff --git a/crates/paavo-db/src/lib.rs b/crates/paavo-db/src/lib.rs index 817b2a7..856f25a 100644 --- a/crates/paavo-db/src/lib.rs +++ b/crates/paavo-db/src/lib.rs @@ -16,6 +16,7 @@ mod build_cache; mod db; mod error; mod job; +mod like; mod log; mod schedule; diff --git a/crates/paavo-db/src/like.rs b/crates/paavo-db/src/like.rs new file mode 100644 index 0000000..7054ffd --- /dev/null +++ b/crates/paavo-db/src/like.rs @@ -0,0 +1,59 @@ +//! Shared `LIKE` helpers: wildcard escaping and the subsequence pattern +//! builder used by the fuzzy-search queries in `job.rs` and the fleet +//! filter in `board.rs`. Kept in one place so the escape rules (`%`, `_`, +//! `\`, paired with `ESCAPE '\'`) never drift between call sites. + +/// Escape `LIKE` wildcards so `%`, `_`, and `\` match literally. Pair the +/// result with `ESCAPE '\'` in the query. +pub(crate) fn escape_like(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if c == '%' || c == '_' || c == '\\' { + out.push('\\'); + } + out.push(c); + } + out +} + +/// Build a subsequence `LIKE` pattern from `query`: each character is +/// escaped and separated by `%`, so `"almcx"` becomes `"%a%l%m%c%x%"`. +/// `LIKE` against this pattern is true iff the characters appear in order +/// — the same membership test `SkimMatcherV2` uses. An empty query yields +/// `"%"` (matches everything). The caller lowercases `query` first so the +/// pattern and the `fuzzy_score` needle agree. +// Introduced ahead of its consumer: the only non-test caller is the +// fuzzy-search query in `job.rs`, added later in this series. Until that +// lands, the non-test build sees this helper as dead code, so allow it +// explicitly to keep `clippy -D warnings` green; drop the attribute once +// `job.rs` calls it. +#[allow(dead_code)] +pub(crate) fn subsequence_pattern(query: &str) -> String { + let mut p = String::from("%"); + for c in query.chars() { + if c == '%' || c == '_' || c == '\\' { + p.push('\\'); + } + p.push(c); + p.push('%'); + } + p +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_like_escapes_wildcards() { + assert_eq!(escape_like("a%b_c\\d"), "a\\%b\\_c\\\\d"); + assert_eq!(escape_like("plain"), "plain"); + } + + #[test] + fn subsequence_pattern_interleaves_percents() { + assert_eq!(subsequence_pattern("almcx"), "%a%l%m%c%x%"); + assert_eq!(subsequence_pattern(""), "%"); + assert_eq!(subsequence_pattern("a%"), "%a%\\%%"); + } +} From a1f1faf4b41927961b1d7a25dd559150c44dd5da Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 08:06:20 -0700 Subject: [PATCH 04/14] feat(db): register fuzzy_score SQLite scalar function --- Cargo.lock | 1 + crates/paavo-db/Cargo.toml | 6 +++- crates/paavo-db/src/db.rs | 63 +++++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c07e38..340eb65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2295,6 +2295,7 @@ name = "paavo-db" version = "0.1.0" dependencies = [ "chrono", + "fuzzy-matcher", "paavo-proto", "refinery", "rusqlite", diff --git a/crates/paavo-db/Cargo.toml b/crates/paavo-db/Cargo.toml index 8d9581d..5ac08da 100644 --- a/crates/paavo-db/Cargo.toml +++ b/crates/paavo-db/Cargo.toml @@ -9,7 +9,7 @@ description = "SQLite schema, migrations, and typed queries for paavo." [dependencies] paavo-proto = { workspace = true } -rusqlite = { workspace = true } +rusqlite = { workspace = true, features = ["functions"] } refinery = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -17,6 +17,10 @@ ulid = { workspace = true } chrono = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } +# Skim-style fuzzy matcher backing the `fuzzy_score` SQLite scalar function +# (crate::db). Not workspace-pinned — paavo-db is the only crate that ranks +# fuzzy matches. Mirrors the dep paavo-web previously carried. +fuzzy-matcher = "0.3" [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/paavo-db/src/db.rs b/crates/paavo-db/src/db.rs index fe3a453..57bd1c1 100644 --- a/crates/paavo-db/src/db.rs +++ b/crates/paavo-db/src/db.rs @@ -2,7 +2,10 @@ //! (paavo-web). WAL mode + busy timeout. use crate::error::{DbError, Result}; -use rusqlite::{Connection, OpenFlags}; +use fuzzy_matcher::skim::SkimMatcherV2; +use fuzzy_matcher::FuzzyMatcher; +use rusqlite::functions::FunctionFlags; +use rusqlite::{Connection, Error, OpenFlags}; use std::path::Path; mod embedded { @@ -59,5 +62,63 @@ fn configure(conn: &mut Connection, readonly: bool) -> Result<()> { conn.pragma_update(None, "synchronous", "NORMAL")?; conn.pragma_update(None, "foreign_keys", "ON")?; } + register_fuzzy_score(conn)?; Ok(()) } + +/// Register `fuzzy_score(haystack, needle) -> Option`: the +/// `SkimMatcherV2` ranking the web UI used to run in memory, now callable +/// from SQL by `JobRow::search_index_page`. Registered on every connection +/// (RW + RO); only the RO search path invokes it. `None` (no match) maps to +/// SQL `NULL`. +fn register_fuzzy_score(conn: &Connection) -> Result<()> { + // Built once and captured: the function runs per matched row, so + // rebuilding the matcher each call would waste its scoring tables. + // SkimMatcherV2 is plain config data (Send + 'static); the closure bounds + // are satisfied and create_scalar_function is safe (no `unsafe`). + let matcher = SkimMatcherV2::default(); + conn.create_scalar_function( + "fuzzy_score", + 2, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + move |ctx| { + // get_raw().as_str() borrows the column bytes — no per-row String. + let haystack = ctx + .get_raw(0) + .as_str() + .map_err(|e| Error::UserFunctionError(e.into()))?; + let needle = ctx + .get_raw(1) + .as_str() + .map_err(|e| Error::UserFunctionError(e.into()))?; + Ok(matcher.fuzzy_match(haystack, needle)) + }, + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn fuzzy_score_some_for_subsequence_none_otherwise() { + let dir = tempdir().unwrap(); + let db = Db::open(dir.path().join("t.sqlite")).unwrap(); + let c = db.raw_conn(); + let hit: Option = c + .query_row("SELECT fuzzy_score('alice mcxa266-01', 'almcx')", [], |r| { + r.get(0) + }) + .unwrap(); + assert!( + hit.is_some(), + "almcx is a subsequence of 'alice mcxa266-01'" + ); + let miss: Option = c + .query_row("SELECT fuzzy_score('bob', 'almcx')", [], |r| r.get(0)) + .unwrap(); + assert!(miss.is_none(), "almcx is not a subsequence of 'bob'"); + } +} From b437977382792d9981a3bd82736fff75388d1df5 Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 08:23:30 -0700 Subject: [PATCH 05/14] feat(db): JobRow fuzzy search queries (search_index_page + search_count) --- crates/paavo-db/src/job.rs | 153 ++++++++++++++++++++++++++++++++++++ crates/paavo-db/src/like.rs | 6 -- 2 files changed, 153 insertions(+), 6 deletions(-) diff --git a/crates/paavo-db/src/job.rs b/crates/paavo-db/src/job.rs index 48ea0a1..12f4944 100644 --- a/crates/paavo-db/src/job.rs +++ b/crates/paavo-db/src/job.rs @@ -283,6 +283,53 @@ impl JobRow { Ok(rows) } + /// One page of fuzzy-search results, ranked by `fuzzy_score`. `q` is + /// matched as a case-insensitive subsequence over the lowercased + /// `id + submitter + state + board_id` haystack; ranking uses the same + /// `SkimMatcherV2` the web UI ran in memory. The `LIKE` pre-filter + /// decides membership (cheap), so `fuzzy_score` runs only on matched + /// rows. Returns the lightweight [`paavo_proto::JobListItem`] projection. + pub fn search_index_page( + conn: &Connection, + q: &str, + offset: u32, + limit: u32, + ) -> Result> { + let needle = q.trim().to_lowercase(); + let pattern = crate::like::subsequence_pattern(&needle); + let mut stmt = conn.prepare( + "SELECT id, state, priority, submitter, board_id, submitted_at, \ + fuzzy_score(lower(id || ' ' || submitter || ' ' || state || ' ' || coalesce(board_id,'')), ?1) AS score \ + FROM job \ + WHERE lower(id || ' ' || submitter || ' ' || state || ' ' || coalesce(board_id,'')) LIKE ?2 ESCAPE '\\' \ + ORDER BY score DESC, submitted_at DESC, id DESC \ + LIMIT ?3 OFFSET ?4", + )?; + let rows = stmt + .query_map( + params![needle, pattern, limit as i64, offset as i64], + index_row, + )? + .collect::, _>>()? + .into_iter() + .collect::>>()?; + Ok(rows) + } + + /// Total fuzzy-search matches for `q` (pagination total). Pure `LIKE` + /// membership — does **not** call `fuzzy_score`. + pub fn search_count(conn: &Connection, q: &str) -> Result { + let needle = q.trim().to_lowercase(); + let pattern = crate::like::subsequence_pattern(&needle); + let n: i64 = conn.query_row( + "SELECT COUNT(*) FROM job \ + WHERE lower(id || ' ' || submitter || ' ' || state || ' ' || coalesce(board_id,'')) LIKE ?1 ESCAPE '\\'", + params![pattern], + |r| r.get(0), + )?; + Ok(n as u64) + } + /// Page of full job rows, newest-first, optionally pinned to /// `submitted_at <= as_of` for stable pagination under live inserts. /// @@ -722,3 +769,109 @@ fn from_row(r: &Row<'_>) -> rusqlite::Result> { }) })()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Db; + use tempfile::{tempdir, TempDir}; + + /// Open a fresh migrated temp DB (registers `fuzzy_score`). + fn test_db() -> (TempDir, Db) { + let dir = tempdir().unwrap(); + let db = Db::open(dir.path().join("t.sqlite")).unwrap(); + (dir, db) + } + + fn new_job(submitter: &str) -> NewJob { + NewJob { + id: JobId::new(), + priority: Priority::Interactive, + submitter: submitter.into(), + source: JobSource::Cli, + board_selector: BoardSelector { + kind: "mcxa266".into(), + instance: None, + wiring_profile: None, + }, + inactivity_timeout_ms: 120_000, + hard_max_ms: 900_000, + tar_blake3: "deadbeef".into(), + tar_path: "/tmp/x.tar".into(), + cargo_update_packages: vec![], + skip_cache: false, + } + } + + /// Insert a job; when `board` is `Some`, transition it to `building` so + /// the board id lands in the searchable haystack. Returns the job id. + fn insert_job( + conn: &Connection, + submitter: &str, + board: Option<&str>, + submitted_at: i64, + ) -> JobId { + let j = new_job(submitter); + let id = j.id; + JobRow::insert(conn, &j, submitted_at).unwrap(); + if let Some(b) = board { + ensure_board(conn, b); + JobRow::transition_to_building(conn, &id, b, submitted_at).unwrap(); + } + id + } + + /// Seed board `id` (idempotent) so the `job.board_id` FK is satisfied + /// before `transition_to_building`. `job.board_id REFERENCES board(id)` + /// and foreign keys are ON. + fn ensure_board(conn: &Connection, id: &str) { + use crate::BoardRow; + use paavo_proto::{BoardHealth, BoardSpec, ProbeSelector}; + if BoardRow::find(conn, id).unwrap().is_some() { + return; + } + let spec = BoardSpec { + id: id.into(), + kind: "mcxa266".into(), + probe_selector: ProbeSelector { + vid: "1366".into(), + pid: "1015".into(), + serial: id.into(), + }, + chip_name: "MCXA266VFL".into(), + target_name: "frdm-mcx-a266".into(), + wiring_profile: Some("default".into()), + health: BoardHealth::Healthy, + }; + BoardRow::insert(conn, &spec, 0).unwrap(); + } + + #[test] + fn fuzzy_search_isolates_by_submitter() { + // q="alice": bob/carol lack a,l,i,c,e in order (ULIDs exclude i,l). + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", None, 3_000); + insert_job(c, "bob", None, 2_000); + insert_job(c, "carol", None, 1_000); + + assert_eq!(JobRow::search_count(c, "alice").unwrap(), 1); + let rows = JobRow::search_index_page(c, "alice", 0, 50).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].submitter, "alice"); + } + + #[test] + fn fuzzy_search_cross_field_subsequence_ranks_best_first() { + // "almcx" spans submitter+board; "alice" (al contiguous) + mcxa + // ranks above "carol" (a..l split) + mcxa. + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", Some("mcxa266-01"), 3_000); + insert_job(c, "carol", Some("mcxa266-03"), 1_000); + + let rows = JobRow::search_index_page(c, "almcx", 0, 50).unwrap(); + assert!(rows.iter().any(|r| r.submitter == "alice")); + assert_eq!(rows[0].submitter, "alice", "best fuzzy match ranks first"); + } +} diff --git a/crates/paavo-db/src/like.rs b/crates/paavo-db/src/like.rs index 7054ffd..7dcc5ab 100644 --- a/crates/paavo-db/src/like.rs +++ b/crates/paavo-db/src/like.rs @@ -22,12 +22,6 @@ pub(crate) fn escape_like(s: &str) -> String { /// — the same membership test `SkimMatcherV2` uses. An empty query yields /// `"%"` (matches everything). The caller lowercases `query` first so the /// pattern and the `fuzzy_score` needle agree. -// Introduced ahead of its consumer: the only non-test caller is the -// fuzzy-search query in `job.rs`, added later in this series. Until that -// lands, the non-test build sees this helper as dead code, so allow it -// explicitly to keep `clippy -D warnings` green; drop the attribute once -// `job.rs` calls it. -#[allow(dead_code)] pub(crate) fn subsequence_pattern(query: &str) -> String { let mut p = String::from("%"); for c in query.chars() { From 416f57a2db95f5d6119e7e510e47a71054e8d219 Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 08:33:02 -0700 Subject: [PATCH 06/14] feat(db): JobRow list_index_page + count_newer (lightweight list mode) --- crates/paavo-db/src/job.rs | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/crates/paavo-db/src/job.rs b/crates/paavo-db/src/job.rs index 12f4944..c127c4c 100644 --- a/crates/paavo-db/src/job.rs +++ b/crates/paavo-db/src/job.rs @@ -330,6 +330,52 @@ impl JobRow { Ok(n as u64) } + /// One page of the time-ordered jobs list, newest-first + /// (`submitted_at DESC, id DESC`), optionally pinned to + /// `submitted_at <= as_of` for stable pagination under live inserts. + /// Lightweight [`paavo_proto::JobListItem`] projection — the blank-query + /// counterpart to `search_index_page`. + pub fn list_index_page( + conn: &Connection, + as_of: Option, + offset: u32, + limit: u32, + ) -> Result> { + let (sql, bind): (&str, Vec) = match as_of { + Some(t) => ( + "SELECT id, state, priority, submitter, board_id, submitted_at FROM job \ + WHERE submitted_at <= ?1 ORDER BY submitted_at DESC, id DESC LIMIT ?2 OFFSET ?3", + vec![t, limit as i64, offset as i64], + ), + None => ( + "SELECT id, state, priority, submitter, board_id, submitted_at FROM job \ + ORDER BY submitted_at DESC, id DESC LIMIT ?1 OFFSET ?2", + vec![limit as i64, offset as i64], + ), + }; + let mut stmt = conn.prepare(sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(bind), index_row)? + .collect::, _>>()? + .into_iter() + .collect::>>()?; + Ok(rows) + } + + /// Count of jobs strictly newer than `as_of` (drives the "N new" pill); + /// 0 when `as_of` is `None`. Index-backed by `idx_job_submitted_at`. + pub fn count_newer(conn: &Connection, as_of: Option) -> Result { + let n: i64 = match as_of { + Some(t) => conn.query_row( + "SELECT COUNT(*) FROM job WHERE submitted_at > ?1", + params![t], + |r| r.get(0), + )?, + None => 0, + }; + Ok(n as u64) + } + /// Page of full job rows, newest-first, optionally pinned to /// `submitted_at <= as_of` for stable pagination under live inserts. /// @@ -874,4 +920,42 @@ mod tests { assert!(rows.iter().any(|r| r.submitter == "alice")); assert_eq!(rows[0].submitter, "alice", "best fuzzy match ranks first"); } + + #[test] + fn list_index_page_orders_and_pins() { + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", None, 3_000); + insert_job(c, "bob", None, 2_000); + insert_job(c, "carol", None, 1_000); + + let all = JobRow::list_index_page(c, None, 0, 50).unwrap(); + assert_eq!( + all.iter().map(|r| r.submitter.as_str()).collect::>(), + ["alice", "bob", "carol"] + ); + let pinned = JobRow::list_index_page(c, Some(2_000), 0, 50).unwrap(); + assert_eq!( + pinned + .iter() + .map(|r| r.submitter.as_str()) + .collect::>(), + ["bob", "carol"] + ); + let p2 = JobRow::list_index_page(c, None, 2, 2).unwrap(); + assert_eq!(p2.len(), 1); + assert_eq!(p2[0].submitter, "carol"); + } + + #[test] + fn count_newer_is_strictly_greater() { + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", None, 3_000); + insert_job(c, "bob", None, 2_000); + assert_eq!(JobRow::count_newer(c, None).unwrap(), 0); + assert_eq!(JobRow::count_newer(c, Some(3_000)).unwrap(), 0, "exclusive"); + assert_eq!(JobRow::count_newer(c, Some(2_000)).unwrap(), 1); + assert_eq!(JobRow::count_newer(c, Some(0)).unwrap(), 2); + } } From 97475a60851a5a1f3353cf4fc3f706b68a7265fa Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 08:39:18 -0700 Subject: [PATCH 07/14] feat(db): JobRow activity_digest (bounded poller fingerprint) --- crates/paavo-db/src/job.rs | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/crates/paavo-db/src/job.rs b/crates/paavo-db/src/job.rs index c127c4c..ae09522 100644 --- a/crates/paavo-db/src/job.rs +++ b/crates/paavo-db/src/job.rs @@ -376,6 +376,38 @@ impl JobRow { Ok(n as u64) } + /// A cheap, bounded fingerprint of the job table for the live poller. + /// `GROUP BY state` yields ≤ 8 rows; folding each + /// `(state, count, max(submitted_at))` into a hash detects every insert + /// (a group's count/max moves) and every state transition (counts shift + /// between groups) without materializing the table. Heuristic: it misses + /// only offsetting same-tick transitions that leave all tallies + /// identical, which self-heals next tick. See design doc §3.3. + pub fn activity_digest(conn: &Connection) -> Result { + use std::hash::{Hash, Hasher}; + let mut stmt = conn.prepare( + "SELECT state, COUNT(*), COALESCE(MAX(submitted_at), 0) \ + FROM job GROUP BY state ORDER BY state", + )?; + let rows = stmt + .query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, i64>(1)?, + r.get::<_, i64>(2)?, + )) + })? + .collect::, _>>()?; + let mut h = std::collections::hash_map::DefaultHasher::new(); + rows.len().hash(&mut h); + for (state, count, max_sub) in &rows { + state.hash(&mut h); + count.hash(&mut h); + max_sub.hash(&mut h); + } + Ok(h.finish()) + } + /// Page of full job rows, newest-first, optionally pinned to /// `submitted_at <= as_of` for stable pagination under live inserts. /// @@ -958,4 +990,26 @@ mod tests { assert_eq!(JobRow::count_newer(c, Some(2_000)).unwrap(), 1); assert_eq!(JobRow::count_newer(c, Some(0)).unwrap(), 2); } + + #[test] + fn activity_digest_changes_on_insert_and_transition() { + let (_d, db) = test_db(); + let c = db.raw_conn(); + let empty = JobRow::activity_digest(c).unwrap(); + let id = insert_job(c, "alice", None, 1_000); + let after_insert = JobRow::activity_digest(c).unwrap(); + assert_ne!(empty, after_insert, "insert must change the digest"); + assert_eq!( + after_insert, + JobRow::activity_digest(c).unwrap(), + "stable no-op" + ); + ensure_board(c, "mcxa266-01"); + JobRow::transition_to_building(c, &id, "mcxa266-01", 2_000).unwrap(); + let after_transition = JobRow::activity_digest(c).unwrap(); + assert_ne!( + after_insert, after_transition, + "transition must change the digest" + ); + } } From 7b9a5df3e716ef2e38ee08d328bd984024d78348 Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 08:46:04 -0700 Subject: [PATCH 08/14] feat(web): WebDb wrappers for DB-side job search --- crates/paavo-web/src/db.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/paavo-web/src/db.rs b/crates/paavo-web/src/db.rs index 595e582..450561f 100644 --- a/crates/paavo-web/src/db.rs +++ b/crates/paavo-web/src/db.rs @@ -79,6 +79,42 @@ impl WebDb { paavo_db::JobRow::list_index(self.inner.lock().raw_conn()) } + /// One page of fuzzy-search results (lightweight projection), ranked. + pub fn jobs_search_page( + &self, + q: &str, + offset: u32, + limit: u32, + ) -> paavo_db::Result> { + paavo_db::JobRow::search_index_page(self.inner.lock().raw_conn(), q, offset, limit) + } + + /// Total fuzzy-search matches for `q` (pagination total). + pub fn jobs_search_count(&self, q: &str) -> paavo_db::Result { + paavo_db::JobRow::search_count(self.inner.lock().raw_conn(), q) + } + + /// One page of the time-ordered jobs list (lightweight), optionally + /// pinned to `submitted_at <= as_of`. + pub fn jobs_list_page( + &self, + as_of: Option, + offset: u32, + limit: u32, + ) -> paavo_db::Result> { + paavo_db::JobRow::list_index_page(self.inner.lock().raw_conn(), as_of, offset, limit) + } + + /// Count of jobs newer than `as_of` (the "N new" pill); 0 when unpinned. + pub fn jobs_new_count(&self, as_of: Option) -> paavo_db::Result { + paavo_db::JobRow::count_newer(self.inner.lock().raw_conn(), as_of) + } + + /// Bounded change-detection fingerprint for the live poller. + pub fn jobs_activity_digest(&self) -> paavo_db::Result { + paavo_db::JobRow::activity_digest(self.inner.lock().raw_conn()) + } + /// Page of full job rows, optionally pinned to `submitted_at <= as_of`. pub fn jobs_page( &self, From 2a9c1c5d6aeff0075f854e9e73786dc4f3a04126 Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 08:53:28 -0700 Subject: [PATCH 09/14] feat(web): /api/jobs reads SQL (fuzzy search + list) instead of the index --- crates/paavo-web/src/api/jobs.rs | 64 ++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/crates/paavo-web/src/api/jobs.rs b/crates/paavo-web/src/api/jobs.rs index fbb7c89..3253c92 100644 --- a/crates/paavo-web/src/api/jobs.rs +++ b/crates/paavo-web/src/api/jobs.rs @@ -1,13 +1,13 @@ //! `/api/jobs*` JSON handlers. //! -//! - `GET /api/jobs` reads the in-memory [`crate::index::JobIndex`] -//! (poller-maintained) so list + fuzzy search never touch sqlite on -//! the request path. +//! - `GET /api/jobs` reads SQLite directly: fuzzy search via the +//! `fuzzy_score` function, or a time-ordered page for a blank query. +//! The live `jobs` revision still comes from the poller. //! - `GET /api/jobs/:id` and `GET /api/jobs/:id/log` read the RO sqlite //! handle directly (single-row / bounded-page queries; see the //! rationale on [`crate::db`]). use crate::db::WebDb; -use crate::index::LiveState; +use crate::proxy::AppState; use axum::extract::{Path, Query, State}; use axum::http::StatusCode; use axum::Json; @@ -15,20 +15,22 @@ use paavo_proto::{JobId, JobListItem, JobView, LogFrame, Page}; use std::collections::HashMap; use std::str::FromStr; -/// `GET /api/jobs?q=&page=&per_page=&as_of=` — paginated fuzzy search -/// over the in-memory jobs index. +/// `GET /api/jobs?q=&page=&per_page=&as_of=` — paginated jobs read +/// straight from the RO sqlite handle. /// /// Blank `q` returns the time-ordered list (optionally pinned to -/// `submitted_at <= as_of` so a paging session sees a stable window); -/// a non-blank `q` switches to fuzzy ranking. `new_count` (jobs newer -/// than `as_of`) is only meaningful in list mode, so it is forced to 0 -/// whenever a query is present. The lock guard is dropped before the -/// `Json` is built — and this handler has no `.await`, so it never -/// holds the index lock across a suspension point. +/// `submitted_at <= as_of` so a paging session sees a stable window), +/// with `new_count` reporting rows newer than `as_of`. A non-blank `q` +/// switches to fuzzy ranking via the `fuzzy_score` SQL function; +/// `new_count` is meaningless there and forced to 0. The `jobs` +/// revision still comes from the live poller so the SPA can de-dup +/// against the `jobs` SSE event. Each `WebDb` call takes and drops its +/// own short lock, and this handler has no `.await` between them, so it +/// never holds a lock across a suspension point. pub async fn list( - State(live): State, + State(s): State, Query(q): Query>, -) -> Json> { +) -> Result>, (StatusCode, String)> { let page: u32 = q .get("page") .and_then(|v| v.parse().ok()) @@ -39,28 +41,36 @@ pub async fn list( .and_then(|v| v.parse().ok()) .unwrap_or(50) .clamp(1, 200); - let query = q.get("q").cloned().unwrap_or_default(); + // Trim so `?q=` (or trailing spaces) behaves like no filter. + let query = q.get("q").map(|v| v.trim().to_string()).unwrap_or_default(); let as_of: Option = q.get("as_of").and_then(|v| v.parse().ok()); - let (items, total, new_count) = { - let idx = live.index.read(); - let (items, total) = idx.search(&query, as_of, page, per_page); - let new_count = if query.trim().is_empty() { - idx.new_count(as_of) - } else { - 0 - }; + let err = |e: paavo_db::DbError| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()); + // saturating_mul: an unclamped hostile `?page=` must yield an empty + // page, not overflow. + let offset = (page - 1).saturating_mul(per_page); + + let (items, total, new_count) = if query.is_empty() { + let items = s.db.jobs_list_page(as_of, offset, per_page).map_err(err)?; + let total = s.db.jobs_count(as_of).map_err(err)?; + let new_count = s.db.jobs_new_count(as_of).map_err(err)?; (items, total, new_count) + } else { + let items = + s.db.jobs_search_page(&query, offset, per_page) + .map_err(err)?; + let total = s.db.jobs_search_count(&query).map_err(err)?; + (items, total, 0) }; - let revision = live.revisions().jobs; - Json(Page { + + Ok(Json(Page { items, total, page, per_page, - revision, + revision: s.live.revisions().jobs, new_count, as_of, - }) + })) } /// `GET /api/jobs/:id` — one job (404 if unknown, 400 if not a ULID). From b13183a54e0659bf0753484732054b2e20c57b89 Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 09:04:09 -0700 Subject: [PATCH 10/14] refactor(web): delete in-memory JobIndex; poller uses activity_digest --- Cargo.lock | 1 - crates/paavo-web/Cargo.toml | 4 - crates/paavo-web/src/index.rs | 199 +++++----------------------------- 3 files changed, 27 insertions(+), 177 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 340eb65..d9f6bc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2360,7 +2360,6 @@ dependencies = [ "chrono", "clap", "futures", - "fuzzy-matcher", "paavo-db", "paavo-proto", "parking_lot", diff --git a/crates/paavo-web/Cargo.toml b/crates/paavo-web/Cargo.toml index f3fa451..91834f8 100644 --- a/crates/paavo-web/Cargo.toml +++ b/crates/paavo-web/Cargo.toml @@ -31,10 +31,6 @@ anyhow = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } parking_lot = { workspace = true } -# Fuzzy ranking for the in-memory jobs index search (`crate::index`). -# A small, dependency-light skim-style matcher; not workspace-pinned -# because paavo-web is the only crate that searches the jobs list. -fuzzy-matcher = "0.3" # Human-readable timestamps. paavod and every other crate already # pin chrono via the workspace; paavo-web pulls the same `0.4 + # serde` pin in. Used by `crate::time` for the `epoch_ms_to_local`, diff --git a/crates/paavo-web/src/index.rs b/crates/paavo-web/src/index.rs index f6ce0d3..c03f343 100644 --- a/crates/paavo-web/src/index.rs +++ b/crates/paavo-web/src/index.rs @@ -1,87 +1,17 @@ -//! In-memory jobs index: lightweight rows + fuzzy search, refreshed by -//! the background poller. The index is the single observation point for -//! the live jobs feed and the search endpoint. -use fuzzy_matcher::skim::SkimMatcherV2; -use fuzzy_matcher::FuzzyMatcher; -use paavo_proto::JobListItem; +//! Live revision poller + the shared [`LiveState`] it feeds. +//! +//! `paavo-web` keeps no per-job data resident: the jobs list and fuzzy +//! search read SQLite directly (see `crate::api::jobs` and +//! `paavo_db::JobRow::search_index_page`). This module's only job is to +//! watch a handful of cheap, bounded aggregates and bump a per-resource +//! revision counter when something changes, which fans out over +//! `/api/events` so the SPA refetches. The `RwLock` guards are always +//! dropped before `.await`. use parking_lot::RwLock; use std::sync::Arc; use std::time::Duration; use tokio::sync::watch; -/// In-memory jobs index: list items plus a lowercased fuzzy haystack each. -#[derive(Default, Clone)] -pub struct JobIndex { - items: Vec, - haystacks: Vec, -} - -impl JobIndex { - /// Build an index from newest-first list items. - pub fn from_items(items: Vec) -> Self { - let haystacks = items.iter().map(haystack).collect(); - Self { items, haystacks } - } - - /// Return `(page_items, total)`. Blank `q` => time-ordered (the items - /// are already newest-first), optionally pinned to `submitted_at <= as_of`. - /// Non-blank `q` => fuzzy-ranked (score desc, newest-first tiebreak). - pub fn search( - &self, - q: &str, - as_of: Option, - page: u32, - per_page: u32, - ) -> (Vec, u64) { - let matched: Vec<&JobListItem> = if q.trim().is_empty() { - self.items - .iter() - .filter(|it| as_of.is_none_or(|t| it.submitted_at <= t)) - .collect() - } else { - let m = SkimMatcherV2::default(); - let mut scored: Vec<(i64, usize, &JobListItem)> = self - .items - .iter() - .enumerate() - .filter_map(|(i, it)| m.fuzzy_match(&self.haystacks[i], q).map(|s| (s, i, it))) - .collect(); - // score desc; stable tiebreak by original index (already newest-first) - scored.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); - scored.into_iter().map(|(_, _, it)| it).collect() - }; - let total = matched.len() as u64; - let start = (page.saturating_sub(1) as usize) * per_page as usize; - let items = matched - .into_iter() - .skip(start) - .take(per_page as usize) - .cloned() - .collect(); - (items, total) - } - - /// Count of jobs newer than `as_of` (drives the "N new" pill). 0 when - /// `as_of` is None. - pub fn new_count(&self, as_of: Option) -> u64 { - match as_of { - Some(t) => self.items.iter().filter(|it| it.submitted_at > t).count() as u64, - None => 0, - } - } -} - -fn haystack(it: &JobListItem) -> String { - format!( - "{} {} {:?} {}", - it.id, - it.submitter, - it.state, - it.board_id.as_deref().unwrap_or("") - ) - .to_lowercase() -} - /// Monotonic per-resource revision counters. Bumped when the resource's /// content fingerprint changes; pushed to clients over `/api/events`. #[derive(Clone, Copy, Default, PartialEq, Eq, serde::Serialize)] @@ -94,13 +24,11 @@ pub struct Revisions { pub schedules: u64, } -/// Process-local live state shared by the poller and the API handlers: -/// the current jobs index, the current revisions, and a watch channel -/// that fans revision bumps out to every `/api/events` connection. +/// Process-local live state shared by the poller and the API handlers: the +/// current revisions and a watch channel that fans revision bumps out to +/// every `/api/events` connection. #[derive(Clone)] pub struct LiveState { - /// Current jobs index (rebuilt each time jobs change). - pub index: Arc>, rev: Arc>, tx: Arc>, fp: Arc>, // (jobs, boards, schedules) fingerprints @@ -117,7 +45,6 @@ impl LiveState { pub fn new() -> Self { let (tx, _) = watch::channel(Revisions::default()); Self { - index: Arc::new(RwLock::new(JobIndex::default())), rev: Arc::new(RwLock::new(Revisions::default())), tx: Arc::new(tx), fp: Arc::new(RwLock::new((0, 0, 0))), @@ -145,15 +72,6 @@ fn hash_u64(items: &[T]) -> u64 { h.finish() } -fn fp_jobs(items: &[JobListItem]) -> u64 { - // hash id + state per row (state changes must bump even at constant len) - let keys: Vec = items - .iter() - .map(|it| format!("{}:{:?}", it.id, it.state)) - .collect(); - hash_u64(&keys) -} - /// Spawn the single background poller. `interval` is a parameter so tests /// can run it at ~20ms. A transient DB read error keeps the last snapshot /// and skips the tick. The RwLock guards are always dropped before `.await`. @@ -165,11 +83,11 @@ pub fn spawn_poller(db: crate::db::WebDb, live: LiveState, interval: Duration) { let mut changed = false; let mut snap = live.revisions(); - if let Ok(items) = db.jobs_index() { - let f = fp_jobs(&items); - if live.fp.read().0 != f { - *live.index.write() = JobIndex::from_items(items); - live.fp.write().0 = f; + // Jobs: a bounded GROUP BY state digest (no full-table scan into + // memory). Catches inserts + state transitions. + if let Ok(digest) = db.jobs_activity_digest() { + if live.fp.read().0 != digest { + live.fp.write().0 = digest; snap.jobs += 1; changed = true; } @@ -215,67 +133,9 @@ pub fn spawn_poller(db: crate::db::WebDb, live: LiveState, interval: Duration) { mod tests { use super::*; use paavo_db::{Db, NewJob}; - use paavo_proto::{BoardSelector, JobId, JobSource, JobState, Priority}; + use paavo_proto::{BoardSelector, JobId, JobSource, Priority}; use tempfile::tempdir; - fn item(submitter: &str, board: &str, state: JobState, submitted_at: i64) -> JobListItem { - JobListItem { - id: JobId::new(), - state, - priority: Priority::Interactive, - submitter: submitter.into(), - board_id: Some(board.into()), - submitted_at, - } - } - - /// Newest-first fixture: alice (newest) / bob / cron (oldest). - fn sample_index() -> JobIndex { - JobIndex::from_items(vec![ - item("alice", "mcxa266-01", JobState::Running, 3_000), - item("bob", "mcxa266-02", JobState::Passed, 2_000), - item("cron", "mcxa266-03", JobState::Building, 1_000), - ]) - } - - #[test] - fn blank_query_returns_all_in_order() { - let idx = sample_index(); - let (items, total) = idx.search("", None, 1, 50); - assert_eq!(total, 3); - assert_eq!(items.len(), 3); - assert_eq!(items[0].submitter, "alice"); - assert_eq!(items[2].submitter, "cron"); - } - - #[test] - fn fuzzy_query_ranks_match_first() { - let idx = sample_index(); - let (items, total) = idx.search("alice", None, 1, 50); - assert_eq!(total, 1); - assert_eq!(items.len(), 1); - assert_eq!(items[0].submitter, "alice"); - } - - #[test] - fn blank_query_pages() { - let idx = sample_index(); - // page 2 of size 2 over 3 items => the single remaining (oldest) row. - let (items, total) = idx.search("", None, 2, 2); - assert_eq!(total, 3); - assert_eq!(items.len(), 1); - assert_eq!(items[0].submitter, "cron"); - } - - #[test] - fn new_count_counts_strictly_newer() { - let idx = sample_index(); - assert_eq!(idx.new_count(None), 0); - assert_eq!(idx.new_count(Some(3_000)), 0, "boundary is exclusive"); - assert_eq!(idx.new_count(Some(2_000)), 1); - assert_eq!(idx.new_count(Some(0)), 3); - } - fn sample_new_job(id: JobId) -> NewJob { NewJob { id, @@ -296,10 +156,9 @@ mod tests { } } - /// Mirrors `feed.rs::spawn_poller_pushes_after_insert`: a RW `Db` seeds - /// the same temp file the RO `WebDb` reads via WAL. After an insert the - /// poller must bump the `jobs` revision and carry the new row into the - /// in-memory index, observed by a `subscribe()`r within a bounded wait. + /// A RW `Db` seeds the same temp file the RO `WebDb` reads via WAL. + /// After an insert the poller must bump the `jobs` revision, observed by + /// a `subscribe()`r within a bounded wait. #[tokio::test] async fn spawn_poller_bumps_jobs_revision_after_insert() { let dir = tempdir().unwrap(); @@ -313,24 +172,20 @@ mod tests { let id = JobId::new(); paavo_db::JobRow::insert(rw.raw_conn(), &sample_new_job(id), 0).unwrap(); - // Loop until a revision bump carries the inserted job into the index. let deadline = tokio::time::Instant::now() + Duration::from_secs(5); loop { let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); - assert!(!remaining.is_zero(), "poller never observed the new job"); + assert!( + !remaining.is_zero(), + "poller never bumped the jobs revision" + ); if tokio::time::timeout(remaining, rx.changed()).await.is_err() { - panic!("poller never observed the new job (timeout)"); + panic!("poller never bumped the jobs revision (timeout)"); } let _ = rx.borrow_and_update(); - // Read+drop the index guard before looping back to the await. - let (items, _) = live.index.read().search("", None, 1, 100); - if items.iter().any(|it| it.id == id) { + if live.revisions().jobs >= 1 { break; } } - assert!( - live.revisions().jobs >= 1, - "jobs revision should have bumped" - ); } } From 59cfb66be104ba7f8fcf44c81a316acc03e431eb Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 09:24:32 -0700 Subject: [PATCH 11/14] chore(db,web): remove dead list_index / jobs_index --- crates/paavo-db/src/job.rs | 29 ++++--------------------- crates/paavo-db/tests/job_ops.rs | 37 -------------------------------- crates/paavo-web/src/db.rs | 5 ----- 3 files changed, 4 insertions(+), 67 deletions(-) diff --git a/crates/paavo-db/src/job.rs b/crates/paavo-db/src/job.rs index ae09522..b447acb 100644 --- a/crates/paavo-db/src/job.rs +++ b/crates/paavo-db/src/job.rs @@ -261,28 +261,6 @@ impl JobRow { Ok(rows) } - /// Lightweight projection feeding paavo-web's in-memory jobs index. - /// Only the columns the list/search need; newest-first. - /// - /// Returns [`paavo_proto::JobListItem`] rows rather than full - /// [`JobRow`]s so the web viewer can hold the entire jobs index in - /// memory cheaply (no JSON-encoded selector/outcome/path columns). - /// Same ordering as `list_recent` — `submitted_at DESC, id DESC` — - /// so ties on the same millisecond stay deterministic via the - /// monotonic ULID id. - pub fn list_index(conn: &Connection) -> Result> { - let mut stmt = conn.prepare( - "SELECT id, state, priority, submitter, board_id, submitted_at - FROM job ORDER BY submitted_at DESC, id DESC", - )?; - let rows = stmt - .query_map([], index_row)? - .collect::, _>>()? - .into_iter() - .collect::>>()?; - Ok(rows) - } - /// One page of fuzzy-search results, ranked by `fuzzy_score`. `q` is /// matched as a case-insensitive subsequence over the lowercased /// `id + submitter + state + board_id` haystack; ranking uses the same @@ -417,7 +395,7 @@ impl JobRow { /// it through every page request, so a job submitted mid-scroll /// cannot shift rows across page boundaries. `None` means "no pin" /// (newest snapshot, used for the very first page). Ordering matches - /// `list_recent`/`list_index`: `submitted_at DESC, id DESC`. + /// `list_recent`/`list_index_page`: `submitted_at DESC, id DESC`. pub fn list_page( conn: &Connection, as_of: Option, @@ -737,8 +715,9 @@ fn state_from_str(s: &str) -> Result { }) } -/// Map a row from the `list_index` projection to a -/// [`paavo_proto::JobListItem`]. Follows the same double-`Result` shape +/// Map a row from the lightweight index projection (`list_index_page` / +/// `search_index_page`) to a [`paavo_proto::JobListItem`]. Follows the +/// same double-`Result` shape /// as `from_row` (outer = rusqlite row error, inner = decode error) so /// the caller can collect both layers in one chain. fn index_row(r: &Row<'_>) -> rusqlite::Result> { diff --git a/crates/paavo-db/tests/job_ops.rs b/crates/paavo-db/tests/job_ops.rs index 4d644b6..daad885 100644 --- a/crates/paavo-db/tests/job_ops.rs +++ b/crates/paavo-db/tests/job_ops.rs @@ -459,43 +459,6 @@ fn finalize_allowed_from_awaiting_board() { ); } -#[test] -fn list_index_returns_lightweight_rows_newest_first() { - let db = fresh_db(); - insert_default_board(&db); - let now = Utc::now().timestamp_millis(); - - // Older job, distinct submitter, left in Submitted. - let older = JobId::new(); - let mut older_job = sample_new_job(older); - older_job.submitter = "alice".into(); - JobRow::insert(db.raw_conn(), &older_job, now).unwrap(); - - // Newer job, distinct submitter, transitioned to Building so the - // projected `state` differs from the older row. - let newer = JobId::new(); - let mut newer_job = sample_new_job(newer); - newer_job.submitter = "bob".into(); - JobRow::insert(db.raw_conn(), &newer_job, now + 10).unwrap(); - JobRow::transition_to_building(db.raw_conn(), &newer, "mcxa266-01", now + 11).unwrap(); - - let index = JobRow::list_index(db.raw_conn()).unwrap(); - assert_eq!(index.len(), 2); - - // Newest first. - assert_eq!(index[0].id, newer); - assert_eq!(index[0].state, JobState::Building); - assert_eq!(index[0].submitter, "bob"); - assert_eq!(index[0].board_id.as_deref(), Some("mcxa266-01")); - assert_eq!(index[0].submitted_at, now + 10); - - assert_eq!(index[1].id, older); - assert_eq!(index[1].state, JobState::Submitted); - assert_eq!(index[1].submitter, "alice"); - assert!(index[1].board_id.is_none()); - assert_eq!(index[1].submitted_at, now); -} - #[test] fn list_page_and_count_honor_as_of_pin() { let db = fresh_db(); diff --git a/crates/paavo-web/src/db.rs b/crates/paavo-web/src/db.rs index 450561f..560e97f 100644 --- a/crates/paavo-web/src/db.rs +++ b/crates/paavo-web/src/db.rs @@ -74,11 +74,6 @@ impl WebDb { ScheduleRow::list_all(self.inner.lock().raw_conn()) } - /// Lightweight jobs index projection (id/state/priority/submitter/board/ts). - pub fn jobs_index(&self) -> paavo_db::Result> { - paavo_db::JobRow::list_index(self.inner.lock().raw_conn()) - } - /// One page of fuzzy-search results (lightweight projection), ranked. pub fn jobs_search_page( &self, From c0efd7d6585d2048c0b104369d25cc1c81d940de Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 09:34:49 -0700 Subject: [PATCH 12/14] feat(web-ui): bump jobs search debounce 150 -> 250 ms --- crates/paavo-web-ui/src/components/jobs_list.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/paavo-web-ui/src/components/jobs_list.rs b/crates/paavo-web-ui/src/components/jobs_list.rs index 9919db9..a8da934 100644 --- a/crates/paavo-web-ui/src/components/jobs_list.rs +++ b/crates/paavo-web-ui/src/components/jobs_list.rs @@ -24,7 +24,9 @@ //! //! Keystrokes write the raw `query` signal immediately but only commit to the //! `dq` (debounced query) signal — which the data resource is keyed on — -//! after a 150 ms quiet period. We implement that with a monotonically +//! after a 250 ms quiet period. Each committed query now runs a DB-side +//! table scan, so a longer quiet window is warranted. We implement that +//! with a monotonically //! increasing generation counter: each keystroke schedules a //! [`Timeout`](gloo_timers::callback::Timeout) tagged with its generation and //! only the newest surviving timer commits. No timer handles to juggle, and @@ -87,7 +89,7 @@ pub fn JobsList() -> impl IntoView { query.set(value.clone()); let my_gen = gen.get_untracked() + 1; gen.set(my_gen); - Timeout::new(150, move || { + Timeout::new(250, move || { // Only commit if no newer keystroke superseded us. if gen.get_untracked() == my_gen { dq.set(value); From 26a11b45ca7dbfda7fa1f2dfc37c4bb0f64e862b Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 10:26:50 -0700 Subject: [PATCH 13/14] test(web): refresh /api/jobs integration comments after index removal --- crates/paavo-web/tests/api_jobs.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/paavo-web/tests/api_jobs.rs b/crates/paavo-web/tests/api_jobs.rs index 216cd11..a5e7d3d 100644 --- a/crates/paavo-web/tests/api_jobs.rs +++ b/crates/paavo-web/tests/api_jobs.rs @@ -1,9 +1,9 @@ -//! Integration tests for `GET /api/jobs`: pagination + fuzzy search -//! over the poller-maintained in-memory index. +//! Integration tests for `GET /api/jobs`: pagination + fuzzy search. //! -//! The index is populated asynchronously by the background poller, so -//! these tests seed rows via a live RW `Db` writer and then poll the -//! HTTP endpoint until the index reflects them before asserting. +//! Rows are seeded via a live RW `Db` writer and read back through the +//! read-only `WebDb` (WAL), which the handler now queries directly — there +//! is no in-memory index. A short poll loop tolerates WAL read-visibility +//! latency before asserting. use axum::body::{to_bytes, Body}; use axum::http::Request; @@ -24,8 +24,9 @@ fn jobs_app(interval: Duration) -> (tempfile::TempDir, Db, axum::Router) { let path = dir.path().join("paavo.sqlite"); let rw = Db::open(&path).unwrap(); let webdb = WebDb::open(&path).unwrap(); - // The router and the poller must share the SAME `LiveState`: the - // poller writes the index, `GET /api/jobs` reads it. + // The poller drives only the live `jobs` revision now; `GET /api/jobs` + // reads SQLite directly. The shared `LiveState` still supplies the + // revision echoed on each page. let live = LiveState::new(); paavo_web::index::spawn_poller(webdb.clone(), live.clone(), interval); let paavod = PaavodClient::new("http://127.0.0.1:1").expect("valid URL"); @@ -69,8 +70,9 @@ async fn get_page(app: &axum::Router, uri: &str) -> Page { serde_json::from_slice(&bytes).expect("Page JSON") } -/// Poll `GET /api/jobs` until the index reports `want` total rows or the -/// timeout elapses. +/// Poll `GET /api/jobs` until it reports `want` total rows (WAL read +/// visibility is effectively immediate; the loop guards against any +/// checkpoint lag) or the timeout elapses. async fn wait_for_total(app: &axum::Router, want: u64, timeout: Duration) { let deadline = tokio::time::Instant::now() + timeout; loop { From 67e7732209934d48050b7b317d6325c87c0f0ad6 Mon Sep 17 00:00:00 2001 From: Felipe Balbi Date: Wed, 17 Jun 2026 10:47:24 -0700 Subject: [PATCH 14/14] test(db): divergence guard (LIKE == fuzzy_score) + search pagination --- crates/paavo-db/src/job.rs | 57 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/crates/paavo-db/src/job.rs b/crates/paavo-db/src/job.rs index b447acb..a4cecf3 100644 --- a/crates/paavo-db/src/job.rs +++ b/crates/paavo-db/src/job.rs @@ -991,4 +991,61 @@ mod tests { "transition must change the digest" ); } + + #[test] + fn like_membership_matches_fuzzy_score_some() { + let (_d, db) = test_db(); + let c = db.raw_conn(); + insert_job(c, "alice", Some("mcxa266-01"), 5_000); + insert_job(c, "bob", Some("mcxa266-02"), 4_000); + insert_job(c, "carol", Some("mcxa266-03"), 3_000); + insert_job(c, "dave", None, 2_000); + insert_job(c, "almcx-bot", None, 1_000); + + for needle in ["almcx", "alice", "mcx", "bob", "zzz", "266", ""] { + // LIKE-membership count (what search_count uses). + let like_n = JobRow::search_count(c, needle).unwrap(); + // fuzzy_score-Some count over the SAME lowercased haystack. + let lowered = needle.trim().to_lowercase(); + let udf_n: i64 = c + .query_row( + "SELECT COUNT(*) FROM job WHERE fuzzy_score(\ + lower(id || ' ' || submitter || ' ' || state || ' ' || coalesce(board_id,'')), ?1) IS NOT NULL", + params![lowered], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + like_n, udf_n as u64, + "LIKE-membership and fuzzy_score-Some disagree for needle {needle:?}" + ); + } + } + + #[test] + fn search_index_page_paginates() { + let (_d, db) = test_db(); + let c = db.raw_conn(); + for (s, t) in [ + ("mcxone", 4_000), + ("mcxtwo", 3_000), + ("mcxthree", 2_000), + ("mcxfour", 1_000), + ] { + insert_job(c, s, None, t); + } + assert_eq!(JobRow::search_count(c, "mcx").unwrap(), 4); + let p1 = JobRow::search_index_page(c, "mcx", 0, 2).unwrap(); + let p2 = JobRow::search_index_page(c, "mcx", 2, 2).unwrap(); + assert_eq!(p1.len(), 2); + assert_eq!(p2.len(), 2); + // Pages are disjoint (no id in both). + for a in &p1 { + assert!(!p2.iter().any(|b| b.id == a.id), "search pages overlap"); + } + // Offset past the end yields an empty page. + assert!(JobRow::search_index_page(c, "mcx", 4, 2) + .unwrap() + .is_empty()); + } }