Build San José Legistar decision pipeline

apps/scraper/README.md

@@ -4,13 +4,14 @@ Pulls in government content like bills, court cases, and White House content and
 
 ## Active data sources
 
-Only these five are registered and run by `all`:
+These sources are registered and run by `all`:
 
 | CLI name            | Source and data fetched                                                                    | Stored/used as                                                                                        |
 | ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- |
-| `federalregister`   | Federal Register API presidential documents, then each document's body HTML                | `government_content`; AI article and summary enrichment                                               |
-| `congress`          | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI enrichment                                                 |
-| `scotus`            | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court        | `court_case`; powers court content and AI enrichment                                                  |
+| `federalregister`   | Federal Register API presidential documents, then each document's body HTML                | `government_content`; AI article/summary and feed-image enrichment                                    |
+| `congress`          | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment                                            |
+| `legistar`          | San José Legistar meetings, matters, attachments, histories, and structured votes          | Normalized `local_*` decision, occurrence, document, vote, and ingestion-run tables                   |
+| `scotus`            | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court        | `court_case`; powers court content and AI/feed enrichment                                             |
 | `scc-cvig`          | Hand-configured Santa Clara County voter-guide PDFs                                        | Candidate statements in `CivicApiCache`; the API matches statements to candidates                     |
 | `ca-sos-statements` | California SOS statewide-office candidate-statement pages                                  | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source |
 
@@ -47,13 +48,13 @@ work:
 | Variable                                     | Required by                             | Why it matters                                                                                                     |
 | -------------------------------------------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------ |
 | `POSTGRES_URL`                               | Every active scraper                    | Your Postgres connection. If inserting credentials manually, percent-encode only the username/password components. |
-| `OPENROUTER_API_KEY`                         | `federalregister`, `congress`, `scotus` | Preferred provider for article, summary, image-keyword, and web-research generation.                               |
+| `OPENROUTER_API_KEY`                         | `federalregister`, `congress`, `scotus` | Preferred provider for article, summary, image-keyword, feed-copy, and web-research generation.                    |
 | `OPENROUTER_MODEL`                           | Optional                                | OpenRouter model slug; defaults to `deepseek/deepseek-v4-flash`.                                                   |
 | `LOCAL_LLM_BASE_URL` / `LOCAL_LLM_MODEL`     | Local fallback                          | OpenAI-compatible local text endpoint and model; the Big Mac deployment uses bounded-context Qwen.                 |
 | `DEEPSEEK_API_KEY`                           | Deprecated fallback                     | Keeps direct DeepSeek generation working during the OpenRouter credential migration.                               |
 | `CONGRESS_API_KEY`                           | `congress`                              | Free at [api.congress.gov/sign-up](https://api.congress.gov/sign-up/).                                             |
-| `BFL_API_KEY`                                | Optional                                | Reserved for image workflows; the disabled video feed does not use it.                                             |
-| `LOCAL_FLUX_BASE_URL` / `LOCAL_FLUX_MODEL`   | Optional                                | Local FLUX HTTP fallback for explicit image jobs.                                                                  |
+| `BFL_API_KEY`                                | Optional                                | FLUX feed images; raw content and AI text still persist without it.                                                |
+| `LOCAL_FLUX_BASE_URL` / `LOCAL_FLUX_MODEL`   | Optional                                | Local FLUX HTTP fallback; the Big Mac deployment uses FLUX.2 Klein.                                                |
 | `COURTLISTENER_API_KEY`                      | Optional                                | Higher CourtListener limits for `scotus`.                                                                          |
 | `GOOGLE_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` | Optional pair                           | Google Custom Search article thumbnails.                                                                           |
 | `GOOGLE_GENERATIVE_AI_API_KEY`               | Optional                                | Gemini vision fallback for `scc-cvig` PDF extraction.                                                              |
@@ -79,8 +80,8 @@ pnpm --filter @acme/scraper build
 Vite writes the scraper CLI to `dist/main.js`, the dual-lens backfill to
 `dist/retroactive-lenses.js`, the incomplete-content repair job to
 `dist/reprocess-content.js`, the missing bill-description repair job to
-`dist/backfill-bill-descriptions.js`. The build can also emit shared chunks;
-deploy the
+`dist/backfill-bill-descriptions.js`, and the retroactive-video job to
+`dist/retroactive-videos.js`. The build can also emit shared chunks; deploy the
 whole `dist/` directory rather than copying only an entry file. Linked
 `@acme/*` workspace source is included in the build, while normal third-party
 packages remain runtime dependencies.
@@ -139,7 +140,7 @@ bills that require a generated description are deferred before insertion;
 other content may still be stored raw for later backfill.
 
 For a large, explicitly bounded seed, set `SCRAPER_SKIP_DUAL_LENS=1` to write
-each bill as soon as its required summary and brief are complete.
+each bill as soon as its required summary, brief, and header art are complete.
 The optional lenses can then be filled by `retroactive-lenses` without holding
 up the source backfill.
 

apps/scraper/src/scraper-contracts.ts

@@ -3,11 +3,13 @@ import type { ScraperEnvContract } from "@acme/env";
 import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js";
 import { congressConfig } from "./scrapers/congress.config.js";
 import { federalregisterConfig } from "./scrapers/federalregister.config.js";
+import { legistarConfig } from "./scrapers/legistar.config.js";
 import { sccCvigConfig } from "./scrapers/scc-cvig.config.js";
 import { scotusConfig } from "./scrapers/scotus.config.js";
 
 export const scraperContracts: readonly ScraperEnvContract[] = [
   federalregisterConfig,
+  legistarConfig,
   congressConfig,
   scotusConfig,
   sccCvigConfig,

apps/scraper/src/scrapers.ts

@@ -2,11 +2,13 @@ import type { Scraper } from "./utils/types.js";
 import { caSosStatements } from "./scrapers/ca-sos-statements.js";
 import { congress } from "./scrapers/congress.js";
 import { federalregister } from "./scrapers/federalregister.js";
+import { legistarScraper } from "./scrapers/legistar.js";
 import { openStates } from "./scrapers/open-states.js";
 import { sccCvig } from "./scrapers/scc-cvig.js";
 
 export const scrapers: readonly Scraper[] = [
   federalregister,
+  legistarScraper,
   congress,
   openStates,
   sccCvig,

apps/scraper/src/scrapers/fixtures/legistar/san-jose-sample.json

@@ -0,0 +1,44 @@
+{
+  "body": {
+    "BodyId": 138,
+    "BodyName": "City Council",
+    "BodyTypeName": "Primary Legislative Body",
+    "BodyActiveFlag": 1
+  },
+  "meeting": {
+    "EventId": 7986,
+    "EventBodyId": 138,
+    "EventBodyName": "City Council",
+    "EventDate": "2026-06-23T00:00:00",
+    "EventTime": "1:30 PM",
+    "EventLocation": "Council Chambers"
+  },
+  "item": {
+    "EventItemId": 130822,
+    "EventItemMatterId": 16072,
+    "EventItemTitle": "City Landmark Designation for Property Located at 647 South Sixth Street in Council District 3.",
+    "EventItemActionText": "Approve the designation.",
+    "EventItemAgendaNumber": "4.2"
+  },
+  "matter": {
+    "MatterId": 16072,
+    "MatterTitle": "City Landmark Designation for Property Located at 647 South Sixth Street.",
+    "MatterTypeName": "Land Use",
+    "MatterRequester": "Planning, Building and Code Enforcement",
+    "MatterNotes": null
+  },
+  "staffDocument": {
+    "MatterAttachmentId": 1,
+    "MatterAttachmentName": "Memorandum",
+    "MatterAttachmentDescription": null,
+    "MatterAttachmentIsMinuteOrder": false,
+    "MatterAttachmentIsHyperlink": false
+  },
+  "publicCommentDocument": {
+    "MatterAttachmentId": 2,
+    "MatterAttachmentName": "Letters from the Public",
+    "MatterAttachmentDescription": null,
+    "MatterAttachmentIsMinuteOrder": false,
+    "MatterAttachmentIsHyperlink": false
+  }
+}

apps/scraper/src/scrapers/legistar-policy.test.ts

@@ -0,0 +1,88 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+import type {
+  LegistarAgendaItem,
+  LegistarAttachment,
+  LegistarMatter,
+} from "@acme/api/integrations/legistar";
+
+import {
+  bodyPolicy,
+  classifyDocument,
+  classifyTopic,
+  inferGeographicScope,
+} from "./legistar-policy.js";
+import {
+  nativeTextQuality,
+  needsOcr,
+  parseLegistarMeetingStart,
+} from "./legistar.js";
+
+const fixture = JSON.parse(
+  readFileSync(
+    new URL("./fixtures/legistar/san-jose-sample.json", import.meta.url),
+    "utf8",
+  ),
+) as {
+  item: LegistarAgendaItem;
+  matter: LegistarMatter;
+  staffDocument: LegistarAttachment;
+  publicCommentDocument: LegistarAttachment;
+};
+
+test("curates resident-facing bodies and excludes closed-session buckets", () => {
+  assert.deepEqual(bodyPolicy(138), { included: true, relevanceTier: 1 });
+  assert.deepEqual(bodyPolicy(212), { included: true, relevanceTier: 2 });
+  assert.equal(bodyPolicy(269).included, false);
+  assert.equal(bodyPolicy(244).included, false);
+});
+
+test("classifies topic and conservative geographic scope", () => {
+  assert.equal(classifyTopic(fixture.matter), "housing-land-use");
+  assert.deepEqual(inferGeographicScope(fixture.matter, fixture.item), {
+    kind: "district",
+    districtNumbers: [3],
+    text: "Council District 3",
+  });
+});
+
+test("keeps public comments link-only while extracting staff evidence", () => {
+  assert.deepEqual(classifyDocument(fixture.publicCommentDocument), {
+    category: "public_comment",
+    processingPolicy: "link_only",
+    isPublicComment: true,
+  });
+  assert.deepEqual(classifyDocument(fixture.staffDocument), {
+    category: "staff_report",
+    processingPolicy: "extract_text",
+    isPublicComment: false,
+  });
+});
+
+test("marks image-only PDFs for OCR and accepts dense native text", () => {
+  assert.equal(needsOcr("", 1), true);
+  assert.equal(needsOcr("A".repeat(79), 1), true);
+  assert.equal(needsOcr("A".repeat(80), 1), false);
+  assert.ok(nativeTextQuality("City budget report. ".repeat(50), 1) > 0.9);
+});
+
+test("interprets Legistar local meeting times in San José's timezone", () => {
+  assert.equal(
+    parseLegistarMeetingStart(
+      "2026-06-23T00:00:00",
+      "1:30 PM",
+      "America/Los_Angeles",
+    ).toISOString(),
+    "2026-06-23T20:30:00.000Z",
+  );
+  assert.equal(
+    parseLegistarMeetingStart(
+      "2026-12-15T00:00:00",
+      "1:30 PM",
+      "America/Los_Angeles",
+    ).toISOString(),
+    "2026-12-15T21:30:00.000Z",
+  );
+});