diff --git a/apps/scraper/README.md b/apps/scraper/README.md index 0af871aa..f5a03698 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -4,13 +4,14 @@ Pulls in government content like bills, court cases, and White House content and ## Active data sources -Only these five are registered and run by `all`: +These sources are registered and run by `all`: | CLI name | Source and data fetched | Stored/used as | | ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | -| `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article and summary enrichment | -| `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI enrichment | -| `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI enrichment | +| `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | +| `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | +| `legistar` | San José Legistar meetings, matters, attachments, histories, and structured votes | Normalized `local_*` decision, occurrence, document, vote, and ingestion-run tables | +| `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | | `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | | `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | @@ -47,13 +48,13 @@ work: | Variable | Required by | Why it matters | | -------------------------------------------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | | `POSTGRES_URL` | Every active scraper | Your Postgres connection. If inserting credentials manually, percent-encode only the username/password components. | -| `OPENROUTER_API_KEY` | `federalregister`, `congress`, `scotus` | Preferred provider for article, summary, image-keyword, and web-research generation. | +| `OPENROUTER_API_KEY` | `federalregister`, `congress`, `scotus` | Preferred provider for article, summary, image-keyword, feed-copy, and web-research generation. | | `OPENROUTER_MODEL` | Optional | OpenRouter model slug; defaults to `deepseek/deepseek-v4-flash`. | | `LOCAL_LLM_BASE_URL` / `LOCAL_LLM_MODEL` | Local fallback | OpenAI-compatible local text endpoint and model; the Big Mac deployment uses bounded-context Qwen. | | `DEEPSEEK_API_KEY` | Deprecated fallback | Keeps direct DeepSeek generation working during the OpenRouter credential migration. | | `CONGRESS_API_KEY` | `congress` | Free at [api.congress.gov/sign-up](https://api.congress.gov/sign-up/). | -| `BFL_API_KEY` | Optional | Reserved for image workflows; the disabled video feed does not use it. | -| `LOCAL_FLUX_BASE_URL` / `LOCAL_FLUX_MODEL` | Optional | Local FLUX HTTP fallback for explicit image jobs. | +| `BFL_API_KEY` | Optional | FLUX feed images; raw content and AI text still persist without it. | +| `LOCAL_FLUX_BASE_URL` / `LOCAL_FLUX_MODEL` | Optional | Local FLUX HTTP fallback; the Big Mac deployment uses FLUX.2 Klein. | | `COURTLISTENER_API_KEY` | Optional | Higher CourtListener limits for `scotus`. | | `GOOGLE_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` | Optional pair | Google Custom Search article thumbnails. | | `GOOGLE_GENERATIVE_AI_API_KEY` | Optional | Gemini vision fallback for `scc-cvig` PDF extraction. | @@ -79,8 +80,8 @@ pnpm --filter @acme/scraper build Vite writes the scraper CLI to `dist/main.js`, the dual-lens backfill to `dist/retroactive-lenses.js`, the incomplete-content repair job to `dist/reprocess-content.js`, the missing bill-description repair job to -`dist/backfill-bill-descriptions.js`. The build can also emit shared chunks; -deploy the +`dist/backfill-bill-descriptions.js`, and the retroactive-video job to +`dist/retroactive-videos.js`. The build can also emit shared chunks; deploy the whole `dist/` directory rather than copying only an entry file. Linked `@acme/*` workspace source is included in the build, while normal third-party packages remain runtime dependencies. @@ -139,7 +140,7 @@ bills that require a generated description are deferred before insertion; other content may still be stored raw for later backfill. For a large, explicitly bounded seed, set `SCRAPER_SKIP_DUAL_LENS=1` to write -each bill as soon as its required summary and brief are complete. +each bill as soon as its required summary, brief, and header art are complete. The optional lenses can then be filled by `retroactive-lenses` without holding up the source backfill. diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04f..ea86d429 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -3,11 +3,13 @@ import type { ScraperEnvContract } from "@acme/env"; import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js"; import { congressConfig } from "./scrapers/congress.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; +import { legistarConfig } from "./scrapers/legistar.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; export const scraperContracts: readonly ScraperEnvContract[] = [ federalregisterConfig, + legistarConfig, congressConfig, scotusConfig, sccCvigConfig, diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index 3167e953..fbda1d8c 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -2,11 +2,13 @@ import type { Scraper } from "./utils/types.js"; import { caSosStatements } from "./scrapers/ca-sos-statements.js"; import { congress } from "./scrapers/congress.js"; import { federalregister } from "./scrapers/federalregister.js"; +import { legistarScraper } from "./scrapers/legistar.js"; import { openStates } from "./scrapers/open-states.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; export const scrapers: readonly Scraper[] = [ federalregister, + legistarScraper, congress, openStates, sccCvig, diff --git a/apps/scraper/src/scrapers/fixtures/legistar/san-jose-sample.json b/apps/scraper/src/scrapers/fixtures/legistar/san-jose-sample.json new file mode 100644 index 00000000..1e792607 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/legistar/san-jose-sample.json @@ -0,0 +1,44 @@ +{ + "body": { + "BodyId": 138, + "BodyName": "City Council", + "BodyTypeName": "Primary Legislative Body", + "BodyActiveFlag": 1 + }, + "meeting": { + "EventId": 7986, + "EventBodyId": 138, + "EventBodyName": "City Council", + "EventDate": "2026-06-23T00:00:00", + "EventTime": "1:30 PM", + "EventLocation": "Council Chambers" + }, + "item": { + "EventItemId": 130822, + "EventItemMatterId": 16072, + "EventItemTitle": "City Landmark Designation for Property Located at 647 South Sixth Street in Council District 3.", + "EventItemActionText": "Approve the designation.", + "EventItemAgendaNumber": "4.2" + }, + "matter": { + "MatterId": 16072, + "MatterTitle": "City Landmark Designation for Property Located at 647 South Sixth Street.", + "MatterTypeName": "Land Use", + "MatterRequester": "Planning, Building and Code Enforcement", + "MatterNotes": null + }, + "staffDocument": { + "MatterAttachmentId": 1, + "MatterAttachmentName": "Memorandum", + "MatterAttachmentDescription": null, + "MatterAttachmentIsMinuteOrder": false, + "MatterAttachmentIsHyperlink": false + }, + "publicCommentDocument": { + "MatterAttachmentId": 2, + "MatterAttachmentName": "Letters from the Public", + "MatterAttachmentDescription": null, + "MatterAttachmentIsMinuteOrder": false, + "MatterAttachmentIsHyperlink": false + } +} diff --git a/apps/scraper/src/scrapers/legistar-policy.test.ts b/apps/scraper/src/scrapers/legistar-policy.test.ts new file mode 100644 index 00000000..52c36f46 --- /dev/null +++ b/apps/scraper/src/scrapers/legistar-policy.test.ts @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +import type { + LegistarAgendaItem, + LegistarAttachment, + LegistarMatter, +} from "@acme/api/integrations/legistar"; + +import { + bodyPolicy, + classifyDocument, + classifyTopic, + inferGeographicScope, +} from "./legistar-policy.js"; +import { + nativeTextQuality, + needsOcr, + parseLegistarMeetingStart, +} from "./legistar.js"; + +const fixture = JSON.parse( + readFileSync( + new URL("./fixtures/legistar/san-jose-sample.json", import.meta.url), + "utf8", + ), +) as { + item: LegistarAgendaItem; + matter: LegistarMatter; + staffDocument: LegistarAttachment; + publicCommentDocument: LegistarAttachment; +}; + +test("curates resident-facing bodies and excludes closed-session buckets", () => { + assert.deepEqual(bodyPolicy(138), { included: true, relevanceTier: 1 }); + assert.deepEqual(bodyPolicy(212), { included: true, relevanceTier: 2 }); + assert.equal(bodyPolicy(269).included, false); + assert.equal(bodyPolicy(244).included, false); +}); + +test("classifies topic and conservative geographic scope", () => { + assert.equal(classifyTopic(fixture.matter), "housing-land-use"); + assert.deepEqual(inferGeographicScope(fixture.matter, fixture.item), { + kind: "district", + districtNumbers: [3], + text: "Council District 3", + }); +}); + +test("keeps public comments link-only while extracting staff evidence", () => { + assert.deepEqual(classifyDocument(fixture.publicCommentDocument), { + category: "public_comment", + processingPolicy: "link_only", + isPublicComment: true, + }); + assert.deepEqual(classifyDocument(fixture.staffDocument), { + category: "staff_report", + processingPolicy: "extract_text", + isPublicComment: false, + }); +}); + +test("marks image-only PDFs for OCR and accepts dense native text", () => { + assert.equal(needsOcr("", 1), true); + assert.equal(needsOcr("A".repeat(79), 1), true); + assert.equal(needsOcr("A".repeat(80), 1), false); + assert.ok(nativeTextQuality("City budget report. ".repeat(50), 1) > 0.9); +}); + +test("interprets Legistar local meeting times in San José's timezone", () => { + assert.equal( + parseLegistarMeetingStart( + "2026-06-23T00:00:00", + "1:30 PM", + "America/Los_Angeles", + ).toISOString(), + "2026-06-23T20:30:00.000Z", + ); + assert.equal( + parseLegistarMeetingStart( + "2026-12-15T00:00:00", + "1:30 PM", + "America/Los_Angeles", + ).toISOString(), + "2026-12-15T21:30:00.000Z", + ); +}); diff --git a/apps/scraper/src/scrapers/legistar-policy.ts b/apps/scraper/src/scrapers/legistar-policy.ts new file mode 100644 index 00000000..2bb37cd5 --- /dev/null +++ b/apps/scraper/src/scrapers/legistar-policy.ts @@ -0,0 +1,231 @@ +import type { + LegistarAgendaItem, + LegistarAttachment, + LegistarMatter, +} from "@acme/api/integrations/legistar"; + +export interface BodyPolicy { + included: boolean; + relevanceTier: number; +} + +/** + * San José's active Legistar body list includes internal administration, + * closed sessions, notices, and generic buckets alongside public decision + * makers. This explicit allowlist is the first-release editorial boundary. + */ +export const SAN_JOSE_BODY_POLICY: Readonly> = { + // Primary decision makers and standing policy committees. + 138: { included: true, relevanceTier: 1 }, // City Council + 139: { included: true, relevanceTier: 1 }, // Neighborhood Services & Education + 140: { included: true, relevanceTier: 1 }, // Community & Economic Development + 172: { included: true, relevanceTier: 1 }, // Public Safety & Finance + 223: { included: true, relevanceTier: 1 }, // Transportation & Environment + 231: { included: true, relevanceTier: 1 }, // Rules & Open Government + + // High-impact hearings and resident-facing commissions. + 198: { included: true, relevanceTier: 2 }, // Airport Commission + 199: { included: true, relevanceTier: 2 }, // Appeals Hearing Board + 205: { included: true, relevanceTier: 2 }, // Campaign and political practices + 206: { included: true, relevanceTier: 2 }, // Historic Landmarks + 207: { included: true, relevanceTier: 2 }, // Housing & Community Development + 209: { included: true, relevanceTier: 2 }, // Library & Education + 211: { included: true, relevanceTier: 2 }, // Parks & Recreation + 212: { included: true, relevanceTier: 2 }, // Planning Commission + 224: { included: true, relevanceTier: 2 }, // Treatment Plant Advisory + 230: { included: true, relevanceTier: 2 }, // Bicycle/Pedestrian + 237: { included: true, relevanceTier: 2 }, // Planning Director's Hearing + 246: { included: true, relevanceTier: 2 }, // Arena Authority + 254: { included: true, relevanceTier: 2 }, // Measure T Oversight + 265: { included: true, relevanceTier: 2 }, // Climate Advisory + + // Community constituencies and city quality-of-life bodies. + 200: { included: true, relevanceTier: 3 }, // Arts + 201: { included: true, relevanceTier: 3 }, // Civil Service + 203: { included: true, relevanceTier: 3 }, // Salary Setting + 213: { included: true, relevanceTier: 3 }, // Senior Citizens + 214: { included: true, relevanceTier: 3 }, // Youth + 239: { included: true, relevanceTier: 3 }, // Privacy Taskforce + 242: { included: true, relevanceTier: 3 }, // Smart City + 245: { included: true, relevanceTier: 3 }, // Youth Empowerment Alliance + 256: { included: true, relevanceTier: 3 }, // Small Business +}; + +export function bodyPolicy(sourceBodyId: number): BodyPolicy { + return ( + SAN_JOSE_BODY_POLICY[sourceBodyId] ?? { + included: false, + relevanceTier: 4, + } + ); +} + +const TOPICS: readonly [string, RegExp][] = [ + ["housing-land-use", /housing|zoning|planning|development|land use|permit/i], + [ + "transportation", + /transport|traffic|street|parking|bicycle|pedestrian|transit/i, + ], + ["public-safety", /police|fire|crime|emergency|public safety/i], + [ + "budget-finance", + /budget|appropriation|tax|fee|fiscal|contract|purchase|bond/i, + ], + [ + "environment-utilities", + /climate|environment|water|waste|energy|sewer|utility/i, + ], + [ + "community-services", + /park|library|education|arts|youth|senior|neighborhood/i, + ], + [ + "ethics-government", + /election|campaign|ethic|open government|privacy|audit/i, + ], +]; + +export function classifyTopic(matter: LegistarMatter): string { + const haystack = [ + matter.MatterTitle, + matter.MatterTypeName, + matter.MatterRequester, + matter.MatterNotes, + ] + .filter(Boolean) + .join(" "); + return TOPICS.find(([, pattern]) => pattern.test(haystack))?.[0] ?? "other"; +} + +export interface GeographicScope { + kind: "citywide" | "district" | "place" | "unknown"; + districtNumbers: number[] | null; + text: string | null; +} + +export function inferGeographicScope( + matter: LegistarMatter, + item: LegistarAgendaItem, +): GeographicScope { + const text = [ + matter.MatterTitle, + matter.MatterNotes, + item.EventItemTitle, + item.EventItemActionText, + ] + .filter(Boolean) + .join(" "); + const districts = [ + ...text.matchAll(/(?:council\s+)?district\s+(?:no\.?\s*)?(1[0]|[1-9])\b/gi), + ].map((match) => Number(match[1])); + const districtNumbers = [...new Set(districts)].sort((a, b) => a - b); + if (districtNumbers.length) { + return { + kind: "district", + districtNumbers, + text: `Council District ${districtNumbers.join(", ")}`, + }; + } + + const address = text.match( + /\b\d{1,6}\s+[A-Z][\w.'-]*(?:\s+[A-Z][\w.'-]*){0,4}\s+(?:Street|St\.?|Avenue|Ave\.?|Road|Rd\.?|Boulevard|Blvd\.?|Drive|Dr\.?|Lane|Ln\.?|Way|Court|Ct\.?)\b/i, + )?.[0]; + if (address) return { kind: "place", districtNumbers: null, text: address }; + + if (/citywide|city-wide|municipal code|annual budget/i.test(text)) { + return { kind: "citywide", districtNumbers: null, text: "Citywide" }; + } + return { kind: "unknown", districtNumbers: null, text: null }; +} + +export type DocumentCategory = + | "public_comment" + | "staff_report" + | "ordinance" + | "resolution" + | "fiscal" + | "presentation" + | "minutes_order" + | "reference" + | "other"; + +export interface DocumentPolicy { + category: DocumentCategory; + processingPolicy: "extract_text" | "link_only"; + isPublicComment: boolean; +} + +export function classifyDocument( + attachment: LegistarAttachment, +): DocumentPolicy { + const name = `${attachment.MatterAttachmentName} ${attachment.MatterAttachmentDescription ?? ""}`; + if ( + /letters? from (?:the )?public|public comments?|ecomments?|public correspondence/i.test( + name, + ) + ) { + return { + category: "public_comment", + processingPolicy: "link_only", + isPublicComment: true, + }; + } + if (attachment.MatterAttachmentIsMinuteOrder) + return { + category: "minutes_order", + processingPolicy: "extract_text", + isPublicComment: false, + }; + if (/ordinance/i.test(name)) + return { + category: "ordinance", + processingPolicy: "extract_text", + isPublicComment: false, + }; + if (/resolution/i.test(name)) + return { + category: "resolution", + processingPolicy: "extract_text", + isPublicComment: false, + }; + if (/fiscal|budget|cost|appropriation/i.test(name)) + return { + category: "fiscal", + processingPolicy: "extract_text", + isPublicComment: false, + }; + if (/memorandum|staff report|recommendation|board letter/i.test(name)) + return { + category: "staff_report", + processingPolicy: "extract_text", + isPublicComment: false, + }; + if (/presentation/i.test(name)) + return { + category: "presentation", + processingPolicy: "extract_text", + isPublicComment: false, + }; + if ( + attachment.MatterAttachmentIsHyperlink || + /language access|web(?:site|page)|link/i.test(name) + ) { + return { + category: "reference", + processingPolicy: "link_only", + isPublicComment: false, + }; + } + return { + category: "other", + processingPolicy: "extract_text", + isPublicComment: false, + }; +} + +export function isDecisionItem(item: LegistarAgendaItem): boolean { + if (!item.EventItemMatterId || !item.EventItemTitle?.trim()) return false; + return !/^(language access instructions|please scroll|call to order|pledge of allegiance|orders of the day|closed session)$/i.test( + item.EventItemTitle.trim(), + ); +} diff --git a/apps/scraper/src/scrapers/legistar.config.ts b/apps/scraper/src/scrapers/legistar.config.ts new file mode 100644 index 00000000..dda7e968 --- /dev/null +++ b/apps/scraper/src/scrapers/legistar.config.ts @@ -0,0 +1,20 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const legistarConfig = { + id: "legistar", + name: "San José local decisions", + source: + "Legistar Web API — public meetings, agenda decisions, documents, histories, and votes", + environment: { + required: ["POSTGRES_URL"], + requiredAny: [], + recommended: [], + optional: [ + "LEGISTAR_PAST_DAYS", + "LEGISTAR_FUTURE_DAYS", + "LEGISTAR_MAX_ITEMS", + "LEGISTAR_MAX_DOCUMENT_BYTES", + "LEGISTAR_SKIP_DOCUMENT_TEXT", + ], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/legistar.ts b/apps/scraper/src/scrapers/legistar.ts new file mode 100644 index 00000000..72c8ad52 --- /dev/null +++ b/apps/scraper/src/scrapers/legistar.ts @@ -0,0 +1,935 @@ +import { createHash } from "node:crypto"; +import { getDocumentProxy } from "unpdf"; + +import type { + Jurisdiction, + LegistarAgendaItem, + LegistarAttachment, + LegistarBody, + LegistarMatter, + LegistarMatterHistory, + LegistarMeeting, + LegistarVote, +} from "@acme/api/integrations/legistar"; +import { JURISDICTIONS, LegistarClient } from "@acme/api/integrations/legistar"; +import { and, eq, gte, inArray, isNull, lt, lte, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalBody, + LocalDecision, + LocalDecisionDocument, + LocalDecisionHistory, + LocalDecisionVote, + LocalIngestionRun, + LocalJurisdiction, + LocalMeeting, + LocalMeetingItem, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { getItemLimit } from "../utils/concurrency.js"; +import { incrementTotalProcessed } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { + bodyPolicy, + classifyDocument, + classifyTopic, + inferGeographicScope, + isDecisionItem, +} from "./legistar-policy.js"; +import { legistarConfig } from "./legistar.config.js"; + +const logger = createLogger("Legistar"); +const JURISDICTION: Jurisdiction = "sanjose"; +const DEFAULT_PAST_DAYS = 45; +const DEFAULT_FUTURE_DAYS = 120; +const DEFAULT_MAX_DOCUMENT_BYTES = 15 * 1024 * 1024; +const MIN_NATIVE_CHARACTERS_PER_PAGE = 80; + +interface IngestionCounts extends Record { + bodies: number; + meetings: number; + decisions: number; + items: number; + documents: number; + documentsExtracted: number; + documentsNeedingOcr: number; + publicCommentsLinked: number; + histories: number; + votes: number; +} + +interface ExtractedDocument { + status: "native" | "ocr_required" | "skipped" | "failed"; + text: string | null; + method: "pdf_native" | null; + quality: number | null; + pageCount: number | null; + byteSize: number | null; + mimeType: string | null; + contentHash: string | null; +} + +function positiveInteger(value: string | undefined, fallback: number): number { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function addDays(date: Date, days: number): Date { + const result = new Date(date); + result.setUTCDate(result.getUTCDate() + days); + return result; +} + +function nullableDate(value: string | null | undefined): Date | null { + if (!value) return null; + const date = new Date(value.endsWith("Z") ? value : `${value}Z`); + return Number.isNaN(date.getTime()) ? null : date; +} + +function parseClock(value: string | null): { hour: number; minute: number } { + if (!value) return { hour: 0, minute: 0 }; + const match = value.trim().match(/^(\d{1,2})(?::(\d{2}))?\s*([AP]M)?$/i); + if (!match) return { hour: 0, minute: 0 }; + let hour = Number(match[1]); + const minute = Number(match[2] ?? 0); + const meridiem = match[3]?.toUpperCase(); + if (meridiem === "PM" && hour < 12) hour += 12; + if (meridiem === "AM" && hour === 12) hour = 0; + return { hour, minute }; +} + +/** Convert a timezone-less Legistar date/time pair into an actual instant. */ +export function parseLegistarMeetingStart( + dateValue: string, + timeValue: string | null, + timezone: string, +): Date { + const date = dateValue.slice(0, 10); + const [year, month, day] = date.split("-").map(Number); + const { hour, minute } = parseClock(timeValue); + const wantedUtc = Date.UTC(year!, month! - 1, day!, hour, minute); + const formatter = new Intl.DateTimeFormat("en-US", { + timeZone: timezone, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hourCycle: "h23", + }); + const parts = Object.fromEntries( + formatter + .formatToParts(new Date(wantedUtc)) + .filter((part) => part.type !== "literal") + .map((part) => [part.type, Number(part.value)]), + ); + const renderedUtc = Date.UTC( + parts.year!, + parts.month! - 1, + parts.day!, + parts.hour!, + parts.minute!, + parts.second!, + ); + return new Date(wantedUtc - (renderedUtc - wantedUtc)); +} + +function publicMatterUrl(matter: LegistarMatter): string { + const portal = JURISDICTIONS[JURISDICTION].publicPortalUrl; + return `${portal}/LegislationDetail.aspx?ID=${matter.MatterId}&GUID=${encodeURIComponent(matter.MatterGuid)}`; +} + +function sourcePayload(value: object): Record { + return value as Record; +} + +function isCancelled(meeting: LegistarMeeting): boolean { + return /cancel(?:led|ed)/i.test( + `${meeting.EventAgendaStatusName} ${meeting.EventComment ?? ""}`, + ); +} + +export function nativeTextQuality(text: string, pageCount: number): number { + if (!text.trim() || pageCount <= 0) return 0; + const printable = [...text].filter((character) => + /[\p{L}\p{N}\p{P}\p{Z}\n]/u.test(character), + ).length; + const printableRatio = printable / text.length; + const density = Math.min(1, text.trim().length / pageCount / 500); + return Number((printableRatio * density).toFixed(4)); +} + +export function needsOcr(text: string, pageCount: number): boolean { + return ( + text.trim().length / Math.max(pageCount, 1) < MIN_NATIVE_CHARACTERS_PER_PAGE + ); +} + +async function extractPdf( + url: string, + maxBytes: number, +): Promise { + try { + const response = await fetchWithRetry(url, { timeoutMs: 45_000 }); + const contentLength = Number(response.headers.get("content-length") ?? 0); + if (contentLength > maxBytes) { + return { + status: "skipped", + text: null, + method: null, + quality: null, + pageCount: null, + byteSize: contentLength, + mimeType: response.headers.get("content-type"), + contentHash: null, + }; + } + const bytes = new Uint8Array(await response.arrayBuffer()); + if (bytes.byteLength > maxBytes) { + return { + status: "skipped", + text: null, + method: null, + quality: null, + pageCount: null, + byteSize: bytes.byteLength, + mimeType: response.headers.get("content-type"), + contentHash: null, + }; + } + const mimeType = + response.headers.get("content-type")?.split(";")[0] ?? null; + if (mimeType && mimeType !== "application/pdf") { + return { + status: "skipped", + text: null, + method: null, + quality: null, + pageCount: null, + byteSize: bytes.byteLength, + mimeType, + contentHash: null, + }; + } + + const document = await getDocumentProxy(bytes); + const pages: string[] = []; + for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber++) { + const page = await document.getPage(pageNumber); + const content = await page.getTextContent(); + pages.push( + (content.items as { str?: string }[]) + .map((item) => item.str?.trim()) + .filter(Boolean) + .join(" "), + ); + } + const text = pages.filter(Boolean).join("\n\n").trim(); + const pageCount = document.numPages; + await document.destroy(); + const contentHash = createHash("sha256").update(bytes).digest("hex"); + if (needsOcr(text, pageCount)) { + return { + status: "ocr_required", + text: null, + method: null, + quality: nativeTextQuality(text, pageCount), + pageCount, + byteSize: bytes.byteLength, + mimeType: mimeType ?? "application/pdf", + contentHash, + }; + } + return { + status: "native", + text, + method: "pdf_native", + quality: nativeTextQuality(text, pageCount), + pageCount, + byteSize: bytes.byteLength, + mimeType: mimeType ?? "application/pdf", + contentHash, + }; + } catch (error) { + logger.warn(`Document extraction failed for ${url}`, error); + return { + status: "failed", + text: null, + method: null, + quality: null, + pageCount: null, + byteSize: null, + mimeType: null, + contentHash: null, + }; + } +} + +const client = new LegistarClient((input, init) => + fetchWithRetry(String(input), init), +); + +async function upsertJurisdiction(): Promise { + const config = JURISDICTIONS[JURISDICTION]; + await db + .insert(LocalJurisdiction) + .values({ + key: JURISDICTION, + name: config.name, + state: config.state, + governmentLevel: "city", + timezone: config.timezone, + sourceType: "legistar", + sourceClient: config.client, + sourceBaseUrl: config.baseUrl, + publicPortalUrl: config.publicPortalUrl, + }) + .onConflictDoUpdate({ + target: LocalJurisdiction.key, + set: { + name: config.name, + state: config.state, + timezone: config.timezone, + sourceBaseUrl: config.baseUrl, + publicPortalUrl: config.publicPortalUrl, + active: true, + }, + }); +} + +async function upsertBody(body: LegistarBody, seenAt: Date): Promise { + const policy = bodyPolicy(body.BodyId); + const [row] = await db + .insert(LocalBody) + .values({ + jurisdictionKey: JURISDICTION, + sourceBodyId: body.BodyId, + sourceGuid: body.BodyGuid, + name: body.BodyName, + typeName: body.BodyTypeName, + active: body.BodyActiveFlag === 1, + included: policy.included, + relevanceTier: policy.relevanceTier, + numberOfMembers: body.BodyNumberOfMembers, + description: body.BodyDescription, + contactName: body.BodyContactFullName, + contactEmail: body.BodyContactEmail, + contactPhone: body.BodyContactPhone, + sourceUpdatedAt: nullableDate(body.BodyLastModifiedUtc), + lastSeenAt: seenAt, + sourcePayload: sourcePayload(body), + }) + .onConflictDoUpdate({ + target: [LocalBody.jurisdictionKey, LocalBody.sourceBodyId], + set: { + sourceGuid: body.BodyGuid, + name: body.BodyName, + typeName: body.BodyTypeName, + active: body.BodyActiveFlag === 1, + included: policy.included, + relevanceTier: policy.relevanceTier, + numberOfMembers: body.BodyNumberOfMembers, + description: body.BodyDescription, + contactName: body.BodyContactFullName, + contactEmail: body.BodyContactEmail, + contactPhone: body.BodyContactPhone, + sourceUpdatedAt: nullableDate(body.BodyLastModifiedUtc), + lastSeenAt: seenAt, + sourcePayload: sourcePayload(body), + }, + }) + .returning({ id: LocalBody.id }); + return row!.id; +} + +async function upsertMeeting( + meeting: LegistarMeeting, + bodyId: string, + seenAt: Date, +): Promise { + const timezone = JURISDICTIONS[JURISDICTION].timezone; + const [row] = await db + .insert(LocalMeeting) + .values({ + jurisdictionKey: JURISDICTION, + bodyId, + sourceEventId: meeting.EventId, + sourceGuid: meeting.EventGuid, + startsAt: parseLegistarMeetingStart( + meeting.EventDate, + meeting.EventTime, + timezone, + ), + localDate: meeting.EventDate.slice(0, 10), + timeLabel: meeting.EventTime, + location: meeting.EventLocation, + agendaUrl: meeting.EventAgendaFile, + minutesUrl: meeting.EventMinutesFile, + videoUrl: meeting.EventVideoPath, + sourceUrl: meeting.EventInSiteURL, + agendaStatusName: meeting.EventAgendaStatusName, + minutesStatusName: meeting.EventMinutesStatusName, + comment: meeting.EventComment, + cancelled: isCancelled(meeting), + sourceUpdatedAt: nullableDate(meeting.EventLastModifiedUtc) ?? seenAt, + lastSeenAt: seenAt, + sourceDeletedAt: null, + sourcePayload: sourcePayload(meeting), + }) + .onConflictDoUpdate({ + target: [LocalMeeting.jurisdictionKey, LocalMeeting.sourceEventId], + set: { + bodyId, + sourceGuid: meeting.EventGuid, + startsAt: parseLegistarMeetingStart( + meeting.EventDate, + meeting.EventTime, + timezone, + ), + localDate: meeting.EventDate.slice(0, 10), + timeLabel: meeting.EventTime, + location: meeting.EventLocation, + agendaUrl: meeting.EventAgendaFile, + minutesUrl: meeting.EventMinutesFile, + videoUrl: meeting.EventVideoPath, + sourceUrl: meeting.EventInSiteURL, + agendaStatusName: meeting.EventAgendaStatusName, + minutesStatusName: meeting.EventMinutesStatusName, + comment: meeting.EventComment, + cancelled: isCancelled(meeting), + sourceUpdatedAt: nullableDate(meeting.EventLastModifiedUtc) ?? seenAt, + lastSeenAt: seenAt, + sourceDeletedAt: null, + sourcePayload: sourcePayload(meeting), + }, + }) + .returning({ id: LocalMeeting.id }); + return row!.id; +} + +async function upsertDecision( + matter: LegistarMatter, + item: LegistarAgendaItem, + bodyId: string | null, + seenAt: Date, +): Promise { + const scope = inferGeographicScope(matter, item); + const values = { + primaryBodyId: bodyId, + sourceGuid: matter.MatterGuid, + fileNumber: matter.MatterFile || null, + title: matter.MatterTitle || item.EventItemTitle || "Untitled decision", + name: matter.MatterName, + typeName: matter.MatterTypeName, + statusName: matter.MatterStatusName, + topic: classifyTopic(matter), + scopeKind: scope.kind, + districtNumbers: scope.districtNumbers, + geographicText: scope.text, + introDate: nullableDate(matter.MatterIntroDate), + agendaDate: nullableDate(matter.MatterAgendaDate), + passedDate: nullableDate(matter.MatterPassedDate), + enactmentDate: nullableDate(matter.MatterEnactmentDate), + enactmentNumber: matter.MatterEnactmentNumber, + requester: matter.MatterRequester, + notes: matter.MatterNotes, + sourceUrl: publicMatterUrl(matter), + sourceUpdatedAt: nullableDate(matter.MatterLastModifiedUtc) ?? seenAt, + lastSeenAt: seenAt, + sourceDeletedAt: null, + sourcePayload: sourcePayload(matter), + }; + const [row] = await db + .insert(LocalDecision) + .values({ + jurisdictionKey: JURISDICTION, + sourceMatterId: matter.MatterId, + ...values, + }) + .onConflictDoUpdate({ + target: [LocalDecision.jurisdictionKey, LocalDecision.sourceMatterId], + set: values, + }) + .returning({ id: LocalDecision.id }); + return row!.id; +} + +async function upsertMeetingItem( + item: LegistarAgendaItem, + meetingId: string, + decisionId: string | null, + seenAt: Date, +): Promise { + const values = { + decisionId, + sourceGuid: item.EventItemGuid, + agendaSequence: item.EventItemAgendaSequence, + minutesSequence: item.EventItemMinutesSequence, + agendaNumber: item.EventItemAgendaNumber, + title: item.EventItemTitle, + actionName: item.EventItemActionName, + actionText: item.EventItemActionText, + passedFlagName: item.EventItemPassedFlagName, + tally: item.EventItemTally, + moverName: item.EventItemMover, + seconderName: item.EventItemSeconder, + consent: item.EventItemConsent === 1, + rollCall: item.EventItemRollCallFlag === 1, + agendaNote: item.EventItemAgendaNote, + minutesNote: item.EventItemMinutesNote, + videoIndex: item.EventItemVideoIndex, + sourceUpdatedAt: nullableDate(item.EventItemLastModifiedUtc) ?? seenAt, + lastSeenAt: seenAt, + sourceDeletedAt: null, + sourcePayload: sourcePayload(item), + }; + const [row] = await db + .insert(LocalMeetingItem) + .values({ + meetingId, + sourceEventItemId: item.EventItemId, + ...values, + }) + .onConflictDoUpdate({ + target: [LocalMeetingItem.meetingId, LocalMeetingItem.sourceEventItemId], + set: values, + }) + .returning({ id: LocalMeetingItem.id }); + return row!.id; +} + +async function documentExtraction( + attachment: LegistarAttachment, + processingPolicy: "extract_text" | "link_only", + maxBytes: number, + skipText: boolean, +): Promise { + if (processingPolicy === "link_only" || skipText) { + return { + status: "skipped", + text: null, + method: null, + quality: null, + pageCount: null, + byteSize: null, + mimeType: null, + contentHash: null, + }; + } + + const [existing] = await db + .select({ + sourceUpdatedAt: LocalDecisionDocument.sourceUpdatedAt, + extractionStatus: LocalDecisionDocument.extractionStatus, + extractedText: LocalDecisionDocument.extractedText, + extractionMethod: LocalDecisionDocument.extractionMethod, + extractionQuality: LocalDecisionDocument.extractionQuality, + pageCount: LocalDecisionDocument.pageCount, + byteSize: LocalDecisionDocument.byteSize, + mimeType: LocalDecisionDocument.mimeType, + contentHash: LocalDecisionDocument.contentHash, + }) + .from(LocalDecisionDocument) + .where( + and( + eq(LocalDecisionDocument.jurisdictionKey, JURISDICTION), + eq( + LocalDecisionDocument.sourceAttachmentId, + attachment.MatterAttachmentId, + ), + ), + ) + .limit(1); + const sourceUpdatedAt = nullableDate( + attachment.MatterAttachmentLastModifiedUtc, + ); + if ( + existing && + existing.sourceUpdatedAt?.getTime() === sourceUpdatedAt?.getTime() && + (["native", "ocr_required"].includes(existing.extractionStatus) || + (existing.extractionStatus === "skipped" && + ((existing.byteSize !== null && existing.byteSize > maxBytes) || + (existing.mimeType !== null && + existing.mimeType !== "application/pdf")))) + ) { + return { + status: existing.extractionStatus as ExtractedDocument["status"], + text: existing.extractedText, + method: existing.extractionMethod === "pdf_native" ? "pdf_native" : null, + quality: existing.extractionQuality, + pageCount: existing.pageCount, + byteSize: existing.byteSize, + mimeType: existing.mimeType, + contentHash: existing.contentHash, + }; + } + return extractPdf(attachment.MatterAttachmentHyperlink, maxBytes); +} + +async function upsertDocument( + attachment: LegistarAttachment, + decisionId: string, + seenAt: Date, + counts: IngestionCounts, + maxBytes: number, + skipText: boolean, +): Promise { + const policy = classifyDocument(attachment); + const extraction = await documentExtraction( + attachment, + policy.processingPolicy, + maxBytes, + skipText, + ); + await db + .insert(LocalDecisionDocument) + .values({ + jurisdictionKey: JURISDICTION, + decisionId, + sourceAttachmentId: attachment.MatterAttachmentId, + sourceGuid: attachment.MatterAttachmentGuid, + name: attachment.MatterAttachmentName, + description: attachment.MatterAttachmentDescription, + url: attachment.MatterAttachmentHyperlink, + fileName: attachment.MatterAttachmentFileName, + category: policy.category, + sortOrder: attachment.MatterAttachmentSort, + isSupportingDocument: attachment.MatterAttachmentIsSupportingDocument, + isPublicComment: policy.isPublicComment, + processingPolicy: policy.processingPolicy, + extractionStatus: extraction.status, + extractedText: extraction.text, + extractionMethod: extraction.method, + extractionQuality: extraction.quality, + pageCount: extraction.pageCount, + byteSize: extraction.byteSize, + mimeType: extraction.mimeType, + contentHash: extraction.contentHash, + sourceUpdatedAt: nullableDate(attachment.MatterAttachmentLastModifiedUtc), + lastSeenAt: seenAt, + sourceDeletedAt: null, + sourcePayload: sourcePayload(attachment), + }) + .onConflictDoUpdate({ + target: [ + LocalDecisionDocument.jurisdictionKey, + LocalDecisionDocument.sourceAttachmentId, + ], + set: { + decisionId, + name: attachment.MatterAttachmentName, + description: attachment.MatterAttachmentDescription, + url: attachment.MatterAttachmentHyperlink, + fileName: attachment.MatterAttachmentFileName, + category: policy.category, + sortOrder: attachment.MatterAttachmentSort, + isSupportingDocument: attachment.MatterAttachmentIsSupportingDocument, + isPublicComment: policy.isPublicComment, + processingPolicy: policy.processingPolicy, + extractionStatus: extraction.status, + extractedText: extraction.text, + extractionMethod: extraction.method, + extractionQuality: extraction.quality, + pageCount: extraction.pageCount, + byteSize: extraction.byteSize, + mimeType: extraction.mimeType, + contentHash: extraction.contentHash, + sourceUpdatedAt: nullableDate( + attachment.MatterAttachmentLastModifiedUtc, + ), + lastSeenAt: seenAt, + sourceDeletedAt: null, + sourcePayload: sourcePayload(attachment), + }, + }); + counts.documents++; + if (policy.isPublicComment) counts.publicCommentsLinked++; + if (extraction.status === "native") counts.documentsExtracted++; + if (extraction.status === "ocr_required") counts.documentsNeedingOcr++; +} + +async function upsertHistory( + history: LegistarMatterHistory, + decisionId: string, +): Promise { + const values = { + sourceEventId: history.MatterHistoryEventId, + sourceEventItemId: history.MatterHistoryEventItemId, + bodyName: history.MatterHistoryBodyName, + actionName: history.MatterHistoryActionName, + actionText: history.MatterHistoryDescription, + actionDate: nullableDate(history.MatterHistoryActionDate), + agendaNumber: history.MatterHistoryAgendaNumber, + sourcePayload: sourcePayload(history), + }; + await db + .insert(LocalDecisionHistory) + .values({ + decisionId, + sourceHistoryId: history.MatterHistoryId, + ...values, + }) + .onConflictDoUpdate({ + target: [ + LocalDecisionHistory.decisionId, + LocalDecisionHistory.sourceHistoryId, + ], + set: values, + }); +} + +async function upsertVote( + vote: LegistarVote, + meetingItemId: string, + seenAt: Date, +): Promise { + const values = { + sourcePersonId: vote.VotePersonId, + personName: vote.VotePersonName, + valueName: vote.VoteValueName, + sortOrder: vote.VoteSort, + sourceUpdatedAt: nullableDate(vote.VoteLastModifiedUtc) ?? seenAt, + sourcePayload: sourcePayload(vote), + }; + await db + .insert(LocalDecisionVote) + .values({ meetingItemId, sourceVoteId: vote.VoteId, ...values }) + .onConflictDoUpdate({ + target: [LocalDecisionVote.meetingItemId, LocalDecisionVote.sourceVoteId], + set: values, + }); +} + +async function processMeeting( + meeting: LegistarMeeting, + bodyIds: ReadonlyMap, + seenAt: Date, + counts: IngestionCounts, + maxBytes: number, + skipText: boolean, + seenAttachments: Map>, +): Promise { + const bodyId = bodyIds.get(meeting.EventBodyId); + if (!bodyId) return; + const meetingId = await upsertMeeting(meeting, bodyId, seenAt); + counts.meetings++; + + const items = await client.getAgendaItems(JURISDICTION, meeting.EventId); + const decisionItems = items.filter(isDecisionItem); + for (const item of decisionItems) { + const matter = await client.getMatter( + JURISDICTION, + item.EventItemMatterId!, + ); + const matterBodyId = bodyIds.get(matter.MatterBodyId) ?? bodyId; + const decisionId = await upsertDecision(matter, item, matterBodyId, seenAt); + const meetingItemId = await upsertMeetingItem( + item, + meetingId, + decisionId, + seenAt, + ); + counts.decisions++; + counts.items++; + incrementTotalProcessed(); + + const attachments = item.EventItemMatterAttachments ?? []; + const attachmentIds = seenAttachments.get(decisionId) ?? new Set(); + seenAttachments.set(decisionId, attachmentIds); + for (const attachment of attachments) { + attachmentIds.add(attachment.MatterAttachmentId); + await upsertDocument( + attachment, + decisionId, + seenAt, + counts, + maxBytes, + skipText, + ); + } + + // San José currently returns empty histories for sampled matters, but keep + // the capability live for jurisdictions/records that publish them. Limit + // the extra call to non-agenda-ready records where history is most useful. + if (!/agenda ready/i.test(matter.MatterStatusName)) { + const histories = await client.getMatterHistories( + JURISDICTION, + matter.MatterId, + ); + for (const history of histories) { + await upsertHistory(history, decisionId); + counts.histories++; + } + } + + if ( + item.EventItemRollCallFlag === 1 || + item.EventItemTally || + item.EventItemActionName + ) { + const votes = await client.getVotes(JURISDICTION, item.EventItemId); + for (const vote of votes) { + await upsertVote(vote, meetingItemId, seenAt); + counts.votes++; + } + } + } + + await db + .update(LocalMeetingItem) + .set({ sourceDeletedAt: seenAt }) + .where( + and( + eq(LocalMeetingItem.meetingId, meetingId), + lt(LocalMeetingItem.lastSeenAt, seenAt), + ), + ); +} + +async function runLegistarScrape(maxItems?: number): Promise { + const seenAt = new Date(); + const pastDays = positiveInteger( + process.env.LEGISTAR_PAST_DAYS, + DEFAULT_PAST_DAYS, + ); + const futureDays = positiveInteger( + process.env.LEGISTAR_FUTURE_DAYS, + DEFAULT_FUTURE_DAYS, + ); + const maxBytes = positiveInteger( + process.env.LEGISTAR_MAX_DOCUMENT_BYTES, + DEFAULT_MAX_DOCUMENT_BYTES, + ); + const skipText = process.env.LEGISTAR_SKIP_DOCUMENT_TEXT === "1"; + const windowStart = addDays(seenAt, -pastDays); + const windowEnd = addDays(seenAt, futureDays); + const counts: IngestionCounts = { + bodies: 0, + meetings: 0, + decisions: 0, + items: 0, + documents: 0, + documentsExtracted: 0, + documentsNeedingOcr: 0, + publicCommentsLinked: 0, + histories: 0, + votes: 0, + }; + + await upsertJurisdiction(); + const [run] = await db + .insert(LocalIngestionRun) + .values({ + jurisdictionKey: JURISDICTION, + status: "running", + windowStart, + windowEnd, + counts, + }) + .returning({ id: LocalIngestionRun.id }); + + try { + const bodies = await client.getBodies(JURISDICTION); + const bodyIds = new Map(); + for (const body of bodies) { + const id = await upsertBody(body, seenAt); + if (bodyPolicy(body.BodyId).included) bodyIds.set(body.BodyId, id); + counts.bodies++; + } + + const fetchedMeetings = await client.getMeetings(JURISDICTION, { + start: windowStart, + end: windowEnd, + }); + const relevantMeetings = fetchedMeetings.filter((meeting) => + bodyIds.has(meeting.EventBodyId), + ); + const meetings = relevantMeetings.slice(0, maxItems); + const seenAttachments = new Map>(); + const limit = getItemLimit(); + await Promise.all( + meetings.map((meeting) => + limit(() => + processMeeting( + meeting, + bodyIds, + seenAt, + counts, + maxBytes, + skipText, + seenAttachments, + ), + ), + ), + ); + + // Attachments are returned as a complete snapshot on every processed + // matter occurrence. Keep removed source files for auditability, but hide + // them from normal reads by recording when they disappeared. + for (const [decisionId, attachmentIds] of seenAttachments) { + const predicates = [ + eq(LocalDecisionDocument.decisionId, decisionId), + eq(LocalDecisionDocument.jurisdictionKey, JURISDICTION), + isNull(LocalDecisionDocument.sourceDeletedAt), + ]; + if (attachmentIds.size > 0) { + predicates.push( + notInArray(LocalDecisionDocument.sourceAttachmentId, [ + ...attachmentIds, + ]), + ); + } + await db + .update(LocalDecisionDocument) + .set({ sourceDeletedAt: seenAt }) + .where(and(...predicates)); + } + + // Only a complete window walk can prove a meeting disappeared. Targeted + // --max-items runs intentionally skip tombstoning. + if (maxItems === undefined || relevantMeetings.length <= maxItems) { + await db + .update(LocalMeeting) + .set({ sourceDeletedAt: seenAt }) + .where( + and( + eq(LocalMeeting.jurisdictionKey, JURISDICTION), + inArray(LocalMeeting.bodyId, [...bodyIds.values()]), + gte(LocalMeeting.startsAt, windowStart), + lte(LocalMeeting.startsAt, windowEnd), + lt(LocalMeeting.lastSeenAt, seenAt), + ), + ); + } + + await db + .update(LocalIngestionRun) + .set({ status: "succeeded", completedAt: new Date(), counts }) + .where(eq(LocalIngestionRun.id, run!.id)); + logger.success( + `Stored ${counts.decisions} decision occurrences across ${counts.meetings} meetings`, + ); + } catch (error) { + await db + .update(LocalIngestionRun) + .set({ + status: "failed", + completedAt: new Date(), + counts, + error: error instanceof Error ? error.message : String(error), + }) + .where(eq(LocalIngestionRun.id, run!.id)); + throw error; + } +} + +export const legistarScraper: Scraper = { + ...legistarConfig, + scrape: async ({ maxItems } = {}) => + runLegistarScrape( + maxItems ?? positiveInteger(process.env.LEGISTAR_MAX_ITEMS, 100), + ), +}; diff --git a/docs/api.md b/docs/api.md index 51e0283f..05108139 100644 --- a/docs/api.md +++ b/docs/api.md @@ -18,7 +18,7 @@ The root router (`packages/api/src/root.ts`) composes **nine** sub-routers: | `auth` | `getSession` (Q), `getSecretMessage` (Q 🔒) | | `civic` | `getElections`, `getVoterInfo`, `getRepresentatives`, `getRepresentativesEnriched` (all Q) — Google Civic + measure/candidate cross-validation | | `places` | `autocomplete` (Q), `details` (M) — Google Places address autocomplete for the ballot lookup | -| `legistar` | `getLocalBills`, `getMeetings`, `getAgenda`, `getVotes`, `getBodies`, `getMeetingVotes` (all Q) — local councils | +| `legistar` | `listDecisions`, `getDecision`, `listBodies`, `getIngestionHealth` (Q); deprecated source-format queries remain temporarily — local decisions | | `openStates` | `searchBills`, `getBillDetails`, `getLegislators`, `getBillVotes` (all Q) — CA state legislature (Open States v3) | | `content` | `getAll`, `getByType`, `getById` (all Q) — aggregates bill / government_content / court_case | | `video` | `getInfinite` (Q) — cursor-paginated feed; converts `bytea` images to data URIs | @@ -31,7 +31,7 @@ The `civic` router calls the **Google Civic Information API** (`GOOGLE_CIVIC_API Other live civic integrations: -- **`legistar`** — reads configured Legistar instances through an on-demand API client with a 24-hour database cache. It is not registered in the scraper supervisor. See [Local government decisions and Legistar](./local-government-legistar.md) for the planned ingestion and evidence model. +- **`legistar`** — new clients read durable, normalized local-government decisions populated by the registered scraper. The source transport is stateless and paged; deprecated prototype queries still make read-only source calls during the UI transition. See [Local government decisions and Legistar](./local-government-legistar.md). - **`openStates`** — California bills, legislators, and votes via the Open States v3 API (`OPEN_STATES_API_KEY`). - **`places`** — Google **Places Autocomplete (New)** for the ballot address entry (`packages/api/src/lib/places.ts`). `autocomplete` returns US street-address predictions (biased `includedRegionCodes: ["us"]`, `includedPrimaryTypes: street_address/premise/subpremise`) for queries ≥3 chars; `details` resolves a `placeId` to its full `formattedAddress` (the ZIP the prediction omits, which Civic wants). A **session token** (UUID stable across one address entry) bundles all keystroke calls plus the closing `details` into a single billed unit. Reuses `GOOGLE_PLACES_API_KEY` → `GOOGLE_API_KEY` → `GOOGLE_CIVIC_API_KEY`; with no key it serves a small mock list so the dropdown still works in dev (same fallback pattern as `civic`). diff --git a/docs/data-layer.md b/docs/data-layer.md index 8206ce99..82e03056 100644 --- a/docs/data-layer.md +++ b/docs/data-layer.md @@ -173,7 +173,7 @@ All three content tables share a common pattern: | `polling_location` | Polling places / early-vote sites / drop boxes, geo-located (lat/long), with hours | | `role_description` | Reusable descriptions of offices/roles by level (seeded with ~18 federal→local roles) | -**Local government (Legistar cache)** — `legistar_body`, `legistar_matter`, `legistar_meeting`, `legistar_agenda_item`, `legistar_vote`. These cache San Jose / Santa Clara / Sunnyvale council data (ordinances, meetings, agenda items, votes) keyed by `(jurisdiction, *_id)` with a `fetched_at` timestamp. +**Local government decisions** — `local_jurisdiction`, `local_body`, `local_decision`, `local_meeting`, `local_meeting_item`, `local_decision_document`, `local_decision_history`, `local_decision_vote`, and `local_ingestion_run`. These source-neutral tables normalize Legistar Matters separately from their meeting occurrences, retain raw provenance and document hashes/extraction state, record complete ingestion windows, and soft-delete records that disappear upstream. San José is the first active adapter. See [Local government decisions and Legistar](./local-government-legistar.md). **User engagement & caching:** diff --git a/docs/data-sources-api.md b/docs/data-sources-api.md index 1c6c5b50..a84ab1ee 100644 --- a/docs/data-sources-api.md +++ b/docs/data-sources-api.md @@ -210,12 +210,12 @@ const votes = await legistar.getVotes("sanjose", eventItemId); | ------------------------------------------------------- | ------------------------------------------------------------------------ | | `legistar.getMeetings(jurisdiction, dateRange?)` | `LegistarMeeting[]` — meetings in an optional date range | | `legistar.getLegislation(jurisdiction, query?)` | `LegistarMatter[]` — matters matching optional filters | -| `legistar.getAgendas(jurisdiction, eventId)` | `LegistarAgendaItem[]` — EventItems for one meeting | +| `legistar.getAgendaItems(jurisdiction, eventId)` | `LegistarAgendaItem[]` — complete EventItems for one meeting | | `legistar.getVotes(jurisdiction, eventItemId)` | `LegistarVote[]` — structured votes when the jurisdiction publishes them | | `legistar.getBodies(jurisdiction)` | `LegistarBody[]` — committees and boards | | `legistar.getMatterAttachments(jurisdiction, matterId)` | `LegistarAttachment[]` — PDFs, staff reports, and links | -The current implementation is an on-demand client with a 24-hour database cache, not a scheduled scraper. Read [Local government decisions and Legistar](./local-government-legistar.md) before extending it; client ids, field usage, publication lifecycle, and outcome completeness vary by jurisdiction. +This is the stateless source transport used by the registered San José scraper. New application reads use the durable `legistar.listDecisions`, `getDecision`, `listBodies`, and `getIngestionHealth` tRPC procedures rather than calling the source client. Read [Local government decisions and Legistar](./local-government-legistar.md) before adding a jurisdiction; client IDs, field population, publication lifecycle, and outcome completeness vary. --- diff --git a/docs/local-government-legistar.md b/docs/local-government-legistar.md index 1123b400..fe3ee829 100644 --- a/docs/local-government-legistar.md +++ b/docs/local-government-legistar.md @@ -1,30 +1,17 @@ # Local Government Decisions and Legistar -> Status: Draft RFC +> Status: Backend foundation implemented > > Initial jurisdiction: City of San José > Tracking issue: [#282](https://github.com/billion-app/billion/issues/282) -This document defines how Billion should ingest, normalize, and explain local-government decisions from Legistar. It is intentionally more specific than a data-source setup guide: Legistar is a records-management system with a publication lifecycle, not a feed of interchangeable "local bills." +Legistar is a records-management system, not a feed of interchangeable “local +bills.” A proposal can appear before several bodies and at several meetings, +while the agenda item, attachments, minutes, and vote endpoint each describe a +different part of its lifecycle. Billion therefore treats Legistar as a source +adapter behind a source-neutral local-government model. -The existing integration in `packages/api/src/integrations/legistar.ts` is useful scaffolding. It is an on-demand API client with a 24-hour database cache. It is not yet the ingestion system described here. - -## Why this is a subsystem - -A local decision can be proposed, amended, deferred, heard by multiple bodies, returned at a later meeting, and finally recorded in minutes weeks after the meeting. Different facts may be published in different artifacts: - -- Legistar JSON for meetings, agenda items, matters, bodies, and attachments; -- an agenda or amended-agenda PDF for the item number and recommendation; -- a staff memorandum for fiscal impact and geographic scope; -- meeting minutes for the actual motion, action, tally, and named exceptions; -- a video or transcript for explanatory context; -- a separate GIS service for address and council-district relevance. - -Treating this as another bill API would lose the relationship between a proposal, the meeting at which it was considered, the evidence supporting it, and the outcome actually recorded by the city. - -## Source-system model - -Legistar exposes several related record types. Their names should remain visible in adapter code even when Billion presents friendlier product language. +## Source model and identity ```mermaid erDiagram @@ -34,475 +21,198 @@ erDiagram MATTER ||--o{ ATTACHMENT : has MATTER ||--o{ MATTER_HISTORY : progresses_through EVENT_ITEM ||--o{ VOTE : may_record - EVENT_ITEM ||--o{ ROLL_CALL : may_record - - BODY { - int body_id - string name - } - EVENT { - int event_id - datetime meeting_date - string agenda_status - string minutes_status - } - EVENT_ITEM { - int event_item_id - string agenda_number - int matter_id - string action - string tally - } - MATTER { - int matter_id - string file_number - string title - string status - } -``` - -### Body - -A council, committee, board, commission, hearing body, or other government unit. Bodies are jurisdiction-specific and should not be hard-coded into shared product components. - -### Event - -A scheduled meeting. An Event owns meeting time, location, agenda/minutes publication status, and links to meeting artifacts. - -### EventItem - -One row in an Event's agenda or minutes. This is the closest Legistar record to a decision occurrence, but not every EventItem is a decision. San José also publishes section headings, interpretation instructions, participation instructions, and other boilerplate as EventItems. - -Meeting-specific outcomes—motion, action, tally, mover, seconder, and votes—belong to the EventItem, not to the Matter in the abstract. - -### Matter - -A reusable record for a proposal, report, ordinance, resolution, contract, land-use action, or other file. A Matter may appear at multiple meetings and before multiple bodies. It may also exist before it is scheduled. - -Consequently: - -- `MatterId` is not a unique decision ID; -- Matter modification time is not a reliable decision date; -- the latest modified Matters are not necessarily the decisions residents should see; -- a Matter timeline must be built from its meeting appearances and history. - -### Attachment - -An official document or hyperlink associated with a Matter. Common San José examples include memoranda, staff reports, resolutions, ordinances, presentations, supplemental memoranda, and letters from the public. - -Attachments are first-class source documents. They must be retained and versioned rather than flattened into summary text. - -### MatterHistory, Vote, and RollCall - -Legistar defines structured history and voting endpoints. Availability is jurisdiction-dependent. A missing structured action or vote is unknown data, not evidence that no action or vote occurred. - -## San José observations - -The following are observations from the August 13, 2026 discovery pass and must be captured as fixtures before implementation assumptions become permanent: - -- the public `sanjose` client allowed unauthenticated reads; -- an August 1 through September 30 query returned 43 Events; -- the August 11 City Council Event returned 92 EventItems, of which 33 referenced Matters; -- attachments included memoranda, ordinances, resolutions, presentations, public letters, and supplemental memoranda; -- some API agenda numbers were null even though the published agenda displayed an item number; -- completed meetings did not consistently expose structured actions or votes; -- official minutes PDFs contained richer outcomes, including motions, adopted instruments, tallies, named absences, fiscal amounts, and council districts. - -These observations establish fallback requirements, not universal claims about every San José record. - -## Billion domain model - -The primary product record should be a `LocalDecision`, representing a proposal as considered at a particular meeting. Internally, `jurisdictionId + sourceSystem + sourceEventItemId` is its stable source identity. - -```ts -interface LocalDecision { - id: string; - jurisdictionId: string; - governmentLevel: "city" | "county" | "special_district"; - sourceSystem: "legistar" | string; - sourceEventId: string; - sourceEventItemId: string; - sourceMatterId: string | null; - - title: string; - summary: string | null; - summaryIsAiGenerated: boolean; - topic: LocalDecisionTopic | null; - - meetingBody: string; - meetingStartsAt: string; - agendaNumber: string | null; - fileNumber: string | null; - lifecycleStatus: LocalDecisionStatus; - - recommendation: string | null; - fiscalImpact: string | null; - geographicScope: LocalDecisionScope; - participation: ParticipationInfo | null; - outcome: LocalDecisionOutcome | null; - - sourcePageUrl: string | null; - citations: LocalDecisionCitation[]; - firstObservedAt: string; - lastObservedAt: string; -} -``` - -The interface is illustrative. The storage design may use normalized tables, but the API should deliver an equivalent jurisdiction-neutral contract. - -### Lifecycle status - -Lifecycle status must be derived from cited source state, not guessed from whether a meeting date is in the past. - -Proposed normalized values: - -- `scheduled`: placed on a meeting agenda; -- `amended`: the published proposal or agenda changed materially; -- `deferred`: consideration moved to a later date; -- `withdrawn`: removed by the originating body or staff; -- `cancelled`: the meeting or item was cancelled; -- `awaiting_outcome`: meeting occurred but no official outcome is available; -- `decided`: an official action is published; -- `informational`: heard or filed without an approval decision; -- `unknown`: source state cannot be normalized safely. - -The raw source status and action must also be retained. - -### Geographic scope - -```ts -type LocalDecisionScope = - | { kind: "citywide" } - | { kind: "district"; districtIds: string[] } - | { kind: "addresses"; addresses: string[]; districtIds: string[] } - | { kind: "neighborhood"; names: string[]; districtIds: string[] } - | { kind: "countywide" } - | { kind: "unknown" }; ``` -An absent district reference is not automatically citywide. It remains `unknown` unless the source supports a citywide classification. +- A **Body** is a council, committee, board, commission, or hearing body. +- An **Event** is a meeting and owns its time, location, agenda, minutes, and + video links. +- An **EventItem** is one occurrence on a meeting agenda. Meeting-specific + action, tally, mover, seconder, and votes belong here. +- A **Matter** is the proposal or file. The same Matter can appear at multiple + meetings, so it is the canonical decision identity but not the occurrence. +- An **Attachment** is an official document or link associated with a Matter. -### Matter timeline +The normalized model separates `local_decision` (one source Matter) from +`local_meeting_item` (one EventItem occurrence). This supports one decision +card with a truthful multi-meeting timeline. Similar titles never merge records. -Multiple `LocalDecision` records may point to the same Matter. The product can group these records into a timeline while preserving each meeting occurrence and outcome. - -For example: - -```text -Planning Commission recommendation - → Council first reading - → Council final adoption -``` - -Grouping must use source Matter identity and explicit relations. Similar titles alone are insufficient. - -## Evidence and provenance - -Every material user-facing fact needs a citation. A summary-level "Sources" array is insufficient when different claims come from different documents. - -```ts -interface LocalDecisionCitation { - field: - | "title" - | "summary" - | "recommendation" - | "fiscalImpact" - | "geographicScope" - | "participation" - | "outcome"; - sourceDocumentId: string; - sourceUrl: string; - page: number | null; - section: string | null; - excerpt: string | null; - extractionMethod: "structured" | "deterministic" | "ai" | "manual"; - confidence: "high" | "medium" | "low"; -} -``` - -Requirements: - -- retain the original government URL; -- record retrieval time and content hash; -- retain document versions when content changes at the same URL; -- preserve page boundaries during PDF extraction; -- distinguish source text from Billion-authored explanation; -- never create a factual field from a title alone; -- never infer a vote from transcript sentiment or attendance; -- allow a user to open the exact supporting record. - -## Source precedence - -When sources disagree, use the most specific official record for the fact in question: - -1. Published structured action/vote data for that EventItem. -2. Official adopted or approved meeting minutes. -3. Official amended agenda, agenda, or staff memorandum. -4. Other official attachments and meeting pages. -5. Official video/transcript as contextual evidence only. -6. Third-party sources as optional discovery or gap indicators, never silent replacements. - -The most recently retrieved source does not automatically outrank an adopted record. Precedence is fact-specific and should be encoded in tests. - -## Proposed ingestion pipeline +## Implemented architecture ```mermaid -flowchart TD - discover["Discover Events by jurisdiction and date window"] - items["Fetch EventItems"] - classify["Classify decision candidates and boilerplate"] - hydrate["Hydrate Matters, histories, attachments, votes"] - docs["Download and version official documents"] - extract["Extract page-aware text and structured facts"] - normalize["Normalize LocalDecision records"] - geo["Resolve explicit geographic scope"] - publish["Publish only records meeting quality gates"] - refresh["Re-poll through outcome publication"] - - discover --> items --> classify --> hydrate --> docs --> extract --> normalize - normalize --> geo --> publish - publish --> refresh --> hydrate -``` - -### 1. Discovery - -Fetch Events within bounded windows using OData filtering and paging. Store the query window and completion marker so "some fresh cached rows" cannot be mistaken for a complete window. - -### 2. Candidate classification - -Start with EventItems that reference a Matter, but do not treat that as the final rule. Store excluded items and their reason during the discovery phase so jurisdiction rules can be audited. - -Candidate categories should include: - -- decision; -- informational report; -- procedural item; -- ceremonial item; -- participation/translation boilerplate; -- section heading; -- unknown. - -### 3. Hydration - -For each candidate, fetch the Matter, attachments, history, and structured votes/roll calls. Persist raw source payloads or versioned snapshots needed to reproduce normalization. - -### 4. Document processing - -For each official PDF or supported document: - -1. validate MIME type and size; -2. compute a content hash; -3. preserve the original artifact or a durable reference permitted by source terms; -4. extract text with page boundaries; -5. use OCR only when embedded text is unavailable; -6. detect standard headings and fields deterministically first; -7. use constrained AI extraction only over retrieved text; -8. validate citations against the extracted page text. - -### 5. Normalization and quality gates - -A decision may be published with partial data, but missing fields must be explicit. Minimum proposed requirements for a list-card record: - -- jurisdiction; -- meeting body and date; -- source EventItem identity; -- non-boilerplate title; -- direct official source link. - -AI summary publication additionally requires at least one validated official citation. An outcome requires an official action source. - -### 6. Refresh through outcome - -An ingestion run is not complete when an upcoming agenda is first observed. The scraper must revisit the meeting through minutes publication and capture amendments along the way. - -Proposed initial cadence, subject to measurement: - -- broad Event discovery daily; -- meetings within 14 days every 6 hours; -- meetings within 48 hours every hour; -- completed meetings without outcomes every 6 hours for 14 days; -- then daily through 90 days; -- retain a manual reprocess command for older corrections. - -Use `EventLastModifiedUtc`, `EventItemLastModifiedUtc`, document hashes, and explicit window state to avoid unnecessary reprocessing. Do not assume timestamps capture every attachment replacement; hashes remain necessary. - -## Address and district relevance - -The user's address and a decision's geographic scope are separate pipelines. - -### User jurisdiction - -1. Resolve the saved address to coordinates using the existing Places flow. -2. Query the official San José boundary/district service. -3. Store or cache the least sensitive result needed for ranking, preferably jurisdiction and district rather than duplicating coordinates broadly. -4. Do not show San José decisions as "local" when the point is outside the city. - -### Decision scope - -Extract explicit districts, addresses, APNs, and neighborhood names from official records. Validate addresses against official GIS where practical. Do not assign a district based solely on the sponsoring councilmember. - -Ranking can place a user's district-specific decisions above citywide decisions, but must not hide citywide decisions by default. - -## Participation information - -Participation instructions may be meeting-wide or item-specific. Prefer the current agenda and official meeting page over a generic city participation page. Store: - -- method: in person, Zoom, phone, email, or eComment; -- deadline with timezone when explicitly published; -- meeting/item identifier required in the comment; -- official URL or address; -- retrieval time because instructions can change. - -Never manufacture a deadline from customary practice. - -## Outcomes and votes - -Outcome ingestion uses the following fallback: - -1. structured EventItem action, tally, and Vote/RollCall endpoints; -2. official meeting minutes with page-level citation; -3. `awaiting_outcome` when neither is available. - -Minutes extraction should capture, when present: - -- final action text; -- motion and amendments; -- mover and seconder; -- pass/fail or other disposition; -- tally; -- named yes/no/abstain/absent/recused members; -- adopted ordinance or resolution number. - -A compact tally is not enough to reconstruct individual yes votes unless the minutes explicitly define them. - -## Jurisdiction adapters - -Shared ingestion code should depend on a jurisdiction profile rather than San José conditionals spread across the pipeline. - -```ts -interface LegistarJurisdictionProfile { - jurisdictionId: string; - clientSlug: string; - governmentLevel: "city" | "county" | "special_district"; - canonicalHost: string; - bodyAllowlist?: number[]; - bodyDenylist?: number[]; - boilerplateRules: string[]; - topicRules: TopicRule[]; - documentNameRules: DocumentRule[]; - participationSources: string[]; - geographicResolver: string | null; - outcomeFallback: "minutes_pdf" | "none"; -} +flowchart LR + api["Legistar Web API"] --> transport["Stateless paged adapter"] + transport --> scraper["Scheduled Legistar scraper"] + scraper --> policy["San José policy"] + policy --> db["Normalized local_* tables"] + db --> read["Decision-centric tRPC API"] ``` -Client slugs are configuration, not reliably derivable from a public portal subdomain. - -## Storage implications - -The existing `legistar_*` tables can support a transition but are insufficient for the full system. The design needs durable representations for: - -- completed ingestion windows and cursors; -- raw/versioned source snapshots; -- attachments/source documents and content hashes; -- Matter-to-EventItem appearances; -- normalized LocalDecision records; -- per-field citations; -- extraction attempts, parser versions, and failures; -- geographic scope; -- publication readiness and partial-data reasons. - -Attachments must not disappear when an API response is served from cache. Empty vote results need a cacheable "checked at" state so a legitimate empty response does not trigger a live request on every read. - -## Failure behavior - -- Production must never substitute synthetic Matters for failed government reads. -- Preserve the last known official record and mark it stale when refresh fails. -- Distinguish source unavailable, document unavailable, parse failed, unsupported format, and data genuinely absent. -- Apply bounded retries and per-host rate limits. -- Quarantine malformed records rather than dropping the rest of a meeting. -- Emit metrics for Events discovered, candidates retained/excluded, documents changed, extraction failures, missing outcomes, and publication readiness. - -## Security, privacy, and legal review - -- Treat public-comment letters as potentially containing personal information; do not summarize or index individual commenters by default. -- Confirm source terms for storing government-hosted documents versus retaining hashes and URLs. -- Respect removal or redaction of source documents while retaining an internal audit event appropriate to policy. -- Avoid sending a user's street address to Legistar or an AI provider. -- Ensure AI providers receive only the official document excerpts needed for extraction. - -## Discovery fixtures - -Before schema work, capture immutable test fixtures from at least: - -1. an upcoming San José City Council meeting with attachments and participation instructions; -2. a completed meeting whose structured Legistar outcome is sparse but whose minutes contain actions and tallies; -3. a Matter that appears at multiple meetings; -4. an amended or deferred item; -5. a non-Matter EventItem that should be excluded; -6. a district-specific land-use or contract item; -7. an item with no determinable geographic scope; -8. a meeting with an unavailable or replaced attachment. - -Fixture metadata must record retrieval time, request URL, response headers needed for reproduction, and content hashes. Tests should operate offline. - -## Implementation phases - -### Phase 0: discovery and contracts - -- capture fixtures; -- verify San José field population and publication delays; -- finalize the LocalDecision and citation contracts; -- document source terms and retention policy; -- define quality metrics and expected partial states. - -### Phase 1: official structured ingestion - -- register a Legistar scraper in `apps/scraper` and `apps/supervisor`; -- ingest Events, EventItems, Matters, bodies, histories, attachments, and structured votes; -- implement complete window paging, idempotent upserts, versioning, and replay; -- remove production mock fallbacks. - -### Phase 2: document evidence - -- download/version attachments; -- implement page-aware extraction and citation validation; -- normalize recommendation, fiscal impact, explicit scope, and participation; -- add minutes-based outcome fallback. - -### Phase 3: geographic relevance and API - -- resolve user jurisdiction/district; -- expose list, detail, timeline, and source-document APIs; -- rank district-specific and citywide decisions; -- surface partial/stale/source-failure states. - -### Phase 4: additional jurisdictions - -- extract San José parsing into a jurisdiction profile; -- validate the shared contract against a second Legistar jurisdiction; -- add non-Legistar adapters only after the domain contract survives that comparison. - -## Open questions - -1. Which San José bodies belong in the first release besides City Council and standing committees? -2. Should each reading/adoption be a separate LocalDecision or a timeline step with one canonical card? -3. How long may official documents be retained locally under source terms? -4. Which document types require OCR in practice, and what is the acceptable error rate? -5. How should amended recommendations be compared and presented? -6. What publication delay should trigger a visible "outcome pending" state? -7. When a district is mentioned only in supporting material, what confidence is required before ranking? -8. Should public letters be excluded entirely from AI processing in the first release? -9. Which source fields must be immutable audit history versus latest-state columns? -10. What review sample and accuracy threshold are required before AI-extracted facts ship? - -## Definition of done for the discovery spike - -The discovery phase is complete when: - -- the fixture set above is checked in or stored in an approved fixture location; -- every fixture can be normalized offline into the draft LocalDecision contract; -- every populated factual field has a resolvable citation; -- known unknowns and conflicting-source behavior are represented in tests; -- polling and storage estimates are based on measured San José volumes; -- the team has explicitly approved the domain model, provenance policy, and production failure behavior. +The transport in `packages/api/src/integrations/legistar.ts` performs bounded, +paged source reads and has no cache, mock fallback, or database side effects. +The registered `legistar` scraper: + +1. discovers meetings in a bounded past/future window; +2. restricts them to the explicit San José body policy; +3. fetches complete EventItems, Matters, attachments, histories, and votes; +4. classifies boilerplate, topics, geography, and document policy; +5. extracts native PDF text and hashes official documents; +6. idempotently upserts records and soft-deletes disappeared meetings, + occurrences, and attachments; +7. records each run, its window, counters, failure, and completion state. + +The default window is 45 days back and 120 days forward. Operators can tune the +window, item cap, document-size ceiling, and extraction through `LEGISTAR_*` +variables. A capped run never treats unvisited meetings as deleted. + +### Storage + +The old unused `legistar_*` cache tables are replaced by: + +| Table | Purpose | +| ------------------------- | --------------------------------------------- | +| `local_jurisdiction` | Adapter and public portal identity | +| `local_body` | Source bodies plus inclusion/relevance policy | +| `local_decision` | Canonical Matter, topic, dates, and geography | +| `local_meeting` | Event schedule and official artifact links | +| `local_meeting_item` | Matter occurrence and outcome fields | +| `local_decision_document` | Attachment policy, hash, and extracted text | +| `local_decision_history` | Structured Matter history when published | +| `local_decision_vote` | Named structured votes when published | +| `local_ingestion_run` | Window, status, counters, and errors | + +All foreign keys are indexed. Raw payloads and source modification times are +retained. Source removals are recorded rather than hard-deleted. Row-level +security is enabled on every new public-schema table without anon or +authenticated policies; access is through the server API. + +### Read API + +- `legistar.listDecisions`: upcoming/recent occurrences, text search, topic, + district relevance, and pagination. +- `legistar.getDecision`: canonical detail, timeline, documents, history, + votes, public-comment count, and participation guidance. +- `legistar.listBodies`: included active bodies in editorial priority order. +- `legistar.getIngestionHealth`: latest run and active-decision count. + +Old wire-format endpoints remain temporarily for dormant prototype callers. +They are deprecated, read-only source calls and are not the new UI contract. + +## San José first-release policy + +Live discovery in August 2026 confirmed unauthenticated reads and a large, +mixed body list. EventItems contain both actual Matters and procedural rows. +Attachments include staff memoranda, ordinances, resolutions, presentations, +supplements, and public letters. Sampled completed meetings often had sparse +structured histories and votes, so missing structured data means “unknown,” +never “no action.” + +### Included bodies + +The release uses three editorial tiers: + +1. City Council and the six standing policy committees. +2. High-impact resident-facing bodies, including Planning, Housing, Historic + Landmarks, Appeals, Airport, Bicycle/Pedestrian, Climate, and oversight. +3. Community and quality-of-life bodies, including Arts, Civil Service, + Senior Citizens, Youth, Privacy, Smart City, and Small Business. + +Closed session, notice-only, miscellaneous, internal administrative, and +employee-benefit bodies are excluded. Exact IDs live in +`apps/scraper/src/scrapers/legistar-policy.ts`. The boundary favors decisions +affecting residents’ money, housing, mobility, safety, services, rights, land +use, or participation. + +### Geography + +Scope is conservative. Explicit district references produce district scope; +explicit addresses produce place scope; only explicit citywide language +produces citywide scope. An absent district remains `unknown`. District items +may rank above citywide items, but citywide decisions should not be hidden. + +### Documents and OCR + +A sampled 39-page memorandum contained about 127,000 characters of embedded +text and a 14-page presentation about 17,000. A sampled one-page public letter +had no embedded text. Therefore: + +- official PDFs use native extraction first; +- fewer than 80 extracted characters per page marks `ocr_required`; +- public-comment documents are link-only and never enter OCR or indexing; +- OCR publication should require at least 98% sampled character accuracy (or + equivalent high-confidence validation); +- OCR must not be the sole evidence for votes, money, deadlines, or addresses. + +The current implementation detects and records OCR work; the OCR worker is a +follow-up. Native extraction against live PDFs produced 379,761 characters +across 19 documents with an average quality score of 0.9947. + +## Public-comment privacy decision + +Public-comment letters matter because they show resident engagement and form +part of the official record. They can also expose names, signatures, home +addresses, email addresses, phone numbers, medical circumstances, immigration +details, or other information submitted for a specific civic purpose. + +For the first release Billion shows the official link and aggregate document +count, but does not download, OCR, index, quote, summarize, classify, or build +profiles from individual letters. Public availability is not the same as +consent to make scattered personal details newly searchable or send them to an +AI provider. Aggregate sentiment could be reconsidered only with redaction, +minimum-group thresholds, provenance, and a dedicated privacy review. + +## Evidence, outcomes, and participation + +Fact precedence is field-specific: + +1. structured action/vote data for that EventItem; +2. approved official minutes; +3. amended agenda, agenda, or staff memorandum; +4. other official attachments and meeting pages; +5. official video/transcript as context only. + +Missing votes/history remain unknown. AI-authored explanations require +validated citations and must not infer a vote from attendance or sentiment. +The backend retains the source material for this layer but does not publish AI +summaries yet. Page-aware citations, minutes fallback, and generation remain +quality-gated follow-ups rather than silently shipping uncited text. + +The API currently provides a labeled San José participation fallback link and +warns readers to verify the agenda. A future extractor should prefer current +meeting-specific instructions, method, explicit deadline/timezone, item +identifier, and retrieval time. Never manufacture a deadline. + +## Operations and failure behavior + +- Production never substitutes synthetic records for failed source reads. +- Pagination and retries are bounded. +- Every run records its complete query window and result. +- Last-known records survive transient failures. +- Retrieval, extraction, and deletion are separate states. +- Public reads exclude soft-deleted records. +- Fixtures and policy tests run offline. + +Recommended cadence is daily broad discovery, every six hours within 14 days, +hourly within 48 hours, and continued refresh after meetings until approved +outcomes publish. Scheduler wiring can apply this without changing the scraper. + +## Decisions and remaining work + +Decisions made here: + +- canonical card = Matter; each hearing/reading = timeline occurrence; +- explicit three-tier body allowlist based on resident impact; +- native extraction first, with deterministic OCR and acceptance gates; +- public letters remain links/counts but are excluded from AI/text processing; +- conservative geography with `unknown` as a valid state; +- normalized source-neutral schema instead of unused cache tables; +- no mock fallback and no uncited AI summaries. + +Remaining product-quality work: + +- approved-minutes extraction and page-level citations; +- OCR worker and measured validation set; +- agenda-specific participation extraction; +- official GIS resolution for addresses and user districts; +- cited AI explanations and amended-recommendation comparison; +- measured “outcome pending” thresholds; +- validation against a second jurisdiction before generalizing policy. diff --git a/packages/api/src/integrations/legistar.test.ts b/packages/api/src/integrations/legistar.test.ts new file mode 100644 index 00000000..6b4947e9 --- /dev/null +++ b/packages/api/src/integrations/legistar.test.ts @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { LegistarClient } from "./legistar"; + +function requestUrl(input: Parameters[0]): URL { + if (input instanceof URL) return input; + if (typeof input === "string") return new URL(input); + return new URL(input.url); +} + +void test("paginates list endpoints with explicit top and skip", async () => { + const urls: URL[] = []; + const client = new LegistarClient((input) => { + const url = requestUrl(input); + urls.push(url); + const skip = Number(url.searchParams.get("$skip")); + const rows = + skip === 0 + ? Array.from({ length: 1000 }, (_, index) => ({ BodyId: index + 1 })) + : [{ BodyId: 1001 }]; + return Promise.resolve(Response.json(rows)); + }); + + const bodies = await client.getBodies("sanjose"); + + assert.equal(bodies.length, 1001); + assert.equal(urls.length, 2); + const [first, second] = urls; + assert.ok(first); + assert.ok(second); + assert.equal(first.searchParams.get("$top"), "1000"); + assert.equal(first.searchParams.get("$skip"), "0"); + assert.equal(second.searchParams.get("$skip"), "1000"); +}); + +void test("escapes apostrophes in OData filters", async () => { + let requested: URL | undefined; + const client = new LegistarClient((input) => { + requested = requestUrl(input); + return Promise.resolve(Response.json([])); + }); + + await client.getLegislation("sanjose", { text: "resident's street" }); + + assert.ok(requested); + const filter = requested.searchParams.get("$filter"); + assert.ok(filter); + assert.match(filter, /resident''s street/); +}); + +void test("requests complete agenda-item evidence fields", async () => { + let requested: URL | undefined; + const client = new LegistarClient((input) => { + requested = requestUrl(input); + return Promise.resolve(Response.json([])); + }); + + await client.getAgendaItems("sanjose", 42); + + assert.ok(requested); + assert.equal(requested.pathname, "/v1/sanjose/Events/42/EventItems"); + assert.equal(requested.searchParams.get("AgendaNote"), "1"); + assert.equal(requested.searchParams.get("MinutesNote"), "1"); + assert.equal(requested.searchParams.get("Attachments"), "1"); + assert.equal(requested.searchParams.get("RollCalls"), "1"); +}); diff --git a/packages/api/src/integrations/legistar.ts b/packages/api/src/integrations/legistar.ts index 8e999ef0..5e3ed924 100644 --- a/packages/api/src/integrations/legistar.ts +++ b/packages/api/src/integrations/legistar.ts @@ -1,55 +1,41 @@ -// ============================================================================ -// Cached Client — DB-backed cache with 24h TTL -// ============================================================================ - -import { and, eq, gt } from "@acme/db"; -import { db } from "@acme/db/client"; -import { - LegistarAgendaItem as LegistarAgendaItemRow, - LegistarBody as LegistarBodyRow, - LegistarMatter as LegistarMatterRow, - LegistarMeeting as LegistarMeetingRow, - LegistarVote as LegistarVoteRow, -} from "@acme/db/schema"; - /** - * Legistar Web API Client + * Legistar Web API transport. * - * Integrates with the Legistar API for local government legislation data. - * API docs: https://webapi.legistar.com/Help - * - * Supported jurisdictions: - * - San Jose: sanjose.legistar.com - * - Santa Clara County: sccgov.legistar.com - * - Sunnyvale: sunnyvaleca.legistar.com + * This module intentionally contains no persistence or product policy. The + * scraper owns ingestion and normalization; the API router reads the durable + * local-government tables. Keeping the transport stateless prevents an app + * request from becoming an accidental crawler run. */ -// Jurisdiction configurations -// Note: Client names in Legistar API may differ from subdomain names export const JURISDICTIONS = { sanjose: { client: "sanjose", - name: "City of San Jose", + name: "City of San José", baseUrl: "https://webapi.legistar.com/v1/sanjose", + publicPortalUrl: "https://sanjose.legistar.com", + state: "CA", + timezone: "America/Los_Angeles", }, santaclara: { client: "santaclara", name: "Santa Clara County", baseUrl: "https://webapi.legistar.com/v1/santaclara", + publicPortalUrl: "https://sccgov.legistar.com", + state: "CA", + timezone: "America/Los_Angeles", }, sunnyvale: { client: "sunnyvaleca", name: "City of Sunnyvale", baseUrl: "https://webapi.legistar.com/v1/sunnyvaleca", + publicPortalUrl: "https://sunnyvaleca.legistar.com", + state: "CA", + timezone: "America/Los_Angeles", }, } as const; export type Jurisdiction = keyof typeof JURISDICTIONS; -// ============================================================================ -// Legistar API Types -// ============================================================================ - export interface LegistarMeeting { EventId: number; EventGuid: string; @@ -105,6 +91,27 @@ export interface LegistarMatter { MatterRestrictViewViaWeb: boolean; } +export interface LegistarAttachment { + MatterAttachmentId: number; + MatterAttachmentGuid: string; + MatterAttachmentLastModifiedUtc: string; + MatterAttachmentRowVersion: string; + MatterAttachmentName: string; + MatterAttachmentHyperlink: string; + MatterAttachmentFileName: string | null; + MatterAttachmentMatterVersion: string; + MatterAttachmentIsHyperlink: boolean; + MatterAttachmentBinary: string | null; + MatterAttachmentIsSupportingDocument: boolean; + MatterAttachmentShowOnInternetPage: boolean; + MatterAttachmentIsMinuteOrder: boolean; + MatterAttachmentIsBoardLetter: boolean; + MatterAttachmentAgiloftId: number; + MatterAttachmentDescription: string | null; + MatterAttachmentPrintWithReports: boolean; + MatterAttachmentSort: number; +} + export interface LegistarVote { VoteId: number; VoteGuid: string; @@ -157,27 +164,6 @@ export interface LegistarAgendaItem { EventItemMatterAttachments: LegistarAttachment[] | null; } -export interface LegistarAttachment { - MatterAttachmentId: number; - MatterAttachmentGuid: string; - MatterAttachmentLastModifiedUtc: string; - MatterAttachmentRowVersion: string; - MatterAttachmentName: string; - MatterAttachmentHyperlink: string; - MatterAttachmentFileName: string | null; - MatterAttachmentMatterVersion: string; - MatterAttachmentIsHyperlink: boolean; - MatterAttachmentBinary: string | null; - MatterAttachmentIsSupportingDocument: boolean; - MatterAttachmentShowOnInternetPage: boolean; - MatterAttachmentIsMinuteOrder: boolean; - MatterAttachmentIsBoardLetter: boolean; - MatterAttachmentAgiloftId: number; - MatterAttachmentDescription: string | null; - MatterAttachmentPrintWithReports: boolean; - MatterAttachmentSort: number; -} - export interface LegistarBody { BodyId: number; BodyGuid: string; @@ -201,6 +187,21 @@ export interface LegistarBody { BodyUsedSponsorFlag: number; } +export interface LegistarMatterHistory { + MatterHistoryId: number; + MatterHistoryGuid: string; + MatterHistoryMatterId: number; + MatterHistoryActionDate: string | null; + MatterHistoryActionId: number | null; + MatterHistoryActionName: string | null; + MatterHistoryDescription: string | null; + MatterHistoryBodyId: number | null; + MatterHistoryBodyName: string | null; + MatterHistoryEventId: number | null; + MatterHistoryEventItemId: number | null; + MatterHistoryAgendaNumber: string | null; +} + export interface DateRange { start: Date; end: Date; @@ -215,31 +216,50 @@ export interface LegislationQuery { introDateTo?: Date; } -// ============================================================================ -// Legistar Client -// ============================================================================ +export type FetchLike = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; + +const PAGE_SIZE = 1000; +const MAX_PAGES = 100; + +function isoDate(date: Date): string { + return date.toISOString().slice(0, 10); +} + +function escapeOData(value: string): string { + return value.replaceAll("'", "''"); +} + +export class LegistarError extends Error { + constructor( + message: string, + public statusCode: number, + public jurisdiction: Jurisdiction, + public endpoint: string, + ) { + super(message); + this.name = "LegistarError"; + } +} -class LegistarClient { - private async fetch( +export class LegistarClient { + constructor(private readonly request: FetchLike = fetch) {} + + private async fetchJson( jurisdiction: Jurisdiction, endpoint: string, - params?: Record, + params: Record = {}, ): Promise { - const config = JURISDICTIONS[jurisdiction]; - const url = new URL(`${config.baseUrl}${endpoint}`); - - if (params) { - Object.entries(params).forEach(([key, value]) => { - url.searchParams.set(key, value); - }); + const url = new URL(`${JURISDICTIONS[jurisdiction].baseUrl}${endpoint}`); + for (const [key, value] of Object.entries(params)) { + url.searchParams.set(key, value); } - const response = await fetch(url.toString(), { - headers: { - Accept: "application/json", - }, + const response = await this.request(url, { + headers: { Accept: "application/json" }, }); - if (!response.ok) { throw new LegistarError( `Legistar API error: ${response.status} ${response.statusText}`, @@ -248,928 +268,148 @@ class LegistarClient { endpoint, ); } - return response.json() as Promise; } - /** - * Get meetings for a jurisdiction within a date range. - */ - async getMeetings( + private async fetchAll( + jurisdiction: Jurisdiction, + endpoint: string, + params: Record = {}, + ): Promise { + const rows: T[] = []; + for (let page = 0; page < MAX_PAGES; page++) { + const batch = await this.fetchJson(jurisdiction, endpoint, { + ...params, + $top: String(PAGE_SIZE), + $skip: String(page * PAGE_SIZE), + }); + rows.push(...batch); + if (batch.length < PAGE_SIZE) return rows; + } + throw new LegistarError( + `Legistar pagination exceeded ${MAX_PAGES * PAGE_SIZE} rows`, + 0, + jurisdiction, + endpoint, + ); + } + + getMeetings( jurisdiction: Jurisdiction, dateRange?: DateRange, ): Promise { - const params: Record = {}; - + const params: Record = { $orderby: "EventDate asc" }; if (dateRange) { - // OData filter for date range - const startStr = dateRange.start.toISOString().split("T")[0]; - const endStr = dateRange.end.toISOString().split("T")[0]; - params.$filter = `EventDate ge datetime'${startStr}' and EventDate le datetime'${endStr}'`; + params.$filter = `EventDate ge datetime'${isoDate(dateRange.start)}' and EventDate le datetime'${isoDate(dateRange.end)}'`; } - - params.$orderby = "EventDate desc"; - - return this.fetch(jurisdiction, "/Events", params); + return this.fetchAll(jurisdiction, "/Events", params); } - /** - * Get legislation (matters) for a jurisdiction with optional query filters. - */ - async getLegislation( + getLegislation( jurisdiction: Jurisdiction, - query?: LegislationQuery, + query: LegislationQuery = {}, ): Promise { - const params: Record = {}; const filters: string[] = []; - - if (query) { - if (query.text) { - // Search in title using substringof (OData 2.0 compatible) - filters.push( - `(substringof('${query.text}',MatterTitle) or substringof('${query.text}',MatterFile))`, - ); - } - if (query.matterType) { - filters.push(`MatterTypeName eq '${query.matterType}'`); - } - if (query.status) { - filters.push(`MatterStatusName eq '${query.status}'`); - } - if (query.bodyId) { - filters.push(`MatterBodyId eq ${query.bodyId}`); - } - if (query.introDateFrom) { - const dateStr = query.introDateFrom.toISOString().split("T")[0]; - filters.push(`MatterIntroDate ge datetime'${dateStr}'`); - } - if (query.introDateTo) { - const dateStr = query.introDateTo.toISOString().split("T")[0]; - filters.push(`MatterIntroDate le datetime'${dateStr}'`); - } - } - - if (filters.length > 0) { - params.$filter = filters.join(" and "); + if (query.text) { + const text = escapeOData(query.text); + filters.push( + `(substringof('${text}',MatterTitle) or substringof('${text}',MatterFile))`, + ); } - - params.$orderby = "MatterIntroDate desc"; - params.$top = "100"; - - return this.fetch(jurisdiction, "/Matters", params); + if (query.matterType) + filters.push(`MatterTypeName eq '${escapeOData(query.matterType)}'`); + if (query.status) + filters.push(`MatterStatusName eq '${escapeOData(query.status)}'`); + if (query.bodyId) filters.push(`MatterBodyId eq ${query.bodyId}`); + if (query.introDateFrom) + filters.push( + `MatterIntroDate ge datetime'${isoDate(query.introDateFrom)}'`, + ); + if (query.introDateTo) + filters.push( + `MatterIntroDate le datetime'${isoDate(query.introDateTo)}'`, + ); + return this.fetchAll(jurisdiction, "/Matters", { + ...(filters.length ? { $filter: filters.join(" and ") } : {}), + $orderby: "MatterIntroDate desc", + }); } - /** - * Get votes for a specific event item (agenda item with voting). - * Note: Votes are associated with EventItems, not Matters directly. - */ - async getVotes( + getMeeting( jurisdiction: Jurisdiction, - eventItemId: number, - ): Promise { - return this.fetch( - jurisdiction, - `/EventItems/${eventItemId}/Votes`, - ); + eventId: number, + ): Promise { + return this.fetchJson(jurisdiction, `/Events/${eventId}`); } - /** - * Get roll call votes for all items in a meeting. - * Returns agenda items with their associated votes. - */ - async getMeetingVotes( + getAgendaItems( jurisdiction: Jurisdiction, - meetingId: number, + eventId: number, ): Promise { - return this.fetch( - jurisdiction, - `/Events/${meetingId}/EventItems`, - { RollCalls: "1" }, - ); + return this.fetchAll(jurisdiction, `/Events/${eventId}/EventItems`, { + AgendaNote: "1", + MinutesNote: "1", + Attachments: "1", + RollCalls: "1", + }); } - /** - * Get agenda items for a specific meeting. - */ - async getAgendas( + /** @deprecated Use getAgendaItems. */ + getAgendas( jurisdiction: Jurisdiction, - meetingId: number, + eventId: number, ): Promise { - return this.fetch( - jurisdiction, - `/Events/${meetingId}/EventItems`, - { AgendaNote: "1", MinutesNote: "1", Attachments: "1" }, - ); + return this.getAgendaItems(jurisdiction, eventId); } - /** - * Get a single meeting by ID. - */ - async getMeeting( - jurisdiction: Jurisdiction, - meetingId: number, - ): Promise { - return this.fetch(jurisdiction, `/Events/${meetingId}`, { - EventItems: "1", - EventItemAttachments: "1", - }); - } - - /** - * Get a single matter (legislation) by ID. - */ - async getMatter( + getMatter( jurisdiction: Jurisdiction, matterId: number, ): Promise { - return this.fetch(jurisdiction, `/Matters/${matterId}`); - } - - /** - * Get all bodies (committees, councils, boards) for a jurisdiction. - */ - async getBodies(jurisdiction: Jurisdiction): Promise { - return this.fetch(jurisdiction, "/Bodies", { - $filter: "BodyActiveFlag eq 1", - }); + return this.fetchJson(jurisdiction, `/Matters/${matterId}`); } - /** - * Get attachments for a matter. - */ - async getMatterAttachments( + getMatterAttachments( jurisdiction: Jurisdiction, matterId: number, ): Promise { - return this.fetch( - jurisdiction, - `/Matters/${matterId}/Attachments`, - ); - } - - /** - * Search for matters across all matter types. - */ - async searchMatters( - jurisdiction: Jurisdiction, - searchText: string, - ): Promise { - return this.getLegislation(jurisdiction, { text: searchText }); - } -} - -// ============================================================================ -// Error Handling -// ============================================================================ - -export class LegistarError extends Error { - constructor( - message: string, - public statusCode: number, - public jurisdiction: Jurisdiction, - public endpoint: string, - ) { - super(message); - this.name = "LegistarError"; - } -} - -// ============================================================================ -// Mock Data (used when API is unavailable in development) -// ============================================================================ - -function mockDate(daysAgo: number): string { - const d = new Date(); - d.setDate(d.getDate() - daysAgo); - return d.toISOString(); -} - -const MOCK_MATTERS_SANJOSE: LegistarMatter[] = [ - { - MatterId: 90001, - MatterGuid: "mock-sj-001", - MatterLastModifiedUtc: mockDate(2), - MatterRowVersion: "1", - MatterFile: "RES 2025-101", - MatterName: null, - MatterTitle: - "Approval of Affordable Housing Development at 500 E Santa Clara St", - MatterTypeId: 1, - MatterTypeName: "Resolution", - MatterStatusId: 1, - MatterStatusName: "Approved", - MatterBodyId: 1, - MatterBodyName: "City Council", - MatterIntroDate: mockDate(30), - MatterAgendaDate: mockDate(7), - MatterPassedDate: mockDate(5), - MatterEnactmentDate: null, - MatterEnactmentNumber: null, - MatterRequester: null, - MatterNotes: null, - MatterVersion: "1", - MatterText1: null, - MatterText2: null, - MatterText3: null, - MatterText4: null, - MatterText5: null, - MatterRestrictViewViaWeb: false, - }, - { - MatterId: 90002, - MatterGuid: "mock-sj-002", - MatterLastModifiedUtc: mockDate(5), - MatterRowVersion: "1", - MatterFile: "ORD 2025-045", - MatterName: null, - MatterTitle: - "Amendment to Municipal Code Chapter 20.80 — Protected Trees Ordinance Update", - MatterTypeId: 2, - MatterTypeName: "Ordinance", - MatterStatusId: 2, - MatterStatusName: "Pending", - MatterBodyId: 1, - MatterBodyName: "City Council", - MatterIntroDate: mockDate(20), - MatterAgendaDate: mockDate(10), - MatterPassedDate: null, - MatterEnactmentDate: null, - MatterEnactmentNumber: null, - MatterRequester: null, - MatterNotes: null, - MatterVersion: "1", - MatterText1: null, - MatterText2: null, - MatterText3: null, - MatterText4: null, - MatterText5: null, - MatterRestrictViewViaWeb: false, - }, - { - MatterId: 90003, - MatterGuid: "mock-sj-003", - MatterLastModifiedUtc: mockDate(8), - MatterRowVersion: "1", - MatterFile: "MGR 2025-012", - MatterName: null, - MatterTitle: - "City Manager Report on Downtown Bike Lane Network Expansion Plan", - MatterTypeId: 3, - MatterTypeName: "Report", - MatterStatusId: 1, - MatterStatusName: "Filed", - MatterBodyId: 3, - MatterBodyName: "Transportation & Environment Committee", - MatterIntroDate: mockDate(15), - MatterAgendaDate: mockDate(10), - MatterPassedDate: null, - MatterEnactmentDate: null, - MatterEnactmentNumber: null, - MatterRequester: null, - MatterNotes: null, - MatterVersion: "1", - MatterText1: null, - MatterText2: null, - MatterText3: null, - MatterText4: null, - MatterText5: null, - MatterRestrictViewViaWeb: false, - }, - { - MatterId: 90004, - MatterGuid: "mock-sj-004", - MatterLastModifiedUtc: mockDate(1), - MatterRowVersion: "1", - MatterFile: "RES 2025-118", - MatterName: null, - MatterTitle: - "Authorization for Emergency Water Main Repair on N 1st Street", - MatterTypeId: 1, - MatterTypeName: "Resolution", - MatterStatusId: 1, - MatterStatusName: "Approved", - MatterBodyId: 1, - MatterBodyName: "City Council", - MatterIntroDate: mockDate(3), - MatterAgendaDate: mockDate(2), - MatterPassedDate: mockDate(1), - MatterEnactmentDate: null, - MatterEnactmentNumber: null, - MatterRequester: null, - MatterNotes: null, - MatterVersion: "1", - MatterText1: null, - MatterText2: null, - MatterText3: null, - MatterText4: null, - MatterText5: null, - MatterRestrictViewViaWeb: false, - }, -]; - -const MOCK_MATTERS_SANTACLARA: LegistarMatter[] = [ - { - MatterId: 91001, - MatterGuid: "mock-sc-001", - MatterLastModifiedUtc: mockDate(3), - MatterRowVersion: "1", - MatterFile: "BOS 2025-034", - MatterName: null, - MatterTitle: - "Adoption of Santa Clara County Climate Action Plan 2030 Update", - MatterTypeId: 1, - MatterTypeName: "Board Resolution", - MatterStatusId: 2, - MatterStatusName: "Pending", - MatterBodyId: 1, - MatterBodyName: "Board of Supervisors", - MatterIntroDate: mockDate(25), - MatterAgendaDate: mockDate(7), - MatterPassedDate: null, - MatterEnactmentDate: null, - MatterEnactmentNumber: null, - MatterRequester: null, - MatterNotes: null, - MatterVersion: "1", - MatterText1: null, - MatterText2: null, - MatterText3: null, - MatterText4: null, - MatterText5: null, - MatterRestrictViewViaWeb: false, - }, - { - MatterId: 91002, - MatterGuid: "mock-sc-002", - MatterLastModifiedUtc: mockDate(6), - MatterRowVersion: "1", - MatterFile: "BOS 2025-029", - MatterName: null, - MatterTitle: - "Agreement with Valley Transportation Authority for BART Phase II Funding", - MatterTypeId: 1, - MatterTypeName: "Board Resolution", - MatterStatusId: 1, - MatterStatusName: "Approved", - MatterBodyId: 1, - MatterBodyName: "Board of Supervisors", - MatterIntroDate: mockDate(40), - MatterAgendaDate: mockDate(14), - MatterPassedDate: mockDate(7), - MatterEnactmentDate: null, - MatterEnactmentNumber: null, - MatterRequester: null, - MatterNotes: null, - MatterVersion: "1", - MatterText1: null, - MatterText2: null, - MatterText3: null, - MatterText4: null, - MatterText5: null, - MatterRestrictViewViaWeb: false, - }, - { - MatterId: 91003, - MatterGuid: "mock-sc-003", - MatterLastModifiedUtc: mockDate(4), - MatterRowVersion: "1", - MatterFile: "BOS 2025-041", - MatterName: null, - MatterTitle: - "Ordinance Amending County Code for Short-Term Rental Regulations in Unincorporated Areas", - MatterTypeId: 2, - MatterTypeName: "Ordinance", - MatterStatusId: 2, - MatterStatusName: "Pending", - MatterBodyId: 1, - MatterBodyName: "Board of Supervisors", - MatterIntroDate: mockDate(14), - MatterAgendaDate: mockDate(7), - MatterPassedDate: null, - MatterEnactmentDate: null, - MatterEnactmentNumber: null, - MatterRequester: null, - MatterNotes: null, - MatterVersion: "1", - MatterText1: null, - MatterText2: null, - MatterText3: null, - MatterText4: null, - MatterText5: null, - MatterRestrictViewViaWeb: false, - }, -]; - -class FallbackLegistarClient extends LegistarClient { - override async getLegislation( - jurisdiction: Jurisdiction, - query?: LegislationQuery, - ): Promise { - try { - return await super.getLegislation(jurisdiction, query); - } catch { - if (jurisdiction === "sanjose") return MOCK_MATTERS_SANJOSE; - if (jurisdiction === "santaclara") return MOCK_MATTERS_SANTACLARA; - return []; - } - } -} - -const CACHE_TTL_MS = 24 * 60 * 60 * 1000; - -function parseDate(s: string | null | undefined): Date | null { - if (!s) return null; - const d = new Date(s); - return isNaN(d.getTime()) ? null : d; -} - -class CachedLegistarClient extends FallbackLegistarClient { - override async getLegislation( - jurisdiction: Jurisdiction, - query?: LegislationQuery, - ): Promise { - if ( - !query?.text && - !query?.matterType && - !query?.status && - !query?.bodyId - ) { - const cached = await db - .select() - .from(LegistarMatterRow) - .where( - and( - eq(LegistarMatterRow.jurisdiction, jurisdiction), - gt( - LegistarMatterRow.fetchedAt, - new Date(Date.now() - CACHE_TTL_MS), - ), - ), - ) - .orderBy(LegistarMatterRow.lastModifiedUtc) - .limit(100); - - if (cached.length > 0) { - return cached.map(rowToMatter); - } - } - - const matters = await super.getLegislation(jurisdiction, query); - await this.upsertMatters(jurisdiction, matters); - return matters; - } - - override async getMeetings( - jurisdiction: Jurisdiction, - dateRange?: DateRange, - ): Promise { - const cached = await db - .select() - .from(LegistarMeetingRow) - .where( - and( - eq(LegistarMeetingRow.jurisdiction, jurisdiction), - gt(LegistarMeetingRow.fetchedAt, new Date(Date.now() - CACHE_TTL_MS)), - ), - ) - .orderBy(LegistarMeetingRow.date); - - const filtered = dateRange - ? cached.filter((m) => { - const d = m.date.getTime(); - return d >= dateRange.start.getTime() && d <= dateRange.end.getTime(); - }) - : cached; - - if (filtered.length > 0) return filtered.map(rowToMeeting); - - const meetings = await super.getMeetings(jurisdiction, dateRange); - await this.upsertMeetings(jurisdiction, meetings); - return meetings; - } - - override async getBodies( - jurisdiction: Jurisdiction, - ): Promise { - const cached = await db - .select() - .from(LegistarBodyRow) - .where( - and( - eq(LegistarBodyRow.jurisdiction, jurisdiction), - gt(LegistarBodyRow.fetchedAt, new Date(Date.now() - CACHE_TTL_MS)), - ), - ); - - if (cached.length > 0) return cached.map(rowToBody); - - const bodies = await super.getBodies(jurisdiction); - await this.upsertBodies(jurisdiction, bodies); - return bodies; + return this.fetchAll(jurisdiction, `/Matters/${matterId}/Attachments`); } - override async getAgendas( + getMatterHistories( jurisdiction: Jurisdiction, - meetingId: number, - ): Promise { - const cached = await db - .select() - .from(LegistarAgendaItemRow) - .where( - and( - eq(LegistarAgendaItemRow.jurisdiction, jurisdiction), - eq(LegistarAgendaItemRow.eventId, meetingId), - gt( - LegistarAgendaItemRow.fetchedAt, - new Date(Date.now() - CACHE_TTL_MS), - ), - ), - ) - .orderBy(LegistarAgendaItemRow.agendaSequence); - - if (cached.length > 0) return cached.map(rowToAgendaItem); - - const items = await super.getAgendas(jurisdiction, meetingId); - await this.upsertAgendaItems(jurisdiction, items); - return items; + matterId: number, + ): Promise { + return this.fetchAll(jurisdiction, `/Matters/${matterId}/Histories`); } - override async getVotes( + getVotes( jurisdiction: Jurisdiction, eventItemId: number, ): Promise { - const cached = await db - .select() - .from(LegistarVoteRow) - .where( - and( - eq(LegistarVoteRow.jurisdiction, jurisdiction), - eq(LegistarVoteRow.eventItemId, eventItemId), - gt(LegistarVoteRow.fetchedAt, new Date(Date.now() - CACHE_TTL_MS)), - ), - ) - .orderBy(LegistarVoteRow.sort); - - if (cached.length > 0) return cached.map(rowToVote); - - const votes = await super.getVotes(jurisdiction, eventItemId); - await this.upsertVotes(jurisdiction, votes); - return votes; + return this.fetchAll(jurisdiction, `/EventItems/${eventItemId}/Votes`); } - override async getMeetingVotes( + getMeetingVotes( jurisdiction: Jurisdiction, - meetingId: number, + eventId: number, ): Promise { - const items = await super.getMeetingVotes(jurisdiction, meetingId); - await this.upsertAgendaItems(jurisdiction, items); - return items; - } - - // --- Upsert helpers --- - - private async upsertMatters( - jurisdiction: Jurisdiction, - matters: LegistarMatter[], - ) { - if (matters.length === 0) return; - const now = new Date(); - for (const m of matters) { - await db - .insert(LegistarMatterRow) - .values({ - jurisdiction, - matterId: m.MatterId, - matterGuid: m.MatterGuid, - matterFile: m.MatterFile, - title: m.MatterTitle, - name: m.MatterName, - typeName: m.MatterTypeName, - statusName: m.MatterStatusName, - bodyName: m.MatterBodyName, - bodyId: m.MatterBodyId, - introDate: parseDate(m.MatterIntroDate), - agendaDate: parseDate(m.MatterAgendaDate), - passedDate: parseDate(m.MatterPassedDate), - enactmentDate: parseDate(m.MatterEnactmentDate), - enactmentNumber: m.MatterEnactmentNumber, - requester: m.MatterRequester, - notes: m.MatterNotes, - lastModifiedUtc: new Date(m.MatterLastModifiedUtc), - fetchedAt: now, - }) - .onConflictDoUpdate({ - target: [LegistarMatterRow.jurisdiction, LegistarMatterRow.matterId], - set: { - title: m.MatterTitle, - statusName: m.MatterStatusName, - lastModifiedUtc: new Date(m.MatterLastModifiedUtc), - fetchedAt: now, - }, - }); - } - } - - private async upsertMeetings( - jurisdiction: Jurisdiction, - meetings: LegistarMeeting[], - ) { - if (meetings.length === 0) return; - const now = new Date(); - for (const m of meetings) { - await db - .insert(LegistarMeetingRow) - .values({ - jurisdiction, - eventId: m.EventId, - eventGuid: m.EventGuid, - bodyId: m.EventBodyId, - bodyName: m.EventBodyName, - date: new Date(m.EventDate), - time: m.EventTime, - location: m.EventLocation, - agendaFile: m.EventAgendaFile, - minutesFile: m.EventMinutesFile, - videoPath: m.EventVideoPath, - agendaStatusName: m.EventAgendaStatusName, - minutesStatusName: m.EventMinutesStatusName, - comment: m.EventComment, - inSiteUrl: m.EventInSiteURL, - lastModifiedUtc: new Date(m.EventLastModifiedUtc), - fetchedAt: now, - }) - .onConflictDoUpdate({ - target: [LegistarMeetingRow.jurisdiction, LegistarMeetingRow.eventId], - set: { - agendaFile: m.EventAgendaFile, - minutesFile: m.EventMinutesFile, - videoPath: m.EventVideoPath, - lastModifiedUtc: new Date(m.EventLastModifiedUtc), - fetchedAt: now, - }, - }); - } + return this.getAgendaItems(jurisdiction, eventId); } - private async upsertBodies( - jurisdiction: Jurisdiction, - bodies: LegistarBody[], - ) { - if (bodies.length === 0) return; - const now = new Date(); - for (const b of bodies) { - await db - .insert(LegistarBodyRow) - .values({ - jurisdiction, - bodyId: b.BodyId, - bodyGuid: b.BodyGuid, - name: b.BodyName, - typeName: b.BodyTypeName, - activeFlag: b.BodyActiveFlag === 1, - numberOfMembers: b.BodyNumberOfMembers, - description: b.BodyDescription, - contactName: b.BodyContactFullName, - contactEmail: b.BodyContactEmail, - contactPhone: b.BodyContactPhone, - fetchedAt: now, - }) - .onConflictDoUpdate({ - target: [LegistarBodyRow.jurisdiction, LegistarBodyRow.bodyId], - set: { - name: b.BodyName, - activeFlag: b.BodyActiveFlag === 1, - fetchedAt: now, - }, - }); - } + getBodies(jurisdiction: Jurisdiction): Promise { + return this.fetchAll(jurisdiction, "/Bodies", { + $filter: "BodyActiveFlag eq 1", + $orderby: "BodyName asc", + }); } - private async upsertAgendaItems( + searchMatters( jurisdiction: Jurisdiction, - items: LegistarAgendaItem[], - ) { - if (items.length === 0) return; - const now = new Date(); - for (const i of items) { - await db - .insert(LegistarAgendaItemRow) - .values({ - jurisdiction, - eventItemId: i.EventItemId, - eventId: i.EventItemEventId, - agendaSequence: i.EventItemAgendaSequence, - agendaNumber: i.EventItemAgendaNumber, - title: i.EventItemTitle, - actionName: i.EventItemActionName, - passedFlagName: i.EventItemPassedFlagName, - tally: i.EventItemTally, - moverName: i.EventItemMover, - seconderName: i.EventItemSeconder, - matterId: i.EventItemMatterId, - matterFile: i.EventItemMatterFile, - matterName: i.EventItemMatterName, - matterType: i.EventItemMatterType, - matterStatus: i.EventItemMatterStatus, - consent: i.EventItemConsent === 1, - agendaNote: i.EventItemAgendaNote, - minutesNote: i.EventItemMinutesNote, - lastModifiedUtc: new Date(i.EventItemLastModifiedUtc), - fetchedAt: now, - }) - .onConflictDoUpdate({ - target: [ - LegistarAgendaItemRow.jurisdiction, - LegistarAgendaItemRow.eventItemId, - ], - set: { - actionName: i.EventItemActionName, - passedFlagName: i.EventItemPassedFlagName, - tally: i.EventItemTally, - fetchedAt: now, - }, - }); - } - } - - private async upsertVotes(jurisdiction: Jurisdiction, votes: LegistarVote[]) { - if (votes.length === 0) return; - const now = new Date(); - for (const v of votes) { - await db - .insert(LegistarVoteRow) - .values({ - jurisdiction, - voteId: v.VoteId, - eventItemId: v.VoteEventItemId, - personId: v.VotePersonId, - personName: v.VotePersonName, - valueName: v.VoteValueName, - sort: v.VoteSort, - lastModifiedUtc: new Date(v.VoteLastModifiedUtc), - fetchedAt: now, - }) - .onConflictDoUpdate({ - target: [LegistarVoteRow.jurisdiction, LegistarVoteRow.voteId], - set: { - valueName: v.VoteValueName, - fetchedAt: now, - }, - }); - } + searchText: string, + ): Promise { + return this.getLegislation(jurisdiction, { text: searchText }); } } -// --- Row-to-API-type mappers (for cache reads) --- - -function rowToMatter(r: typeof LegistarMatterRow.$inferSelect): LegistarMatter { - return { - MatterId: r.matterId, - MatterGuid: r.matterGuid ?? "", - MatterLastModifiedUtc: r.lastModifiedUtc.toISOString(), - MatterRowVersion: "1", - MatterFile: r.matterFile ?? "", - MatterName: r.name, - MatterTitle: r.title, - MatterTypeId: 0, - MatterTypeName: r.typeName ?? "", - MatterStatusId: 0, - MatterStatusName: r.statusName ?? "", - MatterBodyId: r.bodyId ?? 0, - MatterBodyName: r.bodyName ?? "", - MatterIntroDate: r.introDate?.toISOString() ?? null, - MatterAgendaDate: r.agendaDate?.toISOString() ?? null, - MatterPassedDate: r.passedDate?.toISOString() ?? null, - MatterEnactmentDate: r.enactmentDate?.toISOString() ?? null, - MatterEnactmentNumber: r.enactmentNumber, - MatterRequester: r.requester, - MatterNotes: r.notes, - MatterVersion: "1", - MatterText1: null, - MatterText2: null, - MatterText3: null, - MatterText4: null, - MatterText5: null, - MatterRestrictViewViaWeb: false, - }; -} - -function rowToMeeting( - r: typeof LegistarMeetingRow.$inferSelect, -): LegistarMeeting { - return { - EventId: r.eventId, - EventGuid: r.eventGuid ?? "", - EventLastModifiedUtc: r.lastModifiedUtc.toISOString(), - EventRowVersion: "1", - EventBodyId: r.bodyId ?? 0, - EventBodyName: r.bodyName ?? "", - EventDate: r.date.toISOString(), - EventTime: r.time, - EventVideoStatus: null, - EventAgendaStatusId: 0, - EventAgendaStatusName: r.agendaStatusName ?? "", - EventMinutesStatusId: 0, - EventMinutesStatusName: r.minutesStatusName ?? "", - EventLocation: r.location, - EventAgendaFile: r.agendaFile, - EventMinutesFile: r.minutesFile, - EventAgendaLastPublishedUTC: null, - EventMinutesLastPublishedUTC: null, - EventComment: r.comment, - EventVideoPath: r.videoPath, - EventInSiteURL: r.inSiteUrl, - EventItems: null, - }; -} - -function rowToBody(r: typeof LegistarBodyRow.$inferSelect): LegistarBody { - return { - BodyId: r.bodyId, - BodyGuid: r.bodyGuid ?? "", - BodyLastModifiedUtc: r.fetchedAt.toISOString(), - BodyRowVersion: "1", - BodyName: r.name, - BodyTypeId: 0, - BodyTypeName: r.typeName ?? "", - BodyMeetFlag: 0, - BodyActiveFlag: r.activeFlag ? 1 : 0, - BodySort: 0, - BodyDescription: r.description, - BodyContactNameId: null, - BodyContactFullName: r.contactName, - BodyContactPhone: r.contactPhone, - BodyContactEmail: r.contactEmail, - BodyUsedControlFlag: 0, - BodyNumberOfMembers: r.numberOfMembers ?? 0, - BodyUsedActingFlag: 0, - BodyUsedTargetFlag: 0, - BodyUsedSponsorFlag: 0, - }; -} - -function rowToAgendaItem( - r: typeof LegistarAgendaItemRow.$inferSelect, -): LegistarAgendaItem { - return { - EventItemId: r.eventItemId, - EventItemGuid: "", - EventItemLastModifiedUtc: r.lastModifiedUtc.toISOString(), - EventItemRowVersion: "1", - EventItemEventId: r.eventId, - EventItemAgendaSequence: r.agendaSequence ?? 0, - EventItemMinutesSequence: null, - EventItemAgendaNumber: r.agendaNumber, - EventItemVideo: null, - EventItemVideoIndex: null, - EventItemVersion: "1", - EventItemAgendaNote: r.agendaNote, - EventItemMinutesNote: r.minutesNote, - EventItemActionId: null, - EventItemActionName: r.actionName, - EventItemActionText: null, - EventItemPassedFlag: null, - EventItemPassedFlagName: r.passedFlagName, - EventItemRollCallFlag: null, - EventItemFlagExtra: null, - EventItemTitle: r.title, - EventItemTally: r.tally, - EventItemAccelaRecordId: null, - EventItemConsent: r.consent ? 1 : 0, - EventItemMoverId: null, - EventItemMover: r.moverName, - EventItemSeconderId: null, - EventItemSeconder: r.seconderName, - EventItemMatterId: r.matterId, - EventItemMatterGuid: null, - EventItemMatterFile: r.matterFile, - EventItemMatterName: r.matterName, - EventItemMatterType: r.matterType, - EventItemMatterStatus: r.matterStatus, - EventItemMatterAttachments: null, - }; -} - -function rowToVote(r: typeof LegistarVoteRow.$inferSelect): LegistarVote { - return { - VoteId: r.voteId, - VoteGuid: "", - VoteLastModifiedUtc: r.lastModifiedUtc.toISOString(), - VoteRowVersion: "1", - VotePersonId: r.personId, - VotePersonName: r.personName, - VoteValueId: 0, - VoteValueName: r.valueName, - VoteSort: r.sort ?? 0, - VoteResult: null, - VoteEventItemId: r.eventItemId, - }; -} - -// ============================================================================ -// Export singleton instance -// ============================================================================ - -export const legistar = new CachedLegistarClient(); - -export { LegistarClient }; +export const legistar = new LegistarClient(); diff --git a/packages/api/src/router/legistar.ts b/packages/api/src/router/legistar.ts index 26f6fe6a..86281283 100644 --- a/packages/api/src/router/legistar.ts +++ b/packages/api/src/router/legistar.ts @@ -2,43 +2,386 @@ import type { TRPCRouterRecord } from "@trpc/server"; import { TRPCError } from "@trpc/server"; import { z } from "zod/v4"; +import type { SQL } from "@acme/db"; +import { and, asc, count, desc, eq, gte, isNull, lte, or, sql } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalBody, + LocalDecision, + LocalDecisionDocument, + LocalDecisionHistory, + LocalDecisionVote, + LocalIngestionRun, + LocalJurisdiction, + LocalMeeting, + LocalMeetingItem, +} from "@acme/db/schema"; + import { JURISDICTIONS, legistar } from "../integrations/legistar"; import { publicProcedure } from "../trpc"; const jurisdictionEnum = z.enum(["sanjose", "santaclara", "sunnyvale"]); +const participation = { + instructionsUrl: + "https://www.sanjoseca.gov/your-government/appointees/city-clerk/council-agendas-minutes", + methods: ["in_person", "email", "ecomment"] as const, + note: "Submission methods and deadlines can change. Verify the official meeting agenda before participating.", +}; + +function apiError(error: unknown, fallback: string): TRPCError { + return new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: error instanceof Error ? error.message : fallback, + cause: error, + }); +} + +const listInput = z + .object({ + jurisdiction: jurisdictionEnum.default("sanjose"), + timeline: z.enum(["upcoming", "recent", "all"]).default("upcoming"), + from: z.date().optional(), + to: z.date().optional(), + topic: z.string().max(80).optional(), + district: z.number().int().min(1).max(10).optional(), + query: z.string().trim().min(2).max(200).optional(), + limit: z.number().int().min(1).max(100).default(30), + offset: z.number().int().min(0).max(10_000).default(0), + }) + .optional(); + +async function listDecisions(input: z.infer) { + const options = input ?? { + jurisdiction: "sanjose" as const, + timeline: "upcoming" as const, + limit: 30, + offset: 0, + }; + const now = new Date(); + const conditions: SQL[] = [ + eq(LocalMeeting.jurisdictionKey, options.jurisdiction), + isNull(LocalMeeting.sourceDeletedAt), + isNull(LocalMeetingItem.sourceDeletedAt), + isNull(LocalDecision.sourceDeletedAt), + ]; + if (options.timeline === "upcoming") + conditions.push(gte(LocalMeeting.startsAt, options.from ?? now)); + if (options.timeline === "recent") + conditions.push(lte(LocalMeeting.startsAt, options.to ?? now)); + if (options.from) conditions.push(gte(LocalMeeting.startsAt, options.from)); + if (options.to) conditions.push(lte(LocalMeeting.startsAt, options.to)); + if (options.topic) conditions.push(eq(LocalDecision.topic, options.topic)); + if (options.district) { + conditions.push( + sql`(${LocalDecision.scopeKind} = 'citywide' or ${options.district} = any(coalesce(${LocalDecision.districtNumbers}, '{}')))`, + ); + } + if (options.query) { + conditions.push( + sql`${LocalDecision.searchVector} @@ websearch_to_tsquery('english', ${options.query})`, + ); + } + + const relevance = options.district + ? sql`case + when ${options.district} = any(coalesce(${LocalDecision.districtNumbers}, '{}')) then 0 + when ${LocalDecision.scopeKind} = 'citywide' then 1 + else 2 + end` + : sql`0`; + const order = + options.timeline === "recent" + ? [asc(relevance), desc(LocalMeeting.startsAt)] + : [asc(relevance), asc(LocalMeeting.startsAt)]; + + return db + .select({ + id: LocalDecision.id, + jurisdiction: LocalJurisdiction.name, + fileNumber: LocalDecision.fileNumber, + title: LocalDecision.title, + type: LocalDecision.typeName, + status: LocalDecision.statusName, + topic: LocalDecision.topic, + scope: LocalDecision.scopeKind, + districtNumbers: LocalDecision.districtNumbers, + geographicText: LocalDecision.geographicText, + sourceUrl: LocalDecision.sourceUrl, + meetingItemId: LocalMeetingItem.id, + sourceEventItemId: LocalMeetingItem.sourceEventItemId, + agendaNumber: LocalMeetingItem.agendaNumber, + proposedAction: LocalMeetingItem.actionText, + outcome: LocalMeetingItem.actionName, + passed: LocalMeetingItem.passedFlagName, + tally: LocalMeetingItem.tally, + meetingId: LocalMeeting.id, + sourceEventId: LocalMeeting.sourceEventId, + meetingStartsAt: LocalMeeting.startsAt, + meetingCancelled: LocalMeeting.cancelled, + body: LocalBody.name, + bodyRelevanceTier: LocalBody.relevanceTier, + }) + .from(LocalMeetingItem) + .innerJoin(LocalDecision, eq(LocalMeetingItem.decisionId, LocalDecision.id)) + .innerJoin(LocalMeeting, eq(LocalMeetingItem.meetingId, LocalMeeting.id)) + .innerJoin(LocalBody, eq(LocalMeeting.bodyId, LocalBody.id)) + .innerJoin( + LocalJurisdiction, + eq(LocalMeeting.jurisdictionKey, LocalJurisdiction.key), + ) + .where(and(...conditions)) + .orderBy(...order) + .limit(options.limit) + .offset(options.offset); +} + export const legistarRouter = { - getLocalBills: publicProcedure.query(async () => { + /** Decision-centric, durable read API for the new frontend. */ + listDecisions: publicProcedure.input(listInput).query(async ({ input }) => { try { - const [sanjose, santaclara] = await Promise.all([ - legistar.getLegislation("sanjose", {}).catch(() => []), - legistar.getLegislation("santaclara", {}).catch(() => []), + return await listDecisions(input); + } catch (error) { + throw apiError(error, "Failed to load local decisions"); + } + }), + + getDecision: publicProcedure + .input(z.object({ id: z.string().uuid() })) + .query(async ({ input }) => { + const [decision] = await db + .select({ + id: LocalDecision.id, + jurisdictionKey: LocalDecision.jurisdictionKey, + jurisdiction: LocalJurisdiction.name, + fileNumber: LocalDecision.fileNumber, + title: LocalDecision.title, + name: LocalDecision.name, + type: LocalDecision.typeName, + status: LocalDecision.statusName, + topic: LocalDecision.topic, + scope: LocalDecision.scopeKind, + districtNumbers: LocalDecision.districtNumbers, + geographicText: LocalDecision.geographicText, + introDate: LocalDecision.introDate, + agendaDate: LocalDecision.agendaDate, + passedDate: LocalDecision.passedDate, + enactmentDate: LocalDecision.enactmentDate, + enactmentNumber: LocalDecision.enactmentNumber, + requester: LocalDecision.requester, + notes: LocalDecision.notes, + sourceUrl: LocalDecision.sourceUrl, + sourceUpdatedAt: LocalDecision.sourceUpdatedAt, + }) + .from(LocalDecision) + .innerJoin( + LocalJurisdiction, + eq(LocalDecision.jurisdictionKey, LocalJurisdiction.key), + ) + .where( + and( + eq(LocalDecision.id, input.id), + isNull(LocalDecision.sourceDeletedAt), + ), + ) + .limit(1); + if (!decision) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Decision not found", + }); + } + + const [occurrences, documents, history] = await Promise.all([ + db + .select({ + id: LocalMeetingItem.id, + sourceEventItemId: LocalMeetingItem.sourceEventItemId, + agendaNumber: LocalMeetingItem.agendaNumber, + title: LocalMeetingItem.title, + proposedAction: LocalMeetingItem.actionText, + action: LocalMeetingItem.actionName, + passed: LocalMeetingItem.passedFlagName, + tally: LocalMeetingItem.tally, + mover: LocalMeetingItem.moverName, + seconder: LocalMeetingItem.seconderName, + consent: LocalMeetingItem.consent, + minutesNote: LocalMeetingItem.minutesNote, + meetingId: LocalMeeting.id, + sourceEventId: LocalMeeting.sourceEventId, + startsAt: LocalMeeting.startsAt, + location: LocalMeeting.location, + cancelled: LocalMeeting.cancelled, + agendaUrl: LocalMeeting.agendaUrl, + minutesUrl: LocalMeeting.minutesUrl, + videoUrl: LocalMeeting.videoUrl, + meetingSourceUrl: LocalMeeting.sourceUrl, + body: LocalBody.name, + }) + .from(LocalMeetingItem) + .innerJoin( + LocalMeeting, + eq(LocalMeetingItem.meetingId, LocalMeeting.id), + ) + .innerJoin(LocalBody, eq(LocalMeeting.bodyId, LocalBody.id)) + .where( + and( + eq(LocalMeetingItem.decisionId, input.id), + isNull(LocalMeetingItem.sourceDeletedAt), + isNull(LocalMeeting.sourceDeletedAt), + ), + ) + .orderBy(asc(LocalMeeting.startsAt)), + db + .select({ + id: LocalDecisionDocument.id, + name: LocalDecisionDocument.name, + description: LocalDecisionDocument.description, + url: LocalDecisionDocument.url, + category: LocalDecisionDocument.category, + isPublicComment: LocalDecisionDocument.isPublicComment, + extractionStatus: LocalDecisionDocument.extractionStatus, + pageCount: LocalDecisionDocument.pageCount, + sourceUpdatedAt: LocalDecisionDocument.sourceUpdatedAt, + }) + .from(LocalDecisionDocument) + .where( + and( + eq(LocalDecisionDocument.decisionId, input.id), + isNull(LocalDecisionDocument.sourceDeletedAt), + ), + ) + .orderBy( + asc(LocalDecisionDocument.isPublicComment), + asc(LocalDecisionDocument.sortOrder), + ), + db + .select({ + id: LocalDecisionHistory.id, + actionDate: LocalDecisionHistory.actionDate, + body: LocalDecisionHistory.bodyName, + action: LocalDecisionHistory.actionName, + description: LocalDecisionHistory.actionText, + agendaNumber: LocalDecisionHistory.agendaNumber, + }) + .from(LocalDecisionHistory) + .where(eq(LocalDecisionHistory.decisionId, input.id)) + .orderBy(asc(LocalDecisionHistory.actionDate)), ]); - const allBills = [ - ...sanjose.map((b) => ({ ...b, jurisdiction: "San Jose" as const })), - ...santaclara.map((b) => ({ - ...b, - jurisdiction: "Santa Clara County" as const, - })), - ]; - - return allBills - .sort( - (a, b) => - new Date(b.MatterLastModifiedUtc).getTime() - - new Date(a.MatterLastModifiedUtc).getTime(), + const occurrenceIds = occurrences.map((occurrence) => occurrence.id); + const votes = occurrenceIds.length + ? await db + .select({ + meetingItemId: LocalDecisionVote.meetingItemId, + personName: LocalDecisionVote.personName, + value: LocalDecisionVote.valueName, + sort: LocalDecisionVote.sortOrder, + }) + .from(LocalDecisionVote) + .where( + or( + ...occurrenceIds.map((id) => + eq(LocalDecisionVote.meetingItemId, id), + ), + ), + ) + .orderBy(asc(LocalDecisionVote.sortOrder)) + : []; + + const publicCommentDocuments = documents.filter( + (document) => document.isPublicComment, + ); + + return { + ...decision, + occurrences, + documents: documents.filter((document) => !document.isPublicComment), + history, + votes, + publicComments: { + documentCount: publicCommentDocuments.length, + officialLinks: publicCommentDocuments.map((document) => ({ + id: document.id, + url: document.url, + label: "Public comments in the official record", + })), + }, + participation, + }; + }), + + listBodies: publicProcedure + .input( + z + .object({ jurisdiction: jurisdictionEnum.default("sanjose") }) + .optional(), + ) + .query(({ input }) => + db + .select({ + id: LocalBody.id, + sourceBodyId: LocalBody.sourceBodyId, + name: LocalBody.name, + type: LocalBody.typeName, + description: LocalBody.description, + relevanceTier: LocalBody.relevanceTier, + }) + .from(LocalBody) + .where( + and( + eq(LocalBody.jurisdictionKey, input?.jurisdiction ?? "sanjose"), + eq(LocalBody.included, true), + eq(LocalBody.active, true), + ), ) - .slice(0, 10); + .orderBy(asc(LocalBody.relevanceTier), asc(LocalBody.name)), + ), + + getIngestionHealth: publicProcedure + .input( + z + .object({ jurisdiction: jurisdictionEnum.default("sanjose") }) + .optional(), + ) + .query(async ({ input }) => { + const jurisdiction = input?.jurisdiction ?? "sanjose"; + const [latestRun] = await db + .select() + .from(LocalIngestionRun) + .where(eq(LocalIngestionRun.jurisdictionKey, jurisdiction)) + .orderBy(desc(LocalIngestionRun.startedAt)) + .limit(1); + const [decisionCount] = await db + .select({ value: count() }) + .from(LocalDecision) + .where( + and( + eq(LocalDecision.jurisdictionKey, jurisdiction), + isNull(LocalDecision.sourceDeletedAt), + ), + ); + return { + jurisdiction, + latestRun: latestRun ?? null, + activeDecisions: decisionCount?.value ?? 0, + }; + }), + + // ----------------------------------------------------------------------- + // Deprecated wire-format compatibility endpoints. These keep the dormant + // prototype components compiling while the new frontend moves to the + // durable decision endpoints above. They do not write to the database. + // ----------------------------------------------------------------------- + getLocalBills: publicProcedure.query(async () => { + try { + const sanjose = await legistar.getLegislation("sanjose", {}); + return sanjose.slice(0, 10).map((matter) => ({ + ...matter, + jurisdiction: "San José" as const, + })); } catch (error) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - error instanceof Error - ? error.message - : "Failed to fetch local bills", - cause: error, - }); + throw apiError(error, "Failed to fetch local bills"); } }), @@ -53,122 +396,42 @@ export const legistarRouter = { ) .query(async ({ input }) => { try { - const jurisdictions = input?.jurisdiction - ? [input.jurisdiction] - : (["sanjose", "santaclara"] as const); + const jurisdiction = input?.jurisdiction ?? "sanjose"; const start = new Date(); - const end = new Date(); + const end = new Date(start); end.setDate(end.getDate() + (input?.daysAhead ?? 30)); - - const results = await Promise.all( - jurisdictions.map(async (j) => { - const meetings = await legistar - .getMeetings(j, { start, end }) - .catch(() => []); - return meetings.map((m) => ({ - ...m, - jurisdiction: JURISDICTIONS[j].name, - })); + return (await legistar.getMeetings(jurisdiction, { start, end })).map( + (meeting) => ({ + ...meeting, + jurisdiction: JURISDICTIONS[jurisdiction].name, }), ); - - return results - .flat() - .sort( - (a, b) => - new Date(a.EventDate).getTime() - new Date(b.EventDate).getTime(), - ); } catch (error) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - error instanceof Error ? error.message : "Failed to fetch meetings", - cause: error, - }); + throw apiError(error, "Failed to fetch meetings"); } }), getAgenda: publicProcedure - .input( - z.object({ - jurisdiction: jurisdictionEnum, - meetingId: z.number(), - }), - ) - .query(async ({ input }) => { - try { - return await legistar.getAgendas(input.jurisdiction, input.meetingId); - } catch (error) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - error instanceof Error ? error.message : "Failed to fetch agenda", - cause: error, - }); - } - }), + .input(z.object({ jurisdiction: jurisdictionEnum, meetingId: z.number() })) + .query(({ input }) => + legistar.getAgendaItems(input.jurisdiction, input.meetingId), + ), getVotes: publicProcedure .input( - z.object({ - jurisdiction: jurisdictionEnum, - eventItemId: z.number(), - }), + z.object({ jurisdiction: jurisdictionEnum, eventItemId: z.number() }), ) - .query(async ({ input }) => { - try { - return await legistar.getVotes(input.jurisdiction, input.eventItemId); - } catch (error) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - error instanceof Error ? error.message : "Failed to fetch votes", - cause: error, - }); - } - }), + .query(({ input }) => + legistar.getVotes(input.jurisdiction, input.eventItemId), + ), getBodies: publicProcedure - .input( - z.object({ - jurisdiction: jurisdictionEnum, - }), - ) - .query(async ({ input }) => { - try { - return await legistar.getBodies(input.jurisdiction); - } catch (error) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - error instanceof Error ? error.message : "Failed to fetch bodies", - cause: error, - }); - } - }), + .input(z.object({ jurisdiction: jurisdictionEnum })) + .query(({ input }) => legistar.getBodies(input.jurisdiction)), getMeetingVotes: publicProcedure - .input( - z.object({ - jurisdiction: jurisdictionEnum, - meetingId: z.number(), - }), - ) - .query(async ({ input }) => { - try { - return await legistar.getMeetingVotes( - input.jurisdiction, - input.meetingId, - ); - } catch (error) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - error instanceof Error - ? error.message - : "Failed to fetch meeting votes", - cause: error, - }); - } - }), + .input(z.object({ jurisdiction: jurisdictionEnum, meetingId: z.number() })) + .query(({ input }) => + legistar.getMeetingVotes(input.jurisdiction, input.meetingId), + ), } satisfies TRPCRouterRecord; diff --git a/packages/db/drizzle/0014_tough_stone_men.sql b/packages/db/drizzle/0014_tough_stone_men.sql new file mode 100644 index 00000000..bc22d41d --- /dev/null +++ b/packages/db/drizzle/0014_tough_stone_men.sql @@ -0,0 +1,253 @@ +CREATE TABLE "local_body" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "jurisdiction_key" varchar(50) NOT NULL, + "source_body_id" integer NOT NULL, + "source_guid" varchar(100), + "name" text NOT NULL, + "type_name" varchar(100), + "active" boolean DEFAULT true NOT NULL, + "included" boolean DEFAULT false NOT NULL, + "relevance_tier" integer DEFAULT 3 NOT NULL, + "number_of_members" integer, + "description" text, + "contact_name" varchar(256), + "contact_email" varchar(256), + "contact_phone" varchar(50), + "source_updated_at" timestamp with time zone, + "last_seen_at" timestamp with time zone DEFAULT now() NOT NULL, + "source_payload" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "local_body_jurisdictionKey_sourceBodyId_unique" UNIQUE("jurisdiction_key","source_body_id") +); +--> statement-breakpoint +CREATE TABLE "local_decision" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "jurisdiction_key" varchar(50) NOT NULL, + "primary_body_id" uuid, + "source_matter_id" integer NOT NULL, + "source_guid" varchar(100), + "file_number" varchar(100), + "title" text NOT NULL, + "name" text, + "type_name" varchar(100), + "status_name" varchar(100), + "topic" varchar(80), + "scope_kind" varchar(30) DEFAULT 'unknown' NOT NULL, + "district_numbers" integer[], + "geographic_text" text, + "intro_date" timestamp with time zone, + "agenda_date" timestamp with time zone, + "passed_date" timestamp with time zone, + "enactment_date" timestamp with time zone, + "enactment_number" varchar(100), + "requester" text, + "notes" text, + "source_url" text, + "source_updated_at" timestamp with time zone NOT NULL, + "last_seen_at" timestamp with time zone DEFAULT now() NOT NULL, + "source_deleted_at" timestamp with time zone, + "source_payload" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + "search_vector" "tsvector" GENERATED ALWAYS AS (( + setweight(to_tsvector('english', coalesce(file_number, '')), 'A') || + setweight(to_tsvector('english', coalesce(title, '')), 'A') || + setweight(to_tsvector('english', coalesce(type_name, '') || ' ' || coalesce(topic, '')), 'B') || + setweight(to_tsvector('english', coalesce(requester, '') || ' ' || coalesce(notes, '') || ' ' || coalesce(geographic_text, '')), 'C') + )) STORED, + CONSTRAINT "local_decision_jurisdictionKey_sourceMatterId_unique" UNIQUE("jurisdiction_key","source_matter_id") +); +--> statement-breakpoint +CREATE TABLE "local_decision_document" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "jurisdiction_key" varchar(50) NOT NULL, + "decision_id" uuid NOT NULL, + "source_attachment_id" integer NOT NULL, + "source_guid" varchar(100), + "name" text NOT NULL, + "description" text, + "url" text NOT NULL, + "file_name" text, + "category" varchar(50) NOT NULL, + "sort_order" integer, + "is_supporting_document" boolean DEFAULT false NOT NULL, + "is_public_comment" boolean DEFAULT false NOT NULL, + "processing_policy" varchar(30) NOT NULL, + "extraction_status" varchar(30) DEFAULT 'pending' NOT NULL, + "extracted_text" text, + "extraction_method" varchar(30), + "extraction_quality" real, + "page_count" integer, + "byte_size" integer, + "mime_type" varchar(100), + "content_hash" varchar(64), + "source_updated_at" timestamp with time zone, + "last_seen_at" timestamp with time zone DEFAULT now() NOT NULL, + "source_deleted_at" timestamp with time zone, + "source_payload" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "local_decision_document_jurisdictionKey_sourceAttachmentId_unique" UNIQUE("jurisdiction_key","source_attachment_id") +); +--> statement-breakpoint +CREATE TABLE "local_decision_history" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "decision_id" uuid NOT NULL, + "source_history_id" integer NOT NULL, + "source_event_id" integer, + "source_event_item_id" integer, + "body_name" varchar(256), + "action_name" varchar(256), + "action_text" text, + "action_date" timestamp with time zone, + "agenda_number" varchar(50), + "source_payload" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "local_decision_history_decisionId_sourceHistoryId_unique" UNIQUE("decision_id","source_history_id") +); +--> statement-breakpoint +CREATE TABLE "local_decision_vote" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "meeting_item_id" uuid NOT NULL, + "source_vote_id" integer NOT NULL, + "source_person_id" integer NOT NULL, + "person_name" varchar(256) NOT NULL, + "value_name" varchar(50) NOT NULL, + "sort_order" integer, + "source_updated_at" timestamp with time zone NOT NULL, + "source_payload" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "local_decision_vote_meetingItemId_sourceVoteId_unique" UNIQUE("meeting_item_id","source_vote_id") +); +--> statement-breakpoint +CREATE TABLE "local_ingestion_run" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "jurisdiction_key" varchar(50) NOT NULL, + "status" varchar(20) NOT NULL, + "window_start" timestamp with time zone NOT NULL, + "window_end" timestamp with time zone NOT NULL, + "started_at" timestamp with time zone DEFAULT now() NOT NULL, + "completed_at" timestamp with time zone, + "counts" jsonb DEFAULT '{}'::jsonb NOT NULL, + "error" text +); +--> statement-breakpoint +CREATE TABLE "local_jurisdiction" ( + "key" varchar(50) PRIMARY KEY NOT NULL, + "name" varchar(256) NOT NULL, + "state" varchar(2) NOT NULL, + "government_level" varchar(30) NOT NULL, + "timezone" varchar(64) NOT NULL, + "source_type" varchar(30) NOT NULL, + "source_client" varchar(100) NOT NULL, + "source_base_url" text NOT NULL, + "public_portal_url" text, + "active" boolean DEFAULT true NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "local_meeting" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "jurisdiction_key" varchar(50) NOT NULL, + "body_id" uuid NOT NULL, + "source_event_id" integer NOT NULL, + "source_guid" varchar(100), + "starts_at" timestamp with time zone NOT NULL, + "local_date" varchar(10) NOT NULL, + "time_label" text, + "location" text, + "agenda_url" text, + "minutes_url" text, + "video_url" text, + "source_url" text, + "agenda_status_name" varchar(100), + "minutes_status_name" varchar(100), + "comment" text, + "cancelled" boolean DEFAULT false NOT NULL, + "source_updated_at" timestamp with time zone NOT NULL, + "last_seen_at" timestamp with time zone DEFAULT now() NOT NULL, + "source_deleted_at" timestamp with time zone, + "source_payload" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "local_meeting_jurisdictionKey_sourceEventId_unique" UNIQUE("jurisdiction_key","source_event_id") +); +--> statement-breakpoint +CREATE TABLE "local_meeting_item" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "meeting_id" uuid NOT NULL, + "decision_id" uuid, + "source_event_item_id" integer NOT NULL, + "source_guid" varchar(100), + "agenda_sequence" integer, + "minutes_sequence" integer, + "agenda_number" varchar(50), + "title" text, + "action_name" varchar(256), + "action_text" text, + "passed_flag_name" varchar(50), + "tally" varchar(50), + "mover_name" varchar(256), + "seconder_name" varchar(256), + "consent" boolean DEFAULT false NOT NULL, + "roll_call" boolean DEFAULT false NOT NULL, + "agenda_note" text, + "minutes_note" text, + "video_index" integer, + "source_updated_at" timestamp with time zone NOT NULL, + "last_seen_at" timestamp with time zone DEFAULT now() NOT NULL, + "source_deleted_at" timestamp with time zone, + "source_payload" jsonb, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "local_meeting_item_meetingId_sourceEventItemId_unique" UNIQUE("meeting_id","source_event_item_id") +); +--> statement-breakpoint +DROP TABLE "legistar_agenda_item" CASCADE;--> statement-breakpoint +DROP TABLE "legistar_body" CASCADE;--> statement-breakpoint +DROP TABLE "legistar_matter" CASCADE;--> statement-breakpoint +DROP TABLE "legistar_meeting" CASCADE;--> statement-breakpoint +DROP TABLE "legistar_vote" CASCADE;--> statement-breakpoint +ALTER TABLE "local_body" ADD CONSTRAINT "local_body_jurisdiction_key_local_jurisdiction_key_fk" FOREIGN KEY ("jurisdiction_key") REFERENCES "public"."local_jurisdiction"("key") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_decision" ADD CONSTRAINT "local_decision_jurisdiction_key_local_jurisdiction_key_fk" FOREIGN KEY ("jurisdiction_key") REFERENCES "public"."local_jurisdiction"("key") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_decision" ADD CONSTRAINT "local_decision_primary_body_id_local_body_id_fk" FOREIGN KEY ("primary_body_id") REFERENCES "public"."local_body"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_decision_document" ADD CONSTRAINT "local_decision_document_jurisdiction_key_local_jurisdiction_key_fk" FOREIGN KEY ("jurisdiction_key") REFERENCES "public"."local_jurisdiction"("key") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_decision_document" ADD CONSTRAINT "local_decision_document_decision_id_local_decision_id_fk" FOREIGN KEY ("decision_id") REFERENCES "public"."local_decision"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_decision_history" ADD CONSTRAINT "local_decision_history_decision_id_local_decision_id_fk" FOREIGN KEY ("decision_id") REFERENCES "public"."local_decision"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_decision_vote" ADD CONSTRAINT "local_decision_vote_meeting_item_id_local_meeting_item_id_fk" FOREIGN KEY ("meeting_item_id") REFERENCES "public"."local_meeting_item"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_ingestion_run" ADD CONSTRAINT "local_ingestion_run_jurisdiction_key_local_jurisdiction_key_fk" FOREIGN KEY ("jurisdiction_key") REFERENCES "public"."local_jurisdiction"("key") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_meeting" ADD CONSTRAINT "local_meeting_jurisdiction_key_local_jurisdiction_key_fk" FOREIGN KEY ("jurisdiction_key") REFERENCES "public"."local_jurisdiction"("key") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_meeting" ADD CONSTRAINT "local_meeting_body_id_local_body_id_fk" FOREIGN KEY ("body_id") REFERENCES "public"."local_body"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_meeting_item" ADD CONSTRAINT "local_meeting_item_meeting_id_local_meeting_id_fk" FOREIGN KEY ("meeting_id") REFERENCES "public"."local_meeting"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "local_meeting_item" ADD CONSTRAINT "local_meeting_item_decision_id_local_decision_id_fk" FOREIGN KEY ("decision_id") REFERENCES "public"."local_decision"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "local_body_jurisdiction_included_idx" ON "local_body" USING btree ("jurisdiction_key","included","relevance_tier");--> statement-breakpoint +CREATE INDEX "local_decision_primary_body_idx" ON "local_decision" USING btree ("primary_body_id");--> statement-breakpoint +CREATE INDEX "local_decision_active_updated_idx" ON "local_decision" USING btree ("jurisdiction_key","source_updated_at") WHERE "local_decision"."source_deleted_at" is null;--> statement-breakpoint +CREATE INDEX "local_decision_search_vector_idx" ON "local_decision" USING gin ("search_vector");--> statement-breakpoint +CREATE INDEX "local_document_decision_category_idx" ON "local_decision_document" USING btree ("decision_id","category");--> statement-breakpoint +CREATE INDEX "local_document_extraction_queue_idx" ON "local_decision_document" USING btree ("extraction_status") WHERE "local_decision_document"."source_deleted_at" is null;--> statement-breakpoint +CREATE INDEX "local_history_decision_action_date_idx" ON "local_decision_history" USING btree ("decision_id","action_date");--> statement-breakpoint +CREATE INDEX "local_vote_meeting_item_idx" ON "local_decision_vote" USING btree ("meeting_item_id");--> statement-breakpoint +CREATE INDEX "local_vote_source_person_idx" ON "local_decision_vote" USING btree ("source_person_id");--> statement-breakpoint +CREATE INDEX "local_ingestion_jurisdiction_started_idx" ON "local_ingestion_run" USING btree ("jurisdiction_key","started_at");--> statement-breakpoint +CREATE INDEX "local_meeting_body_starts_at_idx" ON "local_meeting" USING btree ("body_id","starts_at");--> statement-breakpoint +CREATE INDEX "local_meeting_active_starts_at_idx" ON "local_meeting" USING btree ("jurisdiction_key","starts_at") WHERE "local_meeting"."source_deleted_at" is null;--> statement-breakpoint +CREATE INDEX "local_meeting_item_sequence_idx" ON "local_meeting_item" USING btree ("meeting_id","agenda_sequence");--> statement-breakpoint +CREATE INDEX "local_meeting_item_decision_idx" ON "local_meeting_item" USING btree ("decision_id");--> statement-breakpoint +-- These tables live in Supabase's exposed public schema, but Billion serves +-- them through the server-side tRPC API. Enabling RLS with no anon/authenticated +-- policies keeps the Data API closed while database owners/service roles can +-- still run ingestion and server reads. +ALTER TABLE "local_jurisdiction" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "local_body" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "local_decision" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "local_meeting" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "local_meeting_item" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "local_decision_document" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "local_decision_history" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "local_decision_vote" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "local_ingestion_run" ENABLE ROW LEVEL SECURITY; diff --git a/packages/db/drizzle/meta/0012_snapshot.json b/packages/db/drizzle/meta/0014_snapshot.json similarity index 65% rename from packages/db/drizzle/meta/0012_snapshot.json rename to packages/db/drizzle/meta/0014_snapshot.json index 2aa86989..db6ad1f2 100644 --- a/packages/db/drizzle/meta/0012_snapshot.json +++ b/packages/db/drizzle/meta/0014_snapshot.json @@ -1,6 +1,6 @@ { - "id": "789ac976-2232-4d2d-8d67-ad09e5391a5a", - "prevId": "d27f89da-2614-4e96-9c2f-8fbb2d98ac57", + "id": "5dc2537e-e6b9-4303-93c5-85c4f41904b2", + "prevId": "c6d45e7b-5849-4c8a-adac-3600f3d31bf2", "version": "7", "dialect": "postgresql", "tables": { @@ -195,7 +195,10 @@ "bill_billNumber_sourceWebsite_unique": { "name": "bill_billNumber_sourceWebsite_unique", "nullsNotDistinct": false, - "columns": ["bill_number", "source_website"] + "columns": [ + "bill_number", + "source_website" + ] } }, "policies": {}, @@ -267,7 +270,11 @@ "blocked_content_userId_name_type_unique": { "name": "blocked_content_userId_name_type_unique", "nullsNotDistinct": false, - "columns": ["user_id", "name", "type"] + "columns": [ + "user_id", + "name", + "type" + ] } }, "policies": {}, @@ -369,8 +376,12 @@ "name": "brief_change_image_content_brief_id_content_brief_id_fk", "tableFrom": "brief_change_image", "tableTo": "content_brief", - "columnsFrom": ["content_brief_id"], - "columnsTo": ["id"], + "columnsFrom": [ + "content_brief_id" + ], + "columnsTo": [ + "id" + ], "onDelete": "cascade", "onUpdate": "no action" } @@ -380,7 +391,10 @@ "brief_change_image_contentBriefId_changeIndex_unique": { "name": "brief_change_image_contentBriefId_changeIndex_unique", "nullsNotDistinct": false, - "columns": ["content_brief_id", "change_index"] + "columns": [ + "content_brief_id", + "change_index" + ] } }, "policies": {}, @@ -565,7 +579,11 @@ "civic_api_cache_addressHash_endpoint_params_unique": { "name": "civic_api_cache_addressHash_endpoint_params_unique", "nullsNotDistinct": false, - "columns": ["address_hash", "endpoint", "params"] + "columns": [ + "address_hash", + "endpoint", + "params" + ] } }, "policies": {}, @@ -650,7 +668,10 @@ "content_brief_contentType_contentId_unique": { "name": "content_brief_contentType_contentId_unique", "nullsNotDistinct": false, - "columns": ["content_type", "content_id"] + "columns": [ + "content_type", + "content_id" + ] } }, "policies": {}, @@ -735,7 +756,10 @@ "content_lens_contentType_contentId_unique": { "name": "content_lens_contentType_contentId_unique", "nullsNotDistinct": false, - "columns": ["content_type", "content_id"] + "columns": [ + "content_type", + "content_id" + ] } }, "policies": {}, @@ -1044,7 +1068,10 @@ "court_case_caseNumber_court_unique": { "name": "court_case_caseNumber_court_unique", "nullsNotDistinct": false, - "columns": ["case_number", "court"] + "columns": [ + "case_number", + "court" + ] } }, "policies": {}, @@ -1126,7 +1153,10 @@ "election_externalId_source_unique": { "name": "election_externalId_source_unique", "nullsNotDistinct": false, - "columns": ["external_id", "source"] + "columns": [ + "external_id", + "source" + ] } }, "policies": {}, @@ -1260,15 +1290,17 @@ "government_content_url_unique": { "name": "government_content_url_unique", "nullsNotDistinct": false, - "columns": ["url"] + "columns": [ + "url" + ] } }, "policies": {}, "checkConstraints": {}, "isRLSEnabled": false }, - "public.legistar_agenda_item": { - "name": "legistar_agenda_item", + "public.local_body": { + "name": "local_body", "schema": "", "columns": { "id": { @@ -1278,137 +1310,109 @@ "notNull": true, "default": "gen_random_uuid()" }, - "jurisdiction": { - "name": "jurisdiction", + "jurisdiction_key": { + "name": "jurisdiction_key", "type": "varchar(50)", "primaryKey": false, "notNull": true }, - "event_item_id": { - "name": "event_item_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "event_id": { - "name": "event_id", + "source_body_id": { + "name": "source_body_id", "type": "integer", "primaryKey": false, "notNull": true }, - "agenda_sequence": { - "name": "agenda_sequence", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "agenda_number": { - "name": "agenda_number", - "type": "varchar(50)", + "source_guid": { + "name": "source_guid", + "type": "varchar(100)", "primaryKey": false, "notNull": false }, - "title": { - "name": "title", + "name": { + "name": "name", "type": "text", "primaryKey": false, - "notNull": false - }, - "action_name": { - "name": "action_name", - "type": "varchar(256)", - "primaryKey": false, - "notNull": false - }, - "passed_flag_name": { - "name": "passed_flag_name", - "type": "varchar(50)", - "primaryKey": false, - "notNull": false + "notNull": true }, - "tally": { - "name": "tally", - "type": "varchar(50)", + "type_name": { + "name": "type_name", + "type": "varchar(100)", "primaryKey": false, "notNull": false }, - "mover_name": { - "name": "mover_name", - "type": "varchar(256)", + "active": { + "name": "active", + "type": "boolean", "primaryKey": false, - "notNull": false + "notNull": true, + "default": true }, - "seconder_name": { - "name": "seconder_name", - "type": "varchar(256)", + "included": { + "name": "included", + "type": "boolean", "primaryKey": false, - "notNull": false + "notNull": true, + "default": false }, - "matter_id": { - "name": "matter_id", + "relevance_tier": { + "name": "relevance_tier", "type": "integer", "primaryKey": false, - "notNull": false + "notNull": true, + "default": 3 }, - "matter_file": { - "name": "matter_file", - "type": "varchar(100)", + "number_of_members": { + "name": "number_of_members", + "type": "integer", "primaryKey": false, "notNull": false }, - "matter_name": { - "name": "matter_name", + "description": { + "name": "description", "type": "text", "primaryKey": false, "notNull": false }, - "matter_type": { - "name": "matter_type", - "type": "varchar(100)", + "contact_name": { + "name": "contact_name", + "type": "varchar(256)", "primaryKey": false, "notNull": false }, - "matter_status": { - "name": "matter_status", - "type": "varchar(100)", + "contact_email": { + "name": "contact_email", + "type": "varchar(256)", "primaryKey": false, "notNull": false }, - "consent": { - "name": "consent", - "type": "boolean", - "primaryKey": false, - "notNull": false, - "default": false - }, - "agenda_note": { - "name": "agenda_note", - "type": "text", + "contact_phone": { + "name": "contact_phone", + "type": "varchar(50)", "primaryKey": false, "notNull": false }, - "minutes_note": { - "name": "minutes_note", - "type": "text", + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": false }, - "last_modified_utc": { - "name": "last_modified_utc", - "type": "timestamp", - "primaryKey": false, - "notNull": true - }, - "fetched_at": { - "name": "fetched_at", - "type": "timestamp", + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true, "default": "now()" }, + "source_payload": { + "name": "source_payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", - "type": "timestamp", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true, "default": "now()" @@ -1417,15 +1421,28 @@ "name": "updated_at", "type": "timestamp with time zone", "primaryKey": false, - "notNull": false + "notNull": true, + "default": "now()" } }, "indexes": { - "legistar_agenda_item_event_idx": { - "name": "legistar_agenda_item_event_idx", + "local_body_jurisdiction_included_idx": { + "name": "local_body_jurisdiction_included_idx", "columns": [ { - "expression": "event_id", + "expression": "jurisdiction_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "included", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "relevance_tier", "isExpression": false, "asc": true, "nulls": "last" @@ -1437,21 +1454,38 @@ "with": {} } }, - "foreignKeys": {}, + "foreignKeys": { + "local_body_jurisdiction_key_local_jurisdiction_key_fk": { + "name": "local_body_jurisdiction_key_local_jurisdiction_key_fk", + "tableFrom": "local_body", + "tableTo": "local_jurisdiction", + "columnsFrom": [ + "jurisdiction_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, "compositePrimaryKeys": {}, "uniqueConstraints": { - "legistar_agenda_item_jurisdiction_eventItemId_unique": { - "name": "legistar_agenda_item_jurisdiction_eventItemId_unique", + "local_body_jurisdictionKey_sourceBodyId_unique": { + "name": "local_body_jurisdictionKey_sourceBodyId_unique", "nullsNotDistinct": false, - "columns": ["jurisdiction", "event_item_id"] + "columns": [ + "jurisdiction_key", + "source_body_id" + ] } }, "policies": {}, "checkConstraints": {}, "isRLSEnabled": false }, - "public.legistar_body": { - "name": "legistar_body", + "public.local_decision": { + "name": "local_decision", "schema": "", "columns": { "id": { @@ -1461,139 +1495,32 @@ "notNull": true, "default": "gen_random_uuid()" }, - "jurisdiction": { - "name": "jurisdiction", + "jurisdiction_key": { + "name": "jurisdiction_key", "type": "varchar(50)", "primaryKey": false, "notNull": true }, - "body_id": { - "name": "body_id", - "type": "integer", - "primaryKey": false, - "notNull": true - }, - "body_guid": { - "name": "body_guid", - "type": "varchar(100)", - "primaryKey": false, - "notNull": false - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true - }, - "type_name": { - "name": "type_name", - "type": "varchar(100)", - "primaryKey": false, - "notNull": false - }, - "active_flag": { - "name": "active_flag", - "type": "boolean", - "primaryKey": false, - "notNull": false, - "default": true - }, - "number_of_members": { - "name": "number_of_members", - "type": "integer", - "primaryKey": false, - "notNull": false - }, - "description": { - "name": "description", - "type": "text", - "primaryKey": false, - "notNull": false - }, - "contact_name": { - "name": "contact_name", - "type": "varchar(256)", - "primaryKey": false, - "notNull": false - }, - "contact_email": { - "name": "contact_email", - "type": "varchar(256)", - "primaryKey": false, - "notNull": false - }, - "contact_phone": { - "name": "contact_phone", - "type": "varchar(50)", - "primaryKey": false, - "notNull": false - }, - "fetched_at": { - "name": "fetched_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "legistar_body_jurisdiction_bodyId_unique": { - "name": "legistar_body_jurisdiction_bodyId_unique", - "nullsNotDistinct": false, - "columns": ["jurisdiction", "body_id"] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.legistar_matter": { - "name": "legistar_matter", - "schema": "", - "columns": { - "id": { - "name": "id", + "primary_body_id": { + "name": "primary_body_id", "type": "uuid", - "primaryKey": true, - "notNull": true, - "default": "gen_random_uuid()" - }, - "jurisdiction": { - "name": "jurisdiction", - "type": "varchar(50)", "primaryKey": false, - "notNull": true + "notNull": false }, - "matter_id": { - "name": "matter_id", + "source_matter_id": { + "name": "source_matter_id", "type": "integer", "primaryKey": false, "notNull": true }, - "matter_guid": { - "name": "matter_guid", + "source_guid": { + "name": "source_guid", "type": "varchar(100)", "primaryKey": false, "notNull": false }, - "matter_file": { - "name": "matter_file", + "file_number": { + "name": "file_number", "type": "varchar(100)", "primaryKey": false, "notNull": false @@ -1622,39 +1549,52 @@ "primaryKey": false, "notNull": false }, - "body_name": { - "name": "body_name", - "type": "varchar(256)", + "topic": { + "name": "topic", + "type": "varchar(80)", "primaryKey": false, "notNull": false }, - "body_id": { - "name": "body_id", - "type": "integer", + "scope_kind": { + "name": "scope_kind", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "district_numbers": { + "name": "district_numbers", + "type": "integer[]", + "primaryKey": false, + "notNull": false + }, + "geographic_text": { + "name": "geographic_text", + "type": "text", "primaryKey": false, "notNull": false }, "intro_date": { "name": "intro_date", - "type": "timestamp", + "type": "timestamp with time zone", "primaryKey": false, "notNull": false }, "agenda_date": { "name": "agenda_date", - "type": "timestamp", + "type": "timestamp with time zone", "primaryKey": false, "notNull": false }, "passed_date": { "name": "passed_date", - "type": "timestamp", + "type": "timestamp with time zone", "primaryKey": false, "notNull": false }, "enactment_date": { "name": "enactment_date", - "type": "timestamp", + "type": "timestamp with time zone", "primaryKey": false, "notNull": false }, @@ -1676,22 +1616,40 @@ "primaryKey": false, "notNull": false }, - "last_modified_utc": { - "name": "last_modified_utc", - "type": "timestamp", + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true }, - "fetched_at": { - "name": "fetched_at", - "type": "timestamp", + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true, "default": "now()" }, + "source_deleted_at": { + "name": "source_deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source_payload": { + "name": "source_payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", - "type": "timestamp", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true, "default": "now()" @@ -1700,41 +1658,119 @@ "name": "updated_at", "type": "timestamp with time zone", "primaryKey": false, - "notNull": false + "notNull": true, + "default": "now()" + }, + "search_vector": { + "name": "search_vector", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "(\n setweight(to_tsvector('english', coalesce(file_number, '')), 'A') ||\n setweight(to_tsvector('english', coalesce(title, '')), 'A') ||\n setweight(to_tsvector('english', coalesce(type_name, '') || ' ' || coalesce(topic, '')), 'B') ||\n setweight(to_tsvector('english', coalesce(requester, '') || ' ' || coalesce(notes, '') || ' ' || coalesce(geographic_text, '')), 'C')\n )", + "type": "stored" + } } }, "indexes": { - "legistar_matter_file_idx": { - "name": "legistar_matter_file_idx", + "local_decision_primary_body_idx": { + "name": "local_decision_primary_body_idx", + "columns": [ + { + "expression": "primary_body_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "local_decision_active_updated_idx": { + "name": "local_decision_active_updated_idx", "columns": [ { - "expression": "matter_file", + "expression": "jurisdiction_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_updated_at", "isExpression": false, "asc": true, "nulls": "last" } ], "isUnique": false, + "where": "\"local_decision\".\"source_deleted_at\" is null", "concurrently": false, "method": "btree", "with": {} + }, + "local_decision_search_vector_idx": { + "name": "local_decision_search_vector_idx", + "columns": [ + { + "expression": "search_vector", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "local_decision_jurisdiction_key_local_jurisdiction_key_fk": { + "name": "local_decision_jurisdiction_key_local_jurisdiction_key_fk", + "tableFrom": "local_decision", + "tableTo": "local_jurisdiction", + "columnsFrom": [ + "jurisdiction_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "local_decision_primary_body_id_local_body_id_fk": { + "name": "local_decision_primary_body_id_local_body_id_fk", + "tableFrom": "local_decision", + "tableTo": "local_body", + "columnsFrom": [ + "primary_body_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" } }, - "foreignKeys": {}, "compositePrimaryKeys": {}, "uniqueConstraints": { - "legistar_matter_jurisdiction_matterId_unique": { - "name": "legistar_matter_jurisdiction_matterId_unique", + "local_decision_jurisdictionKey_sourceMatterId_unique": { + "name": "local_decision_jurisdictionKey_sourceMatterId_unique", "nullsNotDistinct": false, - "columns": ["jurisdiction", "matter_id"] + "columns": [ + "jurisdiction_key", + "source_matter_id" + ] } }, "policies": {}, "checkConstraints": {}, "isRLSEnabled": false }, - "public.legistar_meeting": { - "name": "legistar_meeting", + "public.local_decision_document": { + "name": "local_decision_document", "schema": "", "columns": { "id": { @@ -1744,233 +1780,1141 @@ "notNull": true, "default": "gen_random_uuid()" }, - "jurisdiction": { - "name": "jurisdiction", + "jurisdiction_key": { + "name": "jurisdiction_key", "type": "varchar(50)", "primaryKey": false, "notNull": true }, - "event_id": { - "name": "event_id", + "decision_id": { + "name": "decision_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_attachment_id": { + "name": "source_attachment_id", "type": "integer", "primaryKey": false, "notNull": true }, - "event_guid": { - "name": "event_guid", + "source_guid": { + "name": "source_guid", "type": "varchar(100)", "primaryKey": false, "notNull": false }, - "body_id": { - "name": "body_id", - "type": "integer", + "name": { + "name": "name", + "type": "text", "primaryKey": false, - "notNull": false + "notNull": true }, - "body_name": { - "name": "body_name", - "type": "varchar(256)", + "description": { + "name": "description", + "type": "text", "primaryKey": false, "notNull": false }, - "date": { - "name": "date", - "type": "timestamp", + "url": { + "name": "url", + "type": "text", "primaryKey": false, "notNull": true }, - "time": { - "name": "time", + "file_name": { + "name": "file_name", "type": "text", "primaryKey": false, "notNull": false }, - "location": { - "name": "location", - "type": "text", + "category": { + "name": "category", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", "primaryKey": false, "notNull": false }, - "agenda_file": { - "name": "agenda_file", + "is_supporting_document": { + "name": "is_supporting_document", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_public_comment": { + "name": "is_public_comment", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "processing_policy": { + "name": "processing_policy", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "extraction_status": { + "name": "extraction_status", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "extracted_text": { + "name": "extracted_text", "type": "text", "primaryKey": false, "notNull": false }, - "minutes_file": { - "name": "minutes_file", - "type": "text", + "extraction_method": { + "name": "extraction_method", + "type": "varchar(30)", "primaryKey": false, "notNull": false }, - "video_path": { - "name": "video_path", - "type": "text", + "extraction_quality": { + "name": "extraction_quality", + "type": "real", "primaryKey": false, "notNull": false }, - "agenda_status_name": { - "name": "agenda_status_name", - "type": "varchar(100)", + "page_count": { + "name": "page_count", + "type": "integer", "primaryKey": false, "notNull": false }, - "minutes_status_name": { - "name": "minutes_status_name", - "type": "varchar(100)", + "byte_size": { + "name": "byte_size", + "type": "integer", "primaryKey": false, "notNull": false }, - "comment": { - "name": "comment", - "type": "text", + "mime_type": { + "name": "mime_type", + "type": "varchar(100)", "primaryKey": false, "notNull": false }, - "in_site_url": { - "name": "in_site_url", - "type": "text", + "content_hash": { + "name": "content_hash", + "type": "varchar(64)", "primaryKey": false, "notNull": false }, - "last_modified_utc": { - "name": "last_modified_utc", - "type": "timestamp", + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", "primaryKey": false, - "notNull": true + "notNull": false }, - "fetched_at": { - "name": "fetched_at", - "type": "timestamp", + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true, "default": "now()" }, + "source_deleted_at": { + "name": "source_deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source_payload": { + "name": "source_payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", - "type": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "local_document_decision_category_idx": { + "name": "local_document_decision_category_idx", + "columns": [ + { + "expression": "decision_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "local_document_extraction_queue_idx": { + "name": "local_document_extraction_queue_idx", + "columns": [ + { + "expression": "extraction_status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"local_decision_document\".\"source_deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "local_decision_document_jurisdiction_key_local_jurisdiction_key_fk": { + "name": "local_decision_document_jurisdiction_key_local_jurisdiction_key_fk", + "tableFrom": "local_decision_document", + "tableTo": "local_jurisdiction", + "columnsFrom": [ + "jurisdiction_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "local_decision_document_decision_id_local_decision_id_fk": { + "name": "local_decision_document_decision_id_local_decision_id_fk", + "tableFrom": "local_decision_document", + "tableTo": "local_decision", + "columnsFrom": [ + "decision_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "local_decision_document_jurisdictionKey_sourceAttachmentId_unique": { + "name": "local_decision_document_jurisdictionKey_sourceAttachmentId_unique", + "nullsNotDistinct": false, + "columns": [ + "jurisdiction_key", + "source_attachment_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_decision_history": { + "name": "local_decision_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "decision_id": { + "name": "decision_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_history_id": { + "name": "source_history_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_event_id": { + "name": "source_event_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source_event_item_id": { + "name": "source_event_item_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "body_name": { + "name": "body_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "action_name": { + "name": "action_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "action_text": { + "name": "action_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action_date": { + "name": "action_date", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "agenda_number": { + "name": "agenda_number", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "source_payload": { + "name": "source_payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "local_history_decision_action_date_idx": { + "name": "local_history_decision_action_date_idx", + "columns": [ + { + "expression": "decision_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "action_date", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "local_decision_history_decision_id_local_decision_id_fk": { + "name": "local_decision_history_decision_id_local_decision_id_fk", + "tableFrom": "local_decision_history", + "tableTo": "local_decision", + "columnsFrom": [ + "decision_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "local_decision_history_decisionId_sourceHistoryId_unique": { + "name": "local_decision_history_decisionId_sourceHistoryId_unique", + "nullsNotDistinct": false, + "columns": [ + "decision_id", + "source_history_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_decision_vote": { + "name": "local_decision_vote", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "meeting_item_id": { + "name": "meeting_item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_vote_id": { + "name": "source_vote_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_person_id": { + "name": "source_person_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "person_name": { + "name": "person_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "value_name": { + "name": "value_name", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "source_payload": { + "name": "source_payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "local_vote_meeting_item_idx": { + "name": "local_vote_meeting_item_idx", + "columns": [ + { + "expression": "meeting_item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "local_vote_source_person_idx": { + "name": "local_vote_source_person_idx", + "columns": [ + { + "expression": "source_person_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "local_decision_vote_meeting_item_id_local_meeting_item_id_fk": { + "name": "local_decision_vote_meeting_item_id_local_meeting_item_id_fk", + "tableFrom": "local_decision_vote", + "tableTo": "local_meeting_item", + "columnsFrom": [ + "meeting_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "local_decision_vote_meetingItemId_sourceVoteId_unique": { + "name": "local_decision_vote_meetingItemId_sourceVoteId_unique", + "nullsNotDistinct": false, + "columns": [ + "meeting_item_id", + "source_vote_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_ingestion_run": { + "name": "local_ingestion_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "jurisdiction_key": { + "name": "jurisdiction_key", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "window_start": { + "name": "window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "window_end": { + "name": "window_end", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "counts": { + "name": "counts", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "local_ingestion_jurisdiction_started_idx": { + "name": "local_ingestion_jurisdiction_started_idx", + "columns": [ + { + "expression": "jurisdiction_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "local_ingestion_run_jurisdiction_key_local_jurisdiction_key_fk": { + "name": "local_ingestion_run_jurisdiction_key_local_jurisdiction_key_fk", + "tableFrom": "local_ingestion_run", + "tableTo": "local_jurisdiction", + "columnsFrom": [ + "jurisdiction_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_jurisdiction": { + "name": "local_jurisdiction", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "varchar(50)", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "varchar(2)", + "primaryKey": false, + "notNull": true + }, + "government_level": { + "name": "government_level", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "timezone": { + "name": "timezone", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "source_client": { + "name": "source_client", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "source_base_url": { + "name": "source_base_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_portal_url": { + "name": "public_portal_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_meeting": { + "name": "local_meeting", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "jurisdiction_key": { + "name": "jurisdiction_key", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "body_id": { + "name": "body_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_event_id": { + "name": "source_event_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_guid": { + "name": "source_guid", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "starts_at": { + "name": "starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "local_date": { + "name": "local_date", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true + }, + "time_label": { + "name": "time_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agenda_url": { + "name": "agenda_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "minutes_url": { + "name": "minutes_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "video_url": { + "name": "video_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agenda_status_name": { + "name": "agenda_status_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "minutes_status_name": { + "name": "minutes_status_name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "comment": { + "name": "comment", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cancelled": { + "name": "cancelled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "source_deleted_at": { + "name": "source_deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source_payload": { + "name": "source_payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "local_meeting_body_starts_at_idx": { + "name": "local_meeting_body_starts_at_idx", + "columns": [ + { + "expression": "body_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "starts_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "local_meeting_active_starts_at_idx": { + "name": "local_meeting_active_starts_at_idx", + "columns": [ + { + "expression": "jurisdiction_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "starts_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"local_meeting\".\"source_deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "local_meeting_jurisdiction_key_local_jurisdiction_key_fk": { + "name": "local_meeting_jurisdiction_key_local_jurisdiction_key_fk", + "tableFrom": "local_meeting", + "tableTo": "local_jurisdiction", + "columnsFrom": [ + "jurisdiction_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "local_meeting_body_id_local_body_id_fk": { + "name": "local_meeting_body_id_local_body_id_fk", + "tableFrom": "local_meeting", + "tableTo": "local_body", + "columnsFrom": [ + "body_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "local_meeting_jurisdictionKey_sourceEventId_unique": { + "name": "local_meeting_jurisdictionKey_sourceEventId_unique", + "nullsNotDistinct": false, + "columns": [ + "jurisdiction_key", + "source_event_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.local_meeting_item": { + "name": "local_meeting_item", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "meeting_id": { + "name": "meeting_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "decision_id": { + "name": "decision_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "source_event_item_id": { + "name": "source_event_item_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_guid": { + "name": "source_guid", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "agenda_sequence": { + "name": "agenda_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "minutes_sequence": { + "name": "minutes_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "agenda_number": { + "name": "agenda_number", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action_name": { + "name": "action_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "action_text": { + "name": "action_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "passed_flag_name": { + "name": "passed_flag_name", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "tally": { + "name": "tally", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "mover_name": { + "name": "mover_name", + "type": "varchar(256)", "primaryKey": false, - "notNull": true, - "default": "now()" + "notNull": false }, - "updated_at": { - "name": "updated_at", - "type": "timestamp with time zone", + "seconder_name": { + "name": "seconder_name", + "type": "varchar(256)", "primaryKey": false, "notNull": false - } - }, - "indexes": { - "legistar_meeting_date_idx": { - "name": "legistar_meeting_date_idx", - "columns": [ - { - "expression": "date", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "concurrently": false, - "method": "btree", - "with": {} - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": { - "legistar_meeting_jurisdiction_eventId_unique": { - "name": "legistar_meeting_jurisdiction_eventId_unique", - "nullsNotDistinct": false, - "columns": ["jurisdiction", "event_id"] - } - }, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - }, - "public.legistar_vote": { - "name": "legistar_vote", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "uuid", - "primaryKey": true, + }, + "consent": { + "name": "consent", + "type": "boolean", + "primaryKey": false, "notNull": true, - "default": "gen_random_uuid()" + "default": false }, - "jurisdiction": { - "name": "jurisdiction", - "type": "varchar(50)", + "roll_call": { + "name": "roll_call", + "type": "boolean", "primaryKey": false, - "notNull": true + "notNull": true, + "default": false }, - "vote_id": { - "name": "vote_id", - "type": "integer", + "agenda_note": { + "name": "agenda_note", + "type": "text", "primaryKey": false, - "notNull": true + "notNull": false }, - "event_item_id": { - "name": "event_item_id", - "type": "integer", + "minutes_note": { + "name": "minutes_note", + "type": "text", "primaryKey": false, - "notNull": true + "notNull": false }, - "person_id": { - "name": "person_id", + "video_index": { + "name": "video_index", "type": "integer", "primaryKey": false, - "notNull": true + "notNull": false }, - "person_name": { - "name": "person_name", - "type": "varchar(256)", + "source_updated_at": { + "name": "source_updated_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true }, - "value_name": { - "name": "value_name", - "type": "varchar(50)", + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", "primaryKey": false, - "notNull": true + "notNull": true, + "default": "now()" }, - "sort": { - "name": "sort", - "type": "integer", + "source_deleted_at": { + "name": "source_deleted_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": false }, - "last_modified_utc": { - "name": "last_modified_utc", - "type": "timestamp", + "source_payload": { + "name": "source_payload", + "type": "jsonb", "primaryKey": false, - "notNull": true + "notNull": false }, - "fetched_at": { - "name": "fetched_at", - "type": "timestamp", + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true, "default": "now()" }, - "created_at": { - "name": "created_at", - "type": "timestamp", + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", "primaryKey": false, "notNull": true, "default": "now()" } }, "indexes": { - "legistar_vote_event_item_idx": { - "name": "legistar_vote_event_item_idx", + "local_meeting_item_sequence_idx": { + "name": "local_meeting_item_sequence_idx", "columns": [ { - "expression": "event_item_id", + "expression": "meeting_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "agenda_sequence", "isExpression": false, "asc": true, "nulls": "last" @@ -1981,11 +2925,11 @@ "method": "btree", "with": {} }, - "legistar_vote_person_idx": { - "name": "legistar_vote_person_idx", + "local_meeting_item_decision_idx": { + "name": "local_meeting_item_decision_idx", "columns": [ { - "expression": "person_id", + "expression": "decision_id", "isExpression": false, "asc": true, "nulls": "last" @@ -1997,13 +2941,43 @@ "with": {} } }, - "foreignKeys": {}, + "foreignKeys": { + "local_meeting_item_meeting_id_local_meeting_id_fk": { + "name": "local_meeting_item_meeting_id_local_meeting_id_fk", + "tableFrom": "local_meeting_item", + "tableTo": "local_meeting", + "columnsFrom": [ + "meeting_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "local_meeting_item_decision_id_local_decision_id_fk": { + "name": "local_meeting_item_decision_id_local_decision_id_fk", + "tableFrom": "local_meeting_item", + "tableTo": "local_decision", + "columnsFrom": [ + "decision_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, "compositePrimaryKeys": {}, "uniqueConstraints": { - "legistar_vote_jurisdiction_voteId_unique": { - "name": "legistar_vote_jurisdiction_voteId_unique", + "local_meeting_item_meetingId_sourceEventItemId_unique": { + "name": "local_meeting_item_meetingId_sourceEventItemId_unique", "nullsNotDistinct": false, - "columns": ["jurisdiction", "vote_id"] + "columns": [ + "meeting_id", + "source_event_item_id" + ] } }, "policies": {}, @@ -2246,7 +3220,10 @@ "role_description_role_level_unique": { "name": "role_description_role_level_unique", "nullsNotDistinct": false, - "columns": ["role", "level"] + "columns": [ + "role", + "level" + ] } }, "policies": {}, @@ -2313,7 +3290,10 @@ "saved_article_userId_contentId_unique": { "name": "saved_article_userId_contentId_unique", "nullsNotDistinct": false, - "columns": ["user_id", "content_id"] + "columns": [ + "user_id", + "content_id" + ] } }, "policies": {}, @@ -2429,7 +3409,10 @@ "compositePrimaryKeys": { "scraper_retry_scraper_key_item_key_pk": { "name": "scraper_retry_scraper_key_item_key_pk", - "columns": ["scraper_key", "item_key"] + "columns": [ + "scraper_key", + "item_key" + ] } }, "uniqueConstraints": {}, @@ -2489,7 +3472,9 @@ "user_preference_userId_unique": { "name": "user_preference_userId_unique", "nullsNotDistinct": false, - "columns": ["user_id"] + "columns": [ + "user_id" + ] } }, "policies": {}, @@ -2569,7 +3554,155 @@ "user_settings_userId_unique": { "name": "user_settings_userId_unique", "nullsNotDistinct": false, - "columns": ["user_id"] + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.video": { + "name": "video", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "content_type": { + "name": "content_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "content_id": { + "name": "content_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "image_data": { + "name": "image_data", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "image_mime_type": { + "name": "image_mime_type", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "image_width": { + "name": "image_width", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "image_height": { + "name": "image_height", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author": { + "name": "author", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "engagement_metrics": { + "name": "engagement_metrics", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{\"likes\":0,\"comments\":0,\"shares\":0}'::jsonb" + }, + "source_content_hash": { + "name": "source_content_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "video_content_id_idx": { + "name": "video_content_id_idx", + "columns": [ + { + "expression": "content_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "video_created_at_idx": { + "name": "video_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "video_contentType_contentId_unique": { + "name": "video_contentType_contentId_unique", + "nullsNotDistinct": false, + "columns": [ + "content_type", + "content_id" + ] } }, "policies": {}, @@ -2665,8 +3798,12 @@ "name": "account_user_id_user_id_fk", "tableFrom": "account", "tableTo": "user", - "columnsFrom": ["user_id"], - "columnsTo": ["id"], + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], "onDelete": "cascade", "onUpdate": "no action" } @@ -2736,8 +3873,12 @@ "name": "session_user_id_user_id_fk", "tableFrom": "session", "tableTo": "user", - "columnsFrom": ["user_id"], - "columnsTo": ["id"], + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], "onDelete": "cascade", "onUpdate": "no action" } @@ -2747,7 +3888,9 @@ "session_token_unique": { "name": "session_token_unique", "nullsNotDistinct": false, - "columns": ["token"] + "columns": [ + "token" + ] } }, "policies": {}, @@ -2808,7 +3951,9 @@ "user_email_unique": { "name": "user_email_unique", "nullsNotDistinct": false, - "columns": ["email"] + "columns": [ + "email" + ] } }, "policies": {}, diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 31c424ca..87ccf561 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -99,6 +99,13 @@ "when": 1787635076539, "tag": "0013_tough_toro", "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1787012515923, + "tag": "0014_tough_stone_men", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index f8ffc279..cd113737 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -521,157 +521,375 @@ export const UserSettings = pgTable( }), ); -// Legistar local government data cache tables +// Local-government decision pipeline. These tables deliberately model the +// product domain rather than Legistar's wire format so another municipal +// records adapter can populate the same read model later. -export const LegistarBody = pgTable( - "legistar_body", +export const LocalJurisdiction = pgTable("local_jurisdiction", (t) => ({ + key: t.varchar({ length: 50 }).notNull().primaryKey(), + name: t.varchar({ length: 256 }).notNull(), + state: t.varchar({ length: 2 }).notNull(), + governmentLevel: t.varchar({ length: 30 }).notNull(), + timezone: t.varchar({ length: 64 }).notNull(), + sourceType: t.varchar({ length: 30 }).notNull(), + sourceClient: t.varchar({ length: 100 }).notNull(), + sourceBaseUrl: t.text().notNull(), + publicPortalUrl: t.text(), + active: t.boolean().notNull().default(true), + createdAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + updatedAt: t + .timestamp({ withTimezone: true }) + .defaultNow() + .$onUpdateFn(() => sql`now()`) + .notNull(), +})); + +export const LocalBody = pgTable( + "local_body", (t) => ({ id: t.uuid().notNull().primaryKey().defaultRandom(), - jurisdiction: t.varchar({ length: 50 }).notNull(), - bodyId: t.integer().notNull(), - bodyGuid: t.varchar({ length: 100 }), + jurisdictionKey: t + .varchar({ length: 50 }) + .notNull() + .references(() => LocalJurisdiction.key, { onDelete: "cascade" }), + sourceBodyId: t.integer().notNull(), + sourceGuid: t.varchar({ length: 100 }), name: t.text().notNull(), typeName: t.varchar({ length: 100 }), - activeFlag: t.boolean().default(true), + active: t.boolean().notNull().default(true), + included: t.boolean().notNull().default(false), + relevanceTier: t.integer().notNull().default(3), numberOfMembers: t.integer(), description: t.text(), contactName: t.varchar({ length: 256 }), contactEmail: t.varchar({ length: 256 }), contactPhone: t.varchar({ length: 50 }), - fetchedAt: t.timestamp().defaultNow().notNull(), - createdAt: t.timestamp().defaultNow().notNull(), + sourceUpdatedAt: t.timestamp({ withTimezone: true }), + lastSeenAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + sourcePayload: t.jsonb().$type>(), + createdAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), updatedAt: t - .timestamp({ mode: "date", withTimezone: true }) - .$onUpdateFn(() => sql`now()`), + .timestamp({ withTimezone: true }) + .defaultNow() + .$onUpdateFn(() => sql`now()`) + .notNull(), }), (table) => ({ - uniqueBody: unique().on(table.jurisdiction, table.bodyId), + uniqueSourceBody: unique().on(table.jurisdictionKey, table.sourceBodyId), + jurisdictionIncludedIdx: index("local_body_jurisdiction_included_idx").on( + table.jurisdictionKey, + table.included, + table.relevanceTier, + ), }), ); -export const LegistarMatter = pgTable( - "legistar_matter", +export const LocalDecision = pgTable( + "local_decision", (t) => ({ id: t.uuid().notNull().primaryKey().defaultRandom(), - jurisdiction: t.varchar({ length: 50 }).notNull(), - matterId: t.integer().notNull(), - matterGuid: t.varchar({ length: 100 }), - matterFile: t.varchar({ length: 100 }), + jurisdictionKey: t + .varchar({ length: 50 }) + .notNull() + .references(() => LocalJurisdiction.key, { onDelete: "cascade" }), + primaryBodyId: t + .uuid() + .references(() => LocalBody.id, { onDelete: "set null" }), + sourceMatterId: t.integer().notNull(), + sourceGuid: t.varchar({ length: 100 }), + fileNumber: t.varchar({ length: 100 }), title: t.text().notNull(), name: t.text(), typeName: t.varchar({ length: 100 }), statusName: t.varchar({ length: 100 }), - bodyName: t.varchar({ length: 256 }), - bodyId: t.integer(), - introDate: t.timestamp(), - agendaDate: t.timestamp(), - passedDate: t.timestamp(), - enactmentDate: t.timestamp(), + topic: t.varchar({ length: 80 }), + scopeKind: t.varchar({ length: 30 }).notNull().default("unknown"), + districtNumbers: t.integer().array(), + geographicText: t.text(), + introDate: t.timestamp({ withTimezone: true }), + agendaDate: t.timestamp({ withTimezone: true }), + passedDate: t.timestamp({ withTimezone: true }), + enactmentDate: t.timestamp({ withTimezone: true }), enactmentNumber: t.varchar({ length: 100 }), requester: t.text(), notes: t.text(), - lastModifiedUtc: t.timestamp().notNull(), - fetchedAt: t.timestamp().defaultNow().notNull(), - createdAt: t.timestamp().defaultNow().notNull(), + sourceUrl: t.text(), + sourceUpdatedAt: t.timestamp({ withTimezone: true }).notNull(), + lastSeenAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + sourceDeletedAt: t.timestamp({ withTimezone: true }), + sourcePayload: t.jsonb().$type>(), + createdAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), updatedAt: t - .timestamp({ mode: "date", withTimezone: true }) - .$onUpdateFn(() => sql`now()`), + .timestamp({ withTimezone: true }) + .defaultNow() + .$onUpdateFn(() => sql`now()`) + .notNull(), + searchVector: tsvector("search_vector").generatedAlwaysAs( + (): SQL => sql`( + setweight(to_tsvector('english', coalesce(file_number, '')), 'A') || + setweight(to_tsvector('english', coalesce(title, '')), 'A') || + setweight(to_tsvector('english', coalesce(type_name, '') || ' ' || coalesce(topic, '')), 'B') || + setweight(to_tsvector('english', coalesce(requester, '') || ' ' || coalesce(notes, '') || ' ' || coalesce(geographic_text, '')), 'C') + )`, + ), }), (table) => ({ - uniqueMatter: unique().on(table.jurisdiction, table.matterId), - matterFileIdx: index("legistar_matter_file_idx").on(table.matterFile), + uniqueSourceMatter: unique().on( + table.jurisdictionKey, + table.sourceMatterId, + ), + primaryBodyIdx: index("local_decision_primary_body_idx").on( + table.primaryBodyId, + ), + activeUpdatedIdx: index("local_decision_active_updated_idx") + .on(table.jurisdictionKey, table.sourceUpdatedAt) + .where(sql`${table.sourceDeletedAt} is null`), + searchVectorIdx: index("local_decision_search_vector_idx").using( + "gin", + table.searchVector, + ), }), ); -export const LegistarMeeting = pgTable( - "legistar_meeting", +export const LocalMeeting = pgTable( + "local_meeting", (t) => ({ id: t.uuid().notNull().primaryKey().defaultRandom(), - jurisdiction: t.varchar({ length: 50 }).notNull(), - eventId: t.integer().notNull(), - eventGuid: t.varchar({ length: 100 }), - bodyId: t.integer(), - bodyName: t.varchar({ length: 256 }), - date: t.timestamp().notNull(), - time: t.text(), + jurisdictionKey: t + .varchar({ length: 50 }) + .notNull() + .references(() => LocalJurisdiction.key, { onDelete: "cascade" }), + bodyId: t + .uuid() + .notNull() + .references(() => LocalBody.id, { onDelete: "cascade" }), + sourceEventId: t.integer().notNull(), + sourceGuid: t.varchar({ length: 100 }), + startsAt: t.timestamp({ withTimezone: true }).notNull(), + localDate: t.varchar({ length: 10 }).notNull(), + timeLabel: t.text(), location: t.text(), - agendaFile: t.text(), - minutesFile: t.text(), - videoPath: t.text(), + agendaUrl: t.text(), + minutesUrl: t.text(), + videoUrl: t.text(), + sourceUrl: t.text(), agendaStatusName: t.varchar({ length: 100 }), minutesStatusName: t.varchar({ length: 100 }), comment: t.text(), - inSiteUrl: t.text(), - lastModifiedUtc: t.timestamp().notNull(), - fetchedAt: t.timestamp().defaultNow().notNull(), - createdAt: t.timestamp().defaultNow().notNull(), + cancelled: t.boolean().notNull().default(false), + sourceUpdatedAt: t.timestamp({ withTimezone: true }).notNull(), + lastSeenAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + sourceDeletedAt: t.timestamp({ withTimezone: true }), + sourcePayload: t.jsonb().$type>(), + createdAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), updatedAt: t - .timestamp({ mode: "date", withTimezone: true }) - .$onUpdateFn(() => sql`now()`), + .timestamp({ withTimezone: true }) + .defaultNow() + .$onUpdateFn(() => sql`now()`) + .notNull(), }), (table) => ({ - uniqueMeeting: unique().on(table.jurisdiction, table.eventId), - meetingDateIdx: index("legistar_meeting_date_idx").on(table.date), + uniqueSourceEvent: unique().on(table.jurisdictionKey, table.sourceEventId), + bodyStartsAtIdx: index("local_meeting_body_starts_at_idx").on( + table.bodyId, + table.startsAt, + ), + activeStartsAtIdx: index("local_meeting_active_starts_at_idx") + .on(table.jurisdictionKey, table.startsAt) + .where(sql`${table.sourceDeletedAt} is null`), }), ); -export const LegistarAgendaItem = pgTable( - "legistar_agenda_item", +export const LocalMeetingItem = pgTable( + "local_meeting_item", (t) => ({ id: t.uuid().notNull().primaryKey().defaultRandom(), - jurisdiction: t.varchar({ length: 50 }).notNull(), - eventItemId: t.integer().notNull(), - eventId: t.integer().notNull(), + meetingId: t + .uuid() + .notNull() + .references(() => LocalMeeting.id, { onDelete: "cascade" }), + decisionId: t + .uuid() + .references(() => LocalDecision.id, { onDelete: "set null" }), + sourceEventItemId: t.integer().notNull(), + sourceGuid: t.varchar({ length: 100 }), agendaSequence: t.integer(), + minutesSequence: t.integer(), agendaNumber: t.varchar({ length: 50 }), title: t.text(), actionName: t.varchar({ length: 256 }), + actionText: t.text(), passedFlagName: t.varchar({ length: 50 }), tally: t.varchar({ length: 50 }), moverName: t.varchar({ length: 256 }), seconderName: t.varchar({ length: 256 }), - matterId: t.integer(), - matterFile: t.varchar({ length: 100 }), - matterName: t.text(), - matterType: t.varchar({ length: 100 }), - matterStatus: t.varchar({ length: 100 }), - consent: t.boolean().default(false), + consent: t.boolean().notNull().default(false), + rollCall: t.boolean().notNull().default(false), agendaNote: t.text(), minutesNote: t.text(), - lastModifiedUtc: t.timestamp().notNull(), - fetchedAt: t.timestamp().defaultNow().notNull(), - createdAt: t.timestamp().defaultNow().notNull(), + videoIndex: t.integer(), + sourceUpdatedAt: t.timestamp({ withTimezone: true }).notNull(), + lastSeenAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + sourceDeletedAt: t.timestamp({ withTimezone: true }), + sourcePayload: t.jsonb().$type>(), + createdAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), updatedAt: t - .timestamp({ mode: "date", withTimezone: true }) - .$onUpdateFn(() => sql`now()`), + .timestamp({ withTimezone: true }) + .defaultNow() + .$onUpdateFn(() => sql`now()`) + .notNull(), }), (table) => ({ - uniqueAgendaItem: unique().on(table.jurisdiction, table.eventItemId), - agendaEventIdx: index("legistar_agenda_item_event_idx").on(table.eventId), + uniqueSourceItem: unique().on(table.meetingId, table.sourceEventItemId), + meetingSequenceIdx: index("local_meeting_item_sequence_idx").on( + table.meetingId, + table.agendaSequence, + ), + decisionIdx: index("local_meeting_item_decision_idx").on(table.decisionId), + }), +); + +export const LocalDecisionDocument = pgTable( + "local_decision_document", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + jurisdictionKey: t + .varchar({ length: 50 }) + .notNull() + .references(() => LocalJurisdiction.key, { onDelete: "cascade" }), + decisionId: t + .uuid() + .notNull() + .references(() => LocalDecision.id, { onDelete: "cascade" }), + sourceAttachmentId: t.integer().notNull(), + sourceGuid: t.varchar({ length: 100 }), + name: t.text().notNull(), + description: t.text(), + url: t.text().notNull(), + fileName: t.text(), + category: t.varchar({ length: 50 }).notNull(), + sortOrder: t.integer(), + isSupportingDocument: t.boolean().notNull().default(false), + isPublicComment: t.boolean().notNull().default(false), + processingPolicy: t.varchar({ length: 30 }).notNull(), + extractionStatus: t.varchar({ length: 30 }).notNull().default("pending"), + extractedText: t.text(), + extractionMethod: t.varchar({ length: 30 }), + extractionQuality: t.real(), + pageCount: t.integer(), + byteSize: t.integer(), + mimeType: t.varchar({ length: 100 }), + contentHash: t.varchar({ length: 64 }), + sourceUpdatedAt: t.timestamp({ withTimezone: true }), + lastSeenAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + sourceDeletedAt: t.timestamp({ withTimezone: true }), + sourcePayload: t.jsonb().$type>(), + createdAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + updatedAt: t + .timestamp({ withTimezone: true }) + .defaultNow() + .$onUpdateFn(() => sql`now()`) + .notNull(), + }), + (table) => ({ + uniqueSourceAttachment: unique().on( + table.jurisdictionKey, + table.sourceAttachmentId, + ), + decisionCategoryIdx: index("local_document_decision_category_idx").on( + table.decisionId, + table.category, + ), + extractionQueueIdx: index("local_document_extraction_queue_idx") + .on(table.extractionStatus) + .where(sql`${table.sourceDeletedAt} is null`), }), ); -export const LegistarVote = pgTable( - "legistar_vote", +export const LocalDecisionHistory = pgTable( + "local_decision_history", (t) => ({ id: t.uuid().notNull().primaryKey().defaultRandom(), - jurisdiction: t.varchar({ length: 50 }).notNull(), - voteId: t.integer().notNull(), - eventItemId: t.integer().notNull(), - personId: t.integer().notNull(), + decisionId: t + .uuid() + .notNull() + .references(() => LocalDecision.id, { onDelete: "cascade" }), + sourceHistoryId: t.integer().notNull(), + sourceEventId: t.integer(), + sourceEventItemId: t.integer(), + bodyName: t.varchar({ length: 256 }), + actionName: t.varchar({ length: 256 }), + actionText: t.text(), + actionDate: t.timestamp({ withTimezone: true }), + agendaNumber: t.varchar({ length: 50 }), + sourcePayload: t.jsonb().$type>(), + createdAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + updatedAt: t + .timestamp({ withTimezone: true }) + .defaultNow() + .$onUpdateFn(() => sql`now()`) + .notNull(), + }), + (table) => ({ + uniqueSourceHistory: unique().on(table.decisionId, table.sourceHistoryId), + decisionActionDateIdx: index("local_history_decision_action_date_idx").on( + table.decisionId, + table.actionDate, + ), + }), +); + +export const LocalDecisionVote = pgTable( + "local_decision_vote", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + meetingItemId: t + .uuid() + .notNull() + .references(() => LocalMeetingItem.id, { onDelete: "cascade" }), + sourceVoteId: t.integer().notNull(), + sourcePersonId: t.integer().notNull(), personName: t.varchar({ length: 256 }).notNull(), valueName: t.varchar({ length: 50 }).notNull(), - sort: t.integer(), - lastModifiedUtc: t.timestamp().notNull(), - fetchedAt: t.timestamp().defaultNow().notNull(), - createdAt: t.timestamp().defaultNow().notNull(), + sortOrder: t.integer(), + sourceUpdatedAt: t.timestamp({ withTimezone: true }).notNull(), + sourcePayload: t.jsonb().$type>(), + createdAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + updatedAt: t + .timestamp({ withTimezone: true }) + .defaultNow() + .$onUpdateFn(() => sql`now()`) + .notNull(), }), (table) => ({ - uniqueVote: unique().on(table.jurisdiction, table.voteId), - voteEventItemIdx: index("legistar_vote_event_item_idx").on( - table.eventItemId, + uniqueSourceVote: unique().on(table.meetingItemId, table.sourceVoteId), + meetingItemIdx: index("local_vote_meeting_item_idx").on( + table.meetingItemId, ), - votePersonIdx: index("legistar_vote_person_idx").on(table.personId), + personIdx: index("local_vote_source_person_idx").on(table.sourcePersonId), + }), +); + +export const LocalIngestionRun = pgTable( + "local_ingestion_run", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + jurisdictionKey: t + .varchar({ length: 50 }) + .notNull() + .references(() => LocalJurisdiction.key, { onDelete: "cascade" }), + status: t.varchar({ length: 20 }).notNull(), + windowStart: t.timestamp({ withTimezone: true }).notNull(), + windowEnd: t.timestamp({ withTimezone: true }).notNull(), + startedAt: t.timestamp({ withTimezone: true }).defaultNow().notNull(), + completedAt: t.timestamp({ withTimezone: true }), + counts: t.jsonb().$type>().notNull().default({}), + error: t.text(), + }), + (table) => ({ + jurisdictionStartedIdx: index( + "local_ingestion_jurisdiction_started_idx", + ).on(table.jurisdictionKey, table.startedAt), }), ); diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 97ca2505..2a751833 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -86,6 +86,7 @@ const scraperSourceLimitDefinitions = [ ["SCC_CVIG_MAX_ITEMS", "Santa Clara voter-guide PDFs per run.", "10"], ["CA_SOS_MAX_ITEMS", "California SOS office pages per run.", "9"], ["OPEN_STATES_MAX_ITEMS", "Open States bills per state per run.", "100"], + ["LEGISTAR_MAX_ITEMS", "Legistar meetings per run.", "100"], ] as const; export const envRegistry = [ @@ -510,6 +511,43 @@ export const envRegistry = [ requirements: { scraper: "optional" }, schema: positiveNumber, }), + define({ + key: "LEGISTAR_PAST_DAYS", + description: "Days of past Legistar meetings refreshed on every run.", + group: "Scraper operations", + secret: false, + defaultValue: "45", + requirements: {}, + schema: positiveInteger, + }), + define({ + key: "LEGISTAR_FUTURE_DAYS", + description: "Days of upcoming Legistar meetings ingested on every run.", + group: "Scraper operations", + secret: false, + defaultValue: "120", + requirements: {}, + schema: positiveInteger, + }), + define({ + key: "LEGISTAR_MAX_DOCUMENT_BYTES", + description: "Largest Legistar PDF downloaded for native text extraction.", + group: "Scraper operations", + secret: false, + defaultValue: "15728640", + requirements: {}, + schema: positiveInteger, + }), + define({ + key: "LEGISTAR_SKIP_DOCUMENT_TEXT", + description: + "Set to 1 to ingest Legistar document metadata without downloading PDFs.", + group: "Scraper operations", + secret: false, + defaultValue: "0", + requirements: {}, + schema: z.enum(["0", "1"]), + }), ...scraperSourceLimitDefinitions.map(([key, description, defaultValue]) => define({ key,