Cap bills during daily scraper refreshes

apps/scraper/package.json

@@ -47,7 +47,8 @@
     "retroactive-briefs": "tsx src/retroactive-briefs-entry.ts",
     "reprocess-content": "tsx src/reprocess-content-entry.ts",
     "retroactive-videos": "tsx src/retroactive-videos-entry.ts",
-    "backfill-bill-descriptions": "tsx src/backfill-bill-descriptions-entry.ts"
+    "backfill-bill-descriptions": "tsx src/backfill-bill-descriptions-entry.ts",
+    "prune-bills": "tsx src/prune-bills-entry.ts"
   },
   "author": "It's not you it's me",
   "license": "ISC"

apps/scraper/src/main.ts

@@ -7,6 +7,8 @@ import {
   validateScraperEnv,
 } from "./env.js";
 import { scrapers } from "./scrapers.js";
+import { DEFAULT_STATES, parseStates } from "./scrapers/open-states.js";
+import { enforceBillRetention } from "./utils/bill-retention.js";
 import { setConcurrency } from "./utils/concurrency.js";
 import { printMetricsSummary, resetMetrics } from "./utils/db/metrics.js";
 import { createLogger } from "./utils/log.js";
@@ -61,6 +63,11 @@ const argv = await yargs(hideBin(process.argv))
     describe:
       "Refresh the N most recently updated bills instead of walking the incremental cursor. Keeps active legislation current rather than pursuing complete historical coverage",
   })
+  .option("retain", {
+    type: "number",
+    describe:
+      "After a successful recent bill refresh, retain only the newest N stored bills in each refreshed jurisdiction",
+  })
   .check((args) => {
     const maxItems = args.maxItems;
     if (
@@ -108,6 +115,20 @@ const argv = await yargs(hideBin(process.argv))
         throw new Error("--recent and --bill select bills different ways");
       }
     }
+    const retain = args.retain;
+    if (retain !== undefined) {
+      if (!Number.isInteger(retain) || retain <= 0) {
+        throw new Error("--retain must be a positive integer");
+      }
+      if (recent === undefined) {
+        throw new Error("--retain requires --recent");
+      }
+      if (args.scraper !== "congress" && args.scraper !== "open-states") {
+        throw new Error(
+          '--retain requires the "congress" or "open-states" scraper',
+        );
+      }
+    }
     if (args.congress !== undefined && !bills?.length) {
       throw new Error("--congress only applies alongside --bill");
     }
@@ -130,6 +151,7 @@ const congressNumber = (argv as { congress?: number }).congress;
 const session = (argv as { session?: string }).session;
 const bulkDir = (argv as { bulkDir?: string }).bulkDir;
 const recent = (argv as { recent?: number }).recent;
+const retain = (argv as { retain?: number }).retain;
 
 function logDatabaseTarget(): void {
   const target = databaseTarget(process.env.POSTGRES_URL!);
@@ -140,6 +162,27 @@ function logDatabaseTarget(): void {
   }
 }
 
+async function applyRetentionAfterRefresh(
+  scraperId: string,
+  keepPerJurisdiction: number,
+) {
+  const jurisdictions =
+    scraperId === "congress"
+      ? ["US"]
+      : (parseStates(process.env.OPEN_STATES_STATES) ?? DEFAULT_STATES).map(
+          (state) => state.toUpperCase(),
+        );
+  const results = await enforceBillRetention(
+    jurisdictions,
+    keepPerJurisdiction,
+  );
+  for (const result of results) {
+    logger.info(
+      `Retention ${result.jurisdiction}: kept ${keepPerJurisdiction}, evicted ${result.bills}`,
+    );
+  }
+}
+
 setConcurrency(concurrency);
 
 async function main() {
@@ -182,6 +225,9 @@ async function main() {
       bulkDir,
       recent,
     });
+    if (retain !== undefined) {
+      await applyRetentionAfterRefresh(arg, retain);
+    }
     printMetricsSummary(scraper.name);
   }
 }

apps/scraper/src/prune-bills-entry.ts

@@ -0,0 +1,4 @@
+import { loadRepoEnv } from "@acme/env/load";
+
+loadRepoEnv();
+await import("./prune-bills.js");

apps/scraper/src/prune-bills.ts

@@ -0,0 +1,135 @@
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+
+import { databaseTarget, databaseTargetMessage } from "./env.js";
+import {
+  billRetentionInventory,
+  enforceBillRetention,
+} from "./utils/bill-retention.js";
+import {
+  createLogger,
+  printFooter,
+  printHeader,
+  printKeyValue,
+} from "./utils/log.js";
+
+const logger = createLogger("bill-retention");
+
+const argv = await yargs(hideBin(process.argv))
+  .option("keep-per-jurisdiction", {
+    type: "number",
+    default: 100,
+    description: "Newest bills to retain independently in each jurisdiction",
+  })
+  .option("apply", {
+    type: "boolean",
+    default: false,
+    description:
+      "Delete selected rows; without this flag the command is read-only",
+  })
+  .option("yes", {
+    type: "boolean",
+    default: false,
+    description: "Acknowledge production deletions",
+  })
+  .check((args) => {
+    const keepPerJurisdiction = args.keepPerJurisdiction;
+    if (
+      typeof keepPerJurisdiction !== "number" ||
+      !Number.isInteger(keepPerJurisdiction) ||
+      keepPerJurisdiction < 1
+    ) {
+      throw new Error("--keep-per-jurisdiction must be a positive integer");
+    }
+    return true;
+  })
+  .strict()
+  .help()
+  .parse();
+
+function printInventory(
+  inventory: Awaited<ReturnType<typeof billRetentionInventory>>,
+) {
+  printHeader("Bill retention inventory");
+  for (const row of inventory) {
+    printKeyValue(
+      row.jurisdiction,
+      `${row.total} total; ${row.evict} selected for eviction`,
+    );
+  }
+  printKeyValue(
+    "Selected bills",
+    inventory.reduce((total, row) => total + row.evict, 0),
+  );
+  printKeyValue("Writes", argv.apply ? "enabled" : "disabled (dry run)");
+  printFooter();
+}
+
+async function main() {
+  const databaseUrl = process.env.POSTGRES_URL;
+  if (!databaseUrl) throw new Error("POSTGRES_URL is required");
+
+  const target = databaseTarget(databaseUrl);
+  if (argv.apply && target.target === "production" && !argv.yes) {
+    throw new Error("Production deletions require both --apply and --yes");
+  }
+  logger[target.target === "production" ? "warn" : "info"](
+    databaseTargetMessage(databaseUrl),
+  );
+
+  const inventory = await billRetentionInventory(argv.keepPerJurisdiction);
+  printInventory(inventory);
+  if (!argv.apply) return;
+
+  const jurisdictions = inventory
+    .filter((row) => row.evict > 0 && /^(US|[A-Z]{2})$/.test(row.jurisdiction))
+    .map((row) => row.jurisdiction);
+  const ignored = inventory.filter(
+    (row) => row.evict > 0 && !/^(US|[A-Z]{2})$/.test(row.jurisdiction),
+  );
+  for (const row of ignored) {
+    logger.warn(
+      `Skipping malformed jurisdiction ${row.jurisdiction} (${row.evict} bill(s))`,
+    );
+  }
+
+  const results = await enforceBillRetention(
+    jurisdictions,
+    argv.keepPerJurisdiction,
+  );
+
+  printHeader("Eviction result");
+  for (const result of results) {
+    printKeyValue(result.jurisdiction, `${result.bills} bills`);
+  }
+  printKeyValue(
+    "Bills",
+    results.reduce((total, result) => total + result.bills, 0),
+  );
+  printKeyValue(
+    "Feed images",
+    results.reduce((total, result) => total + result.videos, 0),
+  );
+  printKeyValue(
+    "Briefs",
+    results.reduce((total, result) => total + result.briefs, 0),
+  );
+  printKeyValue(
+    "Lenses",
+    results.reduce((total, result) => total + result.lenses, 0),
+  );
+  printKeyValue(
+    "Saved references",
+    results.reduce((total, result) => total + result.saves, 0),
+  );
+  printKeyValue(
+    "Brief change images",
+    results.reduce((total, result) => total + result.changeImages, 0),
+  );
+  printFooter();
+  logger.success(
+    `Evicted ${results.reduce((total, result) => total + result.bills, 0)} old bill(s)`,
+  );
+}
+
+await main();

apps/scraper/src/scrapers/open-states.ts

@@ -69,7 +69,7 @@ const PAGE_SIZE = 20;
  */
 const PAGE_DELAY_MS = 1_000;
 
-const DEFAULT_STATES = ["ca"];
+export const DEFAULT_STATES = ["ca"];
 
 interface OpenStatesScraperConfig {
   maxBills?: number;

apps/scraper/src/utils/bill-retention.test.ts

@@ -0,0 +1,18 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { normalizeRetentionJurisdiction } from "./bill-retention.js";
+
+void test("retention jurisdictions are normalized", () => {
+  assert.equal(normalizeRetentionJurisdiction("us"), "US");
+  assert.equal(normalizeRetentionJurisdiction(" ca "), "CA");
+});
+
+void test("retention rejects malformed jurisdictions", () => {
+  for (const jurisdiction of ["", "USA", "C", "C1", "STATE"]) {
+    assert.throws(
+      () => normalizeRetentionJurisdiction(jurisdiction),
+      /Invalid bill-retention jurisdiction/,
+    );
+  }
+});