From 2cb40d854cd8f83bfec88fb628433a8529ecc25e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 23 Sep 2026 00:56:48 -0700 Subject: [PATCH 1/2] fix: count mixed-layout channel data only once --- CHANGELOG.md | 1 + docs/data-imports.md | 2 ++ scripts/channel-storage.mjs | 11 ++++++---- scripts/channel-storage.test.mjs | 35 ++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 scripts/channel-storage.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 86fe16593..5e8cb9aa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,5 +3,6 @@ ## Unreleased - Fix concurrent release QA auth seeding leaking state directories between agents, and roll back partial compatibility patch writes while preserving file permissions. Thanks @petercheng. +- Fix aggregate channel reports counting legacy rows twice when the same channel also has versioned data files. - Fix channel imports, surface imports, and RSS backfills failing to find sample files in TSV lists with CRLF line endings. - Fix surface imports recording incorrect run bounds for unordered samples or accepting invalid intermediate timestamps; normalize imported run timestamps to UTC. diff --git a/docs/data-imports.md b/docs/data-imports.md index 0d0af53c5..60b788577 100644 --- a/docs/data-imports.md +++ b/docs/data-imports.md @@ -35,6 +35,8 @@ pnpm test:docker:npm-telegram-live Current channel folders are `telegram`, `discord`, `slack`, and `whatsapp`. Telegram and Discord still have specialized importers because their source artifact shapes differ; they now share the same storage contract as generic live-transport channels. +Readers also accept legacy `data/channels/.jsonl` files alongside versioned channel directories. Aggregate reports read each source file once, including when both layouts exist for the same channel. + Raw Telegram QA artifacts stay in the OpenClaw repo artifact directory unless explicitly copied here later. Historical Telegram rows imported from the old package RTT wrapper may include `rtt.warmSamples` with every successful sample value. New Telegram imports from `qa-evidence.json` preserve the aggregate dashboard metrics (`canaryMs`, `mentionReplyMs`, `avgMs`, `p50Ms`, `p95Ms`, `maxMs`, `sampleCount`, and `failedSamples`) but do not reconstruct individual sample RTT arrays because the evidence artifact stores aggregate timing. Release RSS backfills only write `resources` onto an existing Telegram or Discord row and its copied `result.json`. The backfill command asserts the stored RTT `p50` and `p95` values are unchanged before it rewrites that version's JSONL file. RSS is process-level data around the sampled command, not isolated channel transport memory. diff --git a/scripts/channel-storage.mjs b/scripts/channel-storage.mjs index 7194c0729..add997834 100644 --- a/scripts/channel-storage.mjs +++ b/scripts/channel-storage.mjs @@ -100,16 +100,19 @@ export async function readAllChannelRows() { throw error; } - const rows = []; + const channelIds = new Set(); for (const entry of entries) { if (entry.isFile() && entry.name.endsWith(".jsonl")) { - rows.push(...(await readJsonl(path.join(CHANNEL_DATA_DIR, entry.name)))); + channelIds.add(entry.name.slice(0, -".jsonl".length)); } if (entry.isDirectory()) { - const channelId = entry.name; - rows.push(...(await readChannelRows(channelId))); + channelIds.add(entry.name); } } + const rows = []; + for (const channelId of channelIds) { + rows.push(...(await readChannelRows(channelId))); + } return rows.sort(compareStartedAt); } diff --git a/scripts/channel-storage.test.mjs b/scripts/channel-storage.test.mjs new file mode 100644 index 000000000..70d31617b --- /dev/null +++ b/scripts/channel-storage.test.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import test from "node:test"; + +const execFileAsync = promisify(execFile); + +test("reads each legacy and versioned channel row once in chronological order", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "rtt-channel-storage-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const data = path.join(root, "data/channels"); + await fs.mkdir(path.join(data, "telegram"), { recursive: true }); + await fs.mkdir(path.join(data, "slack"), { recursive: true }); + for (const [file, id, day] of [ + ["telegram.jsonl", "legacy-telegram", "02"], + ["telegram/2026.9.1.jsonl", "versioned-telegram", "03"], + ["discord.jsonl", "legacy-only", "01"], + ["slack/2026.9.1.jsonl", "versioned-only", "04"], + ]) { + const row = { run: { id, startedAt: `2026-09-${day}T00:00:00.000Z` } }; + await fs.writeFile(path.join(data, file), `${JSON.stringify(row)}\n`); + } + await fs.writeFile(path.join(data, "README.md"), "not a channel"); + const { stdout } = await execFileAsync(process.execPath, [ + "--input-type=module", "--eval", + `import { readAllChannelRows } from ${JSON.stringify(new URL("./channel-storage.mjs", import.meta.url).href)}; + console.log(JSON.stringify((await readAllChannelRows()).map((row) => row.run.id)));`, + ], { cwd: root }); + assert.deepEqual(JSON.parse(stdout), [ + "legacy-only", "legacy-telegram", "versioned-telegram", "versioned-only", + ]); +}); From f004d28de2b54289a590bab076eb3801beac30a1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 23 Sep 2026 01:14:34 -0700 Subject: [PATCH 2/2] docs: keep channel storage release notes mergeable --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e8cb9aa6..4d76752ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,6 @@ ## Unreleased - Fix concurrent release QA auth seeding leaking state directories between agents, and roll back partial compatibility patch writes while preserving file permissions. Thanks @petercheng. -- Fix aggregate channel reports counting legacy rows twice when the same channel also has versioned data files. - Fix channel imports, surface imports, and RSS backfills failing to find sample files in TSV lists with CRLF line endings. - Fix surface imports recording incorrect run bounds for unordered samples or accepting invalid intermediate timestamps; normalize imported run timestamps to UTC. +- Fix aggregate channel reports counting legacy rows twice when the same channel also has versioned data files.