diff --git a/CHANGELOG.md b/CHANGELOG.md index 86fe16593..4d76752ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,3 +5,4 @@ - Fix concurrent release QA auth seeding leaking state directories between agents, and roll back partial compatibility patch writes while preserving file permissions. Thanks @petercheng. - Fix channel imports, surface imports, and RSS backfills failing to find sample files in TSV lists with CRLF line endings. - Fix surface imports recording incorrect run bounds for unordered samples or accepting invalid intermediate timestamps; normalize imported run timestamps to UTC. +- Fix aggregate channel reports counting legacy rows twice when the same channel also has versioned data files. diff --git a/docs/data-imports.md b/docs/data-imports.md index 0d0af53c5..60b788577 100644 --- a/docs/data-imports.md +++ b/docs/data-imports.md @@ -35,6 +35,8 @@ pnpm test:docker:npm-telegram-live Current channel folders are `telegram`, `discord`, `slack`, and `whatsapp`. Telegram and Discord still have specialized importers because their source artifact shapes differ; they now share the same storage contract as generic live-transport channels. +Readers also accept legacy `data/channels/.jsonl` files alongside versioned channel directories. Aggregate reports read each source file once, including when both layouts exist for the same channel. + Raw Telegram QA artifacts stay in the OpenClaw repo artifact directory unless explicitly copied here later. Historical Telegram rows imported from the old package RTT wrapper may include `rtt.warmSamples` with every successful sample value. New Telegram imports from `qa-evidence.json` preserve the aggregate dashboard metrics (`canaryMs`, `mentionReplyMs`, `avgMs`, `p50Ms`, `p95Ms`, `maxMs`, `sampleCount`, and `failedSamples`) but do not reconstruct individual sample RTT arrays because the evidence artifact stores aggregate timing. Release RSS backfills only write `resources` onto an existing Telegram or Discord row and its copied `result.json`. The backfill command asserts the stored RTT `p50` and `p95` values are unchanged before it rewrites that version's JSONL file. RSS is process-level data around the sampled command, not isolated channel transport memory. diff --git a/scripts/channel-storage.mjs b/scripts/channel-storage.mjs index 7194c0729..add997834 100644 --- a/scripts/channel-storage.mjs +++ b/scripts/channel-storage.mjs @@ -100,16 +100,19 @@ export async function readAllChannelRows() { throw error; } - const rows = []; + const channelIds = new Set(); for (const entry of entries) { if (entry.isFile() && entry.name.endsWith(".jsonl")) { - rows.push(...(await readJsonl(path.join(CHANNEL_DATA_DIR, entry.name)))); + channelIds.add(entry.name.slice(0, -".jsonl".length)); } if (entry.isDirectory()) { - const channelId = entry.name; - rows.push(...(await readChannelRows(channelId))); + channelIds.add(entry.name); } } + const rows = []; + for (const channelId of channelIds) { + rows.push(...(await readChannelRows(channelId))); + } return rows.sort(compareStartedAt); } diff --git a/scripts/channel-storage.test.mjs b/scripts/channel-storage.test.mjs new file mode 100644 index 000000000..70d31617b --- /dev/null +++ b/scripts/channel-storage.test.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import test from "node:test"; + +const execFileAsync = promisify(execFile); + +test("reads each legacy and versioned channel row once in chronological order", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "rtt-channel-storage-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const data = path.join(root, "data/channels"); + await fs.mkdir(path.join(data, "telegram"), { recursive: true }); + await fs.mkdir(path.join(data, "slack"), { recursive: true }); + for (const [file, id, day] of [ + ["telegram.jsonl", "legacy-telegram", "02"], + ["telegram/2026.9.1.jsonl", "versioned-telegram", "03"], + ["discord.jsonl", "legacy-only", "01"], + ["slack/2026.9.1.jsonl", "versioned-only", "04"], + ]) { + const row = { run: { id, startedAt: `2026-09-${day}T00:00:00.000Z` } }; + await fs.writeFile(path.join(data, file), `${JSON.stringify(row)}\n`); + } + await fs.writeFile(path.join(data, "README.md"), "not a channel"); + const { stdout } = await execFileAsync(process.execPath, [ + "--input-type=module", "--eval", + `import { readAllChannelRows } from ${JSON.stringify(new URL("./channel-storage.mjs", import.meta.url).href)}; + console.log(JSON.stringify((await readAllChannelRows()).map((row) => row.run.id)));`, + ], { cwd: root }); + assert.deepEqual(JSON.parse(stdout), [ + "legacy-only", "legacy-telegram", "versioned-telegram", "versioned-only", + ]); +});