From 008b2176722bc545ebf09e3295415b1d4a9488a0 Mon Sep 17 00:00:00 2001 From: Xialie Zhuang <62231346+Lieisyourlie@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:08:05 +0800 Subject: [PATCH] fix(realtime): a body cut mid-emoji must not roll back the message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The quote path truncates by UTF-16 code unit — `qr[0].body.slice(0, 240)` — so quoting a message whose 240th code unit is half of a non-BMP character yields a lone surrogate. JSON.stringify emits that as the literal ASCII escape \ud83d, which survives transport byte-for-byte and is then refused by the outbox's cast: ERROR: invalid input syntax for type json DETAIL: Unicode low surrogate must follow a high surrogate. enqueueBroadcast runs INSIDE the caller's transaction, so the failure takes the reply with it: ROLLBACK discards the message row, the sequence bump and the updated_at touch, and the request 500s. Every retry fails identically, so one emoji-bearing message becomes permanently unquotable for every human and every agent in the workspace. This is new in 0.14. The same payload used to go to redis.publish AFTER COMMIT, where a lone surrogate was a cosmetic glyph. Moving it behind a ::jsonb cast inside the transaction turned that into a lost write. messages.body is TEXT and accepts the same bytes, which is why only the outbox half dies. Reproduced against Postgres 16 through enqueueBroadcast itself: SQLSTATE 22P02 on exactly the string router.ts:4133 produces. Scrubbed at the cast rather than at the 24 call sites, so fields added to future events are covered too. The repo already owns the helper — stripLoneSurrogates, whose own comment names body.slice(0, N) as the cause — it had simply never been applied to this boundary. A JSON.stringify replacer visits every string in the tree, so nesting and arrays need no traversal. The regression tests fail with the production error when the scrub is removed. One of them pins the other direction: a well-formed emoji has to survive intact. --- .../__integration__/realtime-outbox.test.ts | 74 ++++++++++++++++++- server/src/realtime-outbox.ts | 28 ++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/server/src/__integration__/realtime-outbox.test.ts b/server/src/__integration__/realtime-outbox.test.ts index 4b1c1f67..87068b20 100644 --- a/server/src/__integration__/realtime-outbox.test.ts +++ b/server/src/__integration__/realtime-outbox.test.ts @@ -2,7 +2,7 @@ import { createServer, type Server } from 'node:http' import { after, before, beforeEach, test } from 'node:test' import assert from 'node:assert/strict' import { pool } from '../db/pool.js' -import { drainRealtimeOutbox } from '../realtime-outbox.js' +import { drainRealtimeOutbox, enqueueBroadcast } from '../realtime-outbox.js' import type { BroadcastEvent } from '../redis.js' import { buildApiTestApp, ensureSchemaOnce, resetAllTables, @@ -154,3 +154,75 @@ test('[integration] document and calendar creates replay without duplicate rows ) assert.equal(events.rows[0]?.count, 2) }) + +// ── a body cut mid-emoji must not roll the transaction back ───────────────── +// +// The quote path truncates by UTF-16 code unit (`qr[0].body.slice(0, 240)`), so +// a message with a non-BMP character straddling that index yields a lone +// surrogate. JSON.stringify emits it as the ASCII escape `\ud83d`, which reaches +// Postgres intact and is refused by the jsonb cast: +// +// ERROR: invalid input syntax for type json +// DETAIL: Unicode low surrogate must follow a high surrogate. +// +// Because the enqueue is inside the caller's transaction, that took the message +// with it. `messages.body` is TEXT and accepts the same bytes, so only the +// outbox half died — which is what made one emoji-bearing message permanently +// un-quotable rather than merely ugly. + +/** Exactly what the quote path produces: 239 ASCII chars then an emoji, cut at 240. */ +const CUT_MID_EMOJI = `${'a'.repeat(239)}\u{1F600}`.slice(0, 240) + +test('[integration] a payload truncated mid-emoji still enqueues', async () => { + assert.equal(CUT_MID_EMOJI.charCodeAt(239), 0xd83d, 'fixture no longer ends in a lone surrogate') + + const id = await enqueueBroadcast(pool, 'message:new', { + type: 'message.new', + conversationId: 'c-outbox', + quoted: { body: CUT_MID_EMOJI }, + } as unknown as BroadcastEvent) + + const { rows } = await pool.query<{ body: string }>( + `SELECT payload->'quoted'->>'body' AS body FROM realtime_outbox WHERE id = $1`, + [id], + ) + assert.equal(rows.length, 1, 'the row was not written') + // The broken half is dropped, not the message: 239 readable characters survive. + assert.equal(rows[0].body, 'a'.repeat(239)) +}) + +test('[integration] a lone surrogate anywhere in the tree is scrubbed, not just at the top', async () => { + // Payload fields grow over time; scrubbing lives at the single cast so a new + // nested field cannot reintroduce this. + const id = await enqueueBroadcast(pool, 'message:new', { + type: 'message.new', + conversationId: 'c-outbox', + quoted: { body: '\ud83d', authorName: 'tail \ud83d' }, + tags: ['\ud83d', 'ok'], + } as unknown as BroadcastEvent) + + const { rows } = await pool.query<{ payload: Record }>( + `SELECT payload FROM realtime_outbox WHERE id = $1`, [id], + ) + const payload = rows[0].payload as { + quoted: { body: string; authorName: string }; tags: string[] + } + assert.equal(payload.quoted.body, '') + assert.equal(payload.quoted.authorName, 'tail ') + assert.deepEqual(payload.tags, ['', 'ok']) +}) + +test('[integration] a well-formed emoji is untouched', async () => { + // The scrub must only remove UNPAIRED halves — a real emoji has to survive + // intact or every message carrying one arrives mangled. + const id = await enqueueBroadcast(pool, 'message:new', { + type: 'message.new', + conversationId: 'c-outbox', + quoted: { body: 'ship it \u{1F680} done' }, + } as unknown as BroadcastEvent) + const { rows } = await pool.query<{ body: string }>( + `SELECT payload->'quoted'->>'body' AS body FROM realtime_outbox WHERE id = $1`, + [id], + ) + assert.equal(rows[0].body, 'ship it \u{1F680} done') +}) diff --git a/server/src/realtime-outbox.ts b/server/src/realtime-outbox.ts index 35f9fbf5..e952d177 100644 --- a/server/src/realtime-outbox.ts +++ b/server/src/realtime-outbox.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto' import type { PoolClient, QueryResult, QueryResultRow } from 'pg' import { pool } from './db/pool.js' +import { stripLoneSurrogates } from './agents/text-safety.js' import { publish, type BroadcastEvent } from './redis.js' /** @@ -60,11 +61,36 @@ export async function enqueueBroadcast( await db.query( `INSERT INTO realtime_outbox (id, channel, payload) VALUES ($1, $2, $3::jsonb)`, - [id, channel, JSON.stringify(payload)], + [id, channel, serializePayload(payload)], ) return id } +/** Serialize an outbox payload so Postgres will accept it as `jsonb`. + * + * A body truncated mid-emoji — `qr[0].body.slice(0, 240)` on the quote path — + * ends in a lone UTF-16 surrogate. JSON.stringify turns that into the literal + * ASCII escape `\ud83d`, which survives transport intact and is then rejected: + * + * ERROR: invalid input syntax for type json + * DETAIL: Unicode low surrogate must follow a high surrogate. + * + * That matters here and not before because the enqueue is INSIDE the caller's + * transaction. The same payload used to go to redis.publish AFTER COMMIT, where + * a lone surrogate was a cosmetic glyph; behind a `::jsonb` cast it rolls the + * message back and 500s, so a quote-reply to one emoji-bearing message fails + * forever. `messages.body` is TEXT and accepts the same bytes, which is why + * only the outbox half dies. + * + * Scrubbing here rather than at each call site is deliberate: every field of + * every future event goes through this one cast. The replacer visits every + * string in the tree, so nesting and arrays need no traversal of our own. */ +function serializePayload(payload: unknown): string { + return JSON.stringify(payload, (_key, value) => + typeof value === 'string' ? stripLoneSurrogates(value) : value, + ) +} + /** Run a mutation and all of its realtime invalidations in one transaction. */ export async function withOutboxTransaction( run: (client: PoolClient) => Promise,