From 43d45c52772c5dcccc9fa0bc23c82435e913e6f1 Mon Sep 17 00:00:00 2001 From: bingqilinweimaotai <111987281+bingqilinweimaotai@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:50:21 +0800 Subject: [PATCH 1/2] fix(markdown): repair CJK link boundaries --- .../__tests__/message-markdown-links.test.ts | 81 +++++++++++++++++ src/components/LinkPreview.tsx | 16 ++-- src/lib/markdownUrls.ts | 78 ++++++++++++++++ src/lib/remarkCumora.ts | 88 ++++++++++++++++++- src/lib/utils.ts | 29 +----- 5 files changed, 254 insertions(+), 38 deletions(-) create mode 100644 server/src/__tests__/message-markdown-links.test.ts create mode 100644 src/lib/markdownUrls.ts diff --git a/server/src/__tests__/message-markdown-links.test.ts b/server/src/__tests__/message-markdown-links.test.ts new file mode 100644 index 00000000..62c5642a --- /dev/null +++ b/server/src/__tests__/message-markdown-links.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import type { Root } from 'mdast' +import remarkGfm from 'remark-gfm' +import remarkParse from 'remark-parse' +import { unified } from 'unified' +import { firstHttpUrlInMarkdown } from '../../../src/lib/markdownUrls' +import { remarkCumora } from '../../../src/lib/remarkCumora' + +async function parse(markdown: string): Promise { + const processor = unified().use(remarkParse).use(remarkGfm).use(remarkCumora) + return await processor.run(processor.parse(markdown)) as Root +} + +describe('chat Markdown links', () => { + it('repairs a bold URL followed immediately by CJK prose', async () => { + const tree = await parse('浏览器打开 **http://127.0.0.1:4182**（本机 preview）') + const paragraph = tree.children[0] + assert.equal(paragraph.type, 'paragraph') + if (paragraph.type !== 'paragraph') return + + assert.deepEqual(paragraph.children.map((node) => node.type), ['text', 'strong', 'text', 'text']) + const strong = paragraph.children[1] + assert.equal(strong.type, 'strong') + if (strong.type !== 'strong') return + const link = strong.children[0] + assert.equal(link.type, 'link') + if (link.type !== 'link') return + assert.equal(link.url, 'http://127.0.0.1:4182') + assert.equal(link.children[0].type === 'text' ? link.children[0].value : null, link.url) + const visibleText = paragraph.children + .filter((node) => node.type === 'text') + .map((node) => node.value) + .join('') + assert.equal(visibleText, '浏览器打开 （本机 preview）') + assert.ok(!visibleText.includes('**')) + }) + + it('keeps a literal URL as one non-nested link', async () => { + const tree = await parse('打开 https://example.com/path') + const paragraph = tree.children[0] + assert.equal(paragraph.type, 'paragraph') + if (paragraph.type !== 'paragraph') return + const link = paragraph.children.find((node) => node.type === 'link') + assert.ok(link && link.type === 'link') + assert.deepEqual(link.children.map((node) => node.type), ['text']) + }) + + it('stops a literal URL at CJK punctuation', async () => { + const tree = await parse('详见 https://example.com/path（说明）') + const paragraph = tree.children[0] + assert.equal(paragraph.type, 'paragraph') + if (paragraph.type !== 'paragraph') return + const link = paragraph.children.find((node) => node.type === 'link') + assert.ok(link && link.type === 'link') + assert.equal(link.url, 'https://example.com/path') + }) + + it('does not alter an explicit Markdown link', async () => { + const tree = await parse('[本机预览](http://127.0.0.1:4182)') + const paragraph = tree.children[0] + assert.equal(paragraph.type, 'paragraph') + if (paragraph.type !== 'paragraph') return + const link = paragraph.children[0] + assert.equal(link.type, 'link') + if (link.type !== 'link') return + assert.equal(link.url, 'http://127.0.0.1:4182') + assert.equal(link.children[0].type === 'text' ? link.children[0].value : null, '本机预览') + }) + + it('uses the same URL boundary for link previews', () => { + assert.equal( + firstHttpUrlInMarkdown('浏览器打开 **http://127.0.0.1:4182**（本机 preview）'), + 'http://127.0.0.1:4182', + ) + assert.equal( + firstHttpUrlInMarkdown('详见 https://example.com/path（说明）'), + 'https://example.com/path', + ) + }) +}) diff --git a/src/components/LinkPreview.tsx b/src/components/LinkPreview.tsx index edf9f854..535305c6 100644 --- a/src/components/LinkPreview.tsx +++ b/src/components/LinkPreview.tsx @@ -1,5 +1,6 @@ -import { useEffect, useState, type ReactNode } from 'react' +import { type ReactNode, useEffect, useState } from 'react' import { http } from '@/api/client' +import { firstHttpUrlInMarkdown } from '@/lib/markdownUrls' /** * Inline OG card rendered under a chat bubble when its body contains a URL. @@ -221,15 +222,8 @@ export function LinkPreview({ url }: { url: string }) { /** Pull the first http(s) URL out of a message body. Returns null when the * body has no link; used by the bubble to decide whether to mount a - * LinkPreview at all. Mirrors the regex in `parseBody` so what we render - * as a link in the inline pass is the same thing we expand into a card. */ + * LinkPreview at all. Uses the renderer's shared URL-boundary policy so the + * address we render is the same one we expand into a card. */ export function firstUrlInBody(body: string): string | null { - const m = body.match(/\bhttps?:\/\/[^\s<>"'`]+/) - if (!m) return null - // Trim sentence punctuation the same way parseBody does. - let url = m[0] - while (/[.,;:!?")\]}>'"]$/.test(url) && url.length > 'https://'.length) { - url = url.slice(0, -1) - } - return url + return firstHttpUrlInMarkdown(body) } diff --git a/src/lib/markdownUrls.ts b/src/lib/markdownUrls.ts new file mode 100644 index 00000000..5592a5b5 --- /dev/null +++ b/src/lib/markdownUrls.ts @@ -0,0 +1,78 @@ +/** + * Shared URL-boundary handling for chat Markdown. + * + * GFM literal autolinks deliberately accept almost every non-whitespace + * character in a URL path. That is surprising in CJK prose, where a URL is + * commonly followed immediately by a full-width punctuation mark, and it can + * also swallow a Markdown closing delimiter when prose follows it without a + * space (for example `**https://example.com**（说明）`). + * + * Keep this policy in one place so the Markdown renderer and link-preview + * extractor agree on the URL that a user can actually open. + */ + +/** Characters that conventionally end a URL in CJK prose. A literal URL + * that really needs one of these characters can still use percent encoding or + * explicit Markdown link syntax. */ +const CJK_PROSE_BOUNDARY_RE = /[，。；：！？、（）【】《》〈〉“”‘’「」『』]/u + +const TRAILING_PUNCTUATION = new Set(['.', ',', ';', ':', '!', '?', '"', "'"]) +const BRACKET_PAIRS: Record = { ')': '(', ']': '[', '}': '{', '>': '<' } + +const HTTP_URL_CANDIDATE_RE = /\bhttps?:\/\/[^\s<>"'`]+/g + +export interface SplitHttpUrlOptions { + /** A Markdown delimiter known to close immediately after the URL. */ + closingMarker?: string +} + +/** Split a permissively captured HTTP(S) URL from prose that was swept into + * the same match. The returned `trail` is always the exact unused suffix. */ +export function splitHttpUrlCandidate( + raw: string, + options: SplitHttpUrlOptions = {}, +): { url: string; trail: string } { + let end = raw.length + + if (options.closingMarker) { + const markerAt = raw.indexOf(options.closingMarker, 'https://'.length) + if (markerAt >= 0) end = Math.min(end, markerAt) + } + + const cjkAt = raw.slice(0, end).search(CJK_PROSE_BOUNDARY_RE) + if (cjkAt >= 0) end = Math.min(end, cjkAt) + + // Trim ordinary sentence punctuation. Keep balanced ASCII brackets inside + // the URL (for example a Wikipedia title ending in `)`), but remove a prose + // closer that has no matching opener in the captured URL. + while (end > 'https://'.length) { + const c = raw[end - 1] + if (TRAILING_PUNCTUATION.has(c)) { + end-- + continue + } + const opener = BRACKET_PAIRS[c] + if (!opener) break + const inside = raw.slice(0, end - 1) + const opens = Array.from(inside).filter((value) => value === opener).length + const closes = Array.from(inside).filter((value) => value === c).length + if (closes >= opens) end-- + else break + } + + return { url: raw.slice(0, end), trail: raw.slice(end) } +} + +const MARKDOWN_URL_WRAPPERS = ['**', '__', '~~', '*', '_'] as const + +/** Return the first HTTP(S) URL represented by a chat Markdown body. */ +export function firstHttpUrlInMarkdown(body: string): string | null { + HTTP_URL_CANDIDATE_RE.lastIndex = 0 + const match = HTTP_URL_CANDIDATE_RE.exec(body) + if (!match) return null + + const prefix = body.slice(0, match.index) + const closingMarker = MARKDOWN_URL_WRAPPERS.find((marker) => prefix.endsWith(marker)) + const { url } = splitHttpUrlCandidate(match[0], { closingMarker }) + return url || null +} diff --git a/src/lib/remarkCumora.ts b/src/lib/remarkCumora.ts index 6c92e692..c3f2a4c8 100644 --- a/src/lib/remarkCumora.ts +++ b/src/lib/remarkCumora.ts @@ -19,8 +19,9 @@ * (bold/italic/code/links), so in practice parseBody only finds the four custom * kinds — but we map every kind it can emit, defensively. */ -import { visit, SKIP } from 'unist-util-visit' -import type { Root, RootContent, Text } from 'mdast' +import type { Link, Parents, Root, RootContent, Text } from 'mdast' +import { SKIP, visit } from 'unist-util-visit' +import { splitHttpUrlCandidate } from './markdownUrls' import { parseBody, type RichToken } from './utils' /** A custom mdast node that mdast-util-to-hast renders as `` because of @@ -53,10 +54,93 @@ function tokenToNode(t: RichToken): RootContent { } } +const URL_WRAPPERS = [ + { marker: '**', type: 'strong' }, + { marker: '__', type: 'strong' }, + { marker: '~~', type: 'delete' }, + { marker: '*', type: 'emphasis' }, + { marker: '_', type: 'emphasis' }, +] as const + +function samePosition(a: RootContent['position'], b: RootContent['position']): boolean { + return !!a && !!b + && a.start.offset === b.start.offset + && a.end.offset === b.end.offset +} + +/** GFM literal autolinks can absorb Markdown closers and CJK punctuation. + * Repair those nodes before applying Cumora's custom inline-token pass. */ +function repairLiteralAutolinks(parent: Parents): void { + for (let index = 0; index < parent.children.length; index++) { + const node = parent.children[index] + if (node.type !== 'link') { + if ('children' in node && Array.isArray(node.children)) repairLiteralAutolinks(node as Parents) + continue + } + + const label = node.children.length === 1 && node.children[0].type === 'text' + ? node.children[0] + : null + // A literal autolink's link and label cover the same source span. This + // excludes `[label](url)` and ``, which already have explicit bounds. + if (!label || label.value !== node.url || !samePosition(node.position, label.position)) continue + + const previous = index > 0 ? parent.children[index - 1] : null + const wrapper = previous?.type === 'text' + ? URL_WRAPPERS.find(({ marker }) => ( + previous.value.endsWith(marker) + && node.url.indexOf(marker, 'https://'.length) >= 0 + )) + : undefined + const split = splitHttpUrlCandidate(node.url, { closingMarker: wrapper?.marker }) + if (!split.url || (split.url === node.url && !wrapper)) continue + + const fixedLink: Link = { + ...node, + url: split.url, + children: [{ type: 'text', value: split.url }], + } + + if (wrapper && previous?.type === 'text') { + const before = previous.value.slice(0, -wrapper.marker.length) + const trail = split.trail.slice(wrapper.marker.length) + const formatted = { + type: wrapper.type, + children: [fixedLink], + } as RootContent + const replacement: RootContent[] = [] + if (before) replacement.push({ type: 'text', value: before }) + replacement.push(formatted) + if (trail) replacement.push({ type: 'text', value: trail }) + parent.children.splice(index - 1, 2, ...replacement) + index = index - 2 + replacement.length + continue + } + + parent.children[index] = fixedLink + if (split.trail) { + parent.children.splice(index + 1, 0, { type: 'text', value: split.trail }) + index++ + } + } +} + export function remarkCumora() { return (tree: Root): void => { + repairLiteralAutolinks(tree) + + // Do not tokenize text that is already a link label. In particular, a + // GFM literal autolink label is the URL itself; converting that text again + // used to produce invalid nested `` elements for every bare URL. + const linkedText = new WeakSet() + visit(tree, ['link', 'linkReference'], (node) => { + visit(node, 'text', (text) => { linkedText.add(text) }) + return SKIP + }) + visit(tree, 'text', (node: Text, index, parent) => { if (!parent || typeof index !== 'number') return + if (linkedText.has(node)) return const tokens = parseBody(node.value) // Nothing Cumora-specific in this text node — leave it untouched. if (tokens.length === 1 && tokens[0].kind === 'text') return diff --git a/src/lib/utils.ts b/src/lib/utils.ts index f48e69e8..f6ca30d9 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,6 +1,6 @@ -import { clsx, type ClassValue } from 'clsx' -import { twMerge } from 'tailwind-merge' +import { type ClassValue, clsx } from 'clsx' import { useEffect, useState } from 'react' +import { twMerge } from 'tailwind-merge' export function cn(...inputs: ClassValue[]): string { return twMerge(clsx(inputs)) @@ -76,6 +76,7 @@ export type RichToken = import emojiRegex from 'emoji-regex' import { findSkypeByShortcode, SKYPE_SHORTCODE_RE } from '@/lib/skypeEmojis' +import { splitHttpUrlCandidate } from './markdownUrls' const DOC_REF_TOKEN_RE = /^doc_[A-Za-z0-9]+$/ const BOARD_REF_TOKEN_RE = /^board-[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*$/ @@ -119,28 +120,6 @@ function splitUnicodeEmoji(seg: string, out: RichToken[]): void { if (last < seg.length) out.push({ kind: 'text', value: seg.slice(last) }) } -/** Strip trailing sentence punctuation from a captured URL match. Keeps - * balanced ()[]{}<> inside the URL — only trims a closing bracket when - * there's no matching opener inside the URL (typical case: the URL was - * inside parens in prose, e.g. "(see https://x.com)"). */ -function trimUrlTrailing(raw: string): { url: string; trail: string } { - const closers: Record = { ')': '(', ']': '[', '}': '{', '>': '<' } - let i = raw.length - while (i > 'https://'.length) { - const c = raw[i - 1] - if (c === '.' || c === ',' || c === ';' || c === ':' || c === '!' || c === '?' || c === '"' || c === "'") { - i-- - } else if (closers[c]) { - const inside = raw.slice(0, i - 1) - const opens = (inside.match(new RegExp(`\\${closers[c]}`, 'g')) ?? []).length - const closes = (inside.match(new RegExp(`\\${c}`, 'g')) ?? []).length - if (closes >= opens) i-- - else break - } else break - } - return { url: raw.slice(0, i), trail: raw.slice(i) } -} - export function parseBody(body: string): RichToken[] { const tokens: RichToken[] = [] // Order matters: inline `code` first so backtick-wrapped @mentions / bold @@ -181,7 +160,7 @@ export function parseBody(body: string): RichToken[] { // Strip trailing punctuation, keeping balanced () [] {} <> inside the // URL (Wikipedia titles, Mediawiki anchors, etc.) and re-emitting the // trimmed tail as plain text so the message keeps its grammar. - const { url, trail } = trimUrlTrailing(t) + const { url, trail } = splitHttpUrlCandidate(t) tokens.push({ kind: 'link', url, text: url }) if (trail) splitEmoji(trail, tokens) } else if (t.startsWith('doc_')) { From 79a25e31926832f031ca0fc94afbc09afbe9117c Mon Sep 17 00:00:00 2001 From: bingqilinweimaotai <111987281+bingqilinweimaotai@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:16:11 +0800 Subject: [PATCH 2/2] test(markdown): keep client tests outside server tree --- package.json | 2 +- .../src/__tests__ => tests}/message-markdown-links.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename {server/src/__tests__ => tests}/message-markdown-links.test.ts (96%) diff --git a/package.json b/package.json index 4d08b7fa..6aa1f400 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "guard:engine-registry": "node scripts/guard-engine-registry.mjs", "lint": "biome lint .", "lint:fix": "biome lint --write .", - "test": "node --import tsx --test \"server/src/__tests__/**/*.test.ts\" \"workers/**/*.test.ts\"", + "test": "node --import tsx --test \"server/src/__tests__/**/*.test.ts\" \"tests/**/*.test.ts\" \"workers/**/*.test.ts\"", "test:integration": "node server/run-integration-tests.mjs", "migrate": "tsx server/src/migrate-bin.ts", "icons": "python3 scripts-make-mac-icon.py && npx -y electron-icon-builder -i build/icon.png -o build --flatten && node scripts-sync-electron-dev-icon.mjs", diff --git a/server/src/__tests__/message-markdown-links.test.ts b/tests/message-markdown-links.test.ts similarity index 96% rename from server/src/__tests__/message-markdown-links.test.ts rename to tests/message-markdown-links.test.ts index 62c5642a..d2374e4d 100644 --- a/server/src/__tests__/message-markdown-links.test.ts +++ b/tests/message-markdown-links.test.ts @@ -4,8 +4,8 @@ import type { Root } from 'mdast' import remarkGfm from 'remark-gfm' import remarkParse from 'remark-parse' import { unified } from 'unified' -import { firstHttpUrlInMarkdown } from '../../../src/lib/markdownUrls' -import { remarkCumora } from '../../../src/lib/remarkCumora' +import { firstHttpUrlInMarkdown } from '../src/lib/markdownUrls' +import { remarkCumora } from '../src/lib/remarkCumora' async function parse(markdown: string): Promise { const processor = unified().use(remarkParse).use(remarkGfm).use(remarkCumora)