Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/markdown-fence-backtick-info-string.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@lexical/markdown': patch
---

[fix] Import a line that starts with three backticks and has more backticks later as a paragraph, not as a code fence, so inline code that Lexical exports at the start of a paragraph no longer becomes a code block that takes the rest of the document

Verify: import `` ``` a``b ``` x `` followed by a blank line and `next`, which gives a paragraph with inline code and then a separate `next` paragraph. A fence whose info string has a backtick, such as `` ```a`b``` `` or `` ```js title=`x` ``, now imports as a paragraph too. Exporting a code block whose stored meta contains a backtick leaves the meta out of the fence line. Typing three backticks and a space at the start of an existing paragraph still makes a code block.
35 changes: 31 additions & 4 deletions packages/lexical-markdown/src/MarkdownTransformers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,14 @@ const CHECK_LIST_REGEX = /^(\s*)(?:[-*+]\s)?\s?(\[(\s|x)?\])\s/i;
const HEADING_REGEX = /^(#{1,6})\s/;
const QUOTE_REGEX = /^>\s/;
const CODE_START_REGEX = /^([ \t]*`{3,})([\w-]+)?[ \t]?/;
// The info string of a backtick fence cannot contain a backtick, so a line
// with one after the fence is not a fence, except the single-line form
// ```code``` whose code has none. This is applied to a whole line of imported
// Markdown, not to `CODE.regExpStart`: the typing shortcut matches that
// against the text of a node that goes on after the caret.
// https://spec.commonmark.org/0.31.2/#fenced-code-blocks
const CODE_IMPORT_START_REGEX =
/^([ \t]*`{3,})(?=[^`]*$|[^`]+`{3,}$)([\w-]+)?[ \t]?/;
const CODE_END_REGEX = /^[ \t]*`{3,}$/;
const CODE_SINGLE_LINE_REGEX =
/^[ \t]*```[^`]+(?:(?:`{1,2}|`{4,})[^`]+)*```(?:[^`]|$)/;
Comment thread
rootnode-bot[bot] marked this conversation as resolved.
Resolved
Expand Down Expand Up @@ -879,7 +887,12 @@ export const CODE: MultilineElementTransformer = {
}
}
const language = node.getLanguage() || '';
const meta = language ? $getState(node, codeMetaState) : '';
// The info string of a backtick fence cannot hold a backtick, so a meta
// that does is left out: written, the fence line would not read back as a
// fence and its closing fence would open a block that takes the rest of
// the document.
const storedMeta = language ? $getState(node, codeMetaState) : '';
const meta = storedMeta.includes('`') ? '' : storedMeta;

return (
fence +
Expand All @@ -897,9 +910,14 @@ export const CODE: MultilineElementTransformer = {
startLineIndex,
startMatch,
}) => {
const currentLine = lines[startLineIndex];
// Only an import gets here, so the rule for the info string is applied
// here rather than in `regExpStart`, which the typing shortcut also uses.
if (!CODE_IMPORT_START_REGEX.test(currentLine)) {
return null;
}
const fence = startMatch[1];
const fenceLength = fence.trim().length;
const currentLine = lines[startLineIndex];
const afterFenceIndex = startMatch.index! + fence.length;
const afterFence = currentLine.slice(afterFenceIndex);

Expand All @@ -923,6 +941,12 @@ export const CODE: MultilineElementTransformer = {
return [true, startLineIndex];
}

// A closing run shorter than the opening one, as in ````code```, leaves a
// backtick in the info string, so the line is not a fence.
if (afterFence.includes('`')) {
return null;
}

const multilineEndRegex = new RegExp(`^[ \\t]*\`{${fenceLength},}$`);

for (let i = startLineIndex + 1; i < lines.length; i++) {
Expand Down Expand Up @@ -1438,14 +1462,17 @@ export function normalizeMarkdown(
lastLine !== undefined && parseMarkdownHardLineBreak(lastLine) !== null;

// Code blocks of ```single line``` don't toggle the inCodeBlock flag
if (CODE_SINGLE_LINE_REGEX.test(line)) {
if (
CODE_IMPORT_START_REGEX.test(line) &&
CODE_SINGLE_LINE_REGEX.test(line)
) {
sanitizedLines.push(line);
continue;
}

if (codeBlockFenceLength === 0) {
// An opening fence may carry an info string (e.g. ```ts)
const openMatch = line.match(CODE_START_REGEX);
const openMatch = line.match(CODE_IMPORT_START_REGEX);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Optional: the lookahead's second alternative ([^`]+`{3,}$) accepts a closing run shorter than the opening fence, so this match sets codeBlockFenceLength = 4 for ````a```, while handleImportAfterStartMatch() now declines that line (line 946) and reads a paragraph. With shouldMergeAdjacentLines, every later line then stays unmerged until a bare fence of four or more backticks; normalizeMarkdown('x\n````a```\ny', true) returns its input. Requiring the closing run to be at least the opening length keeps the two in step (only group 1 is read below):

const CODE_IMPORT_START_REGEX =
  /^([ \t]*(`{3,}))(?=[^`]*$|[^`]+\2`*$)([\w-]+)?[ \t]?/;

if (openMatch) {
codeBlockFenceLength = openMatch[1].trim().length;
sanitizedLines.push(line);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
*/

// @vitest-environment node

import {$createCodeNode, $isCodeNode} from '@lexical/code-core';
import {buildEditorFromExtensions} from '@lexical/extension';
import {
$convertFromMarkdownString,
$convertToMarkdownString,
registerMarkdownShortcuts,
TRANSFORMERS,
} from '@lexical/markdown';
import {
$createParagraphNode,
$createTextNode,
$getRoot,
$getSelection,
$isRangeSelection,
$setState,
} from 'lexical';
import {describe, expect, it} from 'vitest';

import {codeMetaState, normalizeMarkdown} from '../../MarkdownTransformers';
import {MarkdownTestExtension} from '../utils';

function importBlocks(markdown: string): string[] {
using editor = buildEditorFromExtensions([MarkdownTestExtension]);
let blocks: string[] = [];
editor.update(
() => {
$convertFromMarkdownString(markdown, TRANSFORMERS);
blocks = $getRoot()
.getChildren()
.map(
block =>
($isCodeNode(block) ? 'code: ' : 'paragraph: ') +
block.getTextContent(),
);
},
{discrete: true},
);
return blocks;
}

function roundTrip(markdown: string): string {
using editor = buildEditorFromExtensions([MarkdownTestExtension]);
editor.update(
() => {
$convertFromMarkdownString(markdown, TRANSFORMERS);
},
{discrete: true},
);
return editor.read(() => $convertToMarkdownString(TRANSFORMERS));
}

// Exports a paragraph that starts with inline code, then imports the result.
function exportAndImportInlineCode(code: string, after: string): string[] {
using editor = buildEditorFromExtensions([MarkdownTestExtension]);
let markdown = '';
editor.update(
() => {
const paragraph = $createParagraphNode().append(
$createTextNode(code).toggleFormat('code'),
);
if (after) {
paragraph.append($createTextNode(after));
}
$getRoot().clear().append(paragraph);
markdown = $convertToMarkdownString(TRANSFORMERS);
},
{discrete: true},
);
return importBlocks(markdown);
}

// The info string of a backtick fence cannot contain a backtick, so such a
// line is not a fence. https://spec.commonmark.org/0.31.2/#fenced-code-blocks
describe('a line that starts with backticks and has more later (#9404)', () => {
it('reads it as a paragraph that starts with a code span', () => {
expect(importBlocks('``` a``b ``` x')).toEqual(['paragraph: a``b x']);
expect(importBlocks('``` a``b ```')).toEqual(['paragraph: a``b']);
expect(importBlocks('```` ``` ```` x\n\nnext')).toEqual([
'paragraph: ``` x',
'paragraph: next',
]);
});

it("re-imports the editor's own export of inline code", () => {
expect(exportAndImportInlineCode('a``b', ' x')).toEqual([
'paragraph: a``b x',
]);
expect(exportAndImportInlineCode('```', ' x')).toEqual([
'paragraph: ``` x',
]);
expect(exportAndImportInlineCode('a``b', '')).toEqual(['paragraph: a``b']);
});

it('round trips such a line unchanged', () => {
expect(roundTrip('``` a``b ``` x')).toBe('``` a``b ``` x');
expect(roundTrip('```` ``` ```` x\n\nnext')).toBe(
'```` ``` ```` x\n\nnext',
);
});

it('does not take a shorter closing run as the end of a single-line block', () => {
expect(importBlocks('````a```')).toEqual(['paragraph: ````a```']);
});

it('does not stop merging the lines that follow', () => {
expect(normalizeMarkdown('```` ``` ```` x\nnext', true)).toBe(
'```` ``` ```` x next',
);
});

it('still imports fences whose info string has no backtick', () => {
expect(importBlocks('```Single line Code```')).toEqual([
'code: Single line Code',
]);
expect(importBlocks('```a```')).toEqual(['code: a']);
expect(importBlocks('```js\ncode\n```')).toEqual(['code: code']);
expect(importBlocks('```js title="x"\ncode\n```')).toEqual(['code: code']);
expect(importBlocks('````\n```\nin\n```\n````')).toEqual([
'code: ```\nin\n```',
]);
expect(importBlocks('```javascript\nIncomplete')).toEqual([
'code: Incomplete',
]);
});

it('reads a single-line fence whose code has a backtick as a paragraph', () => {
expect(importBlocks('```a`b```')).toEqual(['paragraph: a`b']);
});

it('merges a line like that into the line above it, as for a longer fence', () => {
expect(normalizeMarkdown('a\n``` a``b ``` x', true)).toBe(
'a ``` a``b ``` x',
);
expect(normalizeMarkdown('a\n```` ``` ```` x', true)).toBe(
'a ```` ``` ```` x',
);
});
});

describe('the code fence typing shortcut (#9404)', () => {
function typeFenceAtStartOfParagraph(text: string): string[] {
using editor = buildEditorFromExtensions([MarkdownTestExtension]);
registerMarkdownShortcuts(editor, TRANSFORMERS);
editor.update(
() => {
const textNode = $createTextNode(text);
$getRoot().clear().append($createParagraphNode().append(textNode));
textNode.select(0, 0);
},
{discrete: true},
);
for (const character of '``` ') {
editor.update(
() => {
const selection = $getSelection();
if ($isRangeSelection(selection)) {
selection.insertText(character);
}
},
{discrete: true},
);
}
return editor.read(() =>
$getRoot()
.getChildren()
.map(
block =>
($isCodeNode(block) ? 'code: ' : 'paragraph: ') +
block.getTextContent(),
),
);
}

it('still turns an existing paragraph that has backticks into a code block', () => {
expect(typeFenceAtStartOfParagraph('const x = `y`;')).toEqual([
'code: const x = `y`;',
]);
expect(typeFenceAtStartOfParagraph('a `b` c')).toEqual(['code: a `b` c']);
});
});

describe('exporting the meta of a code block (#9404)', () => {
function exportCode(meta: string): string {
using editor = buildEditorFromExtensions([MarkdownTestExtension]);
editor.update(
() => {
const code = $createCodeNode('js').append($createTextNode('code'));
$setState(code, codeMetaState, meta);
$getRoot().clear().append(code);
},
{discrete: true},
);
return editor.read(() => $convertToMarkdownString(TRANSFORMERS));
}

it('writes a meta that has no backtick', () => {
expect(exportCode('title="x"')).toBe('```js title="x"\ncode\n```');
});

it('leaves out a meta that has a backtick, which would not read back as a fence', () => {
expect(exportCode('title=`x`')).toBe('```js\ncode\n```');
expect(roundTrip(exportCode('title=`x`'))).toBe('```js\ncode\n```');
});
});