Add Ruby Liquid parity test suite for LiquidSyntaxError

packages/theme-check-common/package.json

@@ -43,6 +43,7 @@
     "@types/lodash": "^4.17.20",
     "@types/node": "^22.18.8",
     "@types/postcss-safe-parser": "^5.0.4",
-    "@vitest/expect": "4.1.0"
+    "@vitest/expect": "4.1.0",
+    "yaml": "^2.8.3"
   }
 }

packages/theme-check-common/src/checks/liquid-syntax-error/parity/lint-liquid.ts

@@ -0,0 +1,28 @@
+import { runLiquidCheck } from '../../../test';
+import { LiquidSyntaxError } from '../index';
+
+export interface LintDiagnostic {
+  message: string;
+  line: number;
+}
+
+const DEFAULT_PATH = 'templates/template.liquid';
+
+/**
+ * `themeDocset: undefined` drives tag-name recognition through the real
+ * `builtinTags` fallback in base.ts (the test harness would otherwise inject
+ * a docset whose `tags()` returns `[]`).
+ */
+const NO_DOCSET = { themeDocset: undefined } as const;
+
+export async function lintLiquid(
+  template: string,
+  filePath: string = DEFAULT_PATH,
+): Promise<LintDiagnostic[]> {
+  const fileName = filePath.replace(/^\/+/, '');
+  const offenses = await runLiquidCheck(LiquidSyntaxError, template, fileName, NO_DOCSET);
+  return offenses.map((offense) => ({
+    message: offense.message,
+    line: offense.start.line,
+  }));
+}

packages/theme-check-common/src/checks/liquid-syntax-error/parity/loader.ts

@@ -0,0 +1,28 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { parse } from 'yaml';
+
+export interface Scenario {
+  id: string;
+  template: string;
+  error_path: string;
+  file_path?: string;
+}
+
+export interface SnapshotEntry {
+  id: string;
+  error: string | null;
+  type: string | null;
+}
+
+export function loadScenarios(tag: string): Scenario[] {
+  const path = resolve(__dirname, 'scenarios', `${tag}.yml`);
+  const content = readFileSync(path, 'utf-8');
+  return parse(content) as Scenario[];
+}
+
+export function loadSnapshots(tag: string): SnapshotEntry[] {
+  const path = resolve(__dirname, 'snapshots', `${tag}.snap.yml`);
+  const content = readFileSync(path, 'utf-8');
+  return parse(content) as SnapshotEntry[];
+}

packages/theme-check-common/src/checks/liquid-syntax-error/parity/normalize.ts

@@ -0,0 +1,44 @@
+import type { LintDiagnostic } from './lint-liquid';
+
+export interface NormalizedError {
+  detected: boolean;
+  line: number | null;
+  coreMessage: string;
+}
+
+export function normalizeSnapshot(snap: {
+  error: string | null;
+  type: string | null;
+}): NormalizedError {
+  if (!snap.error) {
+    return { detected: false, line: null, coreMessage: '' };
+  }
+
+  return {
+    detected: true,
+    line: null,
+    coreMessage: snap.error.toLowerCase().trim(),
+  };
+}
+
+export function normalizeThemeCheck(diagnostics: LintDiagnostic[]): NormalizedError {
+  if (diagnostics.length === 0) {
+    return { detected: false, line: null, coreMessage: '' };
+  }
+
+  let msg = diagnostics[0].message.toLowerCase().trim();
+
+  if (msg.startsWith('liquidhtmlsyntaxerror: ')) {
+    msg = msg.slice('liquidhtmlsyntaxerror: '.length).trim();
+  }
+
+  if (msg.startsWith('syntax error: ')) {
+    msg = msg.slice('syntax error: '.length).trim();
+  }
+
+  return {
+    detected: true,
+    line: diagnostics[0].line,
+    coreMessage: msg,
+  };
+}

packages/theme-check-common/src/checks/liquid-syntax-error/parity/parity.spec.ts

@@ -0,0 +1,95 @@
+import { describe, it, afterAll, expect } from 'vitest';
+import { readdirSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { lintLiquid } from './lint-liquid';
+import { type Scenario, loadScenarios, loadSnapshots } from './loader';
+import { normalizeSnapshot, normalizeThemeCheck } from './normalize';
+import { Outcome, ParityReport } from './report';
+
+const report = new ParityReport();
+
+const scenariosDir = resolve(__dirname, 'scenarios');
+const tags = readdirSync(scenariosDir)
+  .filter((f) => f.endsWith('.yml'))
+  .map((f) => f.replace('.yml', ''));
+
+/*
+ * Scenarios where Ruby Liquid reports an error that LiquidSyntaxError does
+ * not. Each entry is asserted to STILL miss, so the day the gap closes this
+ * suite fails and the entry gets deleted rather than quietly masking a
+ * re-introduced regression.
+ *
+ *   section-themecheck-bare-bracket-arg — `{% section 'header', foo: [0] %}`.
+ *     The section syntax check skips BlockArrayLiteral argument values,
+ *     which @shopify/liquid-html-parser produces for `[0]`, so the bare
+ *     bracket never reaches the bare-array-access test. Ruby rejects named
+ *     arguments on {% section %} outright.
+ */
+const KNOWN_GAPS = new Set(['section-themecheck-bare-bracket-arg']);
+
+async function runParityScenario(
+  tag: string,
+  scenario: Scenario,
+  snapshots: ReturnType<typeof loadSnapshots>,
+) {
+  const snap = snapshots.find((s) => s.id === scenario.id);
+  expect(snap, `No snapshot for ${scenario.id}`).toBeDefined();
+
+  const diagnostics = await lintLiquid(scenario.template, scenario.file_path);
+  const snapshotNorm = normalizeSnapshot(snap!);
+  const themeCheckNorm = normalizeThemeCheck(diagnostics);
+
+  if (KNOWN_GAPS.has(scenario.id)) {
+    expect(
+      themeCheckNorm.detected,
+      `[${scenario.id}] is listed in KNOWN_GAPS but theme-check now reports it — remove the entry`,
+    ).toBe(false);
+    return;
+  }
+
+  if (snapshotNorm.detected && themeCheckNorm.detected) {
+    report.record(Outcome.MATCH, { tag, scenarioId: scenario.id });
+    return;
+  }
+
+  if (snapshotNorm.detected && !themeCheckNorm.detected) {
+    report.record(Outcome.UNEXPECTED_MISS, {
+      tag,
+      scenarioId: scenario.id,
+      message: snapshotNorm.coreMessage,
+    });
+    expect.fail(
+      `UNEXPECTED_MISS [${scenario.id}]: snapshot detects error but theme-check does not`,
+    );
+    return;
+  }
+
+  if (!snapshotNorm.detected && themeCheckNorm.detected) {
+    report.record(Outcome.FALSE_POSITIVE, {
+      tag,
+      scenarioId: scenario.id,
+      message: themeCheckNorm.coreMessage,
+    });
+    expect.fail(`FALSE_POSITIVE [${scenario.id}]: theme-check reports error but snapshot has none`);
+    return;
+  }
+
+  report.record(Outcome.AGREE_NO_ERROR, { tag, scenarioId: scenario.id });
+}
+
+for (const tag of tags) {
+  const scenarios = loadScenarios(tag);
+  const snapshots = loadSnapshots(tag);
+
+  describe(`Parity: ${tag}`, () => {
+    it.each(scenarios)('$id', async (scenario) => {
+      await runParityScenario(tag, scenario, snapshots);
+    });
+  });
+}
+
+afterAll(() => {
+  if (process.env.PARITY_REPORT === '1') {
+    report.print();
+  }
+});

packages/theme-check-common/src/checks/liquid-syntax-error/parity/report.ts

@@ -0,0 +1,82 @@
+export const Outcome = {
+  MATCH: 'MATCH',
+  EXPECTED_MISS: 'EXPECTED_MISS',
+  UNEXPECTED_MISS: 'UNEXPECTED_MISS',
+  FALSE_POSITIVE: 'FALSE_POSITIVE',
+  AGREE_NO_ERROR: 'AGREE_NO_ERROR',
+} as const;
+export type Outcome = (typeof Outcome)[keyof typeof Outcome];
+
+export interface RecordEntry {
+  tag: string;
+  scenarioId: string;
+  outcome: Outcome;
+  message?: string;
+}
+
+export class ParityReport {
+  private entries: RecordEntry[] = [];
+
+  record(outcome: Outcome, details: { tag: string; scenarioId: string; message?: string }): void {
+    this.entries.push({
+      tag: details.tag,
+      scenarioId: details.scenarioId,
+      outcome,
+      message: details.message,
+    });
+  }
+
+  print(): void {
+    const counts: Record<Outcome, number> = {
+      [Outcome.MATCH]: 0,
+      [Outcome.EXPECTED_MISS]: 0,
+      [Outcome.UNEXPECTED_MISS]: 0,
+      [Outcome.FALSE_POSITIVE]: 0,
+      [Outcome.AGREE_NO_ERROR]: 0,
+    };
+
+    for (const entry of this.entries) {
+      counts[entry.outcome]++;
+    }
+
+    const total = this.entries.length;
+    const detected = counts[Outcome.MATCH];
+    const coverage = total > 0 ? ((detected / total) * 100).toFixed(1) : '0.0';
+
+    console.log('');
+    console.log('=== Syntax Error Parity Report ===');
+    console.log('');
+    console.log(`Total scenarios: ${total}`);
+    console.log(`  MATCH (both detect):     ${String(counts[Outcome.MATCH]).padStart(3)}`);
+    console.log(`  EXPECTED MISS (known):   ${String(counts[Outcome.EXPECTED_MISS]).padStart(3)}`);
+    console.log(
+      `  UNEXPECTED MISS:         ${String(counts[Outcome.UNEXPECTED_MISS]).padStart(3)}`,
+    );
+    console.log(`  FALSE POSITIVE:          ${String(counts[Outcome.FALSE_POSITIVE]).padStart(3)}`);
+    console.log(`  AGREE NO ERROR:          ${String(counts[Outcome.AGREE_NO_ERROR]).padStart(3)}`);
+    console.log('');
+    console.log(`Coverage: ${coverage}% (${detected}/${total} detected)`);
+
+    const unexpectedMisses = this.entries.filter((e) => e.outcome === Outcome.UNEXPECTED_MISS);
+    if (unexpectedMisses.length > 0) {
+      console.log('');
+      console.log('Unexpected misses (need investigation):');
+      for (const entry of unexpectedMisses) {
+        const suffix = entry.message ? `: ${entry.message}` : '';
+        console.log(`  - ${entry.scenarioId}${suffix}`);
+      }
+    }
+
+    const falsePositives = this.entries.filter((e) => e.outcome === Outcome.FALSE_POSITIVE);
+    if (falsePositives.length > 0) {
+      console.log('');
+      console.log('False positives (theme-check wrong):');
+      for (const entry of falsePositives) {
+        const suffix = entry.message ? `: ${entry.message}` : '';
+        console.log(`  - ${entry.scenarioId}${suffix}`);
+      }
+    }
+
+    console.log('');
+  }
+}