|
| 1 | +import assert from 'node:assert/strict' |
| 2 | +import { test } from 'node:test' |
| 3 | +import { mkdtemp, writeFile, readFile, rm } from 'node:fs/promises' |
| 4 | +import { tmpdir } from 'node:os' |
| 5 | +import { join } from 'node:path' |
| 6 | +import { spawnSync } from 'node:child_process' |
| 7 | +import { profile, schedule, cliWrapper, bridgeInstructions, assertResume } from './protocol.mjs' |
| 8 | +import { compare } from './compare.mjs' |
| 9 | + |
| 10 | +test('pilot has complete balanced pairs and leaves the original campaign at 240 trials', () => { |
| 11 | + const models = ['gpt-6.1-sol', 'gpt-6-luna'] |
| 12 | + const pilot = schedule(profile('bridge-pilot'), models) |
| 13 | + assert.equal(pilot.length, 32) |
| 14 | + assert.equal(new Set(pilot.map(row => row.id)).size, 32) |
| 15 | + assert.equal(pilot[0].arm, 'skill-cli') |
| 16 | + assert.equal(pilot[16].arm, 'skill-mcp-bridge') |
| 17 | + assert.equal(pilot[0].model, models[0]) |
| 18 | + assert.equal(pilot[16].model, models[1]) |
| 19 | + assert.equal(schedule(profile(), models).length, 240) |
| 20 | + assert.throws(() => profile('typo'), /Unknown profile/) |
| 21 | + assert.match(bridgeInstructions('skill-cli', true), /node master-css.mjs suggest/) |
| 22 | + assert.match(bridgeInstructions('skill-mcp-bridge', true), /CLI suggestion command is disabled/) |
| 23 | +}) |
| 24 | + |
| 25 | +test('wrapper records real stdin calls and failures, and prevents suggestions in the MCP arm', async () => { |
| 26 | + const root = await mkdtemp(join(tmpdir(), 'master-bridge-wrapper-')) |
| 27 | + try { |
| 28 | + const binary = join(root, 'fake-cli.mjs') |
| 29 | + await writeFile(binary, 'import fs from \'node:fs\'; console.log(fs.readFileSync(0,\'utf8\')); process.exit(process.argv.includes(\'--fail\')?1:0)') |
| 30 | + const wrapper = join(root, 'master-css.mjs') |
| 31 | + await writeFile(wrapper, cliWrapper(binary, true)) |
| 32 | + for (const fail of [false, true]) { |
| 33 | + const result = spawnSync(process.execPath, [wrapper, 'suggest', '--stdin', ...(fail ? ['--fail'] : [])], { encoding: 'utf8', input: 'display:flex;' }) |
| 34 | + assert.equal(result.status, Number(fail)) |
| 35 | + assert.equal(result.stdout.trim(), 'display:flex;') |
| 36 | + } |
| 37 | + const log = await readFile(join(root, 'bridge-calls.jsonl'), 'utf8') |
| 38 | + assert.deepEqual(log.trim().split('\n').map(line => JSON.parse(line).status), [0, 1]) |
| 39 | + const help = spawnSync(process.execPath, [wrapper, 'suggest', '--help'], { encoding: 'utf8', input: '' }) |
| 40 | + assert.equal(help.status, 0) |
| 41 | + assert.equal(await readFile(join(root, 'bridge-calls.jsonl'), 'utf8'), log) |
| 42 | + await writeFile(wrapper, cliWrapper(binary, false)) |
| 43 | + const blocked = spawnSync(process.execPath, [wrapper, 'suggest'], { encoding: 'utf8' }) |
| 44 | + assert.equal(blocked.status, 1) |
| 45 | + assert.match(blocked.stderr, /disabled/) |
| 46 | + assert.equal(await readFile(join(root, 'bridge-calls.jsonl'), 'utf8'), log) |
| 47 | + } finally { await rm(root, { recursive: true, force: true }) } |
| 48 | +}) |
| 49 | + |
| 50 | +const protocol = { |
| 51 | + profile: 'bridge-pilot', tasks: ['settings-card'], arms: ['skill-cli', 'skill-mcp-bridge'], |
| 52 | + models: ['model'], repeats: 1, revision: 'commit', corpusHash: 'corpus', clientVersion: 'client', toolVersion: 'tool' |
| 53 | +} |
| 54 | +const rows = protocol.arms.map(arm => ({ |
| 55 | + task: 'settings-card', arm, model: 'model', client: 'client', fixtureRevision: 'commit:corpus', toolVersion: 'tool', repeat: 0, |
| 56 | + repairRounds: 0, toolCalls: 2, inputTokens: 100, cachedInputTokens: 60, outputTokens: 10, invalidClasses: 0, |
| 57 | + durationMs: 2000, modelDurationMs: 1000, functional: true, styling: true, accessibility: true, browser: 'passed', |
| 58 | + mcpCalls: arm === 'skill-cli' ? 0 : 2, mcpErrors: 0, bridgeCalls: { cli: arm === 'skill-cli' ? 1 : 0, cliErrors: 0, mcp: arm === 'skill-cli' ? 0 : 1 } |
| 59 | +})) |
| 60 | + |
| 61 | +test('comparison reports paired failures, cache hits and model time without dropping failed rows', () => { |
| 62 | + const report = compare(protocol, [rows[0], { ...rows[1], functional: false, repairRounds: 2, inputTokens: 300 }]) |
| 63 | + assert.deepEqual(report.pairs, { cliWins: 1, mcpWins: 0, bothPass: 0, bothFail: 0 }) |
| 64 | + assert.equal(report.arms['skill-mcp-bridge'].inputTokens, 300) |
| 65 | + assert.equal(report.arms['skill-mcp-bridge'].passed, 0) |
| 66 | + assert.equal(report.arms['skill-cli'].cachedInputTokens, 60) |
| 67 | + assert.equal(report.arms['skill-cli'].medianModelDurationMs, 1000) |
| 68 | + assert.equal(report.failures.length, 1) |
| 69 | +}) |
| 70 | + |
| 71 | +test('comparison rejects missing pairs, mixed versions, missing metrics and crossed tools', () => { |
| 72 | + assert.throws(() => compare(protocol, [rows[0]]), /Incomplete/) |
| 73 | + assert.throws(() => compare(protocol, [rows[0], rows[0]]), /Duplicate/) |
| 74 | + for (const patch of [{ client: 'other' }, { cachedInputTokens: null }, { inputTokens: null }, { cachedInputTokens: 101 }, { mcpCalls: 1 }]) { |
| 75 | + assert.throws(() => compare(protocol, [{ ...rows[0], ...patch }, rows[1]])) |
| 76 | + } |
| 77 | +}) |
| 78 | + |
| 79 | +test('resume rejects changed tools, selection, concurrency and model identifiers', () => { |
| 80 | + assert.doesNotThrow(() => assertResume(protocol, { ...protocol })) |
| 81 | + for (const field of ['artifactHashes', 'harnessHashes', 'profile', 'tasks', 'arms', 'models', 'repeats', 'concurrency', 'commonArgs']) { |
| 82 | + assert.throws(() => assertResume(protocol, { ...protocol, [field]: 'changed' }), new RegExp(field)) |
| 83 | + } |
| 84 | +}) |
0 commit comments