Skip to content

Commit c8e10cd

Browse files
committed
Test(Agent): Add paired CLI and MCP authoring pilot
1 parent d962c74 commit c8e10cd

6 files changed

Lines changed: 277 additions & 20 deletions

File tree

‎.ai/evals/authoring.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,3 +210,35 @@ final HTML/CSS/screenshots, post-run check reports and `analysis.json`. The raw
210210
`e0702e39e86d71a4eeefac4374a9fe31b5b74876a9f3741ccede649e9b10f3ce`.
211211
Onboarding time, voluntary tester retention, autonomous skill discovery and
212212
cross-client behavior remain unmeasured.
213+
214+
## Focused CLI/MCP pilot
215+
216+
The `bridge-pilot` profile compares `skill-cli` with `skill-mcp-bridge`. It runs
217+
settings-card, css-declarations, fallback-order and tailwind-hover with the same
218+
two models, two repetitions, skill, baseline instructions and browser checks:
219+
220+
```sh
221+
node --import tsx scripts/ai-authoring/trials/run.mjs --profile bridge-pilot --limit 4 --concurrency 2
222+
node --import tsx scripts/ai-authoring/trials/run.mjs --profile bridge-pilot --concurrency 2
223+
node scripts/ai-authoring/trials/compare.mjs /absolute/run/path
224+
```
225+
226+
Keep the four-case calibration separate from the 32-case campaign. Arm and model
227+
order reverse on the second repetition. Suggestions are optional in both arms;
228+
this measures the available workflow, not forced tool use. Both can build and use
229+
CLI diagnostics through a protected `master-css.mjs` adapter. Only the CLI arm can
230+
invoke its `suggest` command; the MCP arm has the full server including the bridge.
231+
The adapter audits actual CLI suggestion calls and exit statuses; MCP calls are
232+
counted from client events. This is a full-MCP-surface comparison, not an isolated
233+
measurement of transport overhead. Filesystem access is not a security boundary;
234+
review transcripts for bypasses or audit-log edits before interpreting results.
235+
236+
Protocol v2 hashes the task prompts, runner helpers, skill and built package trees,
237+
including the actual CLI implementation rather than only its entry shim. Resume
238+
with the same profile, models, concurrency and timeout; changed artifacts or
239+
partial trial directories are rejected. This runner does not resume v1 campaigns.
240+
Each result separates model elapsed time from total time including grading and
241+
records cached input tokens, MCP errors and actual suggestion use. The comparison
242+
requires every paired cell, rejects mixed environments and cross-arm tool use,
243+
retains failures and reports per-model/per-task results. Cached input totals are
244+
not a billing estimate, and concurrent local timings are not latency guarantees.

‎scripts/ai-authoring/evaluate.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { readFileSync } from 'node:fs'
22
import { fileURLToPath } from 'node:url'
33

44
export const tasks = JSON.parse(readFileSync(new URL('./tasks.json', import.meta.url), 'utf8'))
5-
export const arms = ['instructions', 'skill', 'mcp', 'skill-mcp', 'skill-mcp-bridge']
5+
export const arms = ['instructions', 'skill', 'mcp', 'skill-mcp', 'skill-mcp-bridge', 'skill-cli']
66

77
/** Summarize observed runs only. Missing attempts and unchecked UI are never successes. */
88
export function summarize(runs) {
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import { readFile, writeFile } from 'node:fs/promises'
2+
import { join, resolve } from 'node:path'
3+
import { fileURLToPath } from 'node:url'
4+
import { summarize } from '../evaluate.mjs'
5+
6+
const passed = row => row.functional && row.styling && row.accessibility && row.browser === 'passed'
7+
const median = values => {
8+
const sorted = [...values].sort((a, b) => a - b)
9+
const middle = Math.floor(sorted.length / 2)
10+
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2
11+
}
12+
13+
/** Require the complete paired design; never silently compare different coverage. */
14+
export function compare(protocol, rows) {
15+
if (protocol.profile !== 'bridge-pilot') throw new Error('Expected bridge-pilot protocol')
16+
summarize(rows) // Validate observations and duplicates before aggregation.
17+
const expected = new Set(protocol.tasks.flatMap(task => protocol.arms.flatMap(arm => protocol.models.flatMap(model =>
18+
Array.from({ length: protocol.repeats }, (_, repeat) => JSON.stringify([task, arm, model, repeat]))))))
19+
for (const row of rows) {
20+
if (!expected.delete(JSON.stringify([row.task, row.arm, row.model, row.repeat]))) throw new Error('Unexpected observation')
21+
if (row.client !== protocol.clientVersion || row.fixtureRevision !== protocol.revision + ':' + protocol.corpusHash || row.toolVersion !== protocol.toolVersion) throw new Error('Mixed experiment environments')
22+
for (const value of [row.cachedInputTokens, row.modelDurationMs, row.mcpCalls, row.mcpErrors, row.bridgeCalls?.cli, row.bridgeCalls?.cliErrors, row.bridgeCalls?.mcp]) {
23+
if (!Number.isSafeInteger(value) || value < 0) throw new Error('Missing transport or timing metric')
24+
}
25+
if (row.inputTokens === null || row.outputTokens === null) throw new Error('Unmeasured token usage')
26+
if (row.cachedInputTokens > row.inputTokens) throw new Error('Invalid cached input count')
27+
if (row.arm === 'skill-cli' ? row.mcpCalls !== 0 : row.bridgeCalls.cli !== 0) throw new Error('Cross-arm tool contamination')
28+
}
29+
if (expected.size) throw new Error(`Incomplete paired coverage: ${expected.size} missing`)
30+
const totals = selection => ({
31+
trials: selection.length, passed: selection.filter(passed).length,
32+
firstPass: selection.filter(row => passed(row) && row.repairRounds === 0).length,
33+
repairs: selection.reduce((n, row) => n + row.repairRounds, 0),
34+
inputTokens: selection.reduce((n, row) => n + row.inputTokens, 0),
35+
cachedInputTokens: selection.reduce((n, row) => n + row.cachedInputTokens, 0),
36+
outputTokens: selection.reduce((n, row) => n + row.outputTokens, 0),
37+
medianDurationMs: median(selection.map(row => row.durationMs)),
38+
medianModelDurationMs: median(selection.map(row => row.modelDurationMs)),
39+
mcpCalls: selection.reduce((n, row) => n + row.mcpCalls, 0),
40+
mcpErrors: selection.reduce((n, row) => n + row.mcpErrors, 0),
41+
cliSuggestions: selection.reduce((n, row) => n + row.bridgeCalls.cli, 0),
42+
cliSuggestionErrors: selection.reduce((n, row) => n + row.bridgeCalls.cliErrors, 0),
43+
mcpSuggestions: selection.reduce((n, row) => n + row.bridgeCalls.mcp, 0),
44+
trialsUsingSuggestions: selection.filter(row => row.bridgeCalls.cli + row.bridgeCalls.mcp > 0).length,
45+
unmatched: selection.every(row => row.invalidClasses !== null) ? selection.reduce((n, row) => n + row.invalidClasses, 0) : null
46+
})
47+
const pairs = { cliWins: 0, mcpWins: 0, bothPass: 0, bothFail: 0 }
48+
for (const cli of rows.filter(row => row.arm === 'skill-cli')) {
49+
const mcp = rows.find(row => row.arm === 'skill-mcp-bridge' && row.task === cli.task && row.model === cli.model && row.repeat === cli.repeat)
50+
pairs[passed(cli) ? passed(mcp) ? 'bothPass' : 'cliWins' : passed(mcp) ? 'mcpWins' : 'bothFail']++
51+
}
52+
return {
53+
version: 1, profile: protocol.profile, revision: protocol.revision,
54+
arms: Object.fromEntries(protocol.arms.map(arm => [arm, totals(rows.filter(row => row.arm === arm))])),
55+
byModel: protocol.models.map(model => ({ model, arms: Object.fromEntries(protocol.arms.map(arm => [arm, totals(rows.filter(row => row.model === model && row.arm === arm))])) })),
56+
byTask: protocol.tasks.map(task => ({ task, arms: Object.fromEntries(protocol.arms.map(arm => [arm, totals(rows.filter(row => row.task === task && row.arm === arm))])) })),
57+
pairs,
58+
failures: rows.filter(row => !passed(row)).map(({ task, arm, model, repeat, execution, artifacts }) => ({ task, arm, model, repeat, execution, artifacts })),
59+
limitation: 'Small fixed-task pilot. CLI versus the full MCP surface, not transport alone. Suggestions are optional. Input tokens include cache hits; no billing estimate. Elapsed times include local concurrency; model time excludes independent grading.'
60+
}
61+
}
62+
63+
if (process.argv[1] === fileURLToPath(import.meta.url)) {
64+
if (!process.argv[2]) throw new Error('Usage: node scripts/ai-authoring/trials/compare.mjs RUN_DIRECTORY')
65+
const root = resolve(process.argv[2])
66+
const protocol = JSON.parse(await readFile(join(root, 'protocol.json'), 'utf8'))
67+
const rows = (await readFile(join(root, 'observations.jsonl'), 'utf8')).trim().split('\n').map(line => JSON.parse(line))
68+
const report = compare(protocol, rows)
69+
await writeFile(join(root, 'comparison.json'), JSON.stringify(report, null, 2))
70+
console.log(JSON.stringify(report, null, 2))
71+
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import { tasks, arms } from '../evaluate.mjs'
2+
3+
export function profile(name = 'full') {
4+
if (name === 'full') return { name, arms: arms.filter(arm => arm !== 'skill-cli'), tasks, repeats: 2 }
5+
if (name === 'bridge-pilot') return {
6+
name, arms: ['skill-cli', 'skill-mcp-bridge'], repeats: 2,
7+
tasks: ['settings-card', 'css-declarations', 'fallback-order', 'tailwind-hover'].map(id => tasks.find(task => task.id === id))
8+
}
9+
throw new Error(`Unknown profile: ${name}`)
10+
}
11+
12+
/** Alternate arm/model order between paired repetitions, keeping tasks identical. */
13+
export function schedule(selection, models) {
14+
const trials = []
15+
for (let repeat = 0; repeat < selection.repeats; repeat++) {
16+
const orderedArms = repeat % 2 ? [...selection.arms].reverse() : selection.arms
17+
const orderedModels = repeat % 2 ? [...models].reverse() : models
18+
for (const task of selection.tasks) for (const arm of orderedArms) for (const model of orderedModels) {
19+
trials.push({ id: `${task.id}-${arm}-${model}-${repeat}`, task, arm, model, repeat })
20+
}
21+
}
22+
return trials
23+
}
24+
25+
export function bridgeInstructions(arm, pilot) {
26+
if (!pilot) return 'Use the declaration-to-class suggestion bridge only if exposed by the connected MCP server; do not invoke an equivalent programmatic API or CLI command otherwise.'
27+
return `For installed CLI commands, run node master-css.mjs from this directory. Do not edit master-css.mjs or bridge-calls.jsonl. ${arm === 'skill-cli'
28+
? 'The declaration suggestion bridge is available through node master-css.mjs suggest (argument or --stdin).'
29+
: 'Use the connected mastercss_suggest_classes tool for declaration suggestions; the CLI suggestion command is disabled for this arm.'} Do not bypass the assigned suggestion interface or invoke its programmatic API. Use suggestions when useful; invoking them is not required to pass.`
30+
}
31+
32+
/** Audits actual CLI invocations, including failures, without changing the public CLI. */
33+
export function cliWrapper(binary, allowSuggest) {
34+
return `import {spawnSync} from 'node:child_process';
35+
import {appendFileSync} from 'node:fs';
36+
const args=process.argv.slice(2);
37+
const suggest=args[0]==='suggest'&&!args.some(arg=>['--help','-h','--version','-V'].includes(arg));
38+
if(suggest&&!${JSON.stringify(allowSuggest)}) {console.error('Suggestion CLI disabled in this trial arm');process.exit(1)}
39+
const result=spawnSync(process.execPath,[${JSON.stringify(binary)},...args],{stdio:'inherit'});
40+
if(suggest) appendFileSync(new URL('bridge-calls.jsonl',import.meta.url),JSON.stringify({status:result.status,signal:result.signal,error:result.error?.message})+'\\n');
41+
if(result.error) console.error(result.error.message);
42+
process.exit(result.status??1);
43+
`
44+
}
45+
46+
export function assertResume(old, current) {
47+
for (const field of ['version', 'revision', 'corpusHash', 'toolVersion', 'clientVersion', 'timeoutMs', 'profile', 'models', 'repeats', 'arms', 'tasks', 'concurrency', 'commonArgs', 'baselineInstructions', 'reasoning', 'maxRepairRounds', 'tailwindVersion', 'harnessHashes', 'artifactHashes']) {
48+
if (JSON.stringify(old[field]) !== JSON.stringify(current[field])) throw new Error(`Cannot resume changed ${field}`)
49+
}
50+
}
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import assert from 'node:assert/strict'
2+
import { test } from 'node:test'
3+
import { mkdtemp, writeFile, readFile, rm } from 'node:fs/promises'
4+
import { tmpdir } from 'node:os'
5+
import { join } from 'node:path'
6+
import { spawnSync } from 'node:child_process'
7+
import { profile, schedule, cliWrapper, bridgeInstructions, assertResume } from './protocol.mjs'
8+
import { compare } from './compare.mjs'
9+
10+
test('pilot has complete balanced pairs and leaves the original campaign at 240 trials', () => {
11+
const models = ['gpt-6.1-sol', 'gpt-6-luna']
12+
const pilot = schedule(profile('bridge-pilot'), models)
13+
assert.equal(pilot.length, 32)
14+
assert.equal(new Set(pilot.map(row => row.id)).size, 32)
15+
assert.equal(pilot[0].arm, 'skill-cli')
16+
assert.equal(pilot[16].arm, 'skill-mcp-bridge')
17+
assert.equal(pilot[0].model, models[0])
18+
assert.equal(pilot[16].model, models[1])
19+
assert.equal(schedule(profile(), models).length, 240)
20+
assert.throws(() => profile('typo'), /Unknown profile/)
21+
assert.match(bridgeInstructions('skill-cli', true), /node master-css.mjs suggest/)
22+
assert.match(bridgeInstructions('skill-mcp-bridge', true), /CLI suggestion command is disabled/)
23+
})
24+
25+
test('wrapper records real stdin calls and failures, and prevents suggestions in the MCP arm', async () => {
26+
const root = await mkdtemp(join(tmpdir(), 'master-bridge-wrapper-'))
27+
try {
28+
const binary = join(root, 'fake-cli.mjs')
29+
await writeFile(binary, 'import fs from \'node:fs\'; console.log(fs.readFileSync(0,\'utf8\')); process.exit(process.argv.includes(\'--fail\')?1:0)')
30+
const wrapper = join(root, 'master-css.mjs')
31+
await writeFile(wrapper, cliWrapper(binary, true))
32+
for (const fail of [false, true]) {
33+
const result = spawnSync(process.execPath, [wrapper, 'suggest', '--stdin', ...(fail ? ['--fail'] : [])], { encoding: 'utf8', input: 'display:flex;' })
34+
assert.equal(result.status, Number(fail))
35+
assert.equal(result.stdout.trim(), 'display:flex;')
36+
}
37+
const log = await readFile(join(root, 'bridge-calls.jsonl'), 'utf8')
38+
assert.deepEqual(log.trim().split('\n').map(line => JSON.parse(line).status), [0, 1])
39+
const help = spawnSync(process.execPath, [wrapper, 'suggest', '--help'], { encoding: 'utf8', input: '' })
40+
assert.equal(help.status, 0)
41+
assert.equal(await readFile(join(root, 'bridge-calls.jsonl'), 'utf8'), log)
42+
await writeFile(wrapper, cliWrapper(binary, false))
43+
const blocked = spawnSync(process.execPath, [wrapper, 'suggest'], { encoding: 'utf8' })
44+
assert.equal(blocked.status, 1)
45+
assert.match(blocked.stderr, /disabled/)
46+
assert.equal(await readFile(join(root, 'bridge-calls.jsonl'), 'utf8'), log)
47+
} finally { await rm(root, { recursive: true, force: true }) }
48+
})
49+
50+
const protocol = {
51+
profile: 'bridge-pilot', tasks: ['settings-card'], arms: ['skill-cli', 'skill-mcp-bridge'],
52+
models: ['model'], repeats: 1, revision: 'commit', corpusHash: 'corpus', clientVersion: 'client', toolVersion: 'tool'
53+
}
54+
const rows = protocol.arms.map(arm => ({
55+
task: 'settings-card', arm, model: 'model', client: 'client', fixtureRevision: 'commit:corpus', toolVersion: 'tool', repeat: 0,
56+
repairRounds: 0, toolCalls: 2, inputTokens: 100, cachedInputTokens: 60, outputTokens: 10, invalidClasses: 0,
57+
durationMs: 2000, modelDurationMs: 1000, functional: true, styling: true, accessibility: true, browser: 'passed',
58+
mcpCalls: arm === 'skill-cli' ? 0 : 2, mcpErrors: 0, bridgeCalls: { cli: arm === 'skill-cli' ? 1 : 0, cliErrors: 0, mcp: arm === 'skill-cli' ? 0 : 1 }
59+
}))
60+
61+
test('comparison reports paired failures, cache hits and model time without dropping failed rows', () => {
62+
const report = compare(protocol, [rows[0], { ...rows[1], functional: false, repairRounds: 2, inputTokens: 300 }])
63+
assert.deepEqual(report.pairs, { cliWins: 1, mcpWins: 0, bothPass: 0, bothFail: 0 })
64+
assert.equal(report.arms['skill-mcp-bridge'].inputTokens, 300)
65+
assert.equal(report.arms['skill-mcp-bridge'].passed, 0)
66+
assert.equal(report.arms['skill-cli'].cachedInputTokens, 60)
67+
assert.equal(report.arms['skill-cli'].medianModelDurationMs, 1000)
68+
assert.equal(report.failures.length, 1)
69+
})
70+
71+
test('comparison rejects missing pairs, mixed versions, missing metrics and crossed tools', () => {
72+
assert.throws(() => compare(protocol, [rows[0]]), /Incomplete/)
73+
assert.throws(() => compare(protocol, [rows[0], rows[0]]), /Duplicate/)
74+
for (const patch of [{ client: 'other' }, { cachedInputTokens: null }, { inputTokens: null }, { cachedInputTokens: 101 }, { mcpCalls: 1 }]) {
75+
assert.throws(() => compare(protocol, [{ ...rows[0], ...patch }, rows[1]]))
76+
}
77+
})
78+
79+
test('resume rejects changed tools, selection, concurrency and model identifiers', () => {
80+
assert.doesNotThrow(() => assertResume(protocol, { ...protocol }))
81+
for (const field of ['artifactHashes', 'harnessHashes', 'profile', 'tasks', 'arms', 'models', 'repeats', 'concurrency', 'commonArgs']) {
82+
assert.throws(() => assertResume(protocol, { ...protocol, [field]: 'changed' }), new RegExp(field))
83+
}
84+
})

0 commit comments

Comments
 (0)