diff --git a/evals/agent-assets.lock.json b/evals/agent-assets.lock.json index d84975edf..11e1455b4 100644 --- a/evals/agent-assets.lock.json +++ b/evals/agent-assets.lock.json @@ -1,7 +1,7 @@ { - "agentAssetsHash": "3609a526c5581390a2ef1ffd02c99942a6d2d9ad563a0741017853e965d827e0", + "agentAssetsHash": "2b4538568281fab8d58110bce85e8e1f30cee9e8f7a82f82fccc917886f507a5", "scope": "azure-debug-generate.agent.md, azure-debug-generate/**, azure-debug-plan.agent.md, azure-debug-plan/**, azure-deploy.agent.md, azure-deploy/**, azure-project-integrate.agent.md, azure-project-integrate/**, azure-project-plan.agent.md, azure-project-plan/**, azure-project-scaffold.agent.md, azure-project-scaffold/**, shared-references/**", - "updatedAt": "2026-09-03T20:36:26.571Z", + "updatedAt": "2026-09-09T17:22:40.732Z", "files": { "azure-debug-generate.agent.md": "c9849fb6271193d104ef30161071bebe95fd29c6abea3a5c6d081416a5ebe8a6", "azure-debug-generate/.metadata.json": "c0ccce18678415f23db0073d48cc646f2de446e7821c99c3130553548d958e0c", @@ -35,7 +35,7 @@ "azure-debug-plan/references/plan-template.md": "e3a1d72d910409ebb7d2ae0a034c7fb69485b033c3b322d3b8431a272649b05d", "azure-debug-plan/references/project-types.md": "bfa27fbbe95be139f6390f2c8fa4719d36c1a577c512885e2eb2c8cf89b8cc58", "azure-debug-plan/references/runtimes.md": "f84769fe13f8464524f8ddeab6e325ea37b91ff2d7ec033d74748fe8a9924904", - "azure-deploy.agent.md": "9de0fb4acdea81c527b0abedd034c584dc2aa2d8cf9056769124f71835c1403d", + "azure-deploy.agent.md": "12b1214797d7ae857e07c53436ba60facd27c77670640bfbf1e6bfe7ed6ad904", "azure-deploy/.metadata.json": "be0709e660b5da6f558d73e464bed2cd39a3382619bc9076796078843d43cb56", "azure-deploy/deploy/instructions.md": "fce3feb85e97b977d1f191ce1f2ee4ef23bf1c467309f4a291b40f9311c5aa99", "azure-deploy/deploy/references/approval-gate-template.md": "ca0b2b5b86c0463f8bbb5972438013fe9a5237dd225953fb89e627769898e049", @@ -83,7 +83,7 @@ "azure-deploy/prereq/references/subagent-starter-scaffold.md": "786e41d77c20df47d8fd6f40b71c2e03fba3bd7504a93fe6db3b272d1a0cd628", "azure-deploy/prereq/references/subscription-resolution.md": "82cf51b5e861fcbef15eb10aa7f3a7d0c9049600ac566778d8046b03cba0e0c7", "azure-deploy/prereq/references/zero-code-path.md": "5979cfd1a7a76d36b290621cebfb13cb6815a8cc5996edbe72f22c5edffbf443", - "azure-deploy/references/approval-gates.md": "229bfe97d3d50ee2532bcec3504effb68098a39f27e07827601f82ddfd9f223c", + "azure-deploy/references/approval-gates.md": "c4eaf58118e7d5a67392afb2dd0c59d79ad9bfc83e380a969fc0a6856114a33c", "azure-deploy/references/azd-template-routing.md": "67bee4fdbfdb42f70faf72871c8ff767051433f9505a271b41c8db665b127797", "azure-deploy/references/handoff-protocol.md": "f492a70fdd3e5a3ebe8b10c90e789946b7a3661c7838a3c2f11c039011316fd2", "azure-deploy/references/iac-resources.md": "49d8cb5820da4ecd2db81b6dc0b0b86fdc099ab7ffc1eaf23021a40ac3cc2379", diff --git a/evals/msbench/check-tool-coverage.ts b/evals/msbench/check-tool-coverage.ts new file mode 100644 index 000000000..a6a0f1894 --- /dev/null +++ b/evals/msbench/check-tool-coverage.ts @@ -0,0 +1,290 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Which of the product's MCP tools can this suite actually invoke? + * + * Every other check here asks whether a *gate* is sound: can it fail, is its artifact reachable + * in the phase it is wired to, does it count the right files. None of them asks the question one + * level up — whether the run exercised the product surface a user actually triggers. + * + * It does not, for most of them. Measured across the 86-run local corpus: + * + * invoked at least once 5 of 15 open_requirements_view (28 runs), open_frontend_preview_view + * (10), open_local_plan_view (5), open_plan_view (2), + * start_project_integrate (1) + * never invoked 10 of 15 including every Phase 3 deploy tool + * + * That is invisible in every report the suite produces, because assertions grade *artifacts* — + * the files an agent wrote — and an agent that writes the right files without going through the + * product's entry point passes every one of them. `scaffold-fullstack` is the clean example: 203 + * tool calls, zero MCP calls, and green artifact gates. + * + * ── Two different reasons a tool is never invoked, and only one is a defect ────────────────── + * + * **Structural.** The four `start_*` hand-off tools cannot be observed here, and this is not an + * oversight — `chain-mechanism-probe.yaml` establishes it by reading the source. `launchAgentChat` + * (src/commands/copilotOnRails/openChatWithAgent.ts) runs: + * + * workbench.action.chat.newChat + * workbench.action.chat.open { mode: agentName, query } + * + * "Fresh chat session per phase hand-off: agents coordinate through the `.azure/*` plan files on + * disk, not chat history." `promptSteps` drives ONE session, so an agent that called a hand-off + * tool would move the work into a session the harness is not watching, and the run would read as + * an agent that stopped. The suite instead enters each phase by staging `.azure/*` state on disk — + * which is how the product coordinates anyway. The tool call is the part that is skipped. + * + * **Coverage.** Everything else. A tool an under-test agent is instructed to call, in a phase the + * suite runs, that no stimulus has ever caused to fire. That is a hole, and it is the thing this + * check exists to make visible. + * + * ── What this proves, and what it does not ─────────────────────────────────────────────────── + * + * This is a **static** check: it reads the registration list, the phase configs and the agent + * instructions, and reports which tools any stimulus could plausibly reach. It cannot prove a tool + * *was* invoked — only the corpus can, and `gate-health` is where that belongs. + * + * So a green here means "a path exists", not "the path was taken". That is deliberately the weaker + * claim, because the stronger one needs paid runs and this needs none. The failure it catches is + * the one that costs the most to discover late: a tool nothing can ever exercise. + */ + +import { readFileSync, readdirSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; + +const HERE = import.meta.dirname; +const REPO = join(HERE, '..', '..'); +const AGENTS = join(REPO, 'resources', 'agents'); +const PHASES = join(HERE, 'config', 'phases'); +const STIMULI = join(HERE, 'config', 'stimuli'); +const REGISTRATION = join(REPO, 'src', 'chat', 'tools', 'copilotOnRails', 'registerCopilotOnRailsTools.ts'); + +/** + * Tools whose absence from the corpus is a property of the harness rather than a gap to close. + * + * This list is deliberately EMPTY, and the reason is worth recording because the obvious entry + * was wrong. + * + * The four `start_*` hand-off tools look unobservable: `launchAgentChat` opens a FRESH chat + * session (`workbench.action.chat.newChat`) because agents coordinate through `.azure/*` files + * rather than chat history, and `promptSteps` drives one session — so the work moves somewhere the + * harness is not watching. All true, and all about the hand-off's *destination*. + * + * The tool *call* is recorded before any of that happens. Two independent proofs: + * `scaffold-autopilot` asserts `COUNT(*) > 0 ... LIKE '%start_project_integrate%'` and the corpus + * shows that tool invoked; `debug-generate-artifacts` asserts the same for + * `start_azure_debug_generate`. A suite would not assert on a tool it could not see. + * + * "The harness cannot follow where this leads" and "the harness cannot see this happen" are + * different claims, and only the first one is true. Waiving on the second would have excused + * exactly the coverage this check exists to demand. + */ +const STRUCTURALLY_UNREACHABLE: Record = {}; + +/** + * A tool has two names and they are not interchangeable, which is worth stating because getting + * it wrong is silent in both directions: + * + * short `open_plan_view` what the agent instructions tell the model to call + * runtime `mcp_copilot_azure_open_plan_view` what appears in the session's `toolCalls` table + * + * Searching the instructions for the runtime name finds nothing and reports full coverage loss; + * searching `toolCalls` for the short name finds nothing and reports the same. Both failures look + * exactly like the real finding they would be hiding. + */ +function shortName(identifier: string): string { + return identifier.replace(/([a-z0-9])([A-Z])/gu, '$1_$2').toLowerCase(); +} + +function runtimeName(identifier: string): string { + return `mcp_copilot_azure_${shortName(identifier)}`; +} + +/** + * The registration file is the source of truth on purpose: adding a tool there and nowhere else is + * exactly the change that should turn this check red. + */ +function registeredTools(): string[] { + const source = readFileSync(REGISTRATION, 'utf8'); + const names: string[] = []; + for (const match of source.matchAll(/registerMcpToolWithTelemetry\(\s*mcpServer\s*,\s*(\w+)\s*\)/gu)) { + names.push(match[1].replace(/Tool$/u, '')); + } + return names; +} + +/** + * The agents the suite actually puts under test. + * + * Two sources, and both are needed. A phase's top-level `chatMode` is the default, but a stimulus + * may override it per step — `debug-generate-artifacts` reaches `azure-debug-generate` that way, + * from a phase whose default is `azure-debug-plan`. Reading only the phase files would report that + * agent's tools as unreachable when a stimulus reaches them every run. + */ +function agentsUnderTest(): Map { + const byAgent = new Map(); + const add = (agent: string, where: string): void => { + const seen = byAgent.get(agent) ?? []; + if (!seen.includes(where)) { + byAgent.set(agent, [...seen, where]); + } + }; + + for (const file of readdirSync(PHASES).filter(name => name.endsWith('.yaml'))) { + const match = /^chatMode:\s*(\S+)/mu.exec(readFileSync(join(PHASES, file), 'utf8')); + if (match) { + add(match[1], file.replace(/\.yaml$/u, '')); + } + } + + for (const file of readdirSync(STIMULI).filter(name => name.endsWith('.yaml'))) { + const text = readFileSync(join(STIMULI, file), 'utf8'); + const stimulus = file.replace(/\.yaml$/u, ''); + // A step override is a LIST ITEM — `- chatMode: azure-debug-generate` — so the `- ` has to + // be optional here. Requiring `chatMode:` immediately after whitespace matched none of + // them, which silently reported azure-debug-generate as never under test while + // debug-generate-artifacts drives it every run. + for (const match of text.matchAll(/^\s*-?\s*chatMode:\s*(\S+)/gmu)) { + add(match[1], `${stimulus} (step override)`); + } + } + return byAgent; +} + +/** Phases that at least one stimulus declares, so "under test" means something was run at it. */ +function phasesWithStimuli(): Set { + const phases = new Set(); + for (const file of readdirSync(STIMULI).filter(name => name.endsWith('.yaml'))) { + const match = /^#\s*phase:\s*(\S+)/mu.exec(readFileSync(join(STIMULI, file), 'utf8')); + phases.add(match ? match[1] : 'plan'); + } + return phases; +} + +/** + * Does this agent's instruction set tell it to call this tool? + * + * An agent's instructions live in TWO places, and reading only one is the difference between a + * true and a false report: `resources/agents/.agent.md` holds the workflow that names most + * tool calls, and `resources/agents//` holds the references it reads. Scanning only the + * directory reported `open_requirements_view` as uncovered while the corpus shows it invoked in 28 + * runs — the check contradicting an observation is what caught it. + */ +function agentReferencesTool(agent: string, tool: string): boolean { + const manifest = join(AGENTS, `${agent}.agent.md`); + if (existsSync(manifest) && readFileSync(manifest, 'utf8').includes(tool)) { + return true; + } + const dir = join(AGENTS, agent); + if (!existsSync(dir)) { + return false; + } + const stack = [dir]; + while (stack.length > 0) { + const current = stack.pop()!; + for (const entry of readdirSync(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (entry.isDirectory()) { + stack.push(path); + } else if (readFileSync(path, 'utf8').includes(tool)) { + return true; + } + } + } + return false; +} + +/** + * Which stimuli assert that this tool was (or was not) called? + * + * This is the question that decides coverage. An agent being *instructed* to call a tool means a + * path exists; a stimulus *asserting* on `toolCalls` is the only thing that turns a missing call + * into a red. The suite already does this for six tools, in both directions — `scaffold-fullstack` + * requires `start_project_integrate` NOT to fire while `scaffold-autopilot` requires that it does, + * which is a falsifiable pair rather than a one-sided check. + */ +function assertingStimuli(shortToolName: string): string[] { + const owners: string[] = []; + for (const file of readdirSync(STIMULI).filter(name => name.endsWith('.yaml'))) { + const text = readFileSync(join(STIMULI, file), 'utf8'); + for (const line of text.split(/\r?\n/u)) { + if (line.includes('FROM toolCalls') && line.includes(shortToolName)) { + owners.push(file.replace(/\.yaml$/u, '')); + break; + } + } + } + return owners; +} + +function main(): void { + const tools = registeredTools(); + const underTest = agentsUnderTest(); + const runnable = phasesWithStimuli(); + + console.log('Tool coverage — can any stimulus reach the product\'s own entry points?\n'); + console.log(`${tools.length} registered tool(s); agents under test: ${[...underTest.keys()].join(', ')}\n`); + + const reachable: string[] = []; + const waived: string[] = []; + const uncovered: { tool: string; detail: string }[] = []; + + for (const identifier of tools) { + const tool = runtimeName(identifier); + const instructionName = shortName(identifier); + const callers = [...underTest.entries()] + .filter(([agent]) => agentReferencesTool(agent, instructionName)) + // A phase counts only if some stimulus declares it; a step override counts always, + // because the override *is* a stimulus naming that agent. + .filter(([, where]) => where.some(w => w.includes('(step override)') || runnable.has(w))); + + // Order matters, and the ordering encodes what "covered" means here. An assertion on + // `toolCalls` is the only thing that makes a missing call fail a run; being named in an + // agent's workflow merely means a path exists. Reporting the second as coverage is how a + // suite ends up green while never touching the surface it claims to test. + const asserted = assertingStimuli(instructionName); + if (asserted.length > 0) { + reachable.push(tool); + console.log(` ASSERTED ${tool}`); + console.log(` by ${asserted.slice(0, 4).join(', ')}${asserted.length > 4 ? ` (+${asserted.length - 4} more)` : ''}`); + } else if (STRUCTURALLY_UNREACHABLE[tool]) { + waived.push(tool); + console.log(` WAIVED ${tool}`); + } else { + const owner = [...underTest.keys()].find(agent => agentReferencesTool(agent, instructionName)); + uncovered.push({ + tool, + detail: callers.length > 0 + ? `reachable via ${callers.map(([a]) => a).join(', ')}, but NO stimulus asserts it is called` + : owner + ? `instructed in ${owner}, but no stimulus runs a phase that uses it, and none asserts it` + : 'no agent under test is instructed to call it, and no stimulus asserts it', + }); + console.log(` UNASSERTED ${tool}`); + } + } + + console.log(`\nasserted ${reachable.length} · waived ${waived.length} · unasserted ${uncovered.length}\n`); + + if (uncovered.length === 0) { + console.log('Every registered tool has a stimulus asserting on its invocation.'); + return; + } + + console.log('UNASSERTED — a registered product surface no stimulus checks the invocation of.'); + console.log('A regression in any of these is invisible: artifact assertions still pass, because'); + console.log('they grade the files an agent wrote and never ask how it was entered.\n'); + for (const { tool, detail } of uncovered) { + console.log(` * ${tool}\n ${detail}`); + } + console.log('\nClose it by asserting on the call — `SELECT COUNT(*) > 0 FROM toolCalls WHERE tool'); + console.log("LIKE '%%'` — in a stimulus that reaches the agent. Six tools already do this,"); + console.log('and four of them in both directions, which is the pattern worth copying: a positive'); + console.log('in one stimulus and a negative in its pair proves the assertion discriminates.'); + process.exitCode = 1; +} + +main(); diff --git a/evals/msbench/config/stimuli/debug-generate-artifacts.yaml b/evals/msbench/config/stimuli/debug-generate-artifacts.yaml index de15fa3be..2354a7282 100644 --- a/evals/msbench/config/stimuli/debug-generate-artifacts.yaml +++ b/evals/msbench/config/stimuli/debug-generate-artifacts.yaml @@ -149,6 +149,28 @@ promptSteps: - comment: launch.json and tasks.json are structurally sound exec: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON /agent/assets/graders/evals/graders/validate-debug-config.ts + # vally: tool-calls required [workflow-tools-open_local_next_steps_view] + # The hand-off surface this turn is contracted to end on. `azure-debug-generate` + # finishes by showing the user what to do next; without the call the artifacts + # exist on disk and the user is told nothing, which every artifact assertion + # above is blind to because the files it grades are all present and correct. + # + # This turn is the one that runs under `chatMode: azure-debug-generate`, so it + # is the only place in the suite where that agent answers and the assertion can + # mean anything. + - comment: Agent should have opened the local next steps view + query: SELECT COUNT(*) > 0 FROM toolCalls WHERE tool LIKE '%open_local_next_steps_view%' + + # vally: tool-calls forbidden [workflow-tools-start_deployment] + # The other end of the same boundary. Generating debug artifacts is where this + # phase stops; `start_deployment` moves the user into the deploy pipeline, and + # nothing in this turn approves that. Asserted negatively because the prompt + # asks only for artifact generation — a positive here would demand a transition + # the stimulus never authorises, which is how a gate ends up red for correct + # behaviour. + - comment: Agent should not have started deployment from the generate phase + query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%start_deployment%' + # vally: program artifacts-conform-to-plan # The conformance check: every plan row marked `[x]` produced exactly one # artifact and every `[ ]` produced none. Catches generation quietly diff --git a/evals/msbench/config/stimuli/deploy-scaffold-iac.yaml b/evals/msbench/config/stimuli/deploy-scaffold-iac.yaml index e360cab86..de36740e6 100644 --- a/evals/msbench/config/stimuli/deploy-scaffold-iac.yaml +++ b/evals/msbench/config/stimuli/deploy-scaffold-iac.yaml @@ -100,6 +100,35 @@ promptSteps: - comment: Agent should not have provisioned any Azure resources (turn 0) query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%deployment%create%' OR tool LIKE '%azd_up%' + # vally: tool-calls required [workflow-tools-open_deploy_plan_view] + # The gate's own UI surface, graded. `azure-deploy.agent.md` is explicit that + # at the scaffold approval gate, "as soon as `prepare-plan.json` is written", + # the agent **MUST** call `open_deploy_plan_view` — that view is how the user + # sees services, SKUs, region and cost before consenting. + # + # Turn 0 is precisely that gate, so this grades the contract the agent was + # actually given rather than a later stage this stimulus never reaches. + - comment: Agent should have opened the deploy plan view at the scaffold approval gate + query: SELECT COUNT(*) > 0 FROM toolCalls WHERE tool LIKE '%open_deploy_plan_view%' + + # vally: tool-calls required [workflow-tools-record_deploy_prerequisites] + # Same gate. The Deployment plan view renders the two CLIs this stage depends + # on, and the agent is instructed to record their status "through **our** MCP + # tool" rather than by narrating it in chat. A run that checks `az`/`azd` by + # hand and describes the result satisfies every artifact assertion here while + # leaving the view empty, which is exactly the substitution this catches. + - comment: Agent should have recorded deploy prerequisites through the MCP tool + query: SELECT COUNT(*) > 0 FROM toolCalls WHERE tool LIKE '%record_deploy_prerequisites%' + + # vally: tool-calls forbidden [workflow-tools-capture_deployment_inventory] + # `capture_deployment_inventory` is contracted to fire with `phase: "baseline"` + # immediately before the first deployment command. The prompt forbids + # provisioning, so its presence means a deployment was about to happen — + # a tool-level companion to the `%deployment%create%` check above, and one + # that fires earlier because the baseline call precedes the deploy itself. + - comment: Agent should not have taken a deployment inventory baseline (turn 0) + query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%capture_deployment_inventory%' + # ─── Turn 1: approve, and grade what the scaffold sub-phase produced ────── # # "Yes" is the literal first option the agent offers at the gate. Everything @@ -150,6 +179,24 @@ promptSteps: - comment: Agent should not have run what-if or terraform plan query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%what_if%' OR tool LIKE '%terraform_plan%' + # vally: tool-calls forbidden [workflow-tools-capture_deployment_inventory] — turn 1 copy. + # Turn 1 is where the agent actually generates infrastructure, so it is the + # turn most likely to slide into deploying it. The baseline inventory call is + # contracted to come immediately before the first deployment command, which + # makes its absence a positive statement that no deployment was attempted. + - comment: Agent should not have taken a deployment inventory baseline (turn 1) + query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%capture_deployment_inventory%' + + # vally: tool-calls forbidden [workflow-tools-open_deploy_result_view] + # `open_deploy_result_view` is contracted to fire exactly once, after + # `deploy-result.json` reaches a terminal status — success or failure alike. + # This stimulus stops at IaC, so a Deployment Results view means the agent + # believed it had finished a deployment. Distinct from the provisioning checks + # above: those catch the act, this catches the agent *reporting* the act, which + # is what a user would actually see. + - comment: Agent should not have opened the deployment results view + query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%open_deploy_result_view%' + # A grader that has only ever returned PASS has not been shown to work — it is # indistinguishable from a grader that cannot fail. The offline certification # suite covers that, but it runs on a developer machine against fixtures; it diff --git a/evals/msbench/config/stimuli/integrate-seam.yaml b/evals/msbench/config/stimuli/integrate-seam.yaml new file mode 100644 index 000000000..3159d4cbd --- /dev/null +++ b/evals/msbench/config/stimuli/integrate-seam.yaml @@ -0,0 +1,110 @@ +# phase: scaffold +# seed: approved-fullstack +# +# Stimulus `integrate-seam` — the first stimulus to grade `azure-project-integrate`. +# +# ── Why this agent had no stimulus until now ────────────────────────────────────── +# +# `integrate` is the only stage of the pipeline with no coverage at all: 35 of the 48 +# stimuli run the plan phase, 4 scaffold, 4 local, 1 deploy, and none reach the +# integrate agent. Two registered MCP tools are owned by it and therefore could not +# be asserted anywhere — `open_scaffold_next_steps_view` and `start_local_development` +# — which is what `check-tool-coverage.ts` reports. +# +# The gap matters beyond tool coverage. Integration is the *seam*: the scaffold phase +# generates a frontend and an API separately, and this agent is what makes them agree. +# Both known defects in that area live here — the pilot run's `integration-plan` gate +# failed because no `integration-plan.md` was written at all, and #1786 is a seam rule +# broken when a page needs data that no planned route provides. +# +# ── Why the workspace is scaffolded by an earlier TURN, not by a seed ───────────── +# +# The obvious approach is a pre-scaffolded seed, and it is wrong here. The only two +# such seeds — `prescaffolded-reference` and `prescaffolded-postgres` — are labelled +# HARNESS SELF-TEST ONLY in `stage-workspace.ts`, because they are fixtures used to +# certify graders offline. Grading the agent against one would measure the fixture. +# +# So turn 0 runs the real scaffold agent and turn 1 switches to integrate, which is +# the mechanism `debug-generate-artifacts` already uses (`- chatMode:` overrides, +# established by `chain-mechanism-probe` rather than assumed). The cost is a real +# scaffold on every run; the benefit is that what integrate receives is genuine agent +# output rather than a hand-maintained tree that cannot drift with the product. +# +# ── The pair that makes the hand-off assertion mean something ───────────────────── +# +# `start_local_development` is asserted twice, in opposite directions, on consecutive +# turns of the SAME run: forbidden on turn 1, where nothing has approved anything, and +# required on turn 2, immediately after the user approves. One positive alone cannot +# tell an agent that hands off correctly from one that hands off always — and this +# suite has already been bitten by exactly that, which is why `scaffold-fullstack` and +# `scaffold-autopilot` are built as a pair. Both degenerate strategies fail one half. + +promptSteps: + # ─── Turn 0: scaffold, so there is something real to integrate ───────────────── + # + # Inherits `chatMode: azure-project-scaffold` from config/phases/scaffold.yaml. + # Nothing here is the subject of this stimulus; it exists to produce the input. + # Graded only for liveness, because a turn that produced nothing would leave every + # assertion below judging an empty workspace. + - text: | + Scaffold the project described by the approved `.azure/project-plan.md`. + assertions: + - comment: Sentinel; this turn must have produced a response or its checks are vacuous + query: SELECT COUNT(*) > 0 FROM llm_responses + + # ─── Turn 1: integrate — the subject of this stimulus ────────────────────────── + - chatMode: azure-project-integrate + text: | + The scaffold is complete. Integrate the frontend and the API: wire the pages to + the routes they need and record the result in `.azure/integration-plan.md`. + assertions: + # Liveness sentinel, turn 1. Load-bearing rather than routine: every assertion + # below is either an absence or a tool call, and both are satisfied by a turn in + # which nothing happened. + - comment: Sentinel; this turn must have produced a response or its checks are vacuous + query: SELECT COUNT(*) > 0 FROM llm_responses + + # vally: program integration-plan --has-frontend + # The artifact the seam is recorded in. Graded with the same validator the + # scaffold suite uses, so a plan that exists but does not describe the frontend + # seam fails here rather than passing on file existence alone. + - comment: integration-plan.md satisfies the hand-off contract, including the frontend seam + exec: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON /agent/assets/graders/evals/graders/validate-integration-plan.ts --has-frontend + + # vally: tool-calls required [workflow-tools-open_scaffold_next_steps_view] + # The surface this agent is contracted to finish on. Without it the integration + # is done and the user is shown nothing — invisible to the artifact assertion + # above, which passes as long as the file is right. + - comment: Agent should have opened the scaffold next steps view + query: SELECT COUNT(*) > 0 FROM toolCalls WHERE tool LIKE '%open_scaffold_next_steps_view%' + + # vally: tool-calls forbidden [workflow-tools-start_local_development] + # First half of the pair. Nothing in this turn approves moving on, so a hand-off + # here is the agent skipping the gate — and it would take the work into a chat + # session the harness is not watching, leaving turn 2 to look like an agent that + # simply stopped. + - comment: Agent should not have started local development before approval + query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%start_local_development%' + + # ─── Turn 2: approve, and require the hand-off ──────────────────────────────── + - chatMode: azure-project-integrate + text: | + Looks good — go ahead and set up local development. + assertions: + - comment: Sentinel; this turn must have produced a response or its checks are vacuous + query: SELECT COUNT(*) > 0 FROM llm_responses + + # vally: tool-calls required [workflow-tools-start_local_development] + # Second half of the pair, and the reason the negative above is evidence rather + # than a formality: the only difference between the two turns is that the user + # approved, so an agent that never hands off fails here while one that always + # hands off fails on turn 1. + - comment: Agent should have started local development once approved + query: SELECT COUNT(*) > 0 FROM toolCalls WHERE tool LIKE '%start_local_development%' + + # Recorded, not asserted. Whether the agent hands off, explains, or asks a + # follow-up are different behaviours that the checks above cannot distinguish, + # and the transcript is what tells them apart on the first red. + - comment: Triage; what the agent did at the local-development hand-off + exec: 'ls -la /workspace/.azure/ 2>&1 || true' + assertZeroExitCode: false diff --git a/evals/msbench/config/stimuli/plan-generation-task-app.yaml b/evals/msbench/config/stimuli/plan-generation-task-app.yaml index d25270e9c..3e8391fa6 100644 --- a/evals/msbench/config/stimuli/plan-generation-task-app.yaml +++ b/evals/msbench/config/stimuli/plan-generation-task-app.yaml @@ -145,6 +145,23 @@ promptSteps: - comment: Agent should not fall back to the chat question tool (turn 1) query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%askQuestion%' + # vally: tool-calls forbidden [workflow-tools-start_project_scaffold] + # The plan-phase approval gate, graded at the tool boundary. + # + # This turn confirms *requirements* and produces a plan; nothing in it + # approves that plan. `azure-project-plan` only hands off to scaffolding once + # the user approves, so a `start_project_scaffold` call here is the agent + # scaffolding without consent — the same failure `Agent should not have + # generated infrastructure before approval` catches one phase later, and the + # exact shape `debug-plan-approval-gate` already asserts for + # `start_azure_debug_generate`. + # + # Worth having as a *tool* check rather than only an artifact check: an agent + # that hands off has already left this session, so the files it would be + # judged on never appear here and every artifact assertion stays green. + - comment: Agent should not hand off to the scaffold agent before the plan is approved + query: SELECT COUNT(*) = 0 FROM toolCalls WHERE tool LIKE '%start_project_scaffold%' + # Triage for the `files`-table assertions above, recorded not asserted. # `assertZeroExitCode: false`, so it adds no check and cannot change the # assertion count — the same shape as the fingerprint below. diff --git a/package-lock.json b/package-lock.json index 58f627e79..4ce5b9901 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7611,9 +7611,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", - "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.7.tgz", + "integrity": "sha512-dOvZVzjdZdz7phd9v6jCbwxrBW3fK6n8Rc0CtdmM4bumzMnxywBYhuph6J819RRw/ku+rLbelwfMunktuzVVHg==", "dev": true, "funding": [ { @@ -8531,9 +8531,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { diff --git a/package.json b/package.json index a74e57ede..dd8e84e88 100644 --- a/package.json +++ b/package.json @@ -1026,6 +1026,7 @@ "all": "npm i && npm run lint && npm run build && npm run package && npm test", "api-extractor": "cd api && npm run api-extractor", "eval:drift": "node evals/check-agent-drift.ts", + "eval:tool-coverage": "node evals/msbench/check-tool-coverage.ts", "eval:cor:graders:certify": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON evals/src/graderCertification.ts", "eval:cor:thresholds:validate": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON evals/src/releaseThresholds.ts evals/release-thresholds.v1.json" }, diff --git a/resources/agents/azure-deploy.agent.md b/resources/agents/azure-deploy.agent.md index 47556ed30..02d8fea76 100644 --- a/resources/agents/azure-deploy.agent.md +++ b/resources/agents/azure-deploy.agent.md @@ -10,6 +10,20 @@ model: ['Claude Opus 4.7 (copilot)', 'Claude Sonnet 4.6 (copilot)'] # Azure Deployment Agent +## Azure Resources MCP Tools + +Every `copilot-azure-resources-extension-tools/*` tool this agent uses is provided by an MCP server declared in this agent's `tools:` frontmatter, so **these tools ARE available in this session.** VS Code does not always surface them directly in your active tool list; that absence does **not** mean the tool is missing or that "the extension does not expose this MCP endpoint." + +When a step tells you to call one of these tools and you do not see it directly available, do **not** give up — load it and call it: + +1. Call `tool_search` with the **exact tool name only** as the query (e.g. `record_deploy_prerequisites`) — a single tool name, never a phrase like "azure mcp deploy prerequisites". +2. If the tool is not already active, enable it with `activate_tools`, then invoke the tool (e.g. `record_deploy_prerequisites`). +3. If the search misses or a call errors, **retry** the search → activate → invoke loop with the exact tool name. Persist until the call succeeds. + +Never claim one of these tools is "not available" or "not exposed", never fall back to a manual work-around (narrating a CLI check in chat, or hand-editing an artifact the tool owns), and never stop, summarize, or announce completion until the required tool call has actually **succeeded**. Treating a required view/state tool as unavailable is a **failure of this agent**, not an acceptable outcome. + +This applies to every tool this agent is contracted to call: `record_deploy_prerequisites`, `open_deploy_plan_view`, `capture_deployment_inventory`, and `open_deploy_result_view`. + You are the deployment phase of the guided Azure project workflow: **Plan → Scaffold → Integrate → Local Dev → Deploy** diff --git a/resources/agents/azure-deploy/references/approval-gates.md b/resources/agents/azure-deploy/references/approval-gates.md index ad8da18a3..43e7ba45d 100644 --- a/resources/agents/azure-deploy/references/approval-gates.md +++ b/resources/agents/azure-deploy/references/approval-gates.md @@ -8,6 +8,16 @@ Display the architecture plan for user approval BEFORE generating any files: > ⛔ **Open the plan view FIRST.** Call `open_deploy_plan_view` before displaying the gate text — it renders the `prepare-plan.json` you just wrote (services, SKUs, region, cost) in a side-by-side view. Call it once per gate; it is a display action only and never replaces the chat approval prompt below. +> ⛔ **Record the CLI prerequisites through the MCP tool — not in chat.** The plan view has a prerequisites section for the two CLIs this stage depends on, and it is populated **only** by `record_deploy_prerequisites`. Before (or right alongside) `open_deploy_plan_view`: +> +> 1. Probe each CLI with its version command in the user's own default shell — **azd** → `azd version`, **az** → `az version`. +> 2. Call `record_deploy_prerequisites` with one entry per tool: `installed: true` when the command returned a version, otherwise `installed: false`. Include the detected `version` when you have it. +> 3. Do **not** pass install links or display names — the view resolves those from its own catalog. +> +> Example: `record_deploy_prerequisites({ tools: [{ id: "azd", installed: true, version: "1.9.2" }, { id: "az", installed: false }] })` +> +> Checking the CLIs by hand and describing the result in chat does **not** satisfy this — the user's plan view stays empty. Never write these into `prepare-plan.json`; that is the vendored pipeline's artifact. If the tool is not directly listed, load it first per "Azure Resources MCP Tools" in [`azure-deploy.agent.md`](../../azure-deploy.agent.md) — do **not** conclude it is unavailable. + > ⛔ **Resource group edit is MANDATORY in the gate display.** Show this exact block: > ``` > 🏢 **Subscription:** {subscriptionName} (`{subscriptionId}`)