From 24b54ac291e5645dd9c140476b6580ce0f4a3166 Mon Sep 17 00:00:00 2001 From: alexweininger Date: Tue, 25 Aug 2026 22:37:46 -0700 Subject: [PATCH 1/5] Add fidelity gates: did the scaffold build what the plan promised? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every scaffold gate today grades the tree against itself — it builds, the frontend is embeddable, the API seam holds. All of them pass a project that dropped a service or wired the wrong datastore, because two working services look exactly like two working services. The plan is the only artifact that knows there should have been three. Two validators, both comparing `.azure/project-plan.md` to the tree: - service fidelity — every planned service exists, nothing was invented, the frontend exists iff the plan says so, and each service's language and framework match what was promised. - datastore fidelity — the wired datastore is the planned one, planned resources are actually referenced, and no unplanned datastore is wired. Both sides of the datastore check normalise to a closed set of families and compare families, never strings. The planned family is resolved from the local connection string's scheme before its marketing name, so "Cosmos DB for MongoDB" resolves to mongodb — the wire protocol is what decides which client the code must speak. Evidence is manifests union imports, because `import sqlite3` and `node:sqlite` never appear in a manifest, and a quiet swap to SQLite is the likeliest form of this bug. On a stack with no analyser these report not-applicable, never a pass, using the shared `NOT_APPLICABLE:` stderr marker agreed with the runtime-gates and gate-health sessions. A Go fixture pins that behaviour as a certified golden expectation, so the escape hatch cannot quietly become green. Every gate has a mutation that turns it red; certification goes 44/44 -> 70/70. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evals/grader-certification/manifest.json | 232 +++++++++ .../.azure/project-plan.md | 64 +++ .../reference-dotnet-api/.env.example | 1 + .../reference-dotnet-api/scenario.json | 27 + .../services/api/Api.csproj | 13 + .../services/api/Program.cs | 26 + .../.azure/project-plan.md | 62 +++ .../reference-go-unsupported/scenario.json | 27 + .../services/api/go.mod | 5 + .../services/api/main.go | 28 ++ .../.azure/project-plan.md | 116 +++++ .../reference-node-multiservice/.env.example | 2 + .../reference-node-multiservice/scenario.json | 21 + .../services/api/package.json | 19 + .../services/api/src/db.ts | 17 + .../services/api/src/index.ts | 24 + .../services/api/src/storage.ts | 11 + .../services/shared/package.json | 7 + .../services/shared/src/types.ts | 5 + .../services/web/index.html | 11 + .../services/web/package.json | 20 + .../services/web/src/App.tsx | 28 ++ .../services/web/src/main.tsx | 9 + .../services/web/vite.config.ts | 10 + .../services/worker/package.json | 18 + .../services/worker/src/index.ts | 17 + .../.azure/project-plan.md | 63 +++ .../reference-python-api/.env.example | 1 + .../reference-python-api/scenario.json | 27 + .../reference-python-api/services/api/main.py | 22 + .../services/api/requirements.txt | 3 + evals/graders/graderHarness.ts | 120 ++++- evals/graders/validate-datastore-fidelity.ts | 37 ++ evals/graders/validate-service-fidelity.ts | 47 ++ .../grader-certification/offline/report.json | 311 +++++++++++- .../grader-certification/offline/report.md | 30 +- evals/src/artifacts/datastoreFidelity.ts | 475 ++++++++++++++++++ evals/src/artifacts/plannedProject.ts | 214 ++++++++ evals/src/artifacts/projectPlan.ts | 3 +- evals/src/artifacts/scaffoldTree.ts | 344 +++++++++++++ evals/src/artifacts/serviceFidelity.ts | 395 +++++++++++++++ evals/src/graderCertification.ts | 27 +- 42 files changed, 2926 insertions(+), 13 deletions(-) create mode 100644 evals/grader-certification/reference-dotnet-api/.azure/project-plan.md create mode 100644 evals/grader-certification/reference-dotnet-api/.env.example create mode 100644 evals/grader-certification/reference-dotnet-api/scenario.json create mode 100644 evals/grader-certification/reference-dotnet-api/services/api/Api.csproj create mode 100644 evals/grader-certification/reference-dotnet-api/services/api/Program.cs create mode 100644 evals/grader-certification/reference-go-unsupported/.azure/project-plan.md create mode 100644 evals/grader-certification/reference-go-unsupported/scenario.json create mode 100644 evals/grader-certification/reference-go-unsupported/services/api/go.mod create mode 100644 evals/grader-certification/reference-go-unsupported/services/api/main.go create mode 100644 evals/grader-certification/reference-node-multiservice/.azure/project-plan.md create mode 100644 evals/grader-certification/reference-node-multiservice/.env.example create mode 100644 evals/grader-certification/reference-node-multiservice/scenario.json create mode 100644 evals/grader-certification/reference-node-multiservice/services/api/package.json create mode 100644 evals/grader-certification/reference-node-multiservice/services/api/src/db.ts create mode 100644 evals/grader-certification/reference-node-multiservice/services/api/src/index.ts create mode 100644 evals/grader-certification/reference-node-multiservice/services/api/src/storage.ts create mode 100644 evals/grader-certification/reference-node-multiservice/services/shared/package.json create mode 100644 evals/grader-certification/reference-node-multiservice/services/shared/src/types.ts create mode 100644 evals/grader-certification/reference-node-multiservice/services/web/index.html create mode 100644 evals/grader-certification/reference-node-multiservice/services/web/package.json create mode 100644 evals/grader-certification/reference-node-multiservice/services/web/src/App.tsx create mode 100644 evals/grader-certification/reference-node-multiservice/services/web/src/main.tsx create mode 100644 evals/grader-certification/reference-node-multiservice/services/web/vite.config.ts create mode 100644 evals/grader-certification/reference-node-multiservice/services/worker/package.json create mode 100644 evals/grader-certification/reference-node-multiservice/services/worker/src/index.ts create mode 100644 evals/grader-certification/reference-python-api/.azure/project-plan.md create mode 100644 evals/grader-certification/reference-python-api/.env.example create mode 100644 evals/grader-certification/reference-python-api/scenario.json create mode 100644 evals/grader-certification/reference-python-api/services/api/main.py create mode 100644 evals/grader-certification/reference-python-api/services/api/requirements.txt create mode 100644 evals/graders/validate-datastore-fidelity.ts create mode 100644 evals/graders/validate-service-fidelity.ts create mode 100644 evals/src/artifacts/datastoreFidelity.ts create mode 100644 evals/src/artifacts/plannedProject.ts create mode 100644 evals/src/artifacts/scaffoldTree.ts create mode 100644 evals/src/artifacts/serviceFidelity.ts diff --git a/evals/grader-certification/manifest.json b/evals/grader-certification/manifest.json index a60ab0ce0..9b8278dad 100644 --- a/evals/grader-certification/manifest.json +++ b/evals/grader-certification/manifest.json @@ -23,6 +23,46 @@ "debug-config", "debug-artifacts" ] + }, + { + "id": "reference-node-multiservice", + "path": "evals/grader-certification/reference-node-multiservice", + "description": "A plan and a scaffold that agree: three declared services (API, web, worker) plus a shared library, wiring the PostgreSQL, Blob Storage and Queue Storage resources the plan's Services Required table promises. The fidelity validators compare a plan to a tree, so they need a fixture whose plan actually describes its own tree.", + "offlineValidators": [ + "service-fidelity", + "datastore-fidelity" + ] + }, + { + "id": "reference-python-api", + "path": "evals/grader-certification/reference-python-api", + "description": "The same contract on a Python stack \u2014 psycopg against requirements.txt. Exists so 'works across ecosystems' is executed rather than asserted: a datastore check that only understands npm would pass this fixture's golden case while being unable to fail any mutation of it.", + "offlineValidators": [ + "service-fidelity", + "datastore-fidelity" + ] + }, + { + "id": "reference-dotnet-api", + "path": "evals/grader-certification/reference-dotnet-api", + "description": "The same contract on a .NET stack \u2014 Npgsql declared in a .csproj and imported with a using directive, neither of which resembles a package.json.", + "offlineValidators": [ + "service-fidelity", + "datastore-fidelity" + ] + }, + { + "id": "reference-go-unsupported", + "path": "evals/grader-certification/reference-go-unsupported", + "description": "A stack no analyser covers. Its golden case must report ecosystemNotSupported rather than passing: a silent pass on an unsupported stack is the vacuous-gate failure, and this is the case that stops the not-applicable escape hatch quietly becoming green.", + "offlineValidators": [ + "service-fidelity", + "datastore-fidelity" + ], + "offlineExpectations": { + "service-fidelity": "ecosystemNotSupported", + "datastore-fidelity": "ecosystemNotSupported" + } } ], "mutations": [ @@ -406,6 +446,198 @@ "replacement": "DATABASE_URL=postgres://golden:******@localhost:5432/golden", "expectedCode": "redactedSecretPlaceholder", "search": "PORT=7071" + }, + { + "id": "fidelity-planned-service-dropped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "file": "services/worker", + "operation": "delete", + "expectedCode": "plannedServiceMissing" + }, + { + "id": "fidelity-service-invented", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "file": ".azure/project-plan.md", + "operation": "replace", + "search": "## 4. Worker \u2014 Background Jobs\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |\n| **Runtime** | Node |\n| **Package Manager** | npm |\n| **Test Runner** | vitest |\n| **Test Command** | npm test |\n| **Orchestration** | docker-compose |\n\n", + "replacement": "", + "expectedCode": "unplannedServiceScaffolded" + }, + { + "id": "fidelity-frontend-missing", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "file": "services/web", + "operation": "delete", + "expectedCode": "frontendMissingFromScaffold" + }, + { + "id": "fidelity-frontend-invented", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "file": ".azure/project-plan.md", + "operation": "replace", + "search": "**App Type**: SPA + API", + "replacement": "**App Type**: API only", + "expectedCode": "frontendNotPlanned" + }, + { + "id": "fidelity-language-swapped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "file": ".azure/project-plan.md", + "operation": "replace", + "search": "## 2. Backend \u2014 Azure Functions\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |", + "replacement": "## 2. Backend \u2014 Azure Functions\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | Python |", + "expectedCode": "serviceLanguageMismatch" + }, + { + "id": "fidelity-framework-swapped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "file": ".azure/project-plan.md", + "operation": "replace", + "search": "| **Framework** | React + Vite |", + "replacement": "| **Framework** | Angular |", + "expectedCode": "serviceFrameworkMismatch" + }, + { + "id": "fidelity-plan-declares-no-services", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "file": ".azure/project-plan.md", + "operation": "replace", + "search": "## 2. Backend \u2014 Azure Functions\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |\n| **Runtime** | Node |\n| **Package Manager** | npm |\n| **Test Runner** | vitest |\n| **Test Command** | npm test |\n| **Orchestration** | docker-compose |\n\n## 3. Frontend \u2014 Web App\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |\n| **Framework** | React + Vite |\n| **Package Manager** | npm |\n| **Test Runner** | vitest |\n| **Test Command** | npm test |\n\n## 4. Worker \u2014 Background Jobs\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |\n| **Runtime** | Node |\n| **Package Manager** | npm |\n| **Test Runner** | vitest |\n| **Test Command** | npm test |\n| **Orchestration** | docker-compose |\n\n", + "replacement": "", + "expectedCode": "planDeclaresNoServices" + }, + { + "id": "fidelity-datastore-import-swapped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "file": "services/api/src/db.ts", + "operation": "replace", + "search": "import { Pool } from 'pg';", + "replacement": "import { Pool } from 'sqlite3';", + "expectedCode": "plannedDatastoreNotWired" + }, + { + "id": "fidelity-datastore-invented", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "file": "services/api/src/db.ts", + "operation": "replace", + "search": "import { Pool } from 'pg';", + "replacement": "import { Pool } from 'pg';\nimport { MongoClient } from 'mongodb';", + "expectedCode": "unplannedDatastoreWired" + }, + { + "id": "fidelity-datastore-dependency-dropped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "file": "services/api/package.json", + "operation": "replace", + "search": ",\n \"pg\": \"^8.13.1\"", + "replacement": "", + "expectedCode": "datastoreDependencyMissing" + }, + { + "id": "fidelity-resource-never-wired", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "file": ".azure/project-plan.md", + "operation": "replace", + "search": "| Blob Storage | Store ticket attachments | STORAGE_CONNECTION_STRING |", + "replacement": "| Blob Storage | Store ticket attachments | ATTACHMENTS_CONNECTION_STRING |", + "expectedCode": "plannedResourceNotWired" + }, + { + "id": "fidelity-services-required-table-unreadable", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "file": ".azure/project-plan.md", + "operation": "replace", + "search": "| Azure Service | Role in App | Environment Variable | Default Value (Local) | Classification |", + "replacement": "| Name | Responsibility | Variable | Local Default | Classification |", + "expectedCode": "plannedResourcesUnreadable" + }, + { + "id": "fidelity-datastore-swapped-python", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "file": "services/api/main.py", + "operation": "replace", + "search": "from psycopg import connect", + "replacement": "from aiosqlite import connect", + "expectedCode": "unplannedDatastoreWired" + }, + { + "id": "fidelity-datastore-unwired-python", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "file": "services/api/main.py", + "operation": "replace", + "search": "from psycopg import connect", + "replacement": "from aiosqlite import connect", + "expectedCode": "plannedDatastoreNotWired" + }, + { + "id": "fidelity-nothing-scaffolded-python", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "file": "services/api/requirements.txt", + "operation": "delete", + "expectedCode": "noServicesScaffolded" + }, + { + "id": "fidelity-datastore-swapped-dotnet", + "tier": "offline", + "fixture": "reference-dotnet-api", + "validator": "datastore-fidelity", + "file": "services/api/Program.cs", + "operation": "replace", + "search": "using Npgsql;", + "replacement": "using Microsoft.Data.Sqlite;", + "expectedCode": "plannedDatastoreNotWired" + }, + { + "id": "fidelity-orm-owns-the-driver-python", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "file": "services/api/main.py", + "operation": "replace", + "search": "from psycopg import connect", + "replacement": "from sqlalchemy import create_engine", + "expectedCode": "passed" + }, + { + "id": "fidelity-orm-owns-the-driver-dotnet", + "tier": "offline", + "fixture": "reference-dotnet-api", + "validator": "datastore-fidelity", + "file": "services/api/Program.cs", + "operation": "replace", + "search": "using Npgsql;", + "replacement": "using Npgsql.EntityFrameworkCore.PostgreSQL;", + "expectedCode": "passed" } ] } diff --git a/evals/grader-certification/reference-dotnet-api/.azure/project-plan.md b/evals/grader-certification/reference-dotnet-api/.azure/project-plan.md new file mode 100644 index 000000000..75eb8bd35 --- /dev/null +++ b/evals/grader-certification/reference-dotnet-api/.azure/project-plan.md @@ -0,0 +1,64 @@ +# Project Plan + +**Status**: Integrated +**Created**: 2026-08-25 +**Mode**: New Project + +## 1. Project Overview + +**Goal**: Build a C# ticket API whose storage layer is independently testable. + +**App Type**: API only + +**Mode**: NEW + +## 2. Backend — ASP.NET Minimal API + +| Component | Technology | +|-----------|-----------| +| **Language** | C# | +| **Runtime** | .NET | +| **Package Manager** | dotnet (NuGet) | +| **Test Runner** | xUnit | +| **Mocking Library** | NSubstitute | +| **Test Command** | dotnet test | +| **Orchestration** | docker-compose | + +## 3. Services Required + +| Azure Service | Role in App | Environment Variable | Default Value (Local) | Classification | +|---------------|------------|---------------------|----------------------|----------------| +| Azure Database for PostgreSQL | Primary data store for tickets | DATABASE_URL | postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets | Essential | + +## 4. Prerequisites + +### Run + +| Tool | Service(s) | Installed | Version | Install | +|------|-----------|-----------|---------|---------| +| .NET SDK | * | ✅ | 9.0 | https://dotnet.microsoft.com/download | + +### Debug + +| Tool | Service(s) | Installed | Version | Install | +|------|-----------|-----------|---------|---------| +| Docker | api | ❓ | — | https://docs.docker.com/get-docker/ | + +## 5. Project Structure + +```text +services/api/Api.csproj +services/api/Program.cs +``` + +## 6. Route Definitions + +| # | Method | Path | Description | Auth | Status Codes | +|---|--------|------|-------------|------|-------------| +| 1 | GET | `/api/health` | Report service health | None | 200 | +| 2 | GET | `/api/tickets` | List tickets | None | 200 | + +## 7. Next Steps + +1. Scaffold the API service. +2. Generate debug artifacts. diff --git a/evals/grader-certification/reference-dotnet-api/.env.example b/evals/grader-certification/reference-dotnet-api/.env.example new file mode 100644 index 000000000..c1cec9fec --- /dev/null +++ b/evals/grader-certification/reference-dotnet-api/.env.example @@ -0,0 +1 @@ +DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets diff --git a/evals/grader-certification/reference-dotnet-api/scenario.json b/evals/grader-certification/reference-dotnet-api/scenario.json new file mode 100644 index 000000000..38b24209d --- /dev/null +++ b/evals/grader-certification/reference-dotnet-api/scenario.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": "1", + "id": "grader-dotnet-api-fidelity", + "prompt": "Build a C# HTTP API for tracking tickets, backed by PostgreSQL.", + "baselinePrompt": "Create a C# minimal-API service for tracking tickets, backed by PostgreSQL. Expose health and list endpoints, and include the project file needed to restore and run it.", + "tags": { + "archetype": "crud", + "frontend": "none", + "backend": "dotnet", + "database": "postgres", + "auth": "none", + "complexity": "small" + }, + "requirementsAnswers": { + "dataStores": [ + "PostgreSQL" + ] + }, + "validation": { + "profile": "minimal", + "build": false, + "test": false, + "lint": "skip", + "timeoutMinutes": 5, + "maxAgentRetries": 0 + } +} diff --git a/evals/grader-certification/reference-dotnet-api/services/api/Api.csproj b/evals/grader-certification/reference-dotnet-api/services/api/Api.csproj new file mode 100644 index 000000000..ead3ca3ad --- /dev/null +++ b/evals/grader-certification/reference-dotnet-api/services/api/Api.csproj @@ -0,0 +1,13 @@ + + + + net9.0 + enable + enable + + + + + + + diff --git a/evals/grader-certification/reference-dotnet-api/services/api/Program.cs b/evals/grader-certification/reference-dotnet-api/services/api/Program.cs new file mode 100644 index 000000000..ad3f053f5 --- /dev/null +++ b/evals/grader-certification/reference-dotnet-api/services/api/Program.cs @@ -0,0 +1,26 @@ +using Npgsql; + +var builder = WebApplication.CreateBuilder(args); +var app = builder.Build(); + +var connectionString = Environment.GetEnvironmentVariable("DATABASE_URL"); + +app.MapGet("/api/health", () => Results.Ok(new { status = "ok" })); + +app.MapGet("/api/tickets", async () => +{ + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand("SELECT id, title, status FROM tickets ORDER BY id", connection); + await using var reader = await command.ExecuteReaderAsync(); + + var tickets = new List(); + while (await reader.ReadAsync()) + { + tickets.Add(new { id = reader.GetString(0), title = reader.GetString(1), status = reader.GetString(2) }); + } + + return Results.Ok(tickets); +}); + +app.Run(); diff --git a/evals/grader-certification/reference-go-unsupported/.azure/project-plan.md b/evals/grader-certification/reference-go-unsupported/.azure/project-plan.md new file mode 100644 index 000000000..ba6fb0d0e --- /dev/null +++ b/evals/grader-certification/reference-go-unsupported/.azure/project-plan.md @@ -0,0 +1,62 @@ +# Project Plan + +**Status**: Integrated +**Created**: 2026-08-25 +**Mode**: New Project + +## 1. Project Overview + +**Goal**: Build a Go ticket API whose storage layer is independently testable. + +**App Type**: API only + +**Mode**: NEW + +## 2. Backend — Go HTTP Service + +| Component | Technology | +|-----------|-----------| +| **Language** | Go | +| **Runtime** | Go | +| **Package Manager** | go modules | +| **Test Runner** | go test | +| **Test Command** | go test ./... | +| **Orchestration** | docker-compose | + +## 3. Services Required + +| Azure Service | Role in App | Environment Variable | Default Value (Local) | Classification | +|---------------|------------|---------------------|----------------------|----------------| +| Azure Database for PostgreSQL | Primary data store for tickets | DATABASE_URL | postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets | Essential | + +## 4. Prerequisites + +### Run + +| Tool | Service(s) | Installed | Version | Install | +|------|-----------|-----------|---------|---------| +| Go | * | ✅ | 1.23 | https://go.dev/dl/ | + +### Debug + +| Tool | Service(s) | Installed | Version | Install | +|------|-----------|-----------|---------|---------| +| Docker | api | ❓ | — | https://docs.docker.com/get-docker/ | + +## 5. Project Structure + +```text +services/api/go.mod +services/api/main.go +``` + +## 6. Route Definitions + +| # | Method | Path | Description | Auth | Status Codes | +|---|--------|------|-------------|------|-------------| +| 1 | GET | `/api/health` | Report service health | None | 200 | + +## 7. Next Steps + +1. Scaffold the API service. +2. Generate debug artifacts. diff --git a/evals/grader-certification/reference-go-unsupported/scenario.json b/evals/grader-certification/reference-go-unsupported/scenario.json new file mode 100644 index 000000000..efa2ef057 --- /dev/null +++ b/evals/grader-certification/reference-go-unsupported/scenario.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": "1", + "id": "grader-go-unsupported-fidelity", + "prompt": "Build a Go HTTP API for tracking tickets, backed by PostgreSQL.", + "baselinePrompt": "Create a Go HTTP service for tracking tickets, backed by PostgreSQL. Expose health and list endpoints, and include the module file needed to build and run it.", + "tags": { + "archetype": "crud", + "frontend": "none", + "backend": "go", + "database": "postgres", + "auth": "none", + "complexity": "small" + }, + "requirementsAnswers": { + "dataStores": [ + "PostgreSQL" + ] + }, + "validation": { + "profile": "minimal", + "build": false, + "test": false, + "lint": "skip", + "timeoutMinutes": 5, + "maxAgentRetries": 0 + } +} diff --git a/evals/grader-certification/reference-go-unsupported/services/api/go.mod b/evals/grader-certification/reference-go-unsupported/services/api/go.mod new file mode 100644 index 000000000..d38e03711 --- /dev/null +++ b/evals/grader-certification/reference-go-unsupported/services/api/go.mod @@ -0,0 +1,5 @@ +module example.com/tickets + +go 1.23 + +require github.com/jackc/pgx/v5 v5.7.2 diff --git a/evals/grader-certification/reference-go-unsupported/services/api/main.go b/evals/grader-certification/reference-go-unsupported/services/api/main.go new file mode 100644 index 000000000..9966ec68c --- /dev/null +++ b/evals/grader-certification/reference-go-unsupported/services/api/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "os" + + "github.com/jackc/pgx/v5" +) + +func main() { + http.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) + }) + + http.HandleFunc("/api/tickets", func(w http.ResponseWriter, r *http.Request) { + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer conn.Close(context.Background()) + _ = json.NewEncoder(w).Encode([]string{}) + }) + + _ = http.ListenAndServe(":8080", nil) +} diff --git a/evals/grader-certification/reference-node-multiservice/.azure/project-plan.md b/evals/grader-certification/reference-node-multiservice/.azure/project-plan.md new file mode 100644 index 000000000..5b3ef225a --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/.azure/project-plan.md @@ -0,0 +1,116 @@ +# Project Plan + +**Status**: Integrated +**Created**: 2026-08-25 +**Mode**: New Project + +## 1. Project Overview + +**Goal**: Build a ticket tracker whose browser UI, HTTP API and background worker are independently testable. + +**App Type**: SPA + API + +**Mode**: NEW + +## 2. Backend — Azure Functions + +| Component | Technology | +|-----------|-----------| +| **Language** | TypeScript | +| **Runtime** | Node | +| **Package Manager** | npm | +| **Test Runner** | vitest | +| **Test Command** | npm test | +| **Orchestration** | docker-compose | + +## 3. Frontend — Web App + +| Component | Technology | +|-----------|-----------| +| **Language** | TypeScript | +| **Framework** | React + Vite | +| **Package Manager** | npm | +| **Test Runner** | vitest | +| **Test Command** | npm test | + +## 4. Worker — Background Jobs + +| Component | Technology | +|-----------|-----------| +| **Language** | TypeScript | +| **Runtime** | Node | +| **Package Manager** | npm | +| **Test Runner** | vitest | +| **Test Command** | npm test | +| **Orchestration** | docker-compose | + +## 5. Services Required + +| Azure Service | Role in App | Environment Variable | Default Value (Local) | Classification | +|---------------|------------|---------------------|----------------------|----------------| +| PostgreSQL | Primary data store for tickets | DATABASE_URL | postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets | Essential | +| Blob Storage | Store ticket attachments | STORAGE_CONNECTION_STRING | UseDevelopmentStorage=true | Essential | +| Queue Storage | Hand queued work to the background worker | QUEUE_CONNECTION_STRING | UseDevelopmentStorage=true | Essential | + +## 6. Prerequisites + +### Run + +| Tool | Service(s) | Installed | Version | Install | +|------|-----------|-----------|---------|---------| +| Node.js | * | ✅ | 22.x | https://nodejs.org | +| npm | * | ✅ | 10.x | https://nodejs.org | + +### Debug + +| Tool | Service(s) | Installed | Version | Install | +|------|-----------|-----------|---------|---------| +| Docker | api, worker | ❓ | — | https://docs.docker.com/get-docker/ | + +## 7. Design System & UI + +**Component Library**: Fluent UI v9 +**Style Direction**: Dense operational console with restrained elevation and scannable ticket rows. +**Typography**: Segoe UI Variable + +### Color Palette + +| Token | Hex | Usage | +|-------|-----|-------| +| `primary` | `#0F6CBD` | Primary actions and active navigation | +| `accent` | `#8764B8` | Priority badges | +| `surface` | `#FFFFFF` | Ticket cards and page background | +| `text` | `#1B1A19` | Ticket titles and body copy | +| `muted` | `#605E5C` | Timestamps and assignee captions | +| `border` | `#E1DFDD` | Row dividers and input borders | + +### Pages + +| Page | Route | Purpose | Layout | +|------|-------|---------|--------| +| Tickets | `/` | Browse and triage open tickets | `header + table + action-bar` | + +## 8. Project Structure + +```text +services/api/src/index.ts +services/api/src/db.ts +services/api/src/storage.ts +services/web/src/App.tsx +services/worker/src/index.ts +services/shared/src/types.ts +``` + +## 9. Route Definitions + +| # | Method | Path | Description | Auth | Status Codes | +|---|--------|------|-------------|------|-------------| +| 1 | GET | `/api/health` | Report service health | None | 200 | +| 2 | GET | `/api/tickets` | List tickets | None | 200 | +| 3 | POST | `/api/tickets` | Create a ticket | None | 201, 400 | + +## 10. Next Steps + +1. Scaffold the three services. +2. Integrate the browser client with the API. +3. Generate debug artifacts. diff --git a/evals/grader-certification/reference-node-multiservice/.env.example b/evals/grader-certification/reference-node-multiservice/.env.example new file mode 100644 index 000000000..8b57f4d21 --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/.env.example @@ -0,0 +1,2 @@ +DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets +QUEUE_CONNECTION_STRING=UseDevelopmentStorage=true diff --git a/evals/grader-certification/reference-node-multiservice/scenario.json b/evals/grader-certification/reference-node-multiservice/scenario.json new file mode 100644 index 000000000..4e8e10920 --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/scenario.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": "1", + "id": "grader-node-multiservice-fidelity", + "prompt": "Build a ticket tracker with a browser UI, an HTTP API and a background worker that processes queued work.", + "baselinePrompt": "Create a ticket tracker as three services: a React browser UI, an HTTP API backed by PostgreSQL, and a background worker that consumes a storage queue. Include build, test and lint configuration for each service.", + "tags": { + "archetype": "crud", + "frontend": "react", + "backend": "node", + "auth": "none", + "complexity": "medium" + }, + "validation": { + "profile": "standard", + "build": false, + "test": false, + "lint": "skip", + "timeoutMinutes": 5, + "maxAgentRetries": 0 + } +} diff --git a/evals/grader-certification/reference-node-multiservice/services/api/package.json b/evals/grader-certification/reference-node-multiservice/services/api/package.json new file mode 100644 index 000000000..89a204a03 --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/api/package.json @@ -0,0 +1,19 @@ +{ + "name": "@app/api", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc -p tsconfig.json", + "start": "node dist/index.js", + "test": "vitest run" + }, + "dependencies": { + "@azure/storage-blob": "^12.26.0", + "pg": "^8.13.1" + }, + "devDependencies": { + "typescript": "^5.9.2", + "vitest": "^2.1.8" + } +} diff --git a/evals/grader-certification/reference-node-multiservice/services/api/src/db.ts b/evals/grader-certification/reference-node-multiservice/services/api/src/db.ts new file mode 100644 index 000000000..d4691f3ac --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/api/src/db.ts @@ -0,0 +1,17 @@ +import { Pool } from 'pg'; +import type { Ticket } from '@app/shared'; + +const pool = new Pool({ connectionString: process.env.DATABASE_URL }); + +export async function listTickets(): Promise { + const result = await pool.query('SELECT id, title, status FROM tickets ORDER BY id'); + return result.rows; +} + +export async function createTicket(title: string): Promise { + const result = await pool.query( + 'INSERT INTO tickets (title, status) VALUES ($1, $2) RETURNING id, title, status', + [title, 'open'], + ); + return result.rows[0]; +} diff --git a/evals/grader-certification/reference-node-multiservice/services/api/src/index.ts b/evals/grader-certification/reference-node-multiservice/services/api/src/index.ts new file mode 100644 index 000000000..903d7684a --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/api/src/index.ts @@ -0,0 +1,24 @@ +import { createServer } from 'node:http'; +import { createTicket, listTickets } from './db.ts'; + +const server = createServer(async (request, response) => { + if (request.url === '/api/health') { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ status: 'ok' })); + return; + } + if (request.url === '/api/tickets' && request.method === 'GET') { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify(await listTickets())); + return; + } + if (request.url === '/api/tickets' && request.method === 'POST') { + const created = await createTicket('Untitled ticket'); + response.writeHead(201, { 'content-type': 'application/json' }); + response.end(JSON.stringify(created)); + return; + } + response.writeHead(404).end(); +}); + +server.listen(Number(process.env.PORT ?? 7071)); diff --git a/evals/grader-certification/reference-node-multiservice/services/api/src/storage.ts b/evals/grader-certification/reference-node-multiservice/services/api/src/storage.ts new file mode 100644 index 000000000..94e8bfe4b --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/api/src/storage.ts @@ -0,0 +1,11 @@ +import { BlobServiceClient } from '@azure/storage-blob'; + +const client = BlobServiceClient.fromConnectionString( + process.env.STORAGE_CONNECTION_STRING ?? 'UseDevelopmentStorage=true', +); + +export async function saveAttachment(ticketId: string, body: Buffer): Promise { + const container = client.getContainerClient('attachments'); + await container.createIfNotExists(); + await container.getBlockBlobClient(`${ticketId}.bin`).uploadData(body); +} diff --git a/evals/grader-certification/reference-node-multiservice/services/shared/package.json b/evals/grader-certification/reference-node-multiservice/services/shared/package.json new file mode 100644 index 000000000..4e31c105e --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/shared/package.json @@ -0,0 +1,7 @@ +{ + "name": "@app/shared", + "version": "1.0.0", + "private": true, + "type": "module", + "main": "src/types.ts" +} diff --git a/evals/grader-certification/reference-node-multiservice/services/shared/src/types.ts b/evals/grader-certification/reference-node-multiservice/services/shared/src/types.ts new file mode 100644 index 000000000..bd29e2d25 --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/shared/src/types.ts @@ -0,0 +1,5 @@ +export interface Ticket { + id: string; + title: string; + status: 'open' | 'triaged' | 'closed'; +} diff --git a/evals/grader-certification/reference-node-multiservice/services/web/index.html b/evals/grader-certification/reference-node-multiservice/services/web/index.html new file mode 100644 index 000000000..4bb37f55f --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/web/index.html @@ -0,0 +1,11 @@ + + + + + Ticket Tracker + + +
+ + + diff --git a/evals/grader-certification/reference-node-multiservice/services/web/package.json b/evals/grader-certification/reference-node-multiservice/services/web/package.json new file mode 100644 index 000000000..781efe032 --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/web/package.json @@ -0,0 +1,20 @@ +{ + "name": "@app/web", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "test": "vitest run" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@vitejs/plugin-react": "^4.3.4", + "vite": "^6.0.7", + "vitest": "^2.1.8" + } +} diff --git a/evals/grader-certification/reference-node-multiservice/services/web/src/App.tsx b/evals/grader-certification/reference-node-multiservice/services/web/src/App.tsx new file mode 100644 index 000000000..e12f6c990 --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/web/src/App.tsx @@ -0,0 +1,28 @@ +import { useEffect, useState } from 'react'; +import type { Ticket } from '@app/shared'; + +export function App(): JSX.Element { + const [tickets, setTickets] = useState([]); + + useEffect(() => { + void fetch('/api/tickets') + .then(response => response.json()) + .then(setTickets); + }, []); + + return ( +
+

Tickets

+ + + {tickets.map(ticket => ( + + + + + ))} + +
{ticket.title}{ticket.status}
+
+ ); +} diff --git a/evals/grader-certification/reference-node-multiservice/services/web/src/main.tsx b/evals/grader-certification/reference-node-multiservice/services/web/src/main.tsx new file mode 100644 index 000000000..766616b9e --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/web/src/main.tsx @@ -0,0 +1,9 @@ +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; +import { App } from './App.tsx'; + +createRoot(document.getElementById('root')!).render( + + + , +); diff --git a/evals/grader-certification/reference-node-multiservice/services/web/vite.config.ts b/evals/grader-certification/reference-node-multiservice/services/web/vite.config.ts new file mode 100644 index 000000000..dcffb1469 --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/web/vite.config.ts @@ -0,0 +1,10 @@ +import react from '@vitejs/plugin-react'; +import { defineConfig } from 'vite'; + +export default defineConfig({ + plugins: [react()], + server: { + host: true, + allowedHosts: true, + }, +}); diff --git a/evals/grader-certification/reference-node-multiservice/services/worker/package.json b/evals/grader-certification/reference-node-multiservice/services/worker/package.json new file mode 100644 index 000000000..ac1b44d13 --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/worker/package.json @@ -0,0 +1,18 @@ +{ + "name": "@app/worker", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc -p tsconfig.json", + "start": "node dist/index.js", + "test": "vitest run" + }, + "dependencies": { + "@azure/storage-queue": "^12.25.0" + }, + "devDependencies": { + "typescript": "^5.9.2", + "vitest": "^2.1.8" + } +} diff --git a/evals/grader-certification/reference-node-multiservice/services/worker/src/index.ts b/evals/grader-certification/reference-node-multiservice/services/worker/src/index.ts new file mode 100644 index 000000000..24568858b --- /dev/null +++ b/evals/grader-certification/reference-node-multiservice/services/worker/src/index.ts @@ -0,0 +1,17 @@ +import { QueueClient } from '@azure/storage-queue'; + +const queue = new QueueClient( + process.env.QUEUE_CONNECTION_STRING ?? 'UseDevelopmentStorage=true', + 'ticket-work', +); + +/** Poll the work queue and mark each queued ticket as triaged. */ +export async function runWorker(): Promise { + await queue.createIfNotExists(); + const messages = await queue.receiveMessages({ numberOfMessages: 8 }); + for (const message of messages.receivedMessageItems) { + await queue.deleteMessage(message.messageId, message.popReceipt); + } +} + +void runWorker(); diff --git a/evals/grader-certification/reference-python-api/.azure/project-plan.md b/evals/grader-certification/reference-python-api/.azure/project-plan.md new file mode 100644 index 000000000..cdd751744 --- /dev/null +++ b/evals/grader-certification/reference-python-api/.azure/project-plan.md @@ -0,0 +1,63 @@ +# Project Plan + +**Status**: Integrated +**Created**: 2026-08-25 +**Mode**: New Project + +## 1. Project Overview + +**Goal**: Build a Python ticket API whose storage layer is independently testable. + +**App Type**: API only + +**Mode**: NEW + +## 2. Backend — FastAPI + +| Component | Technology | +|-----------|-----------| +| **Language** | Python | +| **Runtime** | CPython | +| **Package Manager** | pip | +| **Test Runner** | pytest | +| **Test Command** | pytest | +| **Orchestration** | docker-compose | + +## 3. Services Required + +| Azure Service | Role in App | Environment Variable | Default Value (Local) | Classification | +|---------------|------------|---------------------|----------------------|----------------| +| Azure Database for PostgreSQL | Primary data store for tickets | DATABASE_URL | postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets | Essential | + +## 4. Prerequisites + +### Run + +| Tool | Service(s) | Installed | Version | Install | +|------|-----------|-----------|---------|---------| +| Python | * | ✅ | 3.12 | https://www.python.org/downloads/ | + +### Debug + +| Tool | Service(s) | Installed | Version | Install | +|------|-----------|-----------|---------|---------| +| Docker | api | ❓ | — | https://docs.docker.com/get-docker/ | + +## 5. Project Structure + +```text +services/api/main.py +services/api/requirements.txt +``` + +## 6. Route Definitions + +| # | Method | Path | Description | Auth | Status Codes | +|---|--------|------|-------------|------|-------------| +| 1 | GET | `/api/health` | Report service health | None | 200 | +| 2 | GET | `/api/tickets` | List tickets | None | 200 | + +## 7. Next Steps + +1. Scaffold the API service. +2. Generate debug artifacts. diff --git a/evals/grader-certification/reference-python-api/.env.example b/evals/grader-certification/reference-python-api/.env.example new file mode 100644 index 000000000..c1cec9fec --- /dev/null +++ b/evals/grader-certification/reference-python-api/.env.example @@ -0,0 +1 @@ +DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets diff --git a/evals/grader-certification/reference-python-api/scenario.json b/evals/grader-certification/reference-python-api/scenario.json new file mode 100644 index 000000000..2a8c31cf2 --- /dev/null +++ b/evals/grader-certification/reference-python-api/scenario.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": "1", + "id": "grader-python-api-fidelity", + "prompt": "Build a Python HTTP API for tracking tickets, backed by PostgreSQL.", + "baselinePrompt": "Create a Python HTTP API for tracking tickets, backed by PostgreSQL. Expose health, list and create endpoints, and include the dependency manifest needed to install and run it.", + "tags": { + "archetype": "crud", + "frontend": "none", + "backend": "python", + "database": "postgres", + "auth": "none", + "complexity": "small" + }, + "requirementsAnswers": { + "dataStores": [ + "PostgreSQL" + ] + }, + "validation": { + "profile": "minimal", + "build": false, + "test": false, + "lint": "skip", + "timeoutMinutes": 5, + "maxAgentRetries": 0 + } +} diff --git a/evals/grader-certification/reference-python-api/services/api/main.py b/evals/grader-certification/reference-python-api/services/api/main.py new file mode 100644 index 000000000..6221271ee --- /dev/null +++ b/evals/grader-certification/reference-python-api/services/api/main.py @@ -0,0 +1,22 @@ +import os + +from fastapi import FastAPI +from psycopg import connect + +app = FastAPI() + + +def _connection(): + return connect(os.environ["DATABASE_URL"]) + + +@app.get("/api/health") +def health() -> dict[str, str]: + return {"status": "ok"} + + +@app.get("/api/tickets") +def list_tickets() -> list[dict[str, object]]: + with _connection() as connection, connection.cursor() as cursor: + cursor.execute("SELECT id, title, status FROM tickets ORDER BY id") + return [{"id": row[0], "title": row[1], "status": row[2]} for row in cursor.fetchall()] diff --git a/evals/grader-certification/reference-python-api/services/api/requirements.txt b/evals/grader-certification/reference-python-api/services/api/requirements.txt new file mode 100644 index 000000000..4c977fc0d --- /dev/null +++ b/evals/grader-certification/reference-python-api/services/api/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +psycopg[binary]==3.2.3 diff --git a/evals/graders/graderHarness.ts b/evals/graders/graderHarness.ts index 5d5e100f5..75380be74 100644 --- a/evals/graders/graderHarness.ts +++ b/evals/graders/graderHarness.ts @@ -19,16 +19,90 @@ */ import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; +import { basename, resolve } from 'node:path'; import type { ArtifactValidationIssue } from '../src/artifacts/validationTypes.ts'; export const EXIT_PASS = 0; export const EXIT_PRODUCT_FAILURE = 1; export const EXIT_GRADER_ERROR = 3; +/** + * A gate that has no opinion about this workspace exits **0**, not 3. + * + * Exit 3 means "do not trust this result — the harness broke". A not-applicable verdict is + * the opposite: the gate ran, understood the input, and confidently concluded the property + * it grades is absent here. Collapsing the two would spend the only signal that isolates + * harness faults on cases that are working correctly, and would turn every gate red on + * every stack it does not yet cover — which makes the rational move "wire each gate only + * where it definitely applies", defeating the point of having a not-applicable path at all. + * + * The safety mechanism is therefore NOT the exit code, it is `NOT_APPLICABLE:` on stderr. + * A gate that returns not-applicable *without* emitting the marker is worse than either + * exit code, because it is then genuinely undetectable — it reports a pass forever and + * nobody investigates a passing gate. Emitting the marker is part of every gate's contract. + */ +export const NOT_APPLICABLE_EXIT_CODE = EXIT_PASS; + +/** + * Why a not-applicable verdict happened, in the only distinction that changes what someone + * should *do* about it: + * + * - `outOfScope` — the subject genuinely lacks the property being graded (a backend-only + * project has no frontend to check). A gate that is always `outOfScope` is dead weight: + * delete it or re-target it. + * - `notAttempted` — the gate wanted to run and could not (missing tool, unstaged tree, + * analyser not written yet). A gate that is always `notAttempted` is a **coverage hole**, + * not dead weight: fix the environment or implement the analyser. Deleting it would be + * exactly the wrong response. + * + * The two demand opposite remedies, so a reason code that lands in the wrong bucket sends + * whoever reads the health report in the wrong direction. + */ +export type NotApplicableClass = 'outOfScope' | 'notAttempted'; + +/** + * Every reason code, with its class. + * + * A registry rather than a free string because a reason code must not be able to *default* + * into a bucket: `emitNotApplicable` rejects an unregistered code, so classifying a new + * reason is a required step rather than something you can forget. It is a plain object + * rather than a union type deliberately — adding a member is a new line, which merges + * cleanly across the several sessions adding codes, where a one-line union would conflict. + */ +export const NOT_APPLICABLE_REASONS: Record = { + /** The tree has manifests, but only for an ecosystem no analyser covers yet. */ + ecosystemNotSupported: 'notAttempted', + /** + * No project manifest of any recognised ecosystem anywhere in the tree. + * + * For a gate that has *not* already read an artifact out of the same workspace, this most + * likely means the tree was never staged, which is a harness fault. A gate that reached + * this point after successfully reading, say, `.azure/project-plan.md` from that same + * workspace knows the tree is staged, so for it the same observation means the agent + * shipped nothing — a product failure, and it should say so rather than use this code. + */ + noProjectManifestFound: 'notAttempted', +}; + /** Raised for a bad artifact — anything else thrown is treated as a harness fault. */ export class ProductFailure extends Error { } +/** + * The stable identity of the running gate, used as `gate=` on every verdict line. + * + * Derived from the grader's filename (`validate-service-fidelity.ts` → `service-fidelity`) + * rather than from its prose description, because the description is editorial: reword it + * and the gate silently becomes a *different* gate to anything aggregating history, which + * has already been observed splitting one real gate's record into two partial ones. A + * filename is renamed deliberately and rarely, and it already matches the validator id in + * `grader-certification/manifest.json` — so the same token joins run rows to certification + * results without anyone maintaining a mapping. + */ +export function gateId(): string { + const entry = process.argv[1]; + return entry ? basename(entry).replace(/\.ts$/, '').replace(/^validate-/, '') : 'unknown'; +} + /** * The directory being graded. Vally runs a grader with its cwd already set to the * workspace, so cwd is a legitimate fallback — but when someone runs a grader by hand @@ -63,6 +137,31 @@ export function failWithIssues(summary: string, issues: ArtifactValidationIssue[ throw new ProductFailure(`${summary}\n${issues.map(i => ` • [${i.code}] ${i.path}: ${i.message}`).join('\n')}`); } +/** Thrown to end a grader with a not-applicable verdict; see `NOT_APPLICABLE_EXIT_CODE`. */ +export class NotApplicable extends Error { + readonly reason: string; + readonly detail: string; + /** Extra structured `key=value` pairs, e.g. `{ ecosystem: 'go' }`. Never prose. */ + readonly facts: Record; + + constructor(reason: string, detail: string, facts: Record = {}) { + super(detail); + this.reason = reason; + this.detail = detail; + this.facts = facts; + } +} + +/** + * End the grader with a not-applicable verdict. + * + * `reason` must be registered in `NOT_APPLICABLE_REASONS`; an unregistered code throws, + * which surfaces as a grader error rather than being quietly emitted with a guessed class. + */ +export function notApplicable(reason: string, detail: string, facts: Record = {}): never { + throw new NotApplicable(reason, detail, facts); +} + /** * Run a grader body, mapping its outcome onto the exit-code contract above. * An unexpected throw (TypeError, ReferenceError, …) exits 3 rather than 1 so a @@ -79,7 +178,7 @@ export function runGrader(name: string, body: () => void): void { } catch (error) { exitForError(name, error); } - console.error(`PASS: ${name}`); + console.error(`PASS: gate=${gateId()} — ${name}`); process.exit(EXIT_PASS); } @@ -93,15 +192,26 @@ export async function runGraderAsync(name: string, body: () => Promise): P } catch (error) { exitForError(name, error); } - console.error(`PASS: ${name}`); + console.error(`PASS: gate=${gateId()} — ${name}`); process.exit(EXIT_PASS); } function exitForError(name: string, error: unknown): never { + const gate = gateId(); + if (error instanceof NotApplicable) { + const classification = NOT_APPLICABLE_REASONS[error.reason]; + if (!classification) { + console.error(`GRADER ERROR: gate=${gate} — ${name} reported unregistered not-applicable reason "${error.reason}"`); + process.exit(EXIT_GRADER_ERROR); + } + const facts = Object.entries(error.facts).map(([key, value]) => ` ${key}=${value}`).join(''); + console.error(`NOT_APPLICABLE: gate=${gate} reason=${error.reason} class=${classification}${facts} detail="${error.detail.replace(/"/g, "'")}"`); + process.exit(NOT_APPLICABLE_EXIT_CODE); + } if (error instanceof ProductFailure) { - console.error(`FAIL: ${name} — ${error.message}`); + console.error(`FAIL: gate=${gate} — ${name} — ${error.message}`); process.exit(EXIT_PRODUCT_FAILURE); } - console.error(`GRADER ERROR: ${name} threw ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + console.error(`GRADER ERROR: gate=${gate} — ${name} threw ${error instanceof Error ? error.stack ?? error.message : String(error)}`); process.exit(EXIT_GRADER_ERROR); } diff --git a/evals/graders/validate-datastore-fidelity.ts b/evals/graders/validate-datastore-fidelity.ts new file mode 100644 index 000000000..cdf5bedef --- /dev/null +++ b/evals/graders/validate-datastore-fidelity.ts @@ -0,0 +1,37 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Grades whether the datastore the scaffold actually wires is the one the plan chose. + * + * This is the fidelity failure nothing else can see: a project that plans PostgreSQL and + * wires SQLite installs, builds, starts and passes every other gate in the suite. + * + * The contract lives in `evals/src/artifacts/datastoreFidelity.ts`, shared with grader + * certification, so the certified path and the executed path cannot drift. + * + * On a stack with no dependency analyser this reports **not-applicable**, never a pass — + * a datastore check that silently approves every Go project is indistinguishable from no + * check at all. See `NOT_APPLICABLE_EXIT_CODE` in the harness for why that still exits 0 + * and why the stderr marker, not the exit code, is the safety mechanism. + */ + +import { DATASTORE_NOT_APPLICABLE_CODES, validateDatastoreFidelity } from '../src/artifacts/datastoreFidelity.ts'; +import { failWithIssues, notApplicable, readArtifact, runGraderAsync, workspacePath } from './graderHarness.ts'; + +void runGraderAsync('the wired datastore matches the one the plan chose', async () => { + const planMarkdown = readArtifact('.azure/project-plan.md'); + const result = await validateDatastoreFidelity(workspacePath('.'), planMarkdown); + if (result.valid) { + return; + } + + const blocking = result.issues.filter(value => !(value.code in DATASTORE_NOT_APPLICABLE_CODES)); + if (blocking.length === 0) { + const reason = result.issues[0]; + notApplicable(reason.code, reason.message); + } + failWithIssues('datastore fidelity errors:', blocking); +}); diff --git a/evals/graders/validate-service-fidelity.ts b/evals/graders/validate-service-fidelity.ts new file mode 100644 index 000000000..058522a64 --- /dev/null +++ b/evals/graders/validate-service-fidelity.ts @@ -0,0 +1,47 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Grades whether the scaffold contains the services the plan promised, and only those. + * + * The contract lives in `evals/src/artifacts/serviceFidelity.ts`, shared with grader + * certification, so the certified path and the executed path cannot drift. + * + * Arm-neutral: it compares a plan to a tree and never looks at how either was produced, so + * a baseline (non-Rails) run that wrote a plan can be scored on it too. + */ + +import { readPlannedProject } from '../src/artifacts/plannedProject.ts'; +import { validateServiceFidelity } from '../src/artifacts/serviceFidelity.ts'; +import { failWithIssues, notApplicable, readArtifact, runGraderAsync, workspacePath } from './graderHarness.ts'; + +const NOT_APPLICABLE_CODES = new Set(['ecosystemNotSupported']); + +void runGraderAsync('scaffolded services match the ones the plan declared', async () => { + const planMarkdown = readArtifact('.azure/project-plan.md'); + const result = await validateServiceFidelity(workspacePath('.'), planMarkdown); + if (result.valid) { + return; + } + + const blocking = result.issues.filter(value => !NOT_APPLICABLE_CODES.has(value.code)); + if (blocking.length === 0) { + const reason = result.issues[0]; + notApplicable(reason.code, reason.message, ecosystemFact(planMarkdown)); + } + failWithIssues('service fidelity errors:', blocking); +}); + +/** + * Attach the plan's own languages to a not-applicable verdict, so an unsupported stack + * collapses to one actionable line — "the Go analyser is missing" — rather than a pile of + * individually uninformative skips that nobody can group. + */ +function ecosystemFact(planMarkdown: string): Record { + const languages = [...new Set(readPlannedProject(planMarkdown).services + .map(service => service.language?.trim().toLowerCase()) + .filter((language): language is string => !!language))]; + return languages.length > 0 ? { plannedLanguages: languages.join('+') } : {}; +} diff --git a/evals/results/grader-certification/offline/report.json b/evals/results/grader-certification/offline/report.json index eea07a63c..c86279dc2 100644 --- a/evals/results/grader-certification/offline/report.json +++ b/evals/results/grader-certification/offline/report.json @@ -1,10 +1,14 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-26T03:25:55.311Z", + "generatedAt": "2026-08-26T05:37:34.441Z", "mode": "offline", "fixtures": [ "sample-agent-output", - "reference-node-fullstack" + "reference-node-fullstack", + "reference-node-multiservice", + "reference-python-api", + "reference-dotnet-api", + "reference-go-unsupported" ], "outcome": "passed", "cases": [ @@ -513,6 +517,309 @@ ], "passed": true, "durationMs": 0 + }, + { + "id": "golden-service-fidelity", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "expected": "passed", + "actual": [], + "passed": true, + "durationMs": 0 + }, + { + "id": "golden-datastore-fidelity", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "expected": "passed", + "actual": [], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-planned-service-dropped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "expected": "plannedServiceMissing", + "actual": [ + "plannedServiceMissing" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-service-invented", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "expected": "unplannedServiceScaffolded", + "actual": [ + "unplannedServiceScaffolded" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-frontend-missing", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "expected": "frontendMissingFromScaffold", + "actual": [ + "frontendMissingFromScaffold", + "plannedServiceMissing", + "serviceFrameworkMismatch" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-frontend-invented", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "expected": "frontendNotPlanned", + "actual": [ + "frontendNotPlanned", + "unexpectedFrontendSection" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-language-swapped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "expected": "serviceLanguageMismatch", + "actual": [ + "serviceLanguageMismatch" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-framework-swapped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "expected": "serviceFrameworkMismatch", + "actual": [ + "serviceFrameworkMismatch" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-plan-declares-no-services", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "service-fidelity", + "expected": "planDeclaresNoServices", + "actual": [ + "planDeclaresNoServices" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-datastore-import-swapped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "expected": "plannedDatastoreNotWired", + "actual": [ + "plannedDatastoreNotWired", + "unplannedDatastoreWired" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-datastore-invented", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "expected": "unplannedDatastoreWired", + "actual": [ + "unplannedDatastoreWired" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-datastore-dependency-dropped", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "expected": "datastoreDependencyMissing", + "actual": [ + "datastoreDependencyMissing" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-resource-never-wired", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "expected": "plannedResourceNotWired", + "actual": [ + "plannedResourceNotWired" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-services-required-table-unreadable", + "tier": "offline", + "fixture": "reference-node-multiservice", + "validator": "datastore-fidelity", + "expected": "plannedResourcesUnreadable", + "actual": [ + "plannedResourcesUnreadable" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "golden-service-fidelity", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "service-fidelity", + "expected": "passed", + "actual": [], + "passed": true, + "durationMs": 0 + }, + { + "id": "golden-datastore-fidelity", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "expected": "passed", + "actual": [], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-datastore-swapped-python", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "expected": "unplannedDatastoreWired", + "actual": [ + "plannedDatastoreNotWired", + "unplannedDatastoreWired" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-datastore-unwired-python", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "expected": "plannedDatastoreNotWired", + "actual": [ + "plannedDatastoreNotWired", + "unplannedDatastoreWired" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-nothing-scaffolded-python", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "expected": "noServicesScaffolded", + "actual": [ + "noServicesScaffolded" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-orm-owns-the-driver-python", + "tier": "offline", + "fixture": "reference-python-api", + "validator": "datastore-fidelity", + "expected": "passed", + "actual": [], + "passed": true, + "durationMs": 0 + }, + { + "id": "golden-service-fidelity", + "tier": "offline", + "fixture": "reference-dotnet-api", + "validator": "service-fidelity", + "expected": "passed", + "actual": [], + "passed": true, + "durationMs": 0 + }, + { + "id": "golden-datastore-fidelity", + "tier": "offline", + "fixture": "reference-dotnet-api", + "validator": "datastore-fidelity", + "expected": "passed", + "actual": [], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-datastore-swapped-dotnet", + "tier": "offline", + "fixture": "reference-dotnet-api", + "validator": "datastore-fidelity", + "expected": "plannedDatastoreNotWired", + "actual": [ + "plannedDatastoreNotWired", + "unplannedDatastoreWired" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "fidelity-orm-owns-the-driver-dotnet", + "tier": "offline", + "fixture": "reference-dotnet-api", + "validator": "datastore-fidelity", + "expected": "passed", + "actual": [], + "passed": true, + "durationMs": 0 + }, + { + "id": "golden-service-fidelity", + "tier": "offline", + "fixture": "reference-go-unsupported", + "validator": "service-fidelity", + "expected": "ecosystemNotSupported", + "actual": [ + "ecosystemNotSupported" + ], + "passed": true, + "durationMs": 0 + }, + { + "id": "golden-datastore-fidelity", + "tier": "offline", + "fixture": "reference-go-unsupported", + "validator": "datastore-fidelity", + "expected": "ecosystemNotSupported", + "actual": [ + "ecosystemNotSupported" + ], + "passed": true, + "durationMs": 0 } ] } diff --git a/evals/results/grader-certification/offline/report.md b/evals/results/grader-certification/offline/report.md index 242601bb1..a831904f6 100644 --- a/evals/results/grader-certification/offline/report.md +++ b/evals/results/grader-certification/offline/report.md @@ -1,9 +1,9 @@ # Copilot on Rails Grader Certification - Mode: `offline` -- Fixtures: `sample-agent-output`, `reference-node-fullstack` +- Fixtures: `sample-agent-output`, `reference-node-fullstack`, `reference-node-multiservice`, `reference-python-api`, `reference-dotnet-api`, `reference-go-unsupported` - Outcome: **PASSED** -- Cases: 44/44 passed +- Cases: 70/70 passed | Case | Fixture | Validator | Expected | Actual | Result | |---|---|---|---|---|---| @@ -51,4 +51,30 @@ | `debug-config-duplicate-task-label` | `reference-node-fullstack` | `debug-config` | `duplicateTaskLabels` | `duplicateTaskLabels, dependsOnUnresolved, dependsOnCycle` | PASS | | `debug-artifacts-extension-recommendations` | `reference-node-fullstack` | `debug-artifacts` | `invalidExtensionRecommendations` | `invalidExtensionRecommendations` | PASS | | `debug-artifacts-redacted-secret` | `reference-node-fullstack` | `debug-artifacts` | `redactedSecretPlaceholder` | `redactedSecretPlaceholder` | PASS | +| `golden-service-fidelity` | `reference-node-multiservice` | `service-fidelity` | `passed` | `passed` | PASS | +| `golden-datastore-fidelity` | `reference-node-multiservice` | `datastore-fidelity` | `passed` | `passed` | PASS | +| `fidelity-planned-service-dropped` | `reference-node-multiservice` | `service-fidelity` | `plannedServiceMissing` | `plannedServiceMissing` | PASS | +| `fidelity-service-invented` | `reference-node-multiservice` | `service-fidelity` | `unplannedServiceScaffolded` | `unplannedServiceScaffolded` | PASS | +| `fidelity-frontend-missing` | `reference-node-multiservice` | `service-fidelity` | `frontendMissingFromScaffold` | `frontendMissingFromScaffold, plannedServiceMissing, serviceFrameworkMismatch` | PASS | +| `fidelity-frontend-invented` | `reference-node-multiservice` | `service-fidelity` | `frontendNotPlanned` | `frontendNotPlanned, unexpectedFrontendSection` | PASS | +| `fidelity-language-swapped` | `reference-node-multiservice` | `service-fidelity` | `serviceLanguageMismatch` | `serviceLanguageMismatch` | PASS | +| `fidelity-framework-swapped` | `reference-node-multiservice` | `service-fidelity` | `serviceFrameworkMismatch` | `serviceFrameworkMismatch` | PASS | +| `fidelity-plan-declares-no-services` | `reference-node-multiservice` | `service-fidelity` | `planDeclaresNoServices` | `planDeclaresNoServices` | PASS | +| `fidelity-datastore-import-swapped` | `reference-node-multiservice` | `datastore-fidelity` | `plannedDatastoreNotWired` | `plannedDatastoreNotWired, unplannedDatastoreWired` | PASS | +| `fidelity-datastore-invented` | `reference-node-multiservice` | `datastore-fidelity` | `unplannedDatastoreWired` | `unplannedDatastoreWired` | PASS | +| `fidelity-datastore-dependency-dropped` | `reference-node-multiservice` | `datastore-fidelity` | `datastoreDependencyMissing` | `datastoreDependencyMissing` | PASS | +| `fidelity-resource-never-wired` | `reference-node-multiservice` | `datastore-fidelity` | `plannedResourceNotWired` | `plannedResourceNotWired` | PASS | +| `fidelity-services-required-table-unreadable` | `reference-node-multiservice` | `datastore-fidelity` | `plannedResourcesUnreadable` | `plannedResourcesUnreadable` | PASS | +| `golden-service-fidelity` | `reference-python-api` | `service-fidelity` | `passed` | `passed` | PASS | +| `golden-datastore-fidelity` | `reference-python-api` | `datastore-fidelity` | `passed` | `passed` | PASS | +| `fidelity-datastore-swapped-python` | `reference-python-api` | `datastore-fidelity` | `unplannedDatastoreWired` | `plannedDatastoreNotWired, unplannedDatastoreWired` | PASS | +| `fidelity-datastore-unwired-python` | `reference-python-api` | `datastore-fidelity` | `plannedDatastoreNotWired` | `plannedDatastoreNotWired, unplannedDatastoreWired` | PASS | +| `fidelity-nothing-scaffolded-python` | `reference-python-api` | `datastore-fidelity` | `noServicesScaffolded` | `noServicesScaffolded` | PASS | +| `fidelity-orm-owns-the-driver-python` | `reference-python-api` | `datastore-fidelity` | `passed` | `passed` | PASS | +| `golden-service-fidelity` | `reference-dotnet-api` | `service-fidelity` | `passed` | `passed` | PASS | +| `golden-datastore-fidelity` | `reference-dotnet-api` | `datastore-fidelity` | `passed` | `passed` | PASS | +| `fidelity-datastore-swapped-dotnet` | `reference-dotnet-api` | `datastore-fidelity` | `plannedDatastoreNotWired` | `plannedDatastoreNotWired, unplannedDatastoreWired` | PASS | +| `fidelity-orm-owns-the-driver-dotnet` | `reference-dotnet-api` | `datastore-fidelity` | `passed` | `passed` | PASS | +| `golden-service-fidelity` | `reference-go-unsupported` | `service-fidelity` | `ecosystemNotSupported` | `ecosystemNotSupported` | PASS | +| `golden-datastore-fidelity` | `reference-go-unsupported` | `datastore-fidelity` | `ecosystemNotSupported` | `ecosystemNotSupported` | PASS | diff --git a/evals/src/artifacts/datastoreFidelity.ts b/evals/src/artifacts/datastoreFidelity.ts new file mode 100644 index 000000000..49234579f --- /dev/null +++ b/evals/src/artifacts/datastoreFidelity.ts @@ -0,0 +1,475 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Does the scaffold wire the datastore the plan chose? + * + * This is the fidelity failure with the worst signal-to-consequence ratio: a project that + * plans PostgreSQL and quietly wires SQLite installs, builds, starts, serves traffic and + * passes every other gate we have. It only fails later, in a place where the plan is no + * longer in the room. Nothing else in the suite can see it. + * + * ## How the comparison is made + * + * Both sides are normalised to a closed set of **families** and compared as families, never + * as strings. The plan's wording and the package registry's wording never match — "Azure + * Database for PostgreSQL Flexible Server" versus `pg` — so any string comparison is really + * a table lookup wearing a disguise, and one that silently returns "no match" for every + * spelling nobody thought of. + * + * ## Evidence + * + * A family is *wired* when the source **imports** a driver for it — not when a manifest + * declares one. A dependency that is installed and never imported is not a datastore, it is + * a leftover, and treating declaration as wiring is what would let "swap the import" pass. + * + * Manifest declarations are still read, for the opposite direction: code that imports a + * driver no manifest declares cannot install, which is its own defect. + * + * The one case where an import cannot be required is an ORM: a project using Prisma or + * SQLAlchemy legitimately never imports `pg`. There, a connection string or provider + * setting naming the family counts instead — but *only* when such an ORM is present, so + * this never degrades into "the tree mentions postgres somewhere". + * + * ## Ecosystems + * + * Node, Python and .NET have driver registries below. Anything else reports + * not-applicable rather than passing: a datastore check that silently approves every Go + * project is indistinguishable from no check at all, which is the failure this suite exists + * to prevent. + */ + +import type { PlannedResource } from './plannedProject.ts'; +import { readPlannedProject } from './plannedProject.ts'; +import type { Ecosystem, ScaffoldTree } from './scaffoldTree.ts'; +import { scanScaffoldTree } from './scaffoldTree.ts'; +import type { ArtifactValidationIssue, ArtifactValidationResult } from './validationTypes.ts'; +import { createValidationResult } from './validationTypes.ts'; + +export type DatastoreFamily = + | 'postgres' | 'mysql' | 'mssql' | 'sqlite' | 'mongodb' | 'cosmos' | 'redis' + | 'blob-storage' | 'table-storage' | 'queue-storage' | 'file' | 'in-memory'; + +/** + * Issue codes that mean "this gate has no opinion here", not "the agent did something + * wrong". The grader maps these to a not-applicable verdict; they must never be reported + * as a product failure. + */ +export const DATASTORE_NOT_APPLICABLE_CODES: Record = { + ecosystemNotSupported: 'ecosystemNotSupported', +}; + +interface FamilySignature { + /** Package names as they appear in a manifest. */ + packages: Partial>; + /** Module names as they appear in an import. Defaults to `packages` when omitted. */ + modules?: Partial>; + /** Modules that ship with the runtime and so can never appear in a manifest. */ + stdlibModules?: Partial>; + /** Connection-string schemes and ORM provider values that name this family. */ + connectionPatterns: RegExp[]; + /** Plan wordings, longest matched first so "cosmos db for mongodb" beats "cosmos db". */ + planAliases: string[]; +} + +const FAMILIES: Record = { + postgres: { + packages: { + node: ['pg', 'postgres', 'pg-promise', '@vercel/postgres'], + python: ['psycopg', 'psycopg2', 'psycopg2-binary', 'asyncpg', 'pg8000'], + dotnet: ['npgsql', 'npgsql.entityframeworkcore.postgresql'], + }, + modules: { python: ['psycopg', 'psycopg2', 'asyncpg', 'pg8000'], dotnet: ['npgsql'] }, + connectionPatterns: [/postgres(ql)?:\/\//i, /provider\s*=\s*["']postgresql["']/i, /usenpgsql/i], + planAliases: ['azure database for postgresql', 'postgresql flexible server', 'postgresql', 'postgres'], + }, + mysql: { + packages: { + node: ['mysql', 'mysql2'], + python: ['pymysql', 'mysqlclient', 'aiomysql', 'mysql-connector-python'], + dotnet: ['mysqlconnector', 'pomelo.entityframeworkcore.mysql'], + }, + modules: { python: ['pymysql', 'mysqldb', 'aiomysql', 'mysql'], dotnet: ['mysqlconnector'] }, + connectionPatterns: [/mysql:\/\//i, /mariadb:\/\//i, /provider\s*=\s*["']mysql["']/i], + planAliases: ['azure database for mysql', 'mariadb', 'mysql'], + }, + mssql: { + packages: { + node: ['mssql', 'tedious'], + python: ['pyodbc', 'pymssql'], + dotnet: ['microsoft.data.sqlclient', 'system.data.sqlclient', 'microsoft.entityframeworkcore.sqlserver'], + }, + modules: { dotnet: ['microsoft.data.sqlclient', 'system.data.sqlclient'] }, + connectionPatterns: [/sqlserver:\/\//i, /\bserver\s*=\s*tcp:/i, /initial\s+catalog\s*=/i, /usesqlserver/i], + planAliases: ['azure sql database', 'azure sql', 'sql server', 'sql database', 'mssql'], + }, + sqlite: { + packages: { + node: ['sqlite3', 'better-sqlite3', 'sqlite'], + python: ['aiosqlite'], + dotnet: ['microsoft.data.sqlite', 'microsoft.entityframeworkcore.sqlite'], + }, + modules: { node: ['sqlite3', 'better-sqlite3', 'sqlite'], dotnet: ['microsoft.data.sqlite'] }, + // `sqlite3` is in the Python standard library and `node:sqlite` in Node's, so neither + // can ever appear in a manifest. Requiring a declaration for them would make the most + // common quiet-swap target the one family we structurally cannot catch. + stdlibModules: { python: ['sqlite3'], node: ['node:sqlite'] }, + connectionPatterns: [/sqlite:\/\//i, /provider\s*=\s*["']sqlite["']/i, /\.sqlite3?\b/i], + planAliases: ['sqlite'], + }, + mongodb: { + packages: { + node: ['mongodb', 'mongoose'], + python: ['pymongo', 'motor', 'beanie'], + dotnet: ['mongodb.driver'], + }, + connectionPatterns: [/mongodb(\+srv)?:\/\//i], + planAliases: ['azure cosmos db for mongodb', 'cosmos db for mongodb', 'mongodb', 'mongo'], + }, + cosmos: { + packages: { + node: ['@azure/cosmos'], + python: ['azure-cosmos'], + dotnet: ['microsoft.azure.cosmos'], + }, + connectionPatterns: [/accountendpoint\s*=/i, /documents\.azure\.com/i], + planAliases: ['azure cosmos db for nosql', 'azure cosmos db', 'cosmos db', 'cosmos'], + }, + redis: { + packages: { + node: ['redis', 'ioredis'], + python: ['redis', 'aioredis'], + dotnet: ['stackexchange.redis'], + }, + connectionPatterns: [/rediss?:\/\//i], + planAliases: ['azure cache for redis', 'redis'], + }, + 'blob-storage': { + packages: { + node: ['@azure/storage-blob'], + python: ['azure-storage-blob'], + dotnet: ['azure.storage.blobs'], + }, + connectionPatterns: [/blob\.core\.windows\.net/i], + planAliases: ['azure blob storage', 'blob storage', 'blob'], + }, + 'table-storage': { + packages: { + node: ['@azure/data-tables'], + python: ['azure-data-tables'], + dotnet: ['azure.data.tables'], + }, + connectionPatterns: [/table\.core\.windows\.net/i], + planAliases: ['azure table storage', 'table storage'], + }, + 'queue-storage': { + packages: { + node: ['@azure/storage-queue'], + python: ['azure-storage-queue'], + dotnet: ['azure.storage.queues'], + }, + connectionPatterns: [/queue\.core\.windows\.net/i], + planAliases: ['azure queue storage', 'queue storage', 'storage queue'], + }, + // Families with no driver to import. They are resolvable so the plan side can name them, + // but wiring is never asserted for them — a JSON file on disk has no import signature. + file: { packages: {}, connectionPatterns: [], planAliases: ['file storage', 'file store', 'local file', 'json file'] }, + 'in-memory': { packages: {}, connectionPatterns: [], planAliases: ['in-memory', 'in memory', 'no datastore required'] }, +}; + +/** Families whose wiring cannot be observed in the tree, so absence proves nothing. */ +const UNOBSERVABLE_FAMILIES = new Set(['file', 'in-memory']); + +/** + * ORMs and query builders that own the driver import on the application's behalf. Their + * presence is what licenses connection-string evidence to stand in for an import. + */ +const ORM_PACKAGES: Record = { + node: ['prisma', '@prisma/client', 'sequelize', 'typeorm', 'knex', 'drizzle-orm', 'objection', 'mikro-orm'], + python: ['sqlalchemy', 'django', 'tortoise-orm', 'peewee', 'alembic', 'sqlmodel'], + dotnet: ['microsoft.entityframeworkcore', 'entityframework', 'dapper'], +}; + +export async function validateDatastoreFidelity( + workspaceRoot: string, + planMarkdown: string, +): Promise { + const issues: ArtifactValidationIssue[] = []; + const plan = readPlannedProject(planMarkdown); + const tree = await scanScaffoldTree(workspaceRoot); + + if (tree.unsupported.length > 0) { + // One unsupported manifest blinds the whole comparison: that service's dependencies + // and imports are both invisible, so any "not wired" verdict here would be the + // harness's gap reported as the agent's fault. See serviceFidelity for the same rule. + const languages = [...new Set(tree.unsupported.map(entry => entry.language))].join(', '); + return createValidationResult([issue('ecosystemNotSupported', tree.unsupported[0].file, + `The tree contains ${languages} manifests, which no dependency analyser covers yet.`)]); + } + + if (!plan.resourcesTableRecognised) { + return createValidationResult([issue( + 'plannedResourcesUnreadable', + '$.servicesRequired', + 'The plan has no readable "Services Required" table (expected an "Azure Service" column), so what it promised cannot be compared with what was built.', + )]); + } + + if (tree.manifests.length === 0) { + // The plan was read from this same workspace, so the tree is staged and genuinely + // contains no project. That is a product failure, not a harness one. + return createValidationResult([issue('noServicesScaffolded', '.', + 'The workspace contains no project manifest of any recognised ecosystem, so nothing can be wired to the resources the plan promised.')]); + } + + const planned = plan.resources.map(resource => ({ resource, family: resolveFamily(resource) })); + const plannedFamilies = new Set(planned.flatMap(entry => entry.family ? [entry.family] : [])); + const wired = collectWiring(tree); + + for (const { resource, family } of planned) { + checkPlannedResource(resource, family, tree, wired, issues); + } + checkUnplannedFamilies(plannedFamilies, wired, issues); + + return createValidationResult(issues); +} + +interface Wiring { + /** Families with an import of one of their drivers, by scope, recording the modules seen. */ + imported: Map }>; + /** Families declared in some manifest. */ + declared: Map; + /** Families named by a connection string or ORM provider setting in the tree. */ + configured: Map; + hasOrm: boolean; +} + +function collectWiring(tree: ScaffoldTree): Wiring { + const wiring: Wiring = { imported: new Map(), declared: new Map(), configured: new Map(), hasOrm: false }; + + for (const dependency of tree.dependencies) { + if (isOrm(dependency.name, dependency.ecosystem)) { + wiring.hasOrm = true; + } + const family = familyForPackage(dependency.name, dependency.ecosystem); + if (family) { + push(wiring.declared, family, dependency.manifest); + } + } + + for (const imported of tree.imports) { + if (isOrm(imported.module, imported.ecosystem)) { + wiring.hasOrm = true; + } + const family = familyForModule(imported.module, imported.ecosystem); + if (!family) { + continue; + } + const entry = wiring.imported.get(family) ?? { runtime: [], test: [], modules: new Set() }; + entry[imported.scope].push(imported.file); + entry.modules.add(imported.module); + wiring.imported.set(family, entry); + } + + for (const [file, content] of tree.fileContents) { + for (const [name, signature] of Object.entries(FAMILIES) as Array<[DatastoreFamily, FamilySignature]>) { + if (signature.connectionPatterns.some(pattern => pattern.test(content))) { + push(wiring.configured, name, file); + } + } + } + + return wiring; +} + +/** + * Recognise an ORM from a package name or an import. + * + * Provider packages matter as much as the core one: the canonical PostgreSQL binding for EF + * Core is `Npgsql.EntityFrameworkCore.PostgreSQL` and the MySQL one is + * `Pomelo.EntityFrameworkCore.MySql`, neither of which starts with `microsoft.`. Matching + * only the Microsoft prefix left a textbook EF Core project looking like it had no ORM, so + * its connection-string evidence was refused and the gate failed a correct app. + */ +function isOrm(name: string, ecosystem: Ecosystem): boolean { + if (ecosystem === 'dotnet' && /entityframeworkcore/.test(name)) { + return true; + } + return ORM_PACKAGES[ecosystem].some(orm => name === orm || name.startsWith(`${orm}.`) || name.startsWith(`${orm}/`)); +} + +function checkPlannedResource( + resource: PlannedResource, + family: DatastoreFamily | undefined, + tree: ScaffoldTree, + wired: Wiring, + issues: ArtifactValidationIssue[], +): void { + checkEnvironmentVariable(resource, tree, issues); + + if (!family || UNOBSERVABLE_FAMILIES.has(family)) { + return; + } + if (!hasAnalyserFor(family, tree.ecosystems)) { + // Silence here would be a pass, so say nothing about wiring rather than approve it. + return; + } + + const imported = wired.imported.get(family); + const configured = wired.configured.get(family); + const declared = wired.declared.get(family); + + // Runtime scope only. A driver imported solely from a test file is not the application's + // datastore — an app whose runtime code never touches PostgreSQL has not wired it, however + // thoroughly its test suite does. The unplanned direction applies the same rule, so the two + // halves of this gate cannot disagree about what "wired" means. + if (imported && imported.runtime.length > 0) { + if (!declared && !isStdlibImport(family, imported.modules)) { + issues.push(issue( + 'datastoreDependencyMissing', + imported.runtime[0], + `${resource.azureService} is imported but no manifest declares a ${family} driver, so the project cannot install.`, + )); + } + return; + } + + // An ORM owns the driver import, so a connection string naming the family is the only + // evidence available and is accepted — but only with an ORM present. + if (wired.hasOrm && configured) { + return; + } + + const testOnly = imported && imported.test.length > 0; + issues.push(issue( + 'plannedDatastoreNotWired', + '$.servicesRequired', + `The plan chose ${resource.azureService} (${family}) but no runtime source file imports a ${family} driver${testOnly + ? ` — the only import is from ${imported.test[0]}, which is test scope` + : declared ? ` — ${declared[0]} declares one that is never imported, which is not wiring` : ''}.`, + )); +} + +/** + * A family the plan never named, wired at runtime, is an invented datastore. + * + * Only runtime-scope imports count. A test suite that spins up in-memory SQLite on a + * PostgreSQL project is ordinary practice, and reporting it would train people to ignore + * this gate — which costs more than the case it would catch. + */ +function checkUnplannedFamilies( + plannedFamilies: Set, + wired: Wiring, + issues: ArtifactValidationIssue[], +): void { + for (const [family, files] of wired.imported) { + if (plannedFamilies.has(family) || files.runtime.length === 0) { + continue; + } + issues.push(issue( + 'unplannedDatastoreWired', + files.runtime[0], + `${family} is wired at runtime but the plan's Services Required table never mentions it.`, + )); + } +} + +/** + * The plan's `Environment Variable` column is a contract between the app and the infra that + * provisions it, and it is entirely stack-neutral to check: the name is a literal string + * that must appear somewhere outside the plan. This catches a promised resource that was + * never wired at all, including resources with no driver signature (Service Bus, Key Vault). + */ +function checkEnvironmentVariable( + resource: PlannedResource, + tree: ScaffoldTree, + issues: ArtifactValidationIssue[], +): void { + const variable = resource.environmentVariable; + if (!variable || !/^[A-Z][A-Z0-9_]*$/.test(variable)) { + return; + } + if (!tree.files.some(file => tree.fileContents.get(file)?.includes(variable))) { + issues.push(issue( + 'plannedResourceNotWired', + '$.servicesRequired', + `The plan promised ${resource.azureService} via ${variable}, but that variable appears nowhere in the scaffolded tree.`, + )); + } +} + +function resolveFamily(resource: PlannedResource): DatastoreFamily | undefined { + // Scheme first, name second, and the order matters more than it looks. "Azure Cosmos DB + // for MongoDB" speaks the MongoDB wire protocol, so the code must import a MongoDB + // driver — a name-first reading resolves it to `cosmos`, finds no `@azure/cosmos`, and + // reports a confident failure against a correctly built project. The local connection + // string is the plan's own statement of which protocol it meant. + const local = resource.localDefault; + if (local) { + for (const [family, signature] of Object.entries(FAMILIES) as Array<[DatastoreFamily, FamilySignature]>) { + if (signature.connectionPatterns.some(pattern => pattern.test(local))) { + return family; + } + } + } + + const name = resource.azureService.toLowerCase(); + const aliases = (Object.entries(FAMILIES) as Array<[DatastoreFamily, FamilySignature]>) + .flatMap(([family, signature]) => signature.planAliases.map(alias => ({ family, alias }))) + .sort((left, right) => right.alias.length - left.alias.length); + return aliases.find(entry => name.includes(entry.alias))?.family; +} + +function familyForPackage(name: string, ecosystem: Ecosystem): DatastoreFamily | undefined { + for (const [family, signature] of Object.entries(FAMILIES) as Array<[DatastoreFamily, FamilySignature]>) { + if ((signature.packages[ecosystem] ?? []).includes(name)) { + return family; + } + } + return undefined; +} + +function familyForModule(module: string, ecosystem: Ecosystem): DatastoreFamily | undefined { + for (const [family, signature] of Object.entries(FAMILIES) as Array<[DatastoreFamily, FamilySignature]>) { + const modules = signature.modules?.[ecosystem] ?? signature.packages[ecosystem] ?? []; + const stdlib = signature.stdlibModules?.[ecosystem] ?? []; + if (modules.includes(module) || stdlib.includes(module)) { + return family; + } + } + return undefined; +} + +function hasAnalyserFor(family: DatastoreFamily, ecosystems: Set): boolean { + const signature = FAMILIES[family]; + return [...ecosystems].some(ecosystem => + (signature.packages[ecosystem]?.length ?? 0) > 0 || (signature.stdlibModules?.[ecosystem]?.length ?? 0) > 0); +} + +/** + * Whether the modules that were actually imported are runtime built-ins, and so could not + * appear in any manifest. + * + * Asking "does this family have a stdlib module in some present ecosystem" instead would + * disable the undeclared-driver check for the entire SQLite family in any Node or Python + * tree — including `better-sqlite3`, which absolutely must be declared. The question has to + * be about the specific import that fired. + */ +function isStdlibImport(family: DatastoreFamily, modules: Set): boolean { + const stdlib = FAMILIES[family].stdlibModules; + if (!stdlib) { + return false; + } + const all = new Set(Object.values(stdlib).flat()); + return [...modules].every(module => all.has(module)); +} + +function push(map: Map, key: T, value: string): void { + map.set(key, [...(map.get(key) ?? []), value]); +} + +function issue(code: string, path: string, message: string): ArtifactValidationIssue { + return { code, path, message }; +} diff --git a/evals/src/artifacts/plannedProject.ts b/evals/src/artifacts/plannedProject.ts new file mode 100644 index 000000000..f93e8e328 --- /dev/null +++ b/evals/src/artifacts/plannedProject.ts @@ -0,0 +1,214 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * The typed model of what `.azure/project-plan.md` *promised*, for the fidelity graders. + * + * This is a reader over the production parser, never a second parser. `projectPlan.ts` + * already validates the plan's shape with `parseScaffoldPlanMarkdown`; this file asks the + * same parse tree a different question — not "is the plan well-formed" but "what did it + * commit to building". A second markdown parser for the same document is precisely the + * drift these evals exist to detect, so every field below is read through the webview + * parser's own query helpers. + * + * Two structural facts about the plan template + * (`resources/agents/azure-project-plan/plan.md`) drive the model, and both are easy to + * get backwards: + * + * 1. **`## N. Services Required` is the *Azure resources* table, not the app's services.** + * Its columns are `Azure Service | Role in App | Environment Variable | Default Value + * (Local) | Classification`. Reading it as an inventory of the app's own services + * yields nonsense like a service named "PostgreSQL". + * 2. **The app's services are the per-service stack sections** — `## N. — ` + * — one per service, each carrying a `Language` row. The template names that row as the + * discriminator the plan webview itself uses to decide whether a section is a stack + * card, so this file uses the same test rather than pattern-matching heading text. + */ + +import { + findKeyValue, + findSection, + parseScaffoldPlanMarkdown, + type ScaffoldPlanSection, + type ScaffoldPlanTableContent, +} from '../../../src/webviews/copilotOnRails/views/utils/parseScaffoldPlanMarkdown.ts'; + +/** + * App Types that describe a project with no browser UI. + * + * Exported because `projectPlan.ts` and the fidelity graders must agree on it exactly: one + * deciding that `Background worker` implies no frontend while the other disagrees would + * make a plan simultaneously valid and unfaithful. + */ +export const NON_VISUAL_APP_TYPES: readonly string[] = ['api only', 'background worker']; + +export type PlannedServiceRole = 'backend' | 'frontend' | 'worker' | 'unknown'; + +export interface PlannedService { + /** The part of the heading before the em dash, e.g. `Backend` in `Backend — Azure Functions`. */ + name: string; + /** The full heading text, used verbatim in failure messages so the row is findable in the plan. */ + heading: string; + role: PlannedServiceRole; + language?: string; + runtime?: string; + framework?: string; + packageManager?: string; +} + +export interface PlannedResource { + azureService: string; + roleInApp?: string; + environmentVariable?: string; + /** `Default Value (Local)` — the local connection string, and the most reliable family signal. */ + localDefault?: string; + classification?: string; +} + +export interface PlannedProject { + appType?: string; + /** + * Whether the plan's App Type implies a browser UI. `undefined` when App Type is absent, + * which is a different thing from "no frontend" and must not be collapsed into `false`. + */ + expectsFrontend?: boolean; + services: PlannedService[]; + resources: PlannedResource[]; + /** + * True when a `Services Required` section exists and carries the documented Azure-resource + * table. False when the section is missing or uses some other shape — in which case + * `resources` is empty because nothing was recognised, not because nothing was required. + */ + resourcesTableRecognised: boolean; +} + +export function readPlannedProject(planMarkdown: string): PlannedProject { + const plan = parseScaffoldPlanMarkdown(planMarkdown); + + const overview = findSection(plan, 'project overview'); + const appType = overview && findKeyValue(overview, 'App Type'); + const services = plan.sections.flatMap(section => { + const service = readServiceSection(section); + return service ? [service] : []; + }); + + const requiredSection = findSection(plan, 'services required'); + const resourceTable = requiredSection && findResourceTable(requiredSection); + + return { + appType, + expectsFrontend: appType === undefined + ? undefined + : !NON_VISUAL_APP_TYPES.includes(appType.trim().toLowerCase()), + services, + resources: resourceTable ? readResources(resourceTable) : [], + resourcesTableRecognised: !!resourceTable, + }; +} + +/** + * A section is a service when it carries a `Language` row. + * + * The plan template defines that row as what turns a section into a stack card in the plan + * webview, so it is the plan's own definition of "this section describes a service" rather + * than a heuristic invented here. Matching on heading text instead would miss every service + * whose name the agent chose freely — which is all of them beyond the first. + */ +function readServiceSection(section: ScaffoldPlanSection): PlannedService | undefined { + const table = section.content.find( + (content): content is ScaffoldPlanTableContent => + content.type === 'table' && !!findComponentRow(content, 'Language'), + ); + if (!table) { + return undefined; + } + const name = section.title.split(/\s+[—–-]\s+/)[0].trim(); + return { + name, + heading: section.title, + role: inferRole(section.title, !!findComponentRow(table, 'Framework')), + language: findComponentRow(table, 'Language'), + runtime: findComponentRow(table, 'Runtime'), + framework: findComponentRow(table, 'Framework'), + packageManager: findComponentRow(table, 'Package Manager'), + }; +} + +/** + * Read a `| **Key** | Value |` row out of a two-column stack table. `parseTableRow` has + * already stripped the bold markers, so the comparison is on plain text. + */ +function findComponentRow(table: ScaffoldPlanTableContent, key: string): string | undefined { + const needle = key.toLowerCase(); + const row = table.rows.find(cells => (cells[0] ?? '').trim().toLowerCase() === needle); + const value = row?.[1]?.trim(); + return value ? value : undefined; +} + +/** + * Infer a service's role from its heading, falling back to the presence of a `Framework` + * row. Role rather than name is what the tree can be matched on: an agent may call its + * backend `API`, `Server` or `Support API`, and all three land in the same place. + */ +function inferRole(heading: string, hasFramework: boolean): PlannedServiceRole { + const text = heading.toLowerCase(); + if (/\b(front[\s-]?end|web|ui|client|portal|spa|site)\b/.test(text)) { + return 'frontend'; + } + if (/\b(worker|job|jobs|background|queue|processor|scheduler|consumer)\b/.test(text)) { + return 'worker'; + } + if (/\b(back[\s-]?end|api|server|service|functions?)\b/.test(text)) { + return 'backend'; + } + // Only a frontend stack section carries a Framework row in the template. + return hasFramework ? 'frontend' : 'unknown'; +} + +function findResourceTable(section: ScaffoldPlanSection): ScaffoldPlanTableContent | undefined { + return section.content.find( + (content): content is ScaffoldPlanTableContent => + content.type === 'table' && columnIndex(content, 'azure service') >= 0, + ); +} + +function readResources(table: ScaffoldPlanTableContent): PlannedResource[] { + const service = columnIndex(table, 'azure service'); + const role = columnIndex(table, 'role'); + const variable = columnIndex(table, 'environment variable'); + const local = columnIndex(table, 'default value'); + const classification = columnIndex(table, 'classification'); + + return table.rows.flatMap(cells => { + const azureService = cell(cells, service); + if (!azureService) { + return []; + } + return [{ + azureService, + roleInApp: cell(cells, role), + environmentVariable: cell(cells, variable), + localDefault: cell(cells, local), + classification: cell(cells, classification), + }]; + }); +} + +function columnIndex(table: ScaffoldPlanTableContent, name: string): number { + return table.headers.findIndex(header => header.toLowerCase().trim().includes(name)); +} + +/** + * Read one cell, discarding the template's own `{placeholder}` braces and code ticks so a + * plan that left a placeholder in place reads as empty rather than as a resource literally + * named `{Blob Storage}`. + */ +function cell(cells: string[], index: number): string | undefined { + if (index < 0) { + return undefined; + } + const value = (cells[index] ?? '').trim().replace(/^[{`]+|[}`]+$/g, '').trim(); + return value && value !== '—' && value !== '-' ? value : undefined; +} diff --git a/evals/src/artifacts/projectPlan.ts b/evals/src/artifacts/projectPlan.ts index 6f928da86..694bcd547 100644 --- a/evals/src/artifacts/projectPlan.ts +++ b/evals/src/artifacts/projectPlan.ts @@ -10,6 +10,7 @@ import { } from '../../../src/webviews/copilotOnRails/views/utils/parseScaffoldPlanMarkdown.ts'; import type { ArtifactValidationIssue, ArtifactValidationResult } from './validationTypes.ts'; import { createValidationResult } from './validationTypes.ts'; +import { NON_VISUAL_APP_TYPES } from './plannedProject.ts'; const requiredSections = [ 'project overview', @@ -66,7 +67,7 @@ export function validateProjectPlanArtifact( const projectOverview = findSection(plan, 'project overview'); const appType = projectOverview && findKeyValue(projectOverview, 'App Type'); const designSystem = findSection(plan, 'design system'); - const hasNonVisualAppType = !!appType && ['api only', 'background worker'].includes(appType.toLowerCase()); + const hasNonVisualAppType = !!appType && NON_VISUAL_APP_TYPES.includes(appType.toLowerCase()); const requiresDesignSystem = !hasNonVisualAppType; if (requiresDesignSystem && !designSystem) { issues.push(issue('missingSection', '$', 'Missing required "design system" section.')); diff --git a/evals/src/artifacts/scaffoldTree.ts b/evals/src/artifacts/scaffoldTree.ts new file mode 100644 index 000000000..5b826738a --- /dev/null +++ b/evals/src/artifacts/scaffoldTree.ts @@ -0,0 +1,344 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * A read of the scaffolded tree, shared by the fidelity validators. + * + * The fidelity gates ask "did the agent build what it planned", which means every one of + * them needs the same three facts about the tree: which ecosystems are present, what each + * manifest declares, and what the source actually imports. Gathering those once — and in + * one place — is what stops each gate growing its own half-right notion of, say, whether + * `psycopg2-binary` and `psycopg2` are the same package. + * + * ## Ecosystem coverage + * + * Analysed today: **Node/TypeScript** (`package.json`), **Python** (`requirements.txt`, + * `pyproject.toml`, `Pipfile`) and **.NET** (`*.csproj`, `Directory.Packages.props`). + * + * Recognised-but-not-analysed: Go, Rust, Java, Ruby, PHP. These are listed explicitly + * rather than ignored so a caller can tell "a stack we do not cover" apart from "no project + * here at all" — the first is a coverage hole to report as not-applicable, the second is + * very likely a workspace that was never staged. A gate that cannot tell those apart ends + * up either silently passing every unsupported stack or blaming the agent for the harness. + */ + +import { promises as fs } from 'node:fs'; +import * as path from 'node:path'; + +export type Ecosystem = 'node' | 'python' | 'dotnet'; + +/** Ecosystems recognised by their manifest but with no dependency analyser yet. */ +export const UNSUPPORTED_MANIFESTS: Record = { + 'go.mod': 'go', + 'cargo.toml': 'rust', + 'pom.xml': 'java', + 'build.gradle': 'java', + 'build.gradle.kts': 'java', + gemfile: 'ruby', + 'composer.json': 'php', +}; + +const IGNORED_DIRECTORIES = new Set([ + 'node_modules', 'dist', 'build', '.git', '.next', 'out', 'coverage', + 'bin', 'obj', '__pycache__', '.venv', 'venv', '.pytest_cache', '.azure', +]); + +/** + * Where a piece of evidence came from, which decides how much weight it carries. + * + * A driver imported only from a test file is not the application's datastore — an + * in-memory SQLite in a test suite is normal on a PostgreSQL project — so the two are + * kept apart rather than merged into one "the tree mentions sqlite" signal. + */ +export type EvidenceScope = 'runtime' | 'test'; + +export interface DeclaredDependency { + name: string; + ecosystem: Ecosystem; + scope: EvidenceScope; + manifest: string; +} + +export interface ImportedModule { + module: string; + ecosystem: Ecosystem; + scope: EvidenceScope; + file: string; +} + +export interface ScaffoldTree { + root: string; + /** Every non-ignored file, workspace-relative with `/` separators. */ + files: string[]; + ecosystems: Set; + manifests: string[]; + /** Manifests for stacks with no analyser, mapped to a language id (`go`, `rust`, …). */ + unsupported: Array<{ file: string; language: string }>; + dependencies: DeclaredDependency[]; + imports: ImportedModule[]; + /** + * Text content of every readable text file, keyed by relative path. Cached because + * several gates scan the same small tree for different strings, and re-reading it per + * gate is both slower and a chance for two gates to disagree about what is in it. + */ + fileContents: Map; +} + +const MANIFESTS: Array<{ match: (name: string) => boolean; ecosystem: Ecosystem }> = [ + { match: name => name === 'package.json', ecosystem: 'node' }, + { match: name => name === 'requirements.txt' || name === 'pyproject.toml' || name === 'pipfile', ecosystem: 'python' }, + { match: name => name.endsWith('.csproj') || name === 'directory.packages.props', ecosystem: 'dotnet' }, +]; + +const SOURCE_ECOSYSTEMS: Record = { + '.ts': 'node', '.tsx': 'node', '.js': 'node', '.jsx': 'node', '.mjs': 'node', '.cjs': 'node', + '.py': 'python', + '.cs': 'dotnet', +}; + +/** Extensions that identify a source language, for the plan's per-service `Language` row. */ +export const LANGUAGE_EXTENSIONS: Record = { + typescript: ['.ts', '.tsx'], + javascript: ['.js', '.jsx', '.mjs', '.cjs'], + python: ['.py'], + 'c#': ['.cs'], + csharp: ['.cs'], + go: ['.go'], + java: ['.java'], +}; + +export async function scanScaffoldTree(root: string): Promise { + const files = await listFiles(root, root); + const tree: ScaffoldTree = { + root, + files, + ecosystems: new Set(), + manifests: [], + unsupported: [], + dependencies: [], + imports: [], + fileContents: new Map(), + }; + + for (const file of files) { + if (!isTextFile(file)) { + continue; + } + const content = await readFileSafe(path.join(root, file)); + // Skip anything implausibly large for scaffolded source: a generated bundle or a + // committed data dump would otherwise dominate every content scan. + if (content !== undefined && content.length <= 512 * 1024) { + tree.fileContents.set(file, content); + } + } + + for (const file of files) { + const name = path.basename(file).toLowerCase(); + const unsupported = UNSUPPORTED_MANIFESTS[name]; + if (unsupported) { + tree.unsupported.push({ file, language: unsupported }); + continue; + } + const manifest = MANIFESTS.find(candidate => candidate.match(name)); + if (!manifest) { + continue; + } + tree.manifests.push(file); + tree.ecosystems.add(manifest.ecosystem); + tree.dependencies.push(...readDependencies(tree.fileContents.get(file), file, manifest.ecosystem)); + } + + for (const file of files) { + const ecosystem = SOURCE_ECOSYSTEMS[path.extname(file).toLowerCase()]; + const content = ecosystem && tree.fileContents.get(file); + if (ecosystem && content) { + tree.imports.push(...readImports(file, content, ecosystem)); + } + } + + return tree; +} + +/** Binary-ish and lockfile paths that no gate reads for content. */ +export function isTextFile(file: string): boolean { + return !/\.(png|jpe?g|gif|ico|svg|webp|woff2?|ttf|eot|pdf|zip|gz|tgz)$/i.test(file) + && !/(^|\/)(package-lock\.json|yarn\.lock|pnpm-lock\.yaml|poetry\.lock)$/i.test(file); +} + +async function listFiles(root: string, directory: string): Promise { + const entries = await readDirectorySafe(directory); + const files: string[] = []; + for (const entry of entries) { + if (entry.name.startsWith('.') && entry.name !== '.env.example' && entry.name !== '.env') { + continue; + } + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) { + if (!IGNORED_DIRECTORIES.has(entry.name.toLowerCase())) { + files.push(...await listFiles(root, absolute)); + } + } else if (entry.isFile()) { + files.push(path.relative(root, absolute).split(path.sep).join('/')); + } + } + return files; +} + +/** + * A file is test-scope when its path says so. Kept deliberately broad — a false "this is a + * test" only weakens the *unplanned datastore* direction, whereas a false "this is runtime" + * would report a test dependency as the application's datastore, which is a wrong failure + * shown to a user rather than a missed one. + */ +export function scopeForFile(file: string): EvidenceScope { + const lower = file.toLowerCase(); + return /(^|\/)(tests?|__tests__|spec|e2e)(\/|$)/.test(lower) + || /\.(test|spec)\.[a-z]+$/.test(lower) + || /(^|\/)conftest\.py$/.test(lower) + || /_test\.py$/.test(lower) + ? 'test' + : 'runtime'; +} + +function readDependencies(content: string | undefined, file: string, ecosystem: Ecosystem): DeclaredDependency[] { + if (content === undefined) { + return []; + } + const record = (name: string, scope: EvidenceScope): DeclaredDependency => + ({ name: normalizePackageName(name, ecosystem), ecosystem, scope, manifest: file }); + + if (ecosystem === 'node') { + return readNodeDependencies(content, file, record); + } + if (ecosystem === 'dotnet') { + return [...content.matchAll(/ record(match[1], scopeForFile(file))); + } + return readPythonDependencies(content, file, record); +} + +function readNodeDependencies( + content: string, + file: string, + record: (name: string, scope: EvidenceScope) => DeclaredDependency, +): DeclaredDependency[] { + let parsed: { dependencies?: Record; devDependencies?: Record }; + try { + parsed = JSON.parse(content); + } catch { + return []; + } + const fileScope = scopeForFile(file); + return [ + ...Object.keys(parsed.dependencies ?? {}).map(name => record(name, fileScope)), + // A driver in devDependencies is test tooling until proven otherwise. + ...Object.keys(parsed.devDependencies ?? {}).map(name => record(name, 'test')), + ]; +} + +function readPythonDependencies( + content: string, + file: string, + record: (name: string, scope: EvidenceScope) => DeclaredDependency, +): DeclaredDependency[] { + const dependencies: DeclaredDependency[] = []; + for (const line of content.split('\n')) { + const text = line.trim(); + if (!text || text.startsWith('#')) { + continue; + } + // The distribution name may be terminated by a version specifier, a closing quote + // (PEP 621 arrays), a comma, an environment marker, a comment or end of line. + // Requiring a version operator silently dropped every unpinned dependency, which is + // the normal shape of a generated `pyproject.toml` — and a dropped dependency reads + // downstream as "the driver was never declared", failing a correct project. + const match = /^["']?([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]*\])?\s*(?:[=<>!~^]|["',;#]|\s|$)/.exec(text); + if (match) { + dependencies.push(record(match[1], scopeForFile(file))); + } + } + return dependencies; +} + +/** + * Normalise a package name so a manifest entry and an import can be compared. + * + * Python is the case that matters: PyPI treats `-`, `_` and `.` as equivalent and is + * case-insensitive, so `psycopg2-binary`, `psycopg2_binary` and `Psycopg2-Binary` are one + * package. Comparing raw strings makes a correctly-wired project look unwired. + */ +export function normalizePackageName(name: string, ecosystem: Ecosystem): string { + const trimmed = name.trim(); + if (ecosystem === 'python') { + return trimmed.toLowerCase().replace(/[._]/g, '-'); + } + if (ecosystem === 'dotnet') { + return trimmed.toLowerCase(); + } + return trimmed.toLowerCase(); +} + +function readImports(file: string, content: string, ecosystem: Ecosystem): ImportedModule[] { + const scope = scopeForFile(file); + const modules = new Set(); + + if (ecosystem === 'node') { + for (const match of content.matchAll(/(?:from\s+|require\(\s*|import\s+)['"]([^'"]+)['"]/g)) { + const specifier = match[1]; + if (specifier.startsWith('.') || specifier.startsWith('/')) { + continue; + } + modules.add(nodePackageRoot(specifier)); + } + } else if (ecosystem === 'python') { + // Horizontal whitespace only in the character classes: a `\s` here matches newlines, + // which makes `import os` swallow the rest of the file as one enormous module name. + for (const match of content.matchAll(/^[ \t]*(?:from[ \t]+([A-Za-z0-9_.]+)[ \t]+import|import[ \t]+([A-Za-z0-9_.,\t ]+))/gm)) { + for (const name of (match[1] ?? match[2] ?? '').split(',')) { + // Splitting on whitespace-or-dot reduces `numpy as np` and `os.path` alike. + const root = name.trim().split(/[\s.]/)[0]; + if (root) { + modules.add(normalizePackageName(root, 'python')); + } + } + } + } else { + for (const match of content.matchAll(/^\s*(?:global\s+)?using\s+(?:static\s+)?([A-Za-z0-9_.]+)\s*;/gm)) { + modules.add(match[1].toLowerCase()); + } + } + + return [...modules].map(module => ({ module, ecosystem, scope, file })); +} + +/** + * Reduce an import specifier to the package that would appear in a manifest: + * `@azure/storage-blob/foo` → `@azure/storage-blob`, `pg/lib/x` → `pg`. `node:` builtins + * keep their prefix, because `node:sqlite` is a datastore that no manifest can declare. + */ +function nodePackageRoot(specifier: string): string { + if (specifier.startsWith('node:')) { + return specifier.toLowerCase(); + } + const segments = specifier.split('/'); + const root = specifier.startsWith('@') ? segments.slice(0, 2).join('/') : segments[0]; + return root.toLowerCase(); +} + +export async function readFileSafe(file: string): Promise { + try { + return await fs.readFile(file, 'utf8'); + } catch { + return undefined; + } +} + +export async function readDirectorySafe(directory: string): Promise { + try { + return await fs.readdir(directory, { withFileTypes: true }); + } catch { + return []; + } +} diff --git a/evals/src/artifacts/serviceFidelity.ts b/evals/src/artifacts/serviceFidelity.ts new file mode 100644 index 000000000..687bc56e2 --- /dev/null +++ b/evals/src/artifacts/serviceFidelity.ts @@ -0,0 +1,395 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Does the scaffold contain the services the plan promised — and only those? + * + * Every other scaffold gate grades the tree against itself: it builds, the frontend is + * embeddable, the API seam holds. All of them pass a project that dropped a service, because + * two working services look exactly like two working services. The plan is the only artifact + * that knows there should have been three. + * + * ## Matching is by role, not by name + * + * The plan names services freely — `Backend`, `API`, `Support API`, `Ticket Service` — while + * the tree names directories by convention (`services/api`). Matching those by string is a + * guessing game that fails on the agent's first reasonable naming choice and reports a + * correct scaffold as missing a service. Role (`backend` / `frontend` / `worker`) is derived + * on both sides from evidence, and a name similarity is used only to break ties when several + * services share a role. + * + * ## Both directions + * + * A dropped service and an invented one are different failures with the same cause, and only + * checking the first would let an agent satisfy the gate by scaffolding everything it could + * think of. Library packages (`services/shared` and friends) are excluded from the invented + * side: they are not deployable services and no plan lists them. + * + * Stack coverage matches `scaffoldTree.ts`. Language checks work anywhere a file extension + * identifies a language; framework checks are Node-only today and are skipped — never + * passed — elsewhere. + */ + +import * as path from 'node:path'; +import { discoverFrontendDirectory } from './frontendScaffold.ts'; +import type { PlannedProject, PlannedService, PlannedServiceRole } from './plannedProject.ts'; +import { readPlannedProject } from './plannedProject.ts'; +import type { ScaffoldTree } from './scaffoldTree.ts'; +import { LANGUAGE_EXTENSIONS, scanScaffoldTree } from './scaffoldTree.ts'; +import type { ArtifactValidationIssue, ArtifactValidationResult } from './validationTypes.ts'; +import { createValidationResult } from './validationTypes.ts'; + +/** Directory names that are shared libraries rather than deployable services. */ +const LIBRARY_NAMES = new Set([ + 'shared', 'common', 'types', 'lib', 'libs', 'core', 'utils', 'models', 'contracts', 'sdk', 'config', +]); + +/** Framework names the plan may choose, mapped to the package that proves them. */ +const FRAMEWORK_PACKAGES: Record = { + react: 'react', + vue: 'vue', + angular: '@angular/core', + svelte: 'svelte', + solid: 'solid-js', + next: 'next', + 'next.js': 'next', + nuxt: 'nuxt', + vite: 'vite', + remix: '@remix-run/react', + astro: 'astro', +}; + +interface ActualService { + /** Workspace-relative directory, or `.` for a single-service repository. */ + directory: string; + name: string; + role: PlannedServiceRole | 'library'; +} + +export async function validateServiceFidelity( + workspaceRoot: string, + planMarkdown: string, +): Promise { + const issues: ArtifactValidationIssue[] = []; + const plan = readPlannedProject(planMarkdown); + const tree = await scanScaffoldTree(workspaceRoot); + + const notApplicable = describeUnanalysableTree(tree); + if (notApplicable) { + return createValidationResult([notApplicable]); + } + + if (plan.services.length === 0) { + // Not a silent skip: a plan with no service sections cannot be compared to anything, + // and treating "nothing promised" as "everything delivered" is how a gate ends up + // green for every malformed plan it was built to catch. + return createValidationResult([issue( + 'planDeclaresNoServices', + '$.services', + 'The plan declares no services — expected one "## N. — " section per service, each with a Language row.', + )]); + } + + if (tree.manifests.length === 0) { + // Reachable only because `.azure/project-plan.md` was read from this same workspace, + // so the tree is staged and really does contain no project. A plan promising services + // over an empty tree is the most extreme form of the failure this gate exists for, + // and must be a product failure rather than a not-applicable shrug. + return createValidationResult([issue( + 'noServicesScaffolded', + '.', + `The plan declares ${plan.services.length} service(s), but the workspace contains no project manifest of any recognised ecosystem.`, + )]); + } + + const actual = await discoverActualServices(workspaceRoot, tree); + // Union of two detectors on purpose. `discoverFrontendDirectory` is shared with the + // frontend-scaffold and build gates, so the three always agree about *which* directory is + // the frontend — but it only searches the root and the conventional group folders, so a + // sanctioned flat layout (`frontend/` at the root) is invisible to it. Falling back to + // this file's own role classification stops that layout being reported as a missing UI. + const frontendDirectory = await discoverFrontendDirectory(workspaceRoot); + const frontendService = actual.find(service => service.role === 'frontend'); + const frontendLocation = frontendDirectory + ? relative(workspaceRoot, frontendDirectory) + : frontendService?.directory; + + checkFrontendIntent(plan, frontendLocation, issues); + const pairs = matchServices(plan.services, actual, workspaceRoot, frontendDirectory, issues); + for (const pair of pairs) { + checkLanguage(pair.planned, pair.actual, tree, issues); + checkFramework(pair.planned, pair.actual, tree, issues); + } + + return createValidationResult(issues); +} + +/** + * Report a tree this gate cannot see all of. + * + * A single unsupported manifest anywhere is enough to stop, even when other services are in + * covered ecosystems. A Go service is invisible to `discoverActualServices` (its `go.mod` is + * not in `tree.manifests`) and its imports are invisible to dependency analysis, so a mixed + * repo would otherwise report the Go service as missing and its datastore as unwired — the + * harness blaming the agent for a gap in the harness. Refusing to answer for the whole tree + * is blunt, but a partially-blind gate that reports confident failures is worse than one + * that says it cannot see. + */ +function describeUnanalysableTree(tree: ScaffoldTree): ArtifactValidationIssue | undefined { + if (tree.unsupported.length === 0) { + return undefined; + } + const languages = [...new Set(tree.unsupported.map(entry => entry.language))].join(', '); + return issue('ecosystemNotSupported', tree.unsupported[0].file, + `The tree contains ${languages} manifests, which no analyser covers yet; the rest of the tree cannot be graded without them.`); +} + +/** + * Compare the plan's frontend intent with the tree, in both directions. + * + * `App Type` is the plan's own statement of intent and is used when present. When it is + * absent the presence of a frontend stack section stands in — rather than defaulting to + * "no frontend expected", which would silently excuse a missing UI on every plan whose + * overview lost a row. + */ +function checkFrontendIntent( + plan: PlannedProject, + frontendLocation: string | undefined, + issues: ArtifactValidationIssue[], +): void { + const planHasFrontendSection = plan.services.some(service => service.role === 'frontend'); + const expectsFrontend = plan.expectsFrontend ?? planHasFrontendSection; + + if (expectsFrontend && !frontendLocation) { + issues.push(issue('frontendMissingFromScaffold', 'services/web', + `The plan's App Type "${plan.appType ?? 'unspecified'}" promises a browser UI, but no frontend project was scaffolded.`)); + } + if (!expectsFrontend && frontendLocation) { + issues.push(issue('frontendNotPlanned', frontendLocation, + `A frontend was scaffolded at ${frontendLocation}, but the plan's App Type "${plan.appType ?? 'unspecified'}" describes a project with no UI.`)); + } + if (plan.expectsFrontend === true && !planHasFrontendSection) { + issues.push(issue('plannedFrontendSectionMissing', '$.services', + `App Type "${plan.appType}" implies a UI, but the plan has no frontend service section.`)); + } + if (plan.expectsFrontend === false && planHasFrontendSection) { + issues.push(issue('unexpectedFrontendSection', '$.services', + `App Type "${plan.appType}" describes a project with no UI, but the plan declares a frontend service.`)); + } +} + +interface ServicePair { + planned: PlannedService; + actual: ActualService; +} + +/** + * Pair planned services with scaffolded directories, reporting whatever is left over on + * either side. Matching prefers same-role candidates and falls back to any unclaimed + * directory. + * + * The fallback is what keeps role classification from manufacturing failures. Roles are + * inferred from directory names and file evidence, which is right most of the time and + * confidently wrong the rest — rename `services/worker` to `services/notifications` and it + * reads as a backend. Without the fallback that single misread produces *two* failures on a + * correct project: the planned worker looks missing and the directory looks invented. So a + * count mismatch is reported and a role disagreement is not: surplus and shortfall are facts + * about the tree, whereas a role is this file's opinion about it. + */ +function matchServices( + planned: PlannedService[], + actual: ActualService[], + workspaceRoot: string, + frontendDirectory: string | undefined, + issues: ArtifactValidationIssue[], +): ServicePair[] { + const unclaimed = actual.filter(service => service.role !== 'library'); + const pairs: ServicePair[] = []; + + for (const service of planned) { + const sameRole = unclaimed.filter(candidate => candidate.role === service.role); + const pool = sameRole.length > 0 ? sameRole : unclaimed; + const chosen = [...pool].sort((left, right) => similarity(right.name, service.name) - similarity(left.name, service.name))[0]; + if (!chosen) { + issues.push(issue('plannedServiceMissing', '$.services', + `The plan declares "${service.heading}" (${service.role}), but only ${actual.filter(value => value.role !== 'library').length} service director(ies) were scaffolded.`)); + continue; + } + unclaimed.splice(unclaimed.indexOf(chosen), 1); + pairs.push({ planned: service, actual: chosen }); + } + + for (const leftover of unclaimed) { + // The discovered frontend is never "invented": a plan that promises a UI without a + // frontend stack section is already reported as `plannedFrontendSectionMissing`, and + // reporting the same plan defect twice, once as an invented service, is noise. + if (frontendDirectory && path.resolve(workspaceRoot, leftover.directory) === frontendDirectory) { + continue; + } + issues.push(issue('unplannedServiceScaffolded', leftover.directory, + `${leftover.directory} looks like a deployable ${leftover.role} service, but the plan never declares it.`)); + } + + return pairs; +} + +function checkLanguage( + planned: PlannedService, + actual: ActualService, + tree: ScaffoldTree, + issues: ArtifactValidationIssue[], +): void { + const expected = LANGUAGE_EXTENSIONS[(planned.language ?? '').trim().toLowerCase()]; + if (!expected) { + return; + } + const files = filesUnder(tree, actual.directory); + if (files.some(file => expected.includes(path.extname(file).toLowerCase()))) { + return; + } + const found = [...new Set(files + .map(file => languageForExtension(path.extname(file).toLowerCase())) + .filter((language): language is string => !!language))]; + if (found.length === 0) { + // No source in a recognised language at all: say nothing rather than guess. An empty + // or asset-only directory is a different failure, and `plannedServiceMissing` or the + // build gate is the honest place for it. + return; + } + issues.push(issue('serviceLanguageMismatch', actual.directory, + `The plan builds "${planned.heading}" in ${planned.language}, but ${actual.directory} contains ${found.join(', ')} source.`)); +} + +function checkFramework( + planned: PlannedService, + actual: ActualService, + tree: ScaffoldTree, + issues: ArtifactValidationIssue[], +): void { + if (!planned.framework) { + return; + } + const expected = planned.framework + .split(/[+,/]|\s+and\s+/) + .map(token => FRAMEWORK_PACKAGES[token.trim().toLowerCase()]) + .filter((packageName): packageName is string => !!packageName); + if (expected.length === 0) { + return; + } + // Framework detection reads Node manifests, so a non-Node service is skipped rather than + // passed — an unchecked property must never be reported as a satisfied one. + const manifests = tree.dependencies.filter(dependency => + dependency.ecosystem === 'node' && isUnder(dependency.manifest, actual.directory)); + if (manifests.length === 0) { + return; + } + const declared = new Set(manifests.map(dependency => dependency.name)); + const missing = expected.filter(packageName => !declared.has(packageName)); + if (missing.length > 0) { + issues.push(issue('serviceFrameworkMismatch', actual.directory, + `The plan builds "${planned.heading}" with ${planned.framework}, but ${actual.directory} declares no ${missing.join(', ')}.`)); + } +} + +/** + * Candidate service directories are derived from where manifests actually are, not from a + * fixed list of group folders. + * + * The scaffold agent's own instructions say paths are examples and an existing workspace + * layout wins, so `backend/`, `frontend/`, `worker/` at the root is a sanctioned outcome. A + * `services/`-only search reports every one of those as missing — the gate failing a project + * for following its instructions. + */ +async function discoverActualServices(workspaceRoot: string, tree: ScaffoldTree): Promise { + const manifestDirectories = [...new Set(tree.manifests.map(manifest => { + const directory = path.posix.dirname(manifest); + return directory === '' ? '.' : directory; + }))]; + + // A root manifest alongside others is the monorepo workspace root, not a service of its + // own; and a manifest nested inside another candidate belongs to that candidate. + const roots = manifestDirectories.filter(candidate => + !(candidate === '.' && manifestDirectories.length > 1) + && !manifestDirectories.some(other => other !== candidate && other !== '.' && isUnder(candidate, other))); + + const services: ActualService[] = []; + for (const directory of roots) { + const name = directory === '.' ? path.basename(workspaceRoot) : path.posix.basename(directory); + services.push({ directory, name, role: classifyRole(name, directory, tree) }); + } + return services; +} + +function classifyRole(name: string, directory: string, tree: ScaffoldTree): PlannedServiceRole | 'library' { + if (LIBRARY_NAMES.has(name.toLowerCase())) { + return 'library'; + } + const files = filesUnder(tree, directory); + // Reuse the frontend scorer rather than testing for index.html: it understands Next and + // Angular layouts that have no index.html, and having two disagreeing notions of "this is + // the frontend" inside one file is how a correct Next.js app gets reported as missing. + if (looksLikeFrontend(directory, tree)) { + return 'frontend'; + } + if (/\b(worker|jobs?|background|queue|processor|scheduler|consumer)\b/i.test(name)) { + return 'worker'; + } + // Trigger bindings are the strongest available evidence of a worker: an HTTP-only service + // and a queue-consuming one are otherwise identical from the outside. + const hasTriggerBinding = files.some(file => { + const content = tree.fileContents.get(file); + return !!content && /(queueTrigger|timerTrigger|serviceBusTrigger|blobTrigger|app\.(?:timer|queue|service_bus))/i.test(content); + }); + if (hasTriggerBinding) { + return 'worker'; + } + if (/\b(api|backend|server|service|functions?)\b/i.test(name)) { + return 'backend'; + } + // No positive evidence. `unknown` matches anything during pairing, which is the right + // outcome — an unrecognised name is not grounds for declaring a service missing. + return 'unknown'; +} + +/** Browser-project evidence: a loadable HTML entry point or a frontend framework dependency. */ +function looksLikeFrontend(directory: string, tree: ScaffoldTree): boolean { + const files = filesUnder(tree, directory); + if (files.some(file => /(^|\/)(index\.html|app\/page\.(t|j)sx?)$/i.test(file))) { + return true; + } + const frameworks = new Set(['react', 'react-dom', 'vue', 'svelte', '@angular/core', 'next', 'nuxt', 'astro']); + return tree.dependencies.some(dependency => + dependency.ecosystem === 'node' && isUnder(dependency.manifest, directory) && frameworks.has(dependency.name)); +} + +function filesUnder(tree: ScaffoldTree, directory: string): string[] { + return tree.files.filter(file => isUnder(file, directory)); +} + +function isUnder(file: string, directory: string): boolean { + return directory === '.' ? true : file === directory || file.startsWith(`${directory}/`); +} + +function languageForExtension(extension: string): string | undefined { + return Object.entries(LANGUAGE_EXTENSIONS).find(([, extensions]) => extensions.includes(extension))?.[0]; +} + +/** Crude token overlap, used only to break ties between same-role candidates. */ +function similarity(left: string, right: string): number { + const a = left.toLowerCase(); + const b = right.toLowerCase(); + if (a === b) { + return 3; + } + return a.includes(b) || b.includes(a) ? 2 : 0; +} + +function relative(workspaceRoot: string, target: string): string { + return path.relative(workspaceRoot, target).split(path.sep).join('/') || '.'; +} + +function issue(code: string, path: string, message: string): ArtifactValidationIssue { + return { code, path, message }; +} diff --git a/evals/src/graderCertification.ts b/evals/src/graderCertification.ts index 4b463181c..1a7392702 100644 --- a/evals/src/graderCertification.ts +++ b/evals/src/graderCertification.ts @@ -6,9 +6,11 @@ import { promises as fs } from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; +import { NON_VISUAL_APP_TYPES } from './artifacts/plannedProject.ts'; import type { PlanGateState } from './artifacts/planEvaluation.ts'; import { validatePlanEvaluationContract } from './artifacts/planEvaluation.ts'; import { validateDebugArtifacts } from './artifacts/debugArtifacts.ts'; +import { validateDatastoreFidelity } from './artifacts/datastoreFidelity.ts'; import { validateFrontendScaffold } from './artifacts/frontendScaffold.ts'; import { validateIntegrationPlanArtifact } from './artifacts/integrationPlan.ts'; import { validateDebugLaunchConfiguration } from './artifacts/launchConfig.ts'; @@ -16,6 +18,7 @@ import { validateLocalDebugPlanArtifact } from './artifacts/localDebugPlan.ts'; import { validatePreviewArtifacts } from './artifacts/preview.ts'; import { validateProjectPlanArtifact } from './artifacts/projectPlan.ts'; import { validateRequirementsArtifact } from './artifacts/requirements.ts'; +import { validateServiceFidelity } from './artifacts/serviceFidelity.ts'; import type { ArtifactValidationResult } from './artifacts/validationTypes.ts'; import type { CorEvaluationScenario } from './scenario.ts'; import { validateScenario } from './scenario.ts'; @@ -25,6 +28,16 @@ interface CertificationFixture { path: string; description: string; offlineValidators: string[]; + /** + * Golden-case expectations other than "passed", keyed by validator id. + * + * A gate that answers "not applicable" needs its escape hatch certified like any other + * verdict. Without this, the only expressible golden expectation is a clean pass — so a + * fixture on an unsupported stack would certify green, which is indistinguishable from + * the gate having silently approved it. Pinning the exact code here means that if + * someone later makes unsupported stacks fall through to a pass, certification goes red. + */ + offlineExpectations?: Record; } interface CertificationManifest { @@ -159,7 +172,8 @@ async function certifyFixture( const golden = await runOfflineValidators(root, scenario, fixture.offlineValidators); for (const validator of validators) { const result = golden.get(validator) ?? ['validatorNotExecuted']; - cases.push(createCase(`golden-${validator}`, 'offline', fixture.id, validator, 'passed', result)); + const expected = fixture.offlineExpectations?.[validator] ?? 'passed'; + cases.push(createCase(`golden-${validator}`, 'offline', fixture.id, validator, expected, result)); } for (const mutation of mutations) { cases.push(await withMutatedFixture(root, mutation, async workspace => { @@ -209,6 +223,10 @@ const OFFLINE_VALIDATORS: Record< }, preview: async workspace => validatePreviewArtifacts(path.join(workspace, '.azure', '.preview-temp')), 'frontend-scaffold': async workspace => validateFrontendScaffold(workspace), + 'service-fidelity': async workspace => + validateServiceFidelity(workspace, await readArtifact(workspace, '.azure/project-plan.md')), + 'datastore-fidelity': async workspace => + validateDatastoreFidelity(workspace, await readArtifact(workspace, '.azure/project-plan.md')), 'debug-plan': async workspace => validateLocalDebugPlanArtifact(await readArtifact(workspace, '.azure/vscode-debug-plan.md'), { expectedStatus: 'Implemented', @@ -256,7 +274,7 @@ async function readPlanGateState( ): Promise<{ expectedFrontend: boolean; generatedFrontend: boolean; gate: PlanGateState }> { const expectedFrontend = (scenario.tags.frontend ?? 'none') !== 'none'; const appType = /^\*\*App Type\*\*\s*:\s*(.+)$/im.exec(projectPlan)?.[1].trim().toLowerCase(); - const generatedFrontend = !!appType && !['api only', 'background worker'].includes(appType); + const generatedFrontend = !!appType && !NON_VISUAL_APP_TYPES.includes(appType); return { expectedFrontend, generatedFrontend, @@ -291,7 +309,10 @@ async function withMutatedFixture( if (mutation.file) { const filePath = path.join(workspace, mutation.file); if (mutation.operation === 'delete') { - await fs.rm(filePath); + // Recursive so a mutation can delete a whole service directory — "the plan + // declared three services and the scaffold has two" is not expressible by + // removing a single file. + await fs.rm(filePath, { recursive: true }); } else { const content = await fs.readFile(filePath, 'utf8'); if (mutation.operation === 'replace') { From 738ce96301d834fa20cfb4a841949e43dde7d57b Mon Sep 17 00:00:00 2001 From: alexweininger Date: Tue, 25 Aug 2026 22:42:01 -0700 Subject: [PATCH 2/5] Switch the not-applicable verdict to exit 3 and adopt the shared marker shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit 0 was wrong, and the reason it was wrong is worth recording. It was chosen on the grounds that the stderr marker made an N/A detectable — but detection is not correction. MSBench writes exitCode = 0 as passed: true, `resolved` is computed from it, and the run-analysis site and Kusto publish that number. A separate report saying "not applicable" cannot correct a headline that says green, because nobody investigates green. Exit 3 makes the raw score pessimistic and recoverable; exit 0 makes it optimistic and unrecoverable. A red run carrying a NOT_APPLICABLE marker is explainable in seconds. Also aligns with the marker shape the runtime-gates session landed: NOT_APPLICABLE gate= class= reason= detail="…" `class` replaces the earlier outOfScope/notAttempted split and is now passed explicitly by the caller rather than looked up in a central registry, so each family of gates owns its reason vocabulary and adding a code is never a line two sessions edit at once. The central registry is removed. `ecosystemNotSupported` is classified `environmentGap`, not `outOfScope`: a Go project is not a scenario with nothing to test, it is one we are failing to test. The remedy is "write the analyser", not "unwire the gate", and under exit 3 that distinction is what tells a wiring complaint apart from a true statement that we are not testing something we claim to. Certification is unaffected at 70/70 — it grades issue codes, not exit codes, so the Go fixture still pins the N/A path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evals/graders/graderHarness.ts | 111 ++++++++---------- evals/graders/validate-datastore-fidelity.ts | 10 +- evals/graders/validate-service-fidelity.ts | 23 ++-- .../grader-certification/offline/report.json | 2 +- evals/src/artifacts/datastoreFidelity.ts | 12 +- 5 files changed, 77 insertions(+), 81 deletions(-) diff --git a/evals/graders/graderHarness.ts b/evals/graders/graderHarness.ts index 75380be74..c231713ed 100644 --- a/evals/graders/graderHarness.ts +++ b/evals/graders/graderHarness.ts @@ -27,62 +27,46 @@ export const EXIT_PRODUCT_FAILURE = 1; export const EXIT_GRADER_ERROR = 3; /** - * A gate that has no opinion about this workspace exits **0**, not 3. + * A gate that has no opinion about this workspace exits **3**, and says why on stderr. * - * Exit 3 means "do not trust this result — the harness broke". A not-applicable verdict is - * the opposite: the gate ran, understood the input, and confidently concluded the property - * it grades is absent here. Collapsing the two would spend the only signal that isolates - * harness faults on cases that are working correctly, and would turn every gate red on - * every stack it does not yet cover — which makes the rational move "wire each gate only - * where it definitely applies", defeating the point of having a not-applicable path at all. + * The earlier design exited 0 on the grounds that the marker below made the verdict + * detectable. It is detectable — but detection is not correction. MSBench writes + * `exitCode = 0` as `passed: true`, `resolved` is computed from it, and the run-analysis + * site and the Kusto `resolved` rate publish that number. A separate report saying "not + * applicable" cannot correct a headline that says green, because nobody investigates green. * - * The safety mechanism is therefore NOT the exit code, it is `NOT_APPLICABLE:` on stderr. - * A gate that returns not-applicable *without* emitting the marker is worse than either - * exit code, because it is then genuinely undetectable — it reports a pass forever and - * nobody investigates a passing gate. Emitting the marker is part of every gate's contract. - */ -export const NOT_APPLICABLE_EXIT_CODE = EXIT_PASS; - -/** - * Why a not-applicable verdict happened, in the only distinction that changes what someone - * should *do* about it: + * So the choice is between two kinds of wrong: exit 3 makes the raw score **pessimistic and + * recoverable** — a red run carrying a `NOT_APPLICABLE` marker is explainable in seconds — + * while exit 0 makes it **optimistic and unrecoverable**. A permanently-green gate is the + * vacuous-gate failure wearing a new hat, and an inflated green is permanent and invisible. * - * - `outOfScope` — the subject genuinely lacks the property being graded (a backend-only - * project has no frontend to check). A gate that is always `outOfScope` is dead weight: - * delete it or re-target it. - * - `notAttempted` — the gate wanted to run and could not (missing tool, unstaged tree, - * analyser not written yet). A gate that is always `notAttempted` is a **coverage hole**, - * not dead weight: fix the environment or implement the analyser. Deleting it would be - * exactly the wrong response. + * The objection that exit 3 makes a gate costly to wire broadly is real but points the other + * way: applicability is a *wiring-time* decision, declared in `stacks/.yaml`, not + * something a runtime verdict should be used to paper over. * - * The two demand opposite remedies, so a reason code that lands in the wrong bucket sends - * whoever reads the health report in the wrong direction. + * Which makes the marker more load-bearing, not less: red is now the not-applicable path, so + * the marker is what turns a red from "mysterious failure" into "known gap, here is the fix". */ -export type NotApplicableClass = 'outOfScope' | 'notAttempted'; +export const NOT_APPLICABLE_EXIT_CODE = EXIT_GRADER_ERROR; /** - * Every reason code, with its class. + * What kind of gap a not-applicable verdict describes. The two demand opposite responses, so + * a verdict in the wrong bucket sends whoever reads the run in the wrong direction. + * + * - `outOfScope` — the scenario has nothing for this gate to test. Under exit 3 an + * `outOfScope` red is a complaint about the **wiring**: this gate should not be attached + * to this stack, and the fix is in configuration. + * - `environmentGap` — the gate would apply, but a prerequisite is missing (a tool the + * machine lacks, an analyser nobody has written yet). An `environmentGap` red is a true + * statement that we are not testing something we claim to test, and is *correct* to stay + * red until it is fixed. * - * A registry rather than a free string because a reason code must not be able to *default* - * into a bucket: `emitNotApplicable` rejects an unregistered code, so classifying a new - * reason is a required step rather than something you can forget. It is a plain object - * rather than a union type deliberately — adding a member is a new line, which merges - * cleanly across the several sessions adding codes, where a one-line union would conflict. + * Note what is deliberately absent: "we tried and it did not work" is neither of these. That + * is a product failure and must exit 1. A reason code that quietly means "the thing was + * supposed to work and did not" makes a real bug self-suppressing, and files it under "no + * scenario ever exercised this" — which is the most expensive way to lose a defect. */ -export const NOT_APPLICABLE_REASONS: Record = { - /** The tree has manifests, but only for an ecosystem no analyser covers yet. */ - ecosystemNotSupported: 'notAttempted', - /** - * No project manifest of any recognised ecosystem anywhere in the tree. - * - * For a gate that has *not* already read an artifact out of the same workspace, this most - * likely means the tree was never staged, which is a harness fault. A gate that reached - * this point after successfully reading, say, `.azure/project-plan.md` from that same - * workspace knows the tree is staged, so for it the same observation means the agent - * shipped nothing — a product failure, and it should say so rather than use this code. - */ - noProjectManifestFound: 'notAttempted', -}; +export type NotApplicableClass = 'outOfScope' | 'environmentGap'; /** Raised for a bad artifact — anything else thrown is treated as a harness fault. */ export class ProductFailure extends Error { } @@ -139,27 +123,35 @@ export function failWithIssues(summary: string, issues: ArtifactValidationIssue[ /** Thrown to end a grader with a not-applicable verdict; see `NOT_APPLICABLE_EXIT_CODE`. */ export class NotApplicable extends Error { + readonly gate: string; + readonly classification: NotApplicableClass; readonly reason: string; readonly detail: string; - /** Extra structured `key=value` pairs, e.g. `{ ecosystem: 'go' }`. Never prose. */ - readonly facts: Record; - constructor(reason: string, detail: string, facts: Record = {}) { + constructor(gate: string, classification: NotApplicableClass, reason: string, detail: string) { super(detail); + this.gate = gate; + this.classification = classification; this.reason = reason; this.detail = detail; - this.facts = facts; } } /** * End the grader with a not-applicable verdict. * - * `reason` must be registered in `NOT_APPLICABLE_REASONS`; an unregistered code throws, - * which surfaces as a grader error rather than being quietly emitted with a guessed class. + * `classification` is a required argument rather than something looked up from a shared + * table, so a new reason code cannot be introduced without deciding what should be done + * about it — and so that each family of gates owns its own reason vocabulary instead of the + * several sessions adding codes all editing one registry line. */ -export function notApplicable(reason: string, detail: string, facts: Record = {}): never { - throw new NotApplicable(reason, detail, facts); +export function skipAsNotApplicable( + gate: string, + classification: NotApplicableClass, + reason: string, + detail: string, +): never { + throw new NotApplicable(gate, classification, reason, detail); } /** @@ -199,13 +191,8 @@ export async function runGraderAsync(name: string, body: () => Promise): P function exitForError(name: string, error: unknown): never { const gate = gateId(); if (error instanceof NotApplicable) { - const classification = NOT_APPLICABLE_REASONS[error.reason]; - if (!classification) { - console.error(`GRADER ERROR: gate=${gate} — ${name} reported unregistered not-applicable reason "${error.reason}"`); - process.exit(EXIT_GRADER_ERROR); - } - const facts = Object.entries(error.facts).map(([key, value]) => ` ${key}=${value}`).join(''); - console.error(`NOT_APPLICABLE: gate=${gate} reason=${error.reason} class=${classification}${facts} detail="${error.detail.replace(/"/g, "'")}"`); + console.error(`NOT_APPLICABLE gate=${error.gate} class=${error.classification} reason=${error.reason} detail="${error.detail.replace(/"/g, "'")}"`); + console.error(`SKIP: gate=${error.gate} — ${name} did not apply here; see the NOT_APPLICABLE line above.`); process.exit(NOT_APPLICABLE_EXIT_CODE); } if (error instanceof ProductFailure) { diff --git a/evals/graders/validate-datastore-fidelity.ts b/evals/graders/validate-datastore-fidelity.ts index cdf5bedef..00167b1e2 100644 --- a/evals/graders/validate-datastore-fidelity.ts +++ b/evals/graders/validate-datastore-fidelity.ts @@ -14,12 +14,12 @@ * * On a stack with no dependency analyser this reports **not-applicable**, never a pass — * a datastore check that silently approves every Go project is indistinguishable from no - * check at all. See `NOT_APPLICABLE_EXIT_CODE` in the harness for why that still exits 0 - * and why the stderr marker, not the exit code, is the safety mechanism. + * check at all. That verdict exits 3 and carries a `NOT_APPLICABLE` marker naming the gap; + * see `NOT_APPLICABLE_EXIT_CODE` in the harness for why red-and-explained beats green. */ import { DATASTORE_NOT_APPLICABLE_CODES, validateDatastoreFidelity } from '../src/artifacts/datastoreFidelity.ts'; -import { failWithIssues, notApplicable, readArtifact, runGraderAsync, workspacePath } from './graderHarness.ts'; +import { failWithIssues, gateId, readArtifact, runGraderAsync, skipAsNotApplicable, workspacePath } from './graderHarness.ts'; void runGraderAsync('the wired datastore matches the one the plan chose', async () => { const planMarkdown = readArtifact('.azure/project-plan.md'); @@ -30,8 +30,8 @@ void runGraderAsync('the wired datastore matches the one the plan chose', async const blocking = result.issues.filter(value => !(value.code in DATASTORE_NOT_APPLICABLE_CODES)); if (blocking.length === 0) { - const reason = result.issues[0]; - notApplicable(reason.code, reason.message); + const skipped = result.issues[0]; + skipAsNotApplicable(gateId(), DATASTORE_NOT_APPLICABLE_CODES[skipped.code], skipped.code, skipped.message); } failWithIssues('datastore fidelity errors:', blocking); }); diff --git a/evals/graders/validate-service-fidelity.ts b/evals/graders/validate-service-fidelity.ts index 058522a64..0155a8d0d 100644 --- a/evals/graders/validate-service-fidelity.ts +++ b/evals/graders/validate-service-fidelity.ts @@ -15,9 +15,14 @@ import { readPlannedProject } from '../src/artifacts/plannedProject.ts'; import { validateServiceFidelity } from '../src/artifacts/serviceFidelity.ts'; -import { failWithIssues, notApplicable, readArtifact, runGraderAsync, workspacePath } from './graderHarness.ts'; +import { failWithIssues, gateId, readArtifact, runGraderAsync, skipAsNotApplicable, workspacePath } from './graderHarness.ts'; -const NOT_APPLICABLE_CODES = new Set(['ecosystemNotSupported']); +/** + * This family's reason vocabulary, with the class each code implies. Kept local rather than + * in a shared registry so adding a fidelity reason never collides with another gate family + * doing the same thing — but still a table, so a code cannot reach the marker unclassified. + */ +const FIDELITY_NOT_APPLICABLE = { ecosystemNotSupported: 'environmentGap' } as const; void runGraderAsync('scaffolded services match the ones the plan declared', async () => { const planMarkdown = readArtifact('.azure/project-plan.md'); @@ -26,22 +31,22 @@ void runGraderAsync('scaffolded services match the ones the plan declared', asyn return; } - const blocking = result.issues.filter(value => !NOT_APPLICABLE_CODES.has(value.code)); + const blocking = result.issues.filter(value => !(value.code in FIDELITY_NOT_APPLICABLE)); if (blocking.length === 0) { - const reason = result.issues[0]; - notApplicable(reason.code, reason.message, ecosystemFact(planMarkdown)); + const reason = result.issues[0].code as keyof typeof FIDELITY_NOT_APPLICABLE; + skipAsNotApplicable(gateId(), FIDELITY_NOT_APPLICABLE[reason], reason, describeSkip(planMarkdown, result.issues[0].message)); } failWithIssues('service fidelity errors:', blocking); }); /** - * Attach the plan's own languages to a not-applicable verdict, so an unsupported stack - * collapses to one actionable line — "the Go analyser is missing" — rather than a pile of + * Name the languages the plan asked for alongside the reason, so a coverage hole collapses + * to one actionable line — "the Go analyser is missing" — rather than a pile of * individually uninformative skips that nobody can group. */ -function ecosystemFact(planMarkdown: string): Record { +function describeSkip(planMarkdown: string, message: string): string { const languages = [...new Set(readPlannedProject(planMarkdown).services .map(service => service.language?.trim().toLowerCase()) .filter((language): language is string => !!language))]; - return languages.length > 0 ? { plannedLanguages: languages.join('+') } : {}; + return languages.length > 0 ? `${message} Plan languages: ${languages.join(', ')}.` : message; } diff --git a/evals/results/grader-certification/offline/report.json b/evals/results/grader-certification/offline/report.json index c86279dc2..65501d32a 100644 --- a/evals/results/grader-certification/offline/report.json +++ b/evals/results/grader-certification/offline/report.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-26T05:37:34.441Z", + "generatedAt": "2026-08-26T05:41:49.337Z", "mode": "offline", "fixtures": [ "sample-agent-output", diff --git a/evals/src/artifacts/datastoreFidelity.ts b/evals/src/artifacts/datastoreFidelity.ts index 49234579f..3d4f440a6 100644 --- a/evals/src/artifacts/datastoreFidelity.ts +++ b/evals/src/artifacts/datastoreFidelity.ts @@ -54,11 +54,15 @@ export type DatastoreFamily = /** * Issue codes that mean "this gate has no opinion here", not "the agent did something - * wrong". The grader maps these to a not-applicable verdict; they must never be reported - * as a product failure. + * wrong", mapped to the class each implies. The grader turns these into a not-applicable + * verdict; they must never be reported as a product failure. + * + * `environmentGap` rather than `outOfScope`: a Go project is not a scenario with nothing to + * test, it is one we are failing to test. The remedy is "write the analyser", not "unwire + * the gate", and the class is what tells those apart. */ -export const DATASTORE_NOT_APPLICABLE_CODES: Record = { - ecosystemNotSupported: 'ecosystemNotSupported', +export const DATASTORE_NOT_APPLICABLE_CODES: Record = { + ecosystemNotSupported: 'environmentGap', }; interface FamilySignature { From 31b6d0917be9b4bc7c97ff7f5f490020b3acae5d Mon Sep 17 00:00:00 2001 From: alexweininger Date: Tue, 25 Aug 2026 22:42:36 -0700 Subject: [PATCH 3/5] Make the not-applicable verdict louder, since red is now the N/A path Under exit 3 a human reads this on a failed run, and the expensive mistake is concluding the red says something about the agent's output. Name the class in prose and state plainly that nothing in the verdict is evidence about the generated app. Mirrors the wording the runtime gates use. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evals/graders/graderHarness.ts | 8 +++++++- evals/results/grader-certification/offline/report.json | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/evals/graders/graderHarness.ts b/evals/graders/graderHarness.ts index c231713ed..a48c6a406 100644 --- a/evals/graders/graderHarness.ts +++ b/evals/graders/graderHarness.ts @@ -192,7 +192,13 @@ function exitForError(name: string, error: unknown): never { const gate = gateId(); if (error instanceof NotApplicable) { console.error(`NOT_APPLICABLE gate=${error.gate} class=${error.classification} reason=${error.reason} detail="${error.detail.replace(/"/g, "'")}"`); - console.error(`SKIP: gate=${error.gate} — ${name} did not apply here; see the NOT_APPLICABLE line above.`); + console.error(`SKIP: gate=${error.gate} — ${name} did not apply here.`); + console.error(error.classification === 'environmentGap' + ? ' This gate applies to this project but could not run, so we are not testing something we claim to test.' + : ' This project has nothing for this gate to test, which is a question about how the gate is wired.'); + // Under exit 3 a human reads this on a red run, and the expensive mistake is + // concluding the red says something about the agent's output. It says nothing. + console.error(' Nothing here is evidence about the generated app.'); process.exit(NOT_APPLICABLE_EXIT_CODE); } if (error instanceof ProductFailure) { diff --git a/evals/results/grader-certification/offline/report.json b/evals/results/grader-certification/offline/report.json index 65501d32a..1061a8df2 100644 --- a/evals/results/grader-certification/offline/report.json +++ b/evals/results/grader-certification/offline/report.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-26T05:41:49.337Z", + "generatedAt": "2026-08-26T05:42:30.387Z", "mode": "offline", "fixtures": [ "sample-agent-output", From 99f3fc3465fd95f5378a5b8347ac47bc3539dd4f Mon Sep 17 00:00:00 2001 From: alexweininger Date: Tue, 25 Aug 2026 22:48:01 -0700 Subject: [PATCH 4/5] Rename the not-applicable class environmentGap -> coverageGap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `environmentGap` said the machine was at fault. For the case these gates actually emit it — no analyser for this ecosystem — the fix is unwritten code in this repository, so whoever triaged the red would inspect the container, find nothing wrong, and conclude the marker was broken. A name that sends the reader to the wrong place suppresses its own investigation, which is the same defect the runtime session found when it renamed frontendServerNotStarted. `class=` now answers exactly one question: is this gate dead weight, or a hole worth closing? The test for a new reason code is whether it gets fixed by unwiring the gate or by closing a hole. Who closes it — install a binary vs write an analyser — is a difference of backlog, and `reason=` already carries that as a closed groupable vocabulary. A third class would make `class=` answer two questions at once. The shared harness prose is now owner-neutral for the same reason, with the repository-vs-machine specifics moved into each gate's `detail`, which the gate owns. Baking install-a-binary wording into the harness would have reintroduced the misdirection one layer down. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evals/graders/graderHarness.ts | 27 +++++++++++++++------- evals/graders/validate-service-fidelity.ts | 2 +- evals/src/artifacts/datastoreFidelity.ts | 8 +++---- evals/src/artifacts/serviceFidelity.ts | 2 +- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/evals/graders/graderHarness.ts b/evals/graders/graderHarness.ts index a48c6a406..fa2f55628 100644 --- a/evals/graders/graderHarness.ts +++ b/evals/graders/graderHarness.ts @@ -50,23 +50,34 @@ export const EXIT_GRADER_ERROR = 3; export const NOT_APPLICABLE_EXIT_CODE = EXIT_GRADER_ERROR; /** - * What kind of gap a not-applicable verdict describes. The two demand opposite responses, so - * a verdict in the wrong bucket sends whoever reads the run in the wrong direction. + * What kind of gap a not-applicable verdict describes. `class=` answers exactly one + * question — **is this gate dead weight, or a hole worth closing?** — because that is the + * question that changes what someone does about it. The test for a new reason code is: does + * this get fixed by *unwiring the gate*, or by *closing a hole*? * * - `outOfScope` — the scenario has nothing for this gate to test. Under exit 3 an * `outOfScope` red is a complaint about the **wiring**: this gate should not be attached - * to this stack, and the fix is in configuration. - * - `environmentGap` — the gate would apply, but a prerequisite is missing (a tool the - * machine lacks, an analyser nobody has written yet). An `environmentGap` red is a true + * to this stack, and the fix is in configuration. This must stay reachable — a gate that + * is out of scope on every stack we run should be deleted, and saying so is half the point + * of measuring gate health at all. + * - `coverageGap` — the gate applies here and could not run. A `coverageGap` red is a true * statement that we are not testing something we claim to test, and is *correct* to stay * red until it is fixed. * + * `coverageGap` deliberately does not say *who* closes the gap. Its predecessor was called + * `environmentGap`, which implied the machine was at fault; for a missing analyser the fix + * is unwritten code in this repository, so whoever triaged the red would go inspect the + * container, find nothing wrong, and conclude the marker was broken. A name that sends the + * reader to the wrong place suppresses its own investigation. The owner — install a binary + * versus write an analyser — is a difference of backlog, and `reason=` already carries it as + * a closed, groupable vocabulary. + * * Note what is deliberately absent: "we tried and it did not work" is neither of these. That * is a product failure and must exit 1. A reason code that quietly means "the thing was * supposed to work and did not" makes a real bug self-suppressing, and files it under "no * scenario ever exercised this" — which is the most expensive way to lose a defect. */ -export type NotApplicableClass = 'outOfScope' | 'environmentGap'; +export type NotApplicableClass = 'outOfScope' | 'coverageGap'; /** Raised for a bad artifact — anything else thrown is treated as a harness fault. */ export class ProductFailure extends Error { } @@ -193,8 +204,8 @@ function exitForError(name: string, error: unknown): never { if (error instanceof NotApplicable) { console.error(`NOT_APPLICABLE gate=${error.gate} class=${error.classification} reason=${error.reason} detail="${error.detail.replace(/"/g, "'")}"`); console.error(`SKIP: gate=${error.gate} — ${name} did not apply here.`); - console.error(error.classification === 'environmentGap' - ? ' This gate applies to this project but could not run, so we are not testing something we claim to test.' + console.error(error.classification === 'coverageGap' + ? ' This gate applies here but could not run, so we are not testing something we claim to test. This is a gap to close, not a gate to unwire.' : ' This project has nothing for this gate to test, which is a question about how the gate is wired.'); // Under exit 3 a human reads this on a red run, and the expensive mistake is // concluding the red says something about the agent's output. It says nothing. diff --git a/evals/graders/validate-service-fidelity.ts b/evals/graders/validate-service-fidelity.ts index 0155a8d0d..9e297c497 100644 --- a/evals/graders/validate-service-fidelity.ts +++ b/evals/graders/validate-service-fidelity.ts @@ -22,7 +22,7 @@ import { failWithIssues, gateId, readArtifact, runGraderAsync, skipAsNotApplicab * in a shared registry so adding a fidelity reason never collides with another gate family * doing the same thing — but still a table, so a code cannot reach the marker unclassified. */ -const FIDELITY_NOT_APPLICABLE = { ecosystemNotSupported: 'environmentGap' } as const; +const FIDELITY_NOT_APPLICABLE = { ecosystemNotSupported: 'coverageGap' } as const; void runGraderAsync('scaffolded services match the ones the plan declared', async () => { const planMarkdown = readArtifact('.azure/project-plan.md'); diff --git a/evals/src/artifacts/datastoreFidelity.ts b/evals/src/artifacts/datastoreFidelity.ts index 3d4f440a6..07321b009 100644 --- a/evals/src/artifacts/datastoreFidelity.ts +++ b/evals/src/artifacts/datastoreFidelity.ts @@ -57,12 +57,12 @@ export type DatastoreFamily = * wrong", mapped to the class each implies. The grader turns these into a not-applicable * verdict; they must never be reported as a product failure. * - * `environmentGap` rather than `outOfScope`: a Go project is not a scenario with nothing to + * `coverageGap` rather than `outOfScope`: a Go project is not a scenario with nothing to * test, it is one we are failing to test. The remedy is "write the analyser", not "unwire * the gate", and the class is what tells those apart. */ -export const DATASTORE_NOT_APPLICABLE_CODES: Record = { - ecosystemNotSupported: 'environmentGap', +export const DATASTORE_NOT_APPLICABLE_CODES: Record = { + ecosystemNotSupported: 'coverageGap', }; interface FamilySignature { @@ -210,7 +210,7 @@ export async function validateDatastoreFidelity( // harness's gap reported as the agent's fault. See serviceFidelity for the same rule. const languages = [...new Set(tree.unsupported.map(entry => entry.language))].join(', '); return createValidationResult([issue('ecosystemNotSupported', tree.unsupported[0].file, - `The tree contains ${languages} manifests, which no dependency analyser covers yet.`)]); + `This gate has no dependency analyser for ${languages} yet, so it cannot tell whether the planned datastore was wired. The fix is unwritten code in evals/src/artifacts/datastoreFidelity.ts, not a missing tool on this machine.`)]); } if (!plan.resourcesTableRecognised) { diff --git a/evals/src/artifacts/serviceFidelity.ts b/evals/src/artifacts/serviceFidelity.ts index 687bc56e2..b0834f660 100644 --- a/evals/src/artifacts/serviceFidelity.ts +++ b/evals/src/artifacts/serviceFidelity.ts @@ -143,7 +143,7 @@ function describeUnanalysableTree(tree: ScaffoldTree): ArtifactValidationIssue | } const languages = [...new Set(tree.unsupported.map(entry => entry.language))].join(', '); return issue('ecosystemNotSupported', tree.unsupported[0].file, - `The tree contains ${languages} manifests, which no analyser covers yet; the rest of the tree cannot be graded without them.`); + `This gate has no analyser for ${languages} yet, so it cannot see that part of the tree and will not guess about the rest. The fix is unwritten code in evals/src/artifacts/scaffoldTree.ts, not a missing tool on this machine.`); } /** From 5c073a5b6999b5875e3ea848f0fd24fcf683b969 Mon Sep 17 00:00:00 2001 From: alexweininger Date: Tue, 25 Aug 2026 22:53:25 -0700 Subject: [PATCH 5/5] JSON-encode the marker's detail field so it survives a parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two dialects of one shared field: this branch rewrote `"` to `'`, the runtime gates used JSON.stringify. Only the second round-trips. Rewriting quotes silently corrupts any detail containing one — and details legitimately carry shell commands, so the runtime gates' Functions detail already does — while looking perfectly fine in the log. "detail= is never parsed" was the reason to fix it rather than leave it: the gate-health reader is being written against this line now, and a field that cannot round-trip is a latent bug that fires the moment someone does the obvious thing. JSON.stringify supplies the surrounding quotes, so the field is a JSON string literal and escapes newlines too. Verified against a detail carrying quotes, backticks and a newline. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evals/graders/graderHarness.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/evals/graders/graderHarness.ts b/evals/graders/graderHarness.ts index fa2f55628..f38d1f8ae 100644 --- a/evals/graders/graderHarness.ts +++ b/evals/graders/graderHarness.ts @@ -202,7 +202,12 @@ export async function runGraderAsync(name: string, body: () => Promise): P function exitForError(name: string, error: unknown): never { const gate = gateId(); if (error instanceof NotApplicable) { - console.error(`NOT_APPLICABLE gate=${error.gate} class=${error.classification} reason=${error.reason} detail="${error.detail.replace(/"/g, "'")}"`); + // `detail` is JSON-encoded rather than quote-substituted so the field survives a + // parser: JSON.stringify supplies the surrounding quotes and escapes embedded quotes + // and newlines, where rewriting `"` to `'` silently corrupts any detail that contains + // one — and details legitimately carry shell commands. "This field is never parsed" + // is true right up until someone writes the reader, which is happening now. + console.error(`NOT_APPLICABLE gate=${error.gate} class=${error.classification} reason=${error.reason} detail=${JSON.stringify(error.detail)}`); console.error(`SKIP: gate=${error.gate} — ${name} did not apply here.`); console.error(error.classification === 'coverageGap' ? ' This gate applies here but could not run, so we are not testing something we claim to test. This is a gap to close, not a gate to unwire.'