Add fidelity gates: did the agent build what it planned?

evals/grader-certification/manifest.json

@@ -23,6 +23,46 @@
                 "debug-config",
                 "debug-artifacts"
             ]
+        },
+        {
+            "id": "reference-node-multiservice",
+            "path": "evals/grader-certification/reference-node-multiservice",
+            "description": "A plan and a scaffold that agree: three declared services (API, web, worker) plus a shared library, wiring the PostgreSQL, Blob Storage and Queue Storage resources the plan's Services Required table promises. The fidelity validators compare a plan to a tree, so they need a fixture whose plan actually describes its own tree.",
+            "offlineValidators": [
+                "service-fidelity",
+                "datastore-fidelity"
+            ]
+        },
+        {
+            "id": "reference-python-api",
+            "path": "evals/grader-certification/reference-python-api",
+            "description": "The same contract on a Python stack \u2014 psycopg against requirements.txt. Exists so 'works across ecosystems' is executed rather than asserted: a datastore check that only understands npm would pass this fixture's golden case while being unable to fail any mutation of it.",
+            "offlineValidators": [
+                "service-fidelity",
+                "datastore-fidelity"
+            ]
+        },
+        {
+            "id": "reference-dotnet-api",
+            "path": "evals/grader-certification/reference-dotnet-api",
+            "description": "The same contract on a .NET stack \u2014 Npgsql declared in a .csproj and imported with a using directive, neither of which resembles a package.json.",
+            "offlineValidators": [
+                "service-fidelity",
+                "datastore-fidelity"
+            ]
+        },
+        {
+            "id": "reference-go-unsupported",
+            "path": "evals/grader-certification/reference-go-unsupported",
+            "description": "A stack no analyser covers. Its golden case must report ecosystemNotSupported rather than passing: a silent pass on an unsupported stack is the vacuous-gate failure, and this is the case that stops the not-applicable escape hatch quietly becoming green.",
+            "offlineValidators": [
+                "service-fidelity",
+                "datastore-fidelity"
+            ],
+            "offlineExpectations": {
+                "service-fidelity": "ecosystemNotSupported",
+                "datastore-fidelity": "ecosystemNotSupported"
+            }
         }
     ],
     "mutations": [
@@ -406,6 +446,198 @@
             "replacement": "DATABASE_URL=postgres://golden:******@localhost:5432/golden",
             "expectedCode": "redactedSecretPlaceholder",
             "search": "PORT=7071"
+        },
+        {
+            "id": "fidelity-planned-service-dropped",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "service-fidelity",
+            "file": "services/worker",
+            "operation": "delete",
+            "expectedCode": "plannedServiceMissing"
+        },
+        {
+            "id": "fidelity-service-invented",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "service-fidelity",
+            "file": ".azure/project-plan.md",
+            "operation": "replace",
+            "search": "## 4. Worker \u2014 Background Jobs\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |\n| **Runtime** | Node |\n| **Package Manager** | npm |\n| **Test Runner** | vitest |\n| **Test Command** | npm test |\n| **Orchestration** | docker-compose |\n\n",
+            "replacement": "",
+            "expectedCode": "unplannedServiceScaffolded"
+        },
+        {
+            "id": "fidelity-frontend-missing",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "service-fidelity",
+            "file": "services/web",
+            "operation": "delete",
+            "expectedCode": "frontendMissingFromScaffold"
+        },
+        {
+            "id": "fidelity-frontend-invented",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "service-fidelity",
+            "file": ".azure/project-plan.md",
+            "operation": "replace",
+            "search": "**App Type**: SPA + API",
+            "replacement": "**App Type**: API only",
+            "expectedCode": "frontendNotPlanned"
+        },
+        {
+            "id": "fidelity-language-swapped",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "service-fidelity",
+            "file": ".azure/project-plan.md",
+            "operation": "replace",
+            "search": "## 2. Backend \u2014 Azure Functions\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |",
+            "replacement": "## 2. Backend \u2014 Azure Functions\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | Python |",
+            "expectedCode": "serviceLanguageMismatch"
+        },
+        {
+            "id": "fidelity-framework-swapped",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "service-fidelity",
+            "file": ".azure/project-plan.md",
+            "operation": "replace",
+            "search": "| **Framework** | React + Vite |",
+            "replacement": "| **Framework** | Angular |",
+            "expectedCode": "serviceFrameworkMismatch"
+        },
+        {
+            "id": "fidelity-plan-declares-no-services",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "service-fidelity",
+            "file": ".azure/project-plan.md",
+            "operation": "replace",
+            "search": "## 2. Backend \u2014 Azure Functions\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |\n| **Runtime** | Node |\n| **Package Manager** | npm |\n| **Test Runner** | vitest |\n| **Test Command** | npm test |\n| **Orchestration** | docker-compose |\n\n## 3. Frontend \u2014 Web App\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |\n| **Framework** | React + Vite |\n| **Package Manager** | npm |\n| **Test Runner** | vitest |\n| **Test Command** | npm test |\n\n## 4. Worker \u2014 Background Jobs\n\n| Component | Technology |\n|-----------|-----------|\n| **Language** | TypeScript |\n| **Runtime** | Node |\n| **Package Manager** | npm |\n| **Test Runner** | vitest |\n| **Test Command** | npm test |\n| **Orchestration** | docker-compose |\n\n",
+            "replacement": "",
+            "expectedCode": "planDeclaresNoServices"
+        },
+        {
+            "id": "fidelity-datastore-import-swapped",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "datastore-fidelity",
+            "file": "services/api/src/db.ts",
+            "operation": "replace",
+            "search": "import { Pool } from 'pg';",
+            "replacement": "import { Pool } from 'sqlite3';",
+            "expectedCode": "plannedDatastoreNotWired"
+        },
+        {
+            "id": "fidelity-datastore-invented",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "datastore-fidelity",
+            "file": "services/api/src/db.ts",
+            "operation": "replace",
+            "search": "import { Pool } from 'pg';",
+            "replacement": "import { Pool } from 'pg';\nimport { MongoClient } from 'mongodb';",
+            "expectedCode": "unplannedDatastoreWired"
+        },
+        {
+            "id": "fidelity-datastore-dependency-dropped",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "datastore-fidelity",
+            "file": "services/api/package.json",
+            "operation": "replace",
+            "search": ",\n        \"pg\": \"^8.13.1\"",
+            "replacement": "",
+            "expectedCode": "datastoreDependencyMissing"
+        },
+        {
+            "id": "fidelity-resource-never-wired",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "datastore-fidelity",
+            "file": ".azure/project-plan.md",
+            "operation": "replace",
+            "search": "| Blob Storage | Store ticket attachments | STORAGE_CONNECTION_STRING |",
+            "replacement": "| Blob Storage | Store ticket attachments | ATTACHMENTS_CONNECTION_STRING |",
+            "expectedCode": "plannedResourceNotWired"
+        },
+        {
+            "id": "fidelity-services-required-table-unreadable",
+            "tier": "offline",
+            "fixture": "reference-node-multiservice",
+            "validator": "datastore-fidelity",
+            "file": ".azure/project-plan.md",
+            "operation": "replace",
+            "search": "| Azure Service | Role in App | Environment Variable | Default Value (Local) | Classification |",
+            "replacement": "| Name | Responsibility | Variable | Local Default | Classification |",
+            "expectedCode": "plannedResourcesUnreadable"
+        },
+        {
+            "id": "fidelity-datastore-swapped-python",
+            "tier": "offline",
+            "fixture": "reference-python-api",
+            "validator": "datastore-fidelity",
+            "file": "services/api/main.py",
+            "operation": "replace",
+            "search": "from psycopg import connect",
+            "replacement": "from aiosqlite import connect",
+            "expectedCode": "unplannedDatastoreWired"
+        },
+        {
+            "id": "fidelity-datastore-unwired-python",
+            "tier": "offline",
+            "fixture": "reference-python-api",
+            "validator": "datastore-fidelity",
+            "file": "services/api/main.py",
+            "operation": "replace",
+            "search": "from psycopg import connect",
+            "replacement": "from aiosqlite import connect",
+            "expectedCode": "plannedDatastoreNotWired"
+        },
+        {
+            "id": "fidelity-nothing-scaffolded-python",
+            "tier": "offline",
+            "fixture": "reference-python-api",
+            "validator": "datastore-fidelity",
+            "file": "services/api/requirements.txt",
+            "operation": "delete",
+            "expectedCode": "noServicesScaffolded"
+        },
+        {
+            "id": "fidelity-datastore-swapped-dotnet",
+            "tier": "offline",
+            "fixture": "reference-dotnet-api",
+            "validator": "datastore-fidelity",
+            "file": "services/api/Program.cs",
+            "operation": "replace",
+            "search": "using Npgsql;",
+            "replacement": "using Microsoft.Data.Sqlite;",
+            "expectedCode": "plannedDatastoreNotWired"
+        },
+        {
+            "id": "fidelity-orm-owns-the-driver-python",
+            "tier": "offline",
+            "fixture": "reference-python-api",
+            "validator": "datastore-fidelity",
+            "file": "services/api/main.py",
+            "operation": "replace",
+            "search": "from psycopg import connect",
+            "replacement": "from sqlalchemy import create_engine",
+            "expectedCode": "passed"
+        },
+        {
+            "id": "fidelity-orm-owns-the-driver-dotnet",
+            "tier": "offline",
+            "fixture": "reference-dotnet-api",
+            "validator": "datastore-fidelity",
+            "file": "services/api/Program.cs",
+            "operation": "replace",
+            "search": "using Npgsql;",
+            "replacement": "using Npgsql.EntityFrameworkCore.PostgreSQL;",
+            "expectedCode": "passed"
         }
     ]
 }

evals/grader-certification/reference-dotnet-api/.azure/project-plan.md

@@ -0,0 +1,64 @@
+# Project Plan
+
+**Status**: Integrated
+**Created**: 2026-08-25
+**Mode**: New Project
+
+## 1. Project Overview
+
+**Goal**: Build a C# ticket API whose storage layer is independently testable.
+
+**App Type**: API only
+
+**Mode**: NEW
+
+## 2. Backend — ASP.NET Minimal API
+
+| Component | Technology |
+|-----------|-----------|
+| **Language** | C# |
+| **Runtime** | .NET |
+| **Package Manager** | dotnet (NuGet) |
+| **Test Runner** | xUnit |
+| **Mocking Library** | NSubstitute |
+| **Test Command** | dotnet test |
+| **Orchestration** | docker-compose |
+
+## 3. Services Required
+
+| Azure Service | Role in App | Environment Variable | Default Value (Local) | Classification |
+|---------------|------------|---------------------|----------------------|----------------|
+| Azure Database for PostgreSQL | Primary data store for tickets | DATABASE_URL | postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets | Essential |
+
+## 4. Prerequisites
+
+### Run
+
+| Tool | Service(s) | Installed | Version | Install |
+|------|-----------|-----------|---------|---------|
+| .NET SDK | * | ✅ | 9.0 | https://dotnet.microsoft.com/download |
+
+### Debug
+
+| Tool | Service(s) | Installed | Version | Install |
+|------|-----------|-----------|---------|---------|
+| Docker | api | ❓ | — | https://docs.docker.com/get-docker/ |
+
+## 5. Project Structure
+
+```text
+services/api/Api.csproj
+services/api/Program.cs
+```
+
+## 6. Route Definitions
+
+| # | Method | Path | Description | Auth | Status Codes |
+|---|--------|------|-------------|------|-------------|
+| 1 | GET | `/api/health` | Report service health | None | 200 |
+| 2 | GET | `/api/tickets` | List tickets | None | 200 |
+
+## 7. Next Steps
+
+1. Scaffold the API service.
+2. Generate debug artifacts.

evals/grader-certification/reference-dotnet-api/.env.example

@@ -0,0 +1 @@
+DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/tickets

evals/grader-certification/reference-dotnet-api/scenario.json

@@ -0,0 +1,27 @@
+{
+    "schemaVersion": "1",
+    "id": "grader-dotnet-api-fidelity",
+    "prompt": "Build a C# HTTP API for tracking tickets, backed by PostgreSQL.",
+    "baselinePrompt": "Create a C# minimal-API service for tracking tickets, backed by PostgreSQL. Expose health and list endpoints, and include the project file needed to restore and run it.",
+    "tags": {
+        "archetype": "crud",
+        "frontend": "none",
+        "backend": "dotnet",
+        "database": "postgres",
+        "auth": "none",
+        "complexity": "small"
+    },
+    "requirementsAnswers": {
+        "dataStores": [
+            "PostgreSQL"
+        ]
+    },
+    "validation": {
+        "profile": "minimal",
+        "build": false,
+        "test": false,
+        "lint": "skip",
+        "timeoutMinutes": 5,
+        "maxAgentRetries": 0
+    }
+}

evals/grader-certification/reference-dotnet-api/services/api/Api.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Npgsql" Version="9.0.2" />
+  </ItemGroup>
+
+</Project>

evals/grader-certification/reference-dotnet-api/services/api/Program.cs

@@ -0,0 +1,26 @@
+using Npgsql;
+
+var builder = WebApplication.CreateBuilder(args);
+var app = builder.Build();
+
+var connectionString = Environment.GetEnvironmentVariable("DATABASE_URL");
+
+app.MapGet("/api/health", () => Results.Ok(new { status = "ok" }));
+
+app.MapGet("/api/tickets", async () =>
+{
+    await using var connection = new NpgsqlConnection(connectionString);
+    await connection.OpenAsync();
+    await using var command = new NpgsqlCommand("SELECT id, title, status FROM tickets ORDER BY id", connection);
+    await using var reader = await command.ExecuteReaderAsync();
+
+    var tickets = new List<object>();
+    while (await reader.ReadAsync())
+    {
+        tickets.Add(new { id = reader.GetString(0), title = reader.GetString(1), status = reader.GetString(2) });
+    }
+
+    return Results.Ok(tickets);
+});
+
+app.Run();