From ba3c184a4382d9e4a0bac5e611520109cefce460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hanu=C5=A1?= Date: Thu, 2 Jul 2026 22:49:53 +0200 Subject: [PATCH 1/6] =?UTF-8?q?refactor:=20complete=20multi-Actor=20monore?= =?UTF-8?q?po=20layout=20=E2=80=94=20move=20Dockerfile=20into=20actors/run?= =?UTF-8?q?ner/=20+=20delete=20duplicate=20root=20.actor/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This repo is a monorepo intended to hold multiple Apify Actors under actors/* (the runner today; an orchestrator and possibly analyzer later). For that layout to be coherent, EVERYTHING Actor-specific must live inside the Actor's own directory alongside its .actor/ manifest and src/ tree — not at the repo root where a reader has to guess which Actor it belongs to. Two moves: 1. Dockerfile → actors/runner/Dockerfile The runner's build config lives with the runner. 2. Delete the duplicate root .actor/ Before this PR, root .actor/ mirrored actors/runner/.actor/ — one Actor had two manifests, and it was unclear to a reader which one the platform actually reads. Only actors/runner/.actor/ survives as canonical. Any future actors// Actor gets its own .actor/ alongside its code. Adjustments: - actors/runner/.actor/actor.json — dockerfile path adjusted to point at the new Dockerfile location; dockerContextDir stays "../../.." so the Docker build context is still the repo root (necessary for COPY shared/... to reach the shared/ workspace). - test/run-vm-test.sh, CLAUDE.md, docs/02-decisions.md — path references updated where they clearly meant the runner's build config. Adding actors/orchestrator/ in a future PR becomes purely additive: copy the runner's structure into a sibling directory, no root changes required. Supersedes an earlier attempt on this branch that flattened in the WRONG direction (pulled the runner up to root, would have blocked adding a second Actor). Co-Authored-By: Claude Opus 4.7 --- .actor/actor.json | 22 ----- .actor/dataset_schema.json | 88 ------------------ .actor/input_schema.json | 118 ------------------------- CLAUDE.md | 4 +- actors/runner/.actor/actor.json | 2 +- Dockerfile => actors/runner/Dockerfile | 0 docs/02-decisions.md | 2 +- test/run-vm-test.sh | 2 +- 8 files changed, 5 insertions(+), 233 deletions(-) delete mode 100644 .actor/actor.json delete mode 100644 .actor/dataset_schema.json delete mode 100644 .actor/input_schema.json rename Dockerfile => actors/runner/Dockerfile (100%) diff --git a/.actor/actor.json b/.actor/actor.json deleted file mode 100644 index 4daa037..0000000 --- a/.actor/actor.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "https://apify.com/schemas/v1/actor.ide.json", - "actorSpecification": 1, - "name": "agent-evals-runner", - "title": "Agent Evals Runner", - "description": "Evaluate AI coding agents (Claude Code, Codex, OpenCode) against test scenarios with automatic judging. Write tests in Markdown with deterministic checks (contains, regex, json-schema, bash scripts) and LLM judge. Compare MCP vs CLI tool integration, measure cost/tokens/latency, and track agent trajectory (tool calls, file operations, error recovery). Returns structured results per checkpoint with full metrics.", - "version": "0.2", - "buildTag": "latest", - "categories": ["DEVELOPER_TOOLS"], - "meta": { - "templateId": "ts_empty", - "generatedBy": "Claude Code with Claude Opus 4.6" - }, - "dockerfile": "./Dockerfile", - "defaultMemoryMbytes": 4096, - "minMemoryMbytes": 2048, - "maxMemoryMbytes": 8192, - "usesStandbyMode": false, - "storages": { - "dataset": "./dataset_schema.json" - } -} diff --git a/.actor/dataset_schema.json b/.actor/dataset_schema.json deleted file mode 100644 index 2705c68..0000000 --- a/.actor/dataset_schema.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "actorSpecification": 1, - "fields": {}, - "views": { - "results": { - "title": "Eval Results", - "transformation": { - "fields": [ - "overallVerdict", - "scenarioName", - "testIndex", - "agent", - "model", - "stopReason", - "testPrompt", - "agentOutput", - "verdicts", - "metrics", - "efficiency", - "trajectory", - "discoverability", - "error" - ] - }, - "display": { - "component": "table", - "properties": { - "overallVerdict": { - "label": "Verdict", - "format": "text" - }, - "scenarioName": { - "label": "Scenario", - "format": "text" - }, - "testIndex": { - "label": "Test #", - "format": "number" - }, - "agent": { - "label": "Agent", - "format": "text" - }, - "model": { - "label": "Model", - "format": "text" - }, - "stopReason": { - "label": "Stop Reason", - "format": "text" - }, - "testPrompt": { - "label": "Prompt", - "format": "text" - }, - "agentOutput": { - "label": "Agent Output", - "format": "text" - }, - "verdicts": { - "label": "Verdicts", - "format": "array" - }, - "metrics": { - "label": "Metrics", - "format": "object" - }, - "efficiency": { - "label": "Efficiency", - "format": "object" - }, - "trajectory": { - "label": "Trajectory", - "format": "object" - }, - "discoverability": { - "label": "Discoverability", - "format": "object" - }, - "error": { - "label": "Error", - "format": "text" - } - } - } - } - } -} diff --git a/.actor/input_schema.json b/.actor/input_schema.json deleted file mode 100644 index a822937..0000000 --- a/.actor/input_schema.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "title": "Agent Evals Runner Input", - "type": "object", - "schemaVersion": 1, - "properties": { - "agent": { - "title": "Agent", - "type": "string", - "description": "AI coding agent to evaluate.", - "enum": ["claude-code", "codex", "opencode"], - "enumTitles": ["Claude Code", "Codex CLI", "OpenCode"], - "default": "claude-code" - }, - "model": { - "title": "Model", - "type": "string", - "description": "LLM model for the agent (e.g. claude-sonnet-4-6, claude-opus-4-6). Leave empty for agent's default.", - "editor": "textfield", - "default": "claude-sonnet-4-6", - "nullable": true - }, - "scenario": { - "title": "Testing Scenario", - "type": "string", - "description": "Markdown with YAML frontmatter + ## Test / ## Checkpoint sections separated by ---. Checkpoint supports: contains:, regex:, json-schema:, script: prefixes for deterministic checks, and plain text for LLM judge. Multiple checks in one checkpoint all must pass.", - "editor": "textarea", - "prefill": "---\nname: example-eval\ndescription: Simple eval demonstrating checkpoint types\nabortOnFailure: false\n---\n\n## Test\nWhat is the largest planet in our solar system? Answer in one sentence.\n\n## Checkpoint\ncontains: Jupiter\nThe answer should be scientifically accurate.\n" - }, - "systemPrompt": { - "title": "System Prompt", - "type": "string", - "description": "Instructions for the agent at the start of each test. Default: 'You are an AI agent being evaluated. Always respond in English. Follow the instructions precisely.'", - "editor": "textarea", - "nullable": true - }, - "maxBudgetUsd": { - "title": "Max Budget (USD)", - "type": "number", - "description": "Spending cap for the entire run. Soft limit — checked between agent turns, actual cost may slightly exceed this value.", - "nullable": true, - "minimum": 0.01, - "maximum": 100, - "default": 1.00 - }, - "maxRetries": { - "title": "Max Retries on Failure", - "type": "integer", - "description": "Retry a failed test this many times before marking as failed. Useful for non-deterministic tasks.", - "default": 0, - "minimum": 0, - "maximum": 5 - }, - "maxTurns": { - "title": "Max Turns per Test", - "type": "integer", - "description": "Maximum agent turns (tool calls) per test. Use 3 for simple questions, 10+ for multi-step tasks.", - "default": 10, - "minimum": 1, - "maximum": 100 - }, - "envVariables": { - "title": "Environment Variables", - "type": "object", - "description": "API keys and secrets as JSON object (e.g. {\"GITHUB_TOKEN\": \"ghp_...\", \"APIFY_TOKEN\": \"apify_api_...\"}). Available to agent and init script. Automatically masked in logs.", - "editor": "json", - "isSecret": true, - "nullable": true - }, - "preAuthenticate": { - "title": "Pre-authenticate Apify CLI", - "type": "boolean", - "description": "If true (default), the runner populates ~/.apify/auth.json with APIFY_TOKEN via 'apify login --token' BEFORE the agent starts. This mirrors what every real Apify developer has after running 'apify login' once on their machine — 'apify push' from the agent will work without the agent having to discover auth flow. Set to false for raw-unauthed measurement (the agent has to figure out auth on its own — surfaces F11 in the eval pack's FINDINGS.md).", - "default": true - }, - "initPreset": { - "title": "Tool Setup Preset", - "type": "string", - "description": "Configures what tools the agent has access to. 'None' is a safe default that logs which CLI tools are available and wires up mcpConfigJson if provided. The *_native presets MAKE a surface available without preventing the others; the *_only presets ENFORCE exclusive use of a single surface (PATH-shimming + MCP-config gating + trajectory-based hard reject of cross-surface leaks). Compare _native variants to measure 'does the agent prefer X when X is available'; compare _only variants to measure 'is surface X sufficient on its own'.", - "enum": ["none", "mcp_native", "cli_native", "mcpc", "api_native", "mcp_only", "cli_only", "api_only"], - "enumTitles": [ - "None — logs availability of apify/gh/curl/jq; wires up mcpConfigJson if provided", - "MCP Native — agent gets MCP servers from config JSON (CLI / REST still available)", - "CLI Native — agent uses command-line tools (MCP / REST still available)", - "mcpc — MCP servers via mcpc CLI bridge", - "API Native — agent uses raw HTTPS against api.apify.com (CLI / MCP still available)", - "MCP Only — exclusive MCP surface: CLI binaries shimmed off PATH, REST tooling blocked, in-agent WebFetch/WebSearch hard-rejected", - "CLI Only — exclusive CLI surface: curl/wget shimmed off PATH, MCP not loaded, in-agent WebFetch/WebSearch hard-rejected", - "API Only — exclusive REST surface: apify shimmed off PATH, MCP not loaded, any apify-cli or MCP tool use hard-rejected" - ], - "default": "none" - }, - "initBashScript": { - "title": "Custom Init Script", - "type": "string", - "description": "Bash script to set up the environment before the agent runs. Use to install tools, download validators, or create test fixtures. Runs after the preset. Timeout: 5 minutes.", - "editor": "textarea", - "nullable": true, - "prefill": "" - }, - "mcpConfigJson": { - "title": "MCP Config JSON", - "type": "object", - "description": "MCP server configuration. Required for 'MCP Native' and 'mcpc' presets. Token placeholders like ${APIFY_TOKEN} are resolved from Environment Variables.", - "editor": "json", - "nullable": true, - "prefill": { - "mcpServers": { - "apify": { - "type": "http", - "url": "https://mcp.apify.com", - "headers": { "Authorization": "Bearer ${APIFY_TOKEN}" } - } - } - } - } - }, - "required": ["scenario"] -} diff --git a/CLAUDE.md b/CLAUDE.md index 57024e9..f1150f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -377,8 +377,8 @@ DŮLEŽITÉ: `--max-budget-usd` je **soft limit**. Claude CLI kontroluje budget ## Monorepo deploy -Jeden fat `Dockerfile` v rootu instaluje všechny agent CLI (Claude, Codex, OpenCode, Apify CLI). Agent se vybírá za běhu přes input `agent`. -`actor.json` má `dockerContextDir: "../../.."`, což umožňuje Dockerfile přistupovat k `shared/`. +Jeden fat `Dockerfile` v `actors/runner/` instaluje všechny agent CLI (Claude, Codex, OpenCode, Apify CLI). Agent se vybírá za běhu přes input `agent`. +`actor.json` má `dockerContextDir: "../../.."`, což umožňuje Dockerfile přistupovat k `shared/` v rootu monorepa. Deploy z monorepo rootu (ne z `actors/runner` — `dockerContextDir` by ukazoval mimo upload): ```bash diff --git a/actors/runner/.actor/actor.json b/actors/runner/.actor/actor.json index 8450896..06da26d 100644 --- a/actors/runner/.actor/actor.json +++ b/actors/runner/.actor/actor.json @@ -14,7 +14,7 @@ "templateId": "ts_empty", "generatedBy": "Claude Code with Claude Opus 4.6" }, - "dockerfile": "./Dockerfile", + "dockerfile": "../Dockerfile", "dockerContextDir": "../../..", "defaultMemoryMbytes": 8192, "minMemoryMbytes": 2048, diff --git a/Dockerfile b/actors/runner/Dockerfile similarity index 100% rename from Dockerfile rename to actors/runner/Dockerfile diff --git a/docs/02-decisions.md b/docs/02-decisions.md index e22eec6..7dc5d01 100644 --- a/docs/02-decisions.md +++ b/docs/02-decisions.md @@ -242,7 +242,7 @@ V Apify Docker kontejneru bude auth řešený přes env var `CLAUDE_CODE_OAUTH_T ### Rozhodnutí: Jeden fat Dockerfile se všemi agent CLI -Jeden `Dockerfile` v rootu repa instaluje všechny CLI nástroje (Claude Code, Codex, OpenCode, Apify CLI). Agent se vybírá za běhu přes input parametr `agent`. +Jeden `Dockerfile` v `actors/runner/` instaluje všechny CLI nástroje (Claude Code, Codex, OpenCode, Apify CLI). Agent se vybírá za běhu přes input parametr `agent`. ### Zamítnuto: Separate Dockerfiles per agent, Actor.metamorph() diff --git a/test/run-vm-test.sh b/test/run-vm-test.sh index 6c5474d..fa9d03b 100755 --- a/test/run-vm-test.sh +++ b/test/run-vm-test.sh @@ -27,7 +27,7 @@ fi OPENAI_API_KEY="${OPENAI_API_KEY:-}" echo "=== Building production Docker image ===" -$DOCKER build --network host -t "$IMAGE_NAME" -f "$PROJECT_DIR/Dockerfile" "$PROJECT_DIR" 2>&1 | tail -10 +$DOCKER build --network host -t "$IMAGE_NAME" -f "$PROJECT_DIR/actors/runner/Dockerfile" "$PROJECT_DIR" 2>&1 | tail -10 echo "" echo "=== Running tests inside production image ===" From 32f96455737fa95af38f1c289d89878f3e19df89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hanu=C5=A1?= Date: Fri, 3 Jul 2026 01:36:57 +0200 Subject: [PATCH 2/6] fix(runner): restore simple 5-column dataset view + drop stale root schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rolled into this PR from #22 so the multi-Actor layout + view fix can be tested in a single branch. The Console's dataset "Views" tab is driven by actors/runner/.actor/dataset_schema.json. The shape from b17cb10 (judgeMode + OTel) traded scannability for dot-path metric columns; user feedback was "the previous view was much clearer". Replaces .views with a single Eval Results view — the five columns that answer "what happened on this test": Test # | Verdict | Prompt | Agent Output | Checks verdicts is rendered as an inline array (compact list of check objects) rather than a full unwound table. Per-check drill-down is one click away in the raw Data tab, so the Views tab optimizes for scanning. Also deletes the stale root-level .actor/dataset_schema.json — the pre-flatten copy that survived this PR's restructure. Co-Authored-By: Claude Opus 4.7 --- actors/runner/.actor/dataset_schema.json | 99 ++---------------------- 1 file changed, 8 insertions(+), 91 deletions(-) diff --git a/actors/runner/.actor/dataset_schema.json b/actors/runner/.actor/dataset_schema.json index 3e5222f..de34666 100644 --- a/actors/runner/.actor/dataset_schema.json +++ b/actors/runner/.actor/dataset_schema.json @@ -6,104 +6,21 @@ "title": "Eval Results", "transformation": { "fields": [ - "scenarioName", "testIndex", - "agent", - "model", "overallVerdict", - "stopReason", - "metrics.totalCostUsd", - "metrics.durationMs", - "metrics.numTurns", - "trajectory.toolCallCount", - "efficiency.cacheHitRate", - "retryAttempts", - "error" - ] - }, - "display": { - "component": "table", - "properties": { - "scenarioName": { - "label": "Scenario", - "format": "text" - }, - "testIndex": { - "label": "#", - "format": "number" - }, - "agent": { - "label": "Agent", - "format": "text" - }, - "model": { - "label": "Model", - "format": "text" - }, - "overallVerdict": { - "label": "Verdict", - "format": "text" - }, - "stopReason": { - "label": "Stop", - "format": "text" - }, - "metrics.totalCostUsd": { - "label": "Cost ($)", - "format": "number" - }, - "metrics.durationMs": { - "label": "Duration (ms)", - "format": "number" - }, - "metrics.numTurns": { - "label": "Turns", - "format": "number" - }, - "trajectory.toolCallCount": { - "label": "Tool Calls", - "format": "number" - }, - "efficiency.cacheHitRate": { - "label": "Cache Hit", - "format": "number" - }, - "retryAttempts": { - "label": "Retries", - "format": "number" - }, - "error": { - "label": "Error", - "format": "text" - } - } - } - }, - "verdicts": { - "title": "Checkpoint Verdicts", - "transformation": { - "fields": [ - "scenarioName", - "testIndex", + "testPrompt", + "agentOutput", "verdicts" - ], - "unwind": ["verdicts"] + ] }, "display": { "component": "table", "properties": { - "scenarioName": { - "label": "Scenario", - "format": "text" - }, - "testIndex": { - "label": "#", - "format": "number" - }, - "verdicts": { - "label": "Verdicts", - "format": "object" - } + "testIndex": { "label": "Test #", "format": "number" }, + "overallVerdict": { "label": "Verdict", "format": "text" }, + "testPrompt": { "label": "Prompt", "format": "text" }, + "agentOutput": { "label": "Agent Output", "format": "text" }, + "verdicts": { "label": "Checks", "format": "array" } } } } From 10f7fc72bb4254772da3b55bb1dbea334b13bf95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hanu=C5=A1?= Date: Fri, 3 Jul 2026 02:19:38 +0200 Subject: [PATCH 3/6] =?UTF-8?q?fix(runner):=20tune=20dataset=20views=20?= =?UTF-8?q?=E2=80=94=20drop=20Test=20#=20from=20primary,=20add=20unwound?= =?UTF-8?q?=20Verdict=20Checks=20view?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two adjustments after testing the previous 5-column view against a real run: 1. Primary "Eval Results" view: drop the Test # column. With testPrompt visible, the numeric index was redundant — the prompt itself identifies the test. 2. Add a secondary "Verdict Checks" view that unwinds the verdicts array so each check renders as its own row. Columns: Test | Verdict | Check Type | Check | Evidence. Test # retained here because unwound rows blend across tests otherwise. Users can now scan per-check detail on the Views tab without opening raw JSON. Co-Authored-By: Claude Opus 4.7 --- actors/runner/.actor/dataset_schema.json | 35 ++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/actors/runner/.actor/dataset_schema.json b/actors/runner/.actor/dataset_schema.json index de34666..42b23a1 100644 --- a/actors/runner/.actor/dataset_schema.json +++ b/actors/runner/.actor/dataset_schema.json @@ -6,21 +6,40 @@ "title": "Eval Results", "transformation": { "fields": [ - "testIndex", "overallVerdict", "testPrompt", - "agentOutput", - "verdicts" + "agentOutput" ] }, "display": { "component": "table", "properties": { - "testIndex": { "label": "Test #", "format": "number" }, - "overallVerdict": { "label": "Verdict", "format": "text" }, - "testPrompt": { "label": "Prompt", "format": "text" }, - "agentOutput": { "label": "Agent Output", "format": "text" }, - "verdicts": { "label": "Checks", "format": "array" } + "overallVerdict": { "label": "Verdict", "format": "text" }, + "testPrompt": { "label": "Prompt", "format": "text" }, + "agentOutput": { "label": "Agent Output", "format": "text" } + } + } + }, + "verdicts": { + "title": "Verdict Checks", + "transformation": { + "fields": [ + "testIndex", + "verdicts.verdict", + "verdicts.checkType", + "verdicts.checkValue", + "verdicts.evidence" + ], + "unwind": ["verdicts"] + }, + "display": { + "component": "table", + "properties": { + "testIndex": { "label": "Test", "format": "number" }, + "verdicts.verdict": { "label": "Verdict", "format": "text" }, + "verdicts.checkType": { "label": "Check Type", "format": "text" }, + "verdicts.checkValue": { "label": "Check", "format": "text" }, + "verdicts.evidence": { "label": "Evidence", "format": "text" } } } } From 91e7ba9171c636c8f386ed442ae51e4169c703a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hanu=C5=A1?= Date: Fri, 3 Jul 2026 11:45:05 +0200 Subject: [PATCH 4/6] fix(runner): correct Verdict Checks view unwind syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous attempt used `unwind: ["verdicts"]` + fields `verdicts.checkType` etc., which Apify Console silently ignored — the view showed 2 dataset rows with only testIndex populated instead of unwinding into per-check rows. Corrected per Apify actor-output-schema docs (see PR body). The Verdict Checks view now produces one row per element of the verdicts array, with columns for Test | Verdict | Check Type | Check | Evidence. Ref: run jaN6GrFkd8yOR1GwH exposed the mis-rendering. Co-Authored-By: Claude Opus 4.7 --- actors/runner/.actor/dataset_schema.json | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/actors/runner/.actor/dataset_schema.json b/actors/runner/.actor/dataset_schema.json index 42b23a1..2e590c7 100644 --- a/actors/runner/.actor/dataset_schema.json +++ b/actors/runner/.actor/dataset_schema.json @@ -21,25 +21,20 @@ } }, "verdicts": { - "title": "Verdict Checks", + "title": "Checkpoint Verdicts", "transformation": { - "fields": [ - "testIndex", - "verdicts.verdict", - "verdicts.checkType", - "verdicts.checkValue", - "verdicts.evidence" - ], + "fields": ["scenarioName", "testIndex", "verdicts"], "unwind": ["verdicts"] }, "display": { "component": "table", "properties": { - "testIndex": { "label": "Test", "format": "number" }, - "verdicts.verdict": { "label": "Verdict", "format": "text" }, - "verdicts.checkType": { "label": "Check Type", "format": "text" }, - "verdicts.checkValue": { "label": "Check", "format": "text" }, - "verdicts.evidence": { "label": "Evidence", "format": "text" } + "scenarioName": { "label": "Scenario", "format": "text" }, + "testIndex": { "label": "#", "format": "number" }, + "checkType": { "label": "Check Type", "format": "text" }, + "checkValue": { "label": "Check", "format": "text" }, + "verdict": { "label": "Verdict", "format": "text" }, + "evidence": { "label": "Evidence", "format": "text" } } } } From dc56344be553d8d2d3939b61612e9d7a44197224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hanu=C5=A1?= Date: Fri, 3 Jul 2026 12:30:22 +0200 Subject: [PATCH 5/6] fix(runner): rename Checkpoint Verdicts -> Checkpoints, drop scenarioName column The unwound Checkpoints view only ever contains one scenario per run's dataset, so the Scenario column was pure noise. Title shortened to "Checkpoints" to match the domain vocabulary (checkpoint scripts + checkpoint field on AgentResult). Co-Authored-By: Claude Opus 4.7 --- actors/runner/.actor/dataset_schema.json | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/actors/runner/.actor/dataset_schema.json b/actors/runner/.actor/dataset_schema.json index 2e590c7..8f4eec8 100644 --- a/actors/runner/.actor/dataset_schema.json +++ b/actors/runner/.actor/dataset_schema.json @@ -21,20 +21,19 @@ } }, "verdicts": { - "title": "Checkpoint Verdicts", + "title": "Checkpoints", "transformation": { - "fields": ["scenarioName", "testIndex", "verdicts"], + "fields": ["testIndex", "verdicts"], "unwind": ["verdicts"] }, "display": { "component": "table", "properties": { - "scenarioName": { "label": "Scenario", "format": "text" }, "testIndex": { "label": "#", "format": "number" }, - "checkType": { "label": "Check Type", "format": "text" }, - "checkValue": { "label": "Check", "format": "text" }, - "verdict": { "label": "Verdict", "format": "text" }, - "evidence": { "label": "Evidence", "format": "text" } + "checkType": { "label": "Check Type", "format": "text" }, + "checkValue": { "label": "Check", "format": "text" }, + "verdict": { "label": "Verdict", "format": "text" }, + "evidence": { "label": "Evidence", "format": "text" } } } } From 8138cf8f832a67967596927b2151a9a38bd6d0d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hanu=C5=A1?= Date: Fri, 3 Jul 2026 14:02:29 +0200 Subject: [PATCH 6/6] fix(runner): rename Checkpoints view -> Tests, testIndex -> "Test Index", omit LLM-judge meta fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small tweaks after the last test run: 1. Title "Checkpoints" -> "Tests" — user preference; the view groups check rows by test, and "Tests" reads better as the primary axis. 2. testIndex column label "#" -> "Test Index" — same reason, consistent with the view name. 3. Explicit omit of evalCritique + evalGapSeverity. Apify Console auto-derives columns from every top-level key of the transformed item, so when a verdict happened to have evalCritique populated (LLM judge's meta-feedback about the eval criteria, ~5% of rows in observed data) it leaked into the view as an unformatted column. These fields are for the eval author, not the agent-behaviour triager — they belong in raw JSON only. omit runs after unwind in Apify's transform pipeline (flatten -> pickFields -> unwind -> omit), so this cleanly drops them from the view without affecting anything else. Co-Authored-By: Claude Opus 4.7 --- actors/runner/.actor/dataset_schema.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/actors/runner/.actor/dataset_schema.json b/actors/runner/.actor/dataset_schema.json index 8f4eec8..015bd35 100644 --- a/actors/runner/.actor/dataset_schema.json +++ b/actors/runner/.actor/dataset_schema.json @@ -21,15 +21,16 @@ } }, "verdicts": { - "title": "Checkpoints", + "title": "Tests", "transformation": { "fields": ["testIndex", "verdicts"], - "unwind": ["verdicts"] + "unwind": ["verdicts"], + "omit": ["evalCritique", "evalGapSeverity"] }, "display": { "component": "table", "properties": { - "testIndex": { "label": "#", "format": "number" }, + "testIndex": { "label": "Test Index", "format": "number" }, "checkType": { "label": "Check Type", "format": "text" }, "checkValue": { "label": "Check", "format": "text" }, "verdict": { "label": "Verdict", "format": "text" },