diff --git a/.actor/actor.json b/.actor/actor.json deleted file mode 100644 index 4daa037..0000000 --- a/.actor/actor.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "https://apify.com/schemas/v1/actor.ide.json", - "actorSpecification": 1, - "name": "agent-evals-runner", - "title": "Agent Evals Runner", - "description": "Evaluate AI coding agents (Claude Code, Codex, OpenCode) against test scenarios with automatic judging. Write tests in Markdown with deterministic checks (contains, regex, json-schema, bash scripts) and LLM judge. Compare MCP vs CLI tool integration, measure cost/tokens/latency, and track agent trajectory (tool calls, file operations, error recovery). Returns structured results per checkpoint with full metrics.", - "version": "0.2", - "buildTag": "latest", - "categories": ["DEVELOPER_TOOLS"], - "meta": { - "templateId": "ts_empty", - "generatedBy": "Claude Code with Claude Opus 4.6" - }, - "dockerfile": "./Dockerfile", - "defaultMemoryMbytes": 4096, - "minMemoryMbytes": 2048, - "maxMemoryMbytes": 8192, - "usesStandbyMode": false, - "storages": { - "dataset": "./dataset_schema.json" - } -} diff --git a/.actor/dataset_schema.json b/.actor/dataset_schema.json deleted file mode 100644 index 2705c68..0000000 --- a/.actor/dataset_schema.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "actorSpecification": 1, - "fields": {}, - "views": { - "results": { - "title": "Eval Results", - "transformation": { - "fields": [ - "overallVerdict", - "scenarioName", - "testIndex", - "agent", - "model", - "stopReason", - "testPrompt", - "agentOutput", - "verdicts", - "metrics", - "efficiency", - "trajectory", - "discoverability", - "error" - ] - }, - "display": { - "component": "table", - "properties": { - "overallVerdict": { - "label": "Verdict", - "format": "text" - }, - "scenarioName": { - "label": "Scenario", - "format": "text" - }, - "testIndex": { - "label": "Test #", - "format": "number" - }, - "agent": { - "label": "Agent", - "format": "text" - }, - "model": { - "label": "Model", - "format": "text" - }, - "stopReason": { - "label": "Stop Reason", - "format": "text" - }, - "testPrompt": { - "label": "Prompt", - "format": "text" - }, - "agentOutput": { - "label": "Agent Output", - "format": "text" - }, - "verdicts": { - "label": "Verdicts", - "format": "array" - }, - "metrics": { - "label": "Metrics", - "format": "object" - }, - "efficiency": { - "label": "Efficiency", - "format": "object" - }, - "trajectory": { - "label": "Trajectory", - "format": "object" - }, - "discoverability": { - "label": "Discoverability", - "format": "object" - }, - "error": { - "label": "Error", - "format": "text" - } - } - } - } - } -} diff --git a/.actor/input_schema.json b/.actor/input_schema.json deleted file mode 100644 index a822937..0000000 --- a/.actor/input_schema.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "title": "Agent Evals Runner Input", - "type": "object", - "schemaVersion": 1, - "properties": { - "agent": { - "title": "Agent", - "type": "string", - "description": "AI coding agent to evaluate.", - "enum": ["claude-code", "codex", "opencode"], - "enumTitles": ["Claude Code", "Codex CLI", "OpenCode"], - "default": "claude-code" - }, - "model": { - "title": "Model", - "type": "string", - "description": "LLM model for the agent (e.g. claude-sonnet-4-6, claude-opus-4-6). Leave empty for agent's default.", - "editor": "textfield", - "default": "claude-sonnet-4-6", - "nullable": true - }, - "scenario": { - "title": "Testing Scenario", - "type": "string", - "description": "Markdown with YAML frontmatter + ## Test / ## Checkpoint sections separated by ---. Checkpoint supports: contains:, regex:, json-schema:, script: prefixes for deterministic checks, and plain text for LLM judge. Multiple checks in one checkpoint all must pass.", - "editor": "textarea", - "prefill": "---\nname: example-eval\ndescription: Simple eval demonstrating checkpoint types\nabortOnFailure: false\n---\n\n## Test\nWhat is the largest planet in our solar system? Answer in one sentence.\n\n## Checkpoint\ncontains: Jupiter\nThe answer should be scientifically accurate.\n" - }, - "systemPrompt": { - "title": "System Prompt", - "type": "string", - "description": "Instructions for the agent at the start of each test. Default: 'You are an AI agent being evaluated. Always respond in English. Follow the instructions precisely.'", - "editor": "textarea", - "nullable": true - }, - "maxBudgetUsd": { - "title": "Max Budget (USD)", - "type": "number", - "description": "Spending cap for the entire run. Soft limit — checked between agent turns, actual cost may slightly exceed this value.", - "nullable": true, - "minimum": 0.01, - "maximum": 100, - "default": 1.00 - }, - "maxRetries": { - "title": "Max Retries on Failure", - "type": "integer", - "description": "Retry a failed test this many times before marking as failed. Useful for non-deterministic tasks.", - "default": 0, - "minimum": 0, - "maximum": 5 - }, - "maxTurns": { - "title": "Max Turns per Test", - "type": "integer", - "description": "Maximum agent turns (tool calls) per test. Use 3 for simple questions, 10+ for multi-step tasks.", - "default": 10, - "minimum": 1, - "maximum": 100 - }, - "envVariables": { - "title": "Environment Variables", - "type": "object", - "description": "API keys and secrets as JSON object (e.g. {\"GITHUB_TOKEN\": \"ghp_...\", \"APIFY_TOKEN\": \"apify_api_...\"}). Available to agent and init script. Automatically masked in logs.", - "editor": "json", - "isSecret": true, - "nullable": true - }, - "preAuthenticate": { - "title": "Pre-authenticate Apify CLI", - "type": "boolean", - "description": "If true (default), the runner populates ~/.apify/auth.json with APIFY_TOKEN via 'apify login --token' BEFORE the agent starts. This mirrors what every real Apify developer has after running 'apify login' once on their machine — 'apify push' from the agent will work without the agent having to discover auth flow. Set to false for raw-unauthed measurement (the agent has to figure out auth on its own — surfaces F11 in the eval pack's FINDINGS.md).", - "default": true - }, - "initPreset": { - "title": "Tool Setup Preset", - "type": "string", - "description": "Configures what tools the agent has access to. 'None' is a safe default that logs which CLI tools are available and wires up mcpConfigJson if provided. The *_native presets MAKE a surface available without preventing the others; the *_only presets ENFORCE exclusive use of a single surface (PATH-shimming + MCP-config gating + trajectory-based hard reject of cross-surface leaks). Compare _native variants to measure 'does the agent prefer X when X is available'; compare _only variants to measure 'is surface X sufficient on its own'.", - "enum": ["none", "mcp_native", "cli_native", "mcpc", "api_native", "mcp_only", "cli_only", "api_only"], - "enumTitles": [ - "None — logs availability of apify/gh/curl/jq; wires up mcpConfigJson if provided", - "MCP Native — agent gets MCP servers from config JSON (CLI / REST still available)", - "CLI Native — agent uses command-line tools (MCP / REST still available)", - "mcpc — MCP servers via mcpc CLI bridge", - "API Native — agent uses raw HTTPS against api.apify.com (CLI / MCP still available)", - "MCP Only — exclusive MCP surface: CLI binaries shimmed off PATH, REST tooling blocked, in-agent WebFetch/WebSearch hard-rejected", - "CLI Only — exclusive CLI surface: curl/wget shimmed off PATH, MCP not loaded, in-agent WebFetch/WebSearch hard-rejected", - "API Only — exclusive REST surface: apify shimmed off PATH, MCP not loaded, any apify-cli or MCP tool use hard-rejected" - ], - "default": "none" - }, - "initBashScript": { - "title": "Custom Init Script", - "type": "string", - "description": "Bash script to set up the environment before the agent runs. Use to install tools, download validators, or create test fixtures. Runs after the preset. Timeout: 5 minutes.", - "editor": "textarea", - "nullable": true, - "prefill": "" - }, - "mcpConfigJson": { - "title": "MCP Config JSON", - "type": "object", - "description": "MCP server configuration. Required for 'MCP Native' and 'mcpc' presets. Token placeholders like ${APIFY_TOKEN} are resolved from Environment Variables.", - "editor": "json", - "nullable": true, - "prefill": { - "mcpServers": { - "apify": { - "type": "http", - "url": "https://mcp.apify.com", - "headers": { "Authorization": "Bearer ${APIFY_TOKEN}" } - } - } - } - } - }, - "required": ["scenario"] -} diff --git a/CLAUDE.md b/CLAUDE.md index 57024e9..f1150f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -377,8 +377,8 @@ DŮLEŽITÉ: `--max-budget-usd` je **soft limit**. Claude CLI kontroluje budget ## Monorepo deploy -Jeden fat `Dockerfile` v rootu instaluje všechny agent CLI (Claude, Codex, OpenCode, Apify CLI). Agent se vybírá za běhu přes input `agent`. -`actor.json` má `dockerContextDir: "../../.."`, což umožňuje Dockerfile přistupovat k `shared/`. +Jeden fat `Dockerfile` v `actors/runner/` instaluje všechny agent CLI (Claude, Codex, OpenCode, Apify CLI). Agent se vybírá za běhu přes input `agent`. +`actor.json` má `dockerContextDir: "../../.."`, což umožňuje Dockerfile přistupovat k `shared/` v rootu monorepa. Deploy z monorepo rootu (ne z `actors/runner` — `dockerContextDir` by ukazoval mimo upload): ```bash diff --git a/actors/runner/.actor/actor.json b/actors/runner/.actor/actor.json index 8450896..06da26d 100644 --- a/actors/runner/.actor/actor.json +++ b/actors/runner/.actor/actor.json @@ -14,7 +14,7 @@ "templateId": "ts_empty", "generatedBy": "Claude Code with Claude Opus 4.6" }, - "dockerfile": "./Dockerfile", + "dockerfile": "../Dockerfile", "dockerContextDir": "../../..", "defaultMemoryMbytes": 8192, "minMemoryMbytes": 2048, diff --git a/actors/runner/.actor/dataset_schema.json b/actors/runner/.actor/dataset_schema.json index 3e5222f..015bd35 100644 --- a/actors/runner/.actor/dataset_schema.json +++ b/actors/runner/.actor/dataset_schema.json @@ -6,104 +6,35 @@ "title": "Eval Results", "transformation": { "fields": [ - "scenarioName", - "testIndex", - "agent", - "model", "overallVerdict", - "stopReason", - "metrics.totalCostUsd", - "metrics.durationMs", - "metrics.numTurns", - "trajectory.toolCallCount", - "efficiency.cacheHitRate", - "retryAttempts", - "error" + "testPrompt", + "agentOutput" ] }, "display": { "component": "table", "properties": { - "scenarioName": { - "label": "Scenario", - "format": "text" - }, - "testIndex": { - "label": "#", - "format": "number" - }, - "agent": { - "label": "Agent", - "format": "text" - }, - "model": { - "label": "Model", - "format": "text" - }, - "overallVerdict": { - "label": "Verdict", - "format": "text" - }, - "stopReason": { - "label": "Stop", - "format": "text" - }, - "metrics.totalCostUsd": { - "label": "Cost ($)", - "format": "number" - }, - "metrics.durationMs": { - "label": "Duration (ms)", - "format": "number" - }, - "metrics.numTurns": { - "label": "Turns", - "format": "number" - }, - "trajectory.toolCallCount": { - "label": "Tool Calls", - "format": "number" - }, - "efficiency.cacheHitRate": { - "label": "Cache Hit", - "format": "number" - }, - "retryAttempts": { - "label": "Retries", - "format": "number" - }, - "error": { - "label": "Error", - "format": "text" - } + "overallVerdict": { "label": "Verdict", "format": "text" }, + "testPrompt": { "label": "Prompt", "format": "text" }, + "agentOutput": { "label": "Agent Output", "format": "text" } } } }, "verdicts": { - "title": "Checkpoint Verdicts", + "title": "Tests", "transformation": { - "fields": [ - "scenarioName", - "testIndex", - "verdicts" - ], - "unwind": ["verdicts"] + "fields": ["testIndex", "verdicts"], + "unwind": ["verdicts"], + "omit": ["evalCritique", "evalGapSeverity"] }, "display": { "component": "table", "properties": { - "scenarioName": { - "label": "Scenario", - "format": "text" - }, - "testIndex": { - "label": "#", - "format": "number" - }, - "verdicts": { - "label": "Verdicts", - "format": "object" - } + "testIndex": { "label": "Test Index", "format": "number" }, + "checkType": { "label": "Check Type", "format": "text" }, + "checkValue": { "label": "Check", "format": "text" }, + "verdict": { "label": "Verdict", "format": "text" }, + "evidence": { "label": "Evidence", "format": "text" } } } } diff --git a/Dockerfile b/actors/runner/Dockerfile similarity index 100% rename from Dockerfile rename to actors/runner/Dockerfile diff --git a/docs/02-decisions.md b/docs/02-decisions.md index e22eec6..7dc5d01 100644 --- a/docs/02-decisions.md +++ b/docs/02-decisions.md @@ -242,7 +242,7 @@ V Apify Docker kontejneru bude auth řešený přes env var `CLAUDE_CODE_OAUTH_T ### Rozhodnutí: Jeden fat Dockerfile se všemi agent CLI -Jeden `Dockerfile` v rootu repa instaluje všechny CLI nástroje (Claude Code, Codex, OpenCode, Apify CLI). Agent se vybírá za běhu přes input parametr `agent`. +Jeden `Dockerfile` v `actors/runner/` instaluje všechny CLI nástroje (Claude Code, Codex, OpenCode, Apify CLI). Agent se vybírá za běhu přes input parametr `agent`. ### Zamítnuto: Separate Dockerfiles per agent, Actor.metamorph() diff --git a/test/run-vm-test.sh b/test/run-vm-test.sh index 6c5474d..fa9d03b 100755 --- a/test/run-vm-test.sh +++ b/test/run-vm-test.sh @@ -27,7 +27,7 @@ fi OPENAI_API_KEY="${OPENAI_API_KEY:-}" echo "=== Building production Docker image ===" -$DOCKER build --network host -t "$IMAGE_NAME" -f "$PROJECT_DIR/Dockerfile" "$PROJECT_DIR" 2>&1 | tail -10 +$DOCKER build --network host -t "$IMAGE_NAME" -f "$PROJECT_DIR/actors/runner/Dockerfile" "$PROJECT_DIR" 2>&1 | tail -10 echo "" echo "=== Running tests inside production image ==="