From 595d33f520e92fe5e6628af7d3cef37660ea1805 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 11:37:26 +0200 Subject: [PATCH 01/24] feat(sdk): annotate_trace platform op (code half of the #4999 design) Adds the annotate_trace op to the platform op catalog: self-targeting (trace/span bound from run context, links hidden from the model), additive self-metadata so it defaults to auto-allow. Tests pin the self-targeting binding. Claude-Session: https://claude.ai/code/session_01DGj7GKafjkZeQXMsryWhb2 --- .../agenta/sdk/agents/platform/op_catalog.py | 79 +++++++++++++++++++ .../unit/agents/platform/test_op_catalog.py | 39 +++++++++ 2 files changed, 118 insertions(+) diff --git a/sdks/python/agenta/sdk/agents/platform/op_catalog.py b/sdks/python/agenta/sdk/agents/platform/op_catalog.py index 66ef1349cc..7174321d23 100644 --- a/sdks/python/agenta/sdk/agents/platform/op_catalog.py +++ b/sdks/python/agenta/sdk/agents/platform/op_catalog.py @@ -309,6 +309,70 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: "required": ["workflow_revision"], } +# Annotate trace (mutating, self-targeting): record an annotation (evaluation feedback) on the +# agent's OWN current run trace — "grade myself". ``annotation.links.invocation.trace_id`` / +# ``.span_id`` are bound from run context ($ctx.trace.*) so the agent always annotates its own run +# and can never retarget another trace (the same self-targeting guarantee ``commit_revision`` gives +# via $ctx.workflow.variant.id). Unlike ``commit_revision``, this is additive self-metadata (it does +# not mutate the agent's config), so it defaults to auto-allow / no approval. +_ANNOTATE_TRACE_DESCRIPTION = ( + "Record an annotation (evaluation feedback) on your own current run's trace — grade " + "yourself. Supply `references.evaluator.slug` naming the annotation category (e.g. " + "'self_reflection', 'quality') and the `data.outputs` you are recording (scores, " + "labels, notes). Reuse a stable slug across runs: a new slug auto-creates a simple " + "evaluator in your project. The trace and span you annotate are your own current run, " + "filled automatically — you cannot annotate a different trace." +) +_ANNOTATE_TRACE_INPUT_SCHEMA: Dict[str, Any] = { + "type": "object", + # Closed so the model cannot smuggle `links` past the self-target binding + # ($ctx.trace.trace_id / .span_id); only the cataloged fields are accepted. + "additionalProperties": False, + "properties": { + "references": { + "type": "object", + "additionalProperties": False, + "description": "What this annotation evaluates against.", + "properties": { + "evaluator": { + "type": "object", + "additionalProperties": False, + "description": ( + "Names the annotation category. Auto-created as a simple " + "evaluator if the slug is new, so reuse a stable slug." + ), + "properties": { + "slug": { + "type": "string", + "description": "Stable evaluator slug, e.g. 'self_reflection'.", + } + }, + "required": ["slug"], + } + }, + "required": ["evaluator"], + }, + "data": { + "type": "object", + "additionalProperties": False, + "description": "The annotation payload.", + "properties": { + "outputs": { + "type": "object", + "additionalProperties": True, + "description": ( + "The annotation content you are recording (scores, labels, notes)." + ), + } + }, + "required": ["outputs"], + }, + # links.invocation.{trace_id,span_id} are bound from run context ($ctx.trace.*) and + # never model-supplied; see context_bindings on the op below. + }, + "required": ["references", "data"], +} + _FIND_TRIGGERS_DESCRIPTION = ( "Discover trigger events that fit plain-language use cases. Returns the best-match " "event per use case with event_key, trigger_config schema, sample payload, connection " @@ -500,6 +564,21 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: default_permission="ask", default_needs_approval=True, ), + PlatformOp( + op="annotate_trace", + description=_ANNOTATE_TRACE_DESCRIPTION, + method="POST", + path="/api/annotations/", + input_schema=_ANNOTATE_TRACE_INPUT_SCHEMA, + context_bindings={ + "annotation.links.invocation.trace_id": "$ctx.trace.trace_id", + "annotation.links.invocation.span_id": "$ctx.trace.span_id", + }, + args_into="annotation", + # Additive self-metadata (does not mutate config) -> auto-allow, no approval. + default_permission="allow", + default_needs_approval=False, + ), PlatformOp( op="find_triggers", description=_FIND_TRIGGERS_DESCRIPTION, diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py index c1b3019865..632950962a 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py @@ -39,6 +39,7 @@ def test_catalog_ships_platform_builder_ops(): "find_capabilities", "query_workflows", "commit_revision", + "annotate_trace", "find_triggers", "create_schedule", "create_subscription", @@ -222,6 +223,44 @@ async def test_commit_revision_defaults_to_approval(connection): assert spec.effective_permission() == "ask" +# --- resolver: annotate_trace self-targets its own trace/span ----------------- + + +async def test_annotate_trace_binds_own_trace_and_hides_links(connection): + # "Grade myself": the run's own trace_id/span_id are bound from run context and never + # model-supplied, so the agent can only ever annotate its OWN current trace. + resolution = await _resolver(connection).resolve( + [PlatformToolConfig(op="annotate_trace")] + ) + spec = resolution.tool_specs[0] + assert spec.call.method == "POST" + assert spec.call.path == "/api/annotations/" + assert spec.call.args_into == "annotation" + # Both self-target ids ride as call.context — the runner fills them from runContext at dispatch. + assert spec.call.context == { + "annotation.links.invocation.trace_id": "$ctx.trace.trace_id", + "annotation.links.invocation.span_id": "$ctx.trace.span_id", + } + # The model supplies only the payload (an evaluator slug + the outputs); `links` is never + # exposed, and the schema is closed so the model cannot smuggle its own target. + props = spec.input_schema["properties"] + assert set(props) == {"references", "data"} + assert "links" not in props + assert spec.input_schema["additionalProperties"] is False + assert props["references"]["properties"]["evaluator"]["required"] == ["slug"] + assert props["data"]["properties"]["outputs"]["additionalProperties"] is True + + +async def test_annotate_trace_defaults_to_auto_allow(connection): + # Additive self-metadata (does not mutate config) -> auto-allow, no approval. + resolution = await _resolver(connection).resolve( + [PlatformToolConfig(op="annotate_trace")] + ) + spec = resolution.tool_specs[0] + assert spec.needs_approval is False + assert spec.effective_permission() == "allow" + + async def test_trigger_builder_ops_have_expected_paths_and_defaults(connection): expected_paths = { "find_triggers": ("POST", "/api/triggers/discover"), From 936c1601e25336267c2a9fb891cbad95d53f8a50 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 11:41:07 +0200 Subject: [PATCH 02/24] docs(agent-workflows): approval-boundary workspace: flow explainer, bug, reviews, fix plan Design-only workspace for the approval-boundary bug (auto-approved runs park at the first gated tool): plain-words flow explanation across FE/SDK/service/ runner/harness, verified root cause with current services/runner paths, correctness + code-organization reviews of the permission code, Codex-reviewed fix options, and a phased plan (one resolved permission plan, allow|ask|deny everywhere, delete hasHumanSurface, emit approval events only on park, batch surfaces stop_reason). No code changes. Claude-Session: https://claude.ai/code/session_01DGj7GKafjkZeQXMsryWhb2 --- .../projects/approval-boundary/README.md | 62 +++++ .../code-organization-review.md | 164 +++++++++++ .../projects/approval-boundary/code-review.md | 163 +++++++++++ .../projects/approval-boundary/context.md | 77 ++++++ .../approval-boundary/design-review.md | 149 ++++++++++ .../approval-boundary/how-approvals-work.md | 261 ++++++++++++++++++ .../projects/approval-boundary/plan.md | 235 ++++++++++++++++ .../projects/approval-boundary/status.md | 63 +++++ .../projects/approval-boundary/the-bug.md | 141 ++++++++++ 9 files changed, 1315 insertions(+) create mode 100644 docs/design/agent-workflows/projects/approval-boundary/README.md create mode 100644 docs/design/agent-workflows/projects/approval-boundary/code-organization-review.md create mode 100644 docs/design/agent-workflows/projects/approval-boundary/code-review.md create mode 100644 docs/design/agent-workflows/projects/approval-boundary/context.md create mode 100644 docs/design/agent-workflows/projects/approval-boundary/design-review.md create mode 100644 docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md create mode 100644 docs/design/agent-workflows/projects/approval-boundary/plan.md create mode 100644 docs/design/agent-workflows/projects/approval-boundary/status.md create mode 100644 docs/design/agent-workflows/projects/approval-boundary/the-bug.md diff --git a/docs/design/agent-workflows/projects/approval-boundary/README.md b/docs/design/agent-workflows/projects/approval-boundary/README.md new file mode 100644 index 0000000000..de126a39c1 --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/README.md @@ -0,0 +1,62 @@ +# Approval boundary: how agent tool approvals work, the bug, and the fix plan + +An agent configured to auto-approve tools still stops at the first gated tool and waits for +a human who is not there. This workspace explains the permission/approval system end to end, +pins that bug, reviews the code behind it, and proposes a plan that fixes the bug and the +design weaknesses that produced it. + +Everything here is documentation and planning. No code changes ship with this PR. + +## TL;DR + +- **The system.** Authors set a global permission default plus per-tool + `allow | ask | deny`. Enforcement happens at three gates: Claude Code's own settings file + (rendered by our SDK), the runner's answer to Claude's permission requests, and the + runner's "relay" for tools it executes itself. When a run pauses for an approval (the + code calls this a "park"), the playground shows Approve/Deny buttons and re-sends the + conversation with the answer inside it. +- **The bug.** The runner decides "pause for a human" from whether the request carries a + session id. The SDK now mints a session id for every request, so every fresh gate pauses, + the `auto` policy is dead code, and a headless run (any caller without a chat UI: curl, + agent-as-tool, evaluations, triggers) dies at the first gate. Batch responses hide the + pause entirely: HTTP 200, mid-sentence text. See [the-bug.md](the-bug.md). +- **The fix direction.** Park only on authored intent. One resolved permission plan + (default + per-tool + builtin rules, vocabulary `allow | ask | deny`) is computed by the + SDK and enforced by both runner gates; the session-id inference is deleted; the approval + event fires only when the run actually pauses; batch shows the paused state. Recommended + as one shot (POC, no compat constraints); an independent Codex review concurred. See + [plan.md](plan.md). +- **One expectation to reset.** The fix does not make the original reproducing agent run + unattended under default config. Its `SEND_MESSAGE` tool is a write, and writes default + to `ask` by design, so the run still pauses there until the author marks that tool + `allow`. What changes: the pause happens for the authored reason, it is visible, and the + `auto` policy genuinely governs the tools it applies to. +- **Beyond the bug.** The correctness review found 4 high, 6 medium, and 2 low issues in + the same code (a swallowed reply failure that can hang runs, stale client-tool replay, + and more). The organization review found good invariant discipline but four enforcement + sites with no map, one knob with three names, and a false load-bearing comment. + +## Reading order + +1. [context.md](context.md): why this workspace exists, scope, decisions already taken. +2. [how-approvals-work.md](how-approvals-work.md): the whole flow in plain words. Read + this first if you read only one file. +3. [the-bug.md](the-bug.md): the root cause chain, the history that produced it, why + nothing caught it. +4. [design-review.md](design-review.md): the structural problems and the principles the + fix follows. +5. [code-review.md](code-review.md): correctness findings (H1-H4, M1-M6, L1-L2). +6. [code-organization-review.md](code-organization-review.md): naming, ownership, + docstrings, maintainability. +7. [plan.md](plan.md): options, the recommended design, execution phases, test plan, + behavior deltas. +8. [status.md](status.md): current state and the decisions needed. + +## How this was produced + +Four parallel code-research passes (runner, SDK+service, frontend, API interactions plane) +with every claim cited to current file:line; a dedicated correctness review and a dedicated +organization review of the permission code; an independent design review by OpenAI Codex +(xhigh reasoning) that stress-tested the fix options; and a cold-reader clarity pass over +these documents. All paths verified against the current tree (`services/agent/` was renamed +to `services/runner/`; several older docs cite the stale paths). diff --git a/docs/design/agent-workflows/projects/approval-boundary/code-organization-review.md b/docs/design/agent-workflows/projects/approval-boundary/code-organization-review.md new file mode 100644 index 0000000000..c5ad4bbd26 --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/code-organization-review.md @@ -0,0 +1,164 @@ +# Code organization review + +Scope: the permission/approval code across the runner, the SDK, the service, and the config +schema. This review asks whether the code is well organized, well named, well documented, +and safe for a new engineer to modify. Correctness findings live in +[code-review.md](code-review.md). + +## Verdict + +The code is maintainable by the people who wrote it and reviewable by anyone, but it is not +yet safely modifiable by a newcomer. The comment discipline is exceptional: nearly every +dangerous invariant is documented at the exact point of use, with issue IDs, and pinned by a +named regression test. What is missing is a map. The single concept "a tool needs +permission" is decided in four places, spoken in five vocabularies, and its main knob has +three names across layers. Answering "where do I change permission behavior?" requires +holding the whole system in your head. Worst, the one comment that anchors the central +branch (`hasHumanSurface`) is now false: the code still runs, but the documented reason it +works has drifted out from under it. That drift is exactly how the bug in +[the-bug.md](the-bug.md) survived. + +## What is good (keep it) + +Credit first, because several habits here are worth spreading: + +- **Invariants documented at point of use, with history.** The park semantics block + (`services/runner/src/responder.ts:26-41`) says "Do NOT 'simplify' park back to `deny`" + and explains the F-024 mechanism that makes it dangerous. The park-teardown rationale + (`sandbox_agent.ts:628-634`), the onPark contract (`permissions.ts:8-16`), and the + four-source merge inventory in `claude_settings.py:10-31` are equally strong. +- **Types encode the invariant.** `ResponderOutcome = PermissionDecision | "park"` plus + `decisionToReply(decision: PermissionDecision, ...)` means the compiler enforces "park + never becomes an ACP reply". The F-024 rule lives in the type system, not just a comment. +- **Deliberate security guards.** Approvals key on tool name plus canonical args, never bare + name (approving call A does not approve call B). Allow replies "once", never "always". + Both guarded by tests with names that say why. +- **`effective_permission()`** (`sdks/python/agenta/sdk/agents/tools/models.py:273-292`) is + a genuine single source of truth for the per-tool ladder over the legacy fields. +- **`claude_settings.py` is the model file.** Its docstring lists all four rule sources it + merges, each helper names its layer, and merge order is stated at the merge site. None of + the four looks like the only one. The rest of the subsystem should meet this bar. +- **The golden wire contract** (shared fixtures + compile-time key guard) is the right + cross-language pin. + +## Findings + +### 1. Ownership: four enforcement sites, no map + +"What happens when a tool needs permission" is implemented in four places: + +1. `HITLResponder`/`PolicyResponder` for harness-gated tools + (`services/runner/src/responder.ts:194`). +2. `resolvePermission` for runner-executed relay tools + (`services/runner/src/tools/relay.ts:143`). +3. The rendered Claude `settings.json` rules, which pre-empt the runner entirely + (`sdks/python/agenta/sdk/agents/adapters/claude_settings.py:201`). +4. The sandbox boundary (the network/filesystem isolation config), which renders further + deny rules. + +Changing the meaning of `ask` touches at least three files in two languages. Each site +labels its own layer, which is good, but no document or module lists all four. Related +duplication: the policy collapse `policy === "deny" ? "deny" : "allow"` is written three +times (`responder.ts:83`, `responder.ts:205`, `relay.ts:151`) and can drift. + +Also in this theme: + +- `PolicyResponder` is dead in production. `runSandboxAgent` always constructs + `HITLResponder` (`sandbox_agent.ts:651-656`); `PolicyResponder` survives only in comments + and tests. Delete it or mark it test-only. +- The client-tool park path is wired twice with hand-built, differently shaped payloads + (`permissions.ts:64-75` vs `sandbox_agent.ts:719-737`), and neither site mentions the + other. Only `user_approval` parks record an interaction row; client-tool parks silently + skip the interactions plane with no comment saying so. +- The HITL subsystem has no home: `responder.ts` (top level), `engines/sandbox_agent/ + permissions.ts`, `sessions/interactions.ts`, and ~150 inline lines of park machinery in + `sandbox_agent.ts` are one subsystem spread over four locations with no shared directory + or prefix. + +### 2. Naming: names that lie or collide + +| Name | Problem | Suggestion | +| --- | --- | --- | +| `runner.interactions.headless` / `permission_policy` / `permissionPolicy` | Three names for one knob. The authored name contains neither "permission" nor "policy"; grepping "permission" misses the authoring surface. | One name, in the permission family: `permissions.default` (see design-review.md). | +| `PermissionPolicy` {auto, deny} vs `Permission` {allow, ask, deny} | Two vocabularies where `auto` and `allow` mean the same thing. | Unify on `allow \| ask \| deny`. | +| `hasHumanSurface` | Truthful intent, untruthful derivation: it actually means "the request has a session id", which no longer implies a human. | Delete as a permission input (see plan). | +| `HITLResponder` | It is also the production headless responder; the name suggests `PolicyResponder` handles headless. It does not. | `ApprovalResponder`, and delete `PolicyResponder`. | +| `permissions.ts` | Wires all ACP interaction kinds, including client tools its own comment says are "not really a permission gate". | `acp-interactions.ts`. | +| `SandboxPermission` | An isolation boundary, not an allow/ask/deny permission; collides with the whole Permission family. The template even has `sandbox.permissions` and `harness.permissions` one line apart with unrelated vocabularies. | `SandboxBoundary` / `SandboxIsolation`. | +| `permission_mode` (legacy per-tool alias) vs Claude `PERMISSION_MODES` | Same words, two unrelated vocabularies, adjacent files. | Rename the Claude constant `CLAUDE_DEFAULT_MODES`. | +| `park` | Internal jargon; nothing says "waiting for an approval". | `pending_approval` internally; keep `stopReason: "paused"` on the wire. | +| `parkedCallResultOf` (`responder.ts:353`) | Matches every `tool_result` block, not only parked-call replies. | `toolResultEnvelope`. | + +### 3. Types: stringly seams and one mixed map + +- The wire field is stringly typed: `permissionPolicy?: string` (`protocol.ts:428`) with the + enum only in a comment, while the real `PermissionPolicy` type sits 30 lines away in + `responder.ts:24`. Type it as the enum. +- `ApprovalDecisions = ReadonlyMap` mixes permission decisions (the strings + `"allow"`/`"deny"`) with client-tool outputs (arbitrary values), discriminated at read + time. A client tool whose stored output is the literal string `"allow"` would be + misclassified, and the type permits it silently. Use a tagged union. + +### 4. Comments: one false anchor, one broken pointer + +- **The false load-bearing comment.** `sandbox_agent.ts:621-626` says the headless `/invoke` + path sets no session id. The SDK normalizer has minted one for every request since the + session work landed (`normalizer.py:307`, `shared.py:13-22`), and both golden wire + fixtures carry `sessionId`. A security-relevant branch is anchored to a cross-repo + assumption that exists only in a comment, with no producer-side test keeping it true. The + TS test that "pins" the headless case builds a request shape the Python side no longer + produces. +- **A MUST-sync coupling with a dead pointer.** `claude_settings.py:49-55` couples + `INTERNAL_TOOL_MCP_SERVER = "agenta-tools"` to the TS runner but cites + `services/agent/src/...` paths; the runner lives in `services/runner/src/`. The coupling + is enforced only by this comment, across two languages, and the pointer is broken. Fix the + paths and pin the constant with a shared-fixture test, like the golden wire contract. + +### 5. Tests: strong names, missing table + +Test names state behavior and consequence, and the relay's `resolvePermission` is tested as +a generated truth table. Gaps a maintainer would break silently: + +- The responder's three-branch decision is tested per branch, not as a matrix. Adding a + fourth branch (the `ask` surfacing) will bolt on without forcing review of the whole + table. Convert `responder.test.ts` to a generated table like the relay's. +- Nothing on the Python side asserts what `/invoke` actually puts in `sessionId`, so the + TS-side "headless" premise drifted unchecked. One runner test even pins the bug as + correct ("parks when human surface + no stored decision, even under deny basePolicy"). +- The `agenta-tools` name coupling has no test on either side. + +### 6. File size: `sandbox_agent.ts` carries eight concerns + +The engine file is 867 lines and mixes mount signing, connection env, sandbox reaping, +capability probing, MCP setup, the permission/park machinery (~lines 620-770), usage +resolution, and error detection. Sixteen helpers are already extracted into +`engines/sandbox_agent/`; the park machinery is simply the concern that has not been +extracted yet, and it is the most delicate one (it owns `hasHumanSurface`, `onPark`, the +`parked` flag, the race, and the stop reason, all coupled through closures). Extract a park +controller into the module family and give the state machine a direct unit-test seam. + +### 7. Config schema, by semantic role + +Classifying the permission-ish fields by what they are (policy, config, protocol context): + +| Field | Role | Verdict | +| --- | --- | --- | +| per-tool `permission` | Policy, author-owned | Right home: on the tool spec. Correct. | +| `permission_policy` / `runner.interactions.headless` | Policy (the default tier of the same ladder) | Right concept, wrong home and name: authored under `runner.interactions`, which reads as runtime plumbing, with a vocabulary disjoint from the family it defaults. | +| `needs_approval` | Legacy policy input | Correctly demoted under `effective_permission()`; deprecate at the config edge eventually. | +| `sandbox_permission` | Isolation config, not policy | Misnamed into the policy family; placement fine. | +| `harness.permissions` (Claude rules) | Harness-specific policy passthrough | Correctly quarantined; only `claude_settings.py` knows its shape. | +| `sessionId` as consumed by `hasHumanSurface` | Protocol context used as a policy input | The core smell. Policy inputs must be explicit fields, not inferred from routing metadata. | + +## The five highest-leverage improvements, ordered + +1. Remove the `sessionId` inference behind `hasHumanSurface` (this is also the bug fix; see + [plan.md](plan.md)) and correct the false comment. +2. Extract the park controller out of `sandbox_agent.ts` into the `engines/sandbox_agent/` + module family, giving the park state machine a unit-test seam. +3. Unify the knob's name across layers and type the wire field as an enum. +4. Write the four-site permissions map (one short header or doc listing the enforcement + sites and the "change one, check these" checklist), and fix the dead `services/agent/` + pointers left by the rename. +5. Deduplicate the client-tool interaction emission and the headless policy collapse; delete + the production-dead `PolicyResponder`. diff --git a/docs/design/agent-workflows/projects/approval-boundary/code-review.md b/docs/design/agent-workflows/projects/approval-boundary/code-review.md new file mode 100644 index 0000000000..30faeea19f --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/code-review.md @@ -0,0 +1,163 @@ +# Code review: correctness findings + +Scope: the permission/approval code paths (runner responder, ACP gate wiring, relay, SDK +adapters, service handlers). This review looks for bugs beyond the headline one; the +headline bug itself is in [the-bug.md](the-bug.md). Organization findings live in +[code-organization-review.md](code-organization-review.md). + +Each finding has a concrete failure scenario. "High" means it can hang a run, lose a +decision, or execute the wrong thing. Line numbers are from the current tree +(`services/runner/`, post-rename). + +## High + +### H1. A failed permission reply is swallowed; the run hangs and the interaction is falsely resolved + +`services/runner/src/engines/sandbox_agent/permissions.ts:106-132` (and the client-tool +branch at 76-89). The whole decide-and-reply chain ends in `.catch(() => {})`. If the +responder throws or `session.respondPermission` rejects (daemon hiccup, connection reset mid +gate), no reply reaches the harness, no park is signaled, and no error surfaces. A headless +run then blocks forever inside `session.prompt()`, which is exactly the hang the park +mechanism was built to prevent (F-040), re-opened on the reply path. + +It compounds: `onResolveInteraction(id)` runs at line 126 *before* the reply is sent, so a +reply that then fails has already marked the interaction `resolved`. The gate can no longer +be answered through the interactions plane either. + +Fix direction: on a reply failure, park (so the turn ends cleanly) or fail the run loudly; +move `onResolveInteraction` after the reply succeeds. + +### H2. Every historical tool result is treated as a stored decision; stale client-tool outputs replay instead of re-prompting + +`services/runner/src/responder.ts:353-368`. `parkedCallResultOf` returns `{found: true}` +for *any* `tool_result` block, not only the `{approved: boolean}` approval envelopes. Every +ordinary historical tool output gets stored under its name+args key, and `lookupClientTool` +(`responder.ts:223-231`) returns any such value as a fulfilled client-tool output. + +Failure scenario: `request_connection({integration:"slack"})` parks in turn 1; the browser +fulfills it; the output lands in the transcript. In turn 8 the user asks to reconnect and +the model issues the identical call. The relay finds the turn-1 output stored under the same +name+args and returns the stale "connected" result without ever prompting the browser. +Secondary edge: a tool whose replayed output is literally the string `"allow"` or `"deny"` +becomes a permission decision for later identical calls. + +The existing test ("ignores ordinary tool results that are not approval envelopes") passes +only because its fixtures lack a correlated `tool_call` block; it does not pin the dangerous +case. + +Fix direction: key only `{approved: boolean}` envelopes for permissions; scope client-tool +output replay to results correlated with a parked client-tool interaction (carry the +interaction token), not to any same-name+args result forever. + +### H3. Interaction tokens can collide across turns, making a new gate unanswerable on the interactions plane + +`services/runner/src/engines/sandbox_agent.ts:661-679` uses the raw ACP permission-request +id as the interaction `token`, and the API's create is idempotent on +(project, session, token): on conflict it returns the existing row +(`api/oss/src/dbs/postgres/sessions/interactions/dao.py:59-65`). Every turn is a cold replay +with a fresh harness session. If the sandbox-agent daemon (the ACP server hosting the +harness) numbers permission requests per session (for example `perm-1`), turn N's token +equals turn 1's. The old row was already cancelled at turn +start (`cancelStaleInteractions`), so the create returns a *cancelled* row and no live row +exists for the new gate; the status transition guard then rejects any respond. + +Caveat: the daemon's id scheme is not vendored in this repo, so this is unverified. Verify +it; if ids are per-session counters, namespace the token (for example `turnId:permId`). + +### H4. Duplicate tool-call ids bind an old approval to the wrong call + +`services/runner/src/responder.ts:312-324`. `callShapeById` is a last-write-wins map keyed +by the replayed `toolCallId`. Claude's `toolu_*` ids are globally unique, so this is latent +today, but the code is harness-generic: a harness that mints per-session counters (`call_0` +recurring across cold-replayed turns) would bind turn 1's `{approved: true}` envelope to the +*latest* `call_0`'s name and args, auto-approving a call the user never saw. Bind shape +lookups within one turn, or fail closed on duplicate ids. + +## Medium + +### M1. One approval authorizes unlimited identical calls within the resumed turn + +`responder.ts:201-206`. The decisions map is fixed for the whole turn, so every re-raised +gate with the same name+args matches the stored `allow`. If the model loops or retries the +identical `send_email(...)` call after the approved one runs, each repeat auto-allows with +no new prompt. This quietly defeats the "reply `once`, never `always`" guard: the harness +re-gates, but the stored decision answers every re-gate. Direction: consume a stored +decision on first match (delete from the map), which is what "once" means. + +### M2. Cold-replay argument drift degrades approve-to-resume into a re-prompt loop + +A design consequence of the name+args key (`responder.ts:126-135`): the resumed turn +re-generates the tool call, and the model must reproduce byte-identical arguments for the +stored approval to match. Any drift (an added optional field, a reworded command) misses the +key, parks again, and re-prompts. The user can approve repeatedly while the tool never runs. +The mirror case: a stored deny keeps auto-rejecting the next identical attempt without a +prompt, one silent wrong decision after the user changes their mind. Worth a live test to +measure how often drift happens in practice; the fix options are design work (correlate by +approval id echoed through the replay, or replay the approved result directly). + +### M3. An approval response keyed only by `approvalId` is silently dropped + +`sdks/python/agenta/sdk/agents/adapters/vercel/messages.py:183-192`. The dedicated +`tool-approval-response` part falls back to `approvalId` as the tool-call id. But +`approvalId` is the ACP permission id, never a tool-call id, so the runner finds no +correlated `tool_call` block, computes no key, and drops the decision; the gate re-parks +every turn while the user keeps answering. Any client that uses the dedicated response part +with its natural key hits this. Also: a non-boolean `approved` (the string `"true"`) +produces no decision. Direction: require the frontend to echo `toolCallId` (the approval +request already carries it) and reject uncorrelatable responses loudly instead of dropping +them. + +### M4. Client tools bypass per-tool `permission` entirely + +`services/runner/src/tools/relay.ts:208-229` branches on `kind === "client"` *before* the +permission gate at line 233, and the Claude settings renderer also excludes client tools. So +`permission: "deny"` on a client tool is a no-op everywhere: it still parks, prompts the +browser, and executes via stored output. Check `deny` before the client branch. + +### M5. A late park can flip a completed turn to `paused` + +`sandbox_agent.ts:762-767`. The relay drains in-flight handlers *after* the prompt resolves; +a straggler client-tool request that parks sets `parked = true`, and the `|| parked` clause +then rewrites a genuinely completed turn's stop reason to `paused`. The frontend waits for a +resume of a finished turn. Narrow window, real effect. Latch the stop reason before the +drain, or ignore parks after the prompt resolves. + +### M6. The streaming handler leaks the environment when stream setup throws + +`services/oss/src/agent/app.py:290-291`. `harness.setup()` and `harness.stream(...)` sit +outside the `try`, so a failure in `stream()` (runner unreachable, resolve failure) skips +`harness.cleanup()`. The batch handler has `stream()` inside its `try` and is safe. Move the +call inside the `try`. + +## Low / defensive + +- **L1. A permission request without an id emits an approval prompt that can never be + answered** (`permissions.ts:47, 83, 124`): the event goes out with `approvalId: ""`, then + the reply is skipped. Skip the emit or treat no-id as park/fail-loud. A test currently + pins the silent-drop behavior. +- **L2. MCP rule-namespace collision in the Claude settings renderer** + (`claude_settings.py:141`): a server named `agenta-tools__commit_revision` renders the + same rule string as an internal per-tool rule, so a server-level allow could shadow an + internal tool whose effective permission was `ask`. Reserve the internal server name. + Related: rule strings are unsanitized; a deny rule Claude's matcher cannot parse fails + open at the harness layer (the relay still backstops executable tools; a whole-server + deny has no backstop). + +Checked and found sound: `canonicalJson` (key-order independent, fails closed on +non-JSON values), the double-park guard, the prompt-vs-park race ordering, and usage +recording on parked turns (both handlers record it). + +## Tests that pin behavior the fix will change + +- `services/runner/tests/unit/responder.test.ts:227-237`: asserts "a deny basePolicy must + still PARK", which pins the bug itself. +- `responder.test.ts:184-225, 376-428` and + `sandbox-agent-orchestration.test.ts:883-989`: expect `park` as the no-match outcome + under `basePolicy "auto"` with no disposition on the gate; expected outcomes change with + the fix's default. +- `tool-relay-permission.test.ts:74-77, 123-127`: pin the `ask`-collapses-to-policy + behavior (`TODO(S5)`). +- `sandbox-agent-permissions.test.ts:128-156`: pins the no-id silent drop (L1). + +The plan sequences H1-H4 and M1-M6 alongside the main fix; several of them (H2, M1, M3) sit +in exactly the code the fix rewrites, so fixing them together is cheaper than separately. diff --git a/docs/design/agent-workflows/projects/approval-boundary/context.md b/docs/design/agent-workflows/projects/approval-boundary/context.md new file mode 100644 index 0000000000..ef5ff31bca --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/context.md @@ -0,0 +1,77 @@ +# Context + +## What this workspace is + +This workspace explains how tool permissions and approvals work in the agent runtime, from +the playground form down to the Claude Code harness. It then explains one confirmed bug in +that flow, reviews the code that implements it, and proposes a plan that fixes the bug and +the design weaknesses that produced it. + +Read [how-approvals-work.md](how-approvals-work.md) first. Every other document builds on it. + +## The symptom that started this + +A repo-digest agent (`uc9-digest`) has four tools: three read-only lookups (`LIST_COMMITS`, +`LIST_REPOSITORY_ISSUES`, `LIST_ALL_CHANNELS`) and one side-effecting write (`SEND_MESSAGE`, +which posts to Slack). The agent's permission policy is `auto`, which is documented as +"approve tool prompts automatically, do not wait for a human." + +A one-shot HTTP call to this agent stops before the last tool. The three reads run. Then the +run emits a request for human approval of `SEND_MESSAGE` and ends. Nobody is there to +approve, and a one-shot call has no way to answer, so the run just stops. The batch response +makes it worse: it returns HTTP 200 with a mid-sentence assistant reply and no hint that the +run paused. + +The same agent works in the playground. That is not because the playground avoids the pause. +It hits the same pause, shows an Approve button, and quietly re-sends the conversation after +you click it. The playground papers over a stop that kills every caller without a human and +a resend loop. + +## Why this is a bug and not a policy choice + +The author asked for `auto`. The code that should honor `auto` is never reached. The runner +decides "park and wait for a human" based on whether the request carries a session id, and +the SDK now mints a session id for every request. So the `auto` policy is dead code, and the +run's own comment ("the headless invoke path sets no session id") describes a world that no +longer exists. [the-bug.md](the-bug.md) walks the exact chain. + +## Scope + +In scope: + +- Explaining the whole permission and approval flow in plain words, with current file + references (`services/agent/` was renamed to `services/runner/`; older docs cite stale + paths). +- The bug: root cause, history, and a concrete fix. +- A review of the permission/approval code for correctness and for organization. +- A plan that fixes the bug now and simplifies the design so this class of bug stops + recurring. + +Out of scope: + +- Changing how batch vs streaming invoke returns results. That work lives in the sibling + workspace `../builder-agent-reliability/streaming-invoke/`. +- Wiring the durable interactions plane end to end. The API side exists and is half wired by + design; the product currently resolves approvals through the frontend. We only make sure + the plan does not fight that future work. +- Backward compatibility. The whole big-agents feature is a pre-release POC. We can rename + fields and change behavior freely. + +## Decisions already taken by the owner + +- **Auto means auto everywhere.** When the policy auto-approves a tool, the tool runs and + the human sees that it ran. No prompt, in the playground or anywhere else. Only an + explicit "ask" should wait for a human. +- This PR ships documentation and a plan. Implementation follows in a separate PR once the + plan is reviewed. + +## Related workspaces + +- `../capability-config/` designed the three-layer permission model this doc builds on. Its + proposal already warns against the exact confusion behind this bug. +- `../hitl-fix/` fixed an earlier approval bug (the runner used to answer "reject" when it + wanted to pause, which broke the approval UI). The fix introduced "park", and the park + condition is what this workspace corrects. +- `../builder-agent-reliability/streaming-invoke/` found this bug while investigating + partial batch output. Its `approval-boundary.md` holds the first investigation; this + workspace supersedes it with verified current paths and a broader review. diff --git a/docs/design/agent-workflows/projects/approval-boundary/design-review.md b/docs/design/agent-workflows/projects/approval-boundary/design-review.md new file mode 100644 index 0000000000..c416299566 --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/design-review.md @@ -0,0 +1,149 @@ +# Design review: does the approval boundary make sense? + +Prerequisite: [how-approvals-work.md](how-approvals-work.md). This document judges the +design, names its structural problems, and states the principles a fix should follow. The +concrete plan lives in [plan.md](plan.md). An independent second opinion (OpenAI Codex, +xhigh reasoning) reviewed both this analysis and the fix options; where it sharpened or +overruled a lean, the text says so. + +## The short version + +The intended design is sound. The capability-config proposal +(`../capability-config/proposal.md`) got the model right: per-tool permission +(`allow | ask | deny`) as the policy, a global default for unset tools, and enforcement at +whatever choke point each tool passes. It even warned, in writing, against the exact +mistake that later happened: "collapsing the two — treating `permission_policy` as if it +were a fourth permission — is the mistake to avoid." + +The implementation diverged from that model in one decisive way: **the runner decides +whether to pause from transport metadata (a session id) instead of from the authored +policy.** Everything else on this list is either a contributor to that divergence or a +smaller instance of the same pattern. + +## The structural problems + +### 1. Intent is inferred, not declared + +Whether a tool call needs a human is a fact about the tool and its config. The runner +instead infers it from `sessionId` presence (`sandbox_agent.ts:627`). Transport metadata is +owned by other layers for other reasons, so it changes for other reasons; when the SDK +started minting session ids for every request, the inference silently inverted and no test +noticed. Policy inputs must be explicit fields with one owner, never proxies. + +### 2. The policy vocabulary cannot express the design + +The global policy is `auto | deny`. The per-tool vocabulary is `allow | ask | deny`. The +global one has no `ask`, so "pause for a human" is not expressible as policy at all. That +missing word is why the code reached for a proxy: the runner wanted to say "someone might +want to answer this" and had no policy value to say it with, so it looked at the session id. +A design that cannot express its central concept in its own vocabulary will express it as a +hack. + +### 3. Three enforcers, no single computer + +The per-tool disposition is computed independently in three places: + +- Python, when rendering Claude's settings file (`claude_settings.py:151-189`); +- TypeScript, in the relay for runner-executed tools (`relay.ts:143-152`); +- and never, in the one place that decides about parking (the ACP responder). + +Each site consults a different subset of the config. The responder sees none of it, the +relay collapses `ask` onto the policy (its own `TODO(S5)` admits this), and the settings +renderer erases the ask/unset distinction before the gate is ever raised. Three enforcement +points are fine (tools genuinely pass different choke points), but three independent +*computations* of the same policy, in two languages, guarantee disagreement. One layer +should compute; every enforcement point should look up. + +### 4. The approval event fires before the decision + +`attachPermissionResponder` emits `interaction_request(user_approval)` before it consults +the responder (`permissions.ts:93-107`). So the event does not mean "a human must act"; it +means "a gate was raised", and the consumer must guess which. Under any fix where `auto` +answers gates in place, an auto-approved run would still emit an approval request that no +one should act on. Events must mean what their consumers will do with them: emit the +approval request only when the run actually pauses for an answer. The "human sees what ran" +need is already served by the `tool_call`/`tool_result` events. + +### 5. Batch erases the terminal state + +`_agent_batch` returns only the final assistant text; `stop_reason` is read nowhere +(`app.py:303-321`). A run that paused for an approval is indistinguishable from a run that +finished. Whatever the permission semantics become, a caller must be able to see "this run +paused, here is the pending interaction". This is visibility, not permission logic, but it +is what turned a behavior bug into a silent one. + +### 6. Two planes, stitched halfway + +The messages plane (decision rides in the conversation) is the real product path. The +interactions plane (durable rows plus respond endpoint) is deliberately future work, but its +current half-wiring creates traps: the runner writes rows only for committed revisions, the +respond endpoint re-invokes instead of resuming, and the product UI never reads the plane. +None of this blocks the fix, but the fix must not deepen the split: parking should keep +producing interaction rows (they are the seed of the durable flow), and the paused state +that batch surfaces should carry the interaction identity so the future respond flow has +something to grab. + +### 7. The playground UX leans on the bug's behavior + +Today the playground gets its Approve/Deny prompt because *everything* parks. Under correct +`auto` semantics, the prompt appears only for tools that resolve to `ask`. That is the +intended product behavior (the owner confirmed: auto means auto everywhere), but it is worth +stating as a design consequence: the UI's gate experience becomes a function of authored +config, not of the surface you run from. The tool editor's defaults (read-only tools default +to `allow`, mutating tools to `ask` via catalog hints) become the main source of prompts. + +## What makes this area genuinely tricky + +These are constraints any design must respect, not flaws: + +- **Claude gates first.** Claude Code consults its own settings before asking anyone. + The only way to pre-approve a tool on Claude is a rendered allow rule; the only way to be + asked is to leave the gate raised. The settings file is therefore part of the permission + system whether we like it or not (F-046 proved this). +- **Park must not answer.** Replying "reject" to pause breaks the UI, because Claude turns + the reject into a failed tool call that overwrites the approval prompt (F-024). Pausing + means: no reply, tear down, resume by cold replay. This mechanism is correct and stays. +- **Resume is a cold replay.** Every turn is a fresh session; approvals must re-match a + re-raised gate by tool name plus canonical args. The keying is deliberate security + hardening (bare names would over-authorize) and constrains any alternative resume design. +- **Harnesses are asymmetric.** Pi never raises gates, so per-tool `ask` on Pi is + enforceable only at the relay, which today cannot park mid-prompt. Claude builtins have no + tool spec, so their dispositions exist only as authored settings rules. + +## Principles for the fix + +1. **Declare intent; never infer it from transport.** Park only when the resolved + disposition says `ask`. Delete `hasHumanSurface`. Session ids stay what they are: + correlation for persistence and tracing. +2. **One computer, many enforcers.** The SDK already owns `effective_permission()` and the + settings rendering; make it compute one resolved permission plan (default disposition + plus per-tool dispositions, including structured rules for Claude builtins) and ship it + on the run request. The responder and the relay become lookups into the same plan. + (Codex pushed hard on this point: a fix that only patches the responder "fixes `auto` + while silently breaking explicit builtin `ask`", because authored ask-rules for Claude + builtins are visible only inside the settings file.) +3. **One vocabulary.** `allow | ask | deny` everywhere. The global default becomes + `default_permission` in the same vocabulary (`auto` maps to `allow`; a global `ask` is + now expressible and legitimate: approval-by-default). Retire the + `runner.interactions.headless` authoring path in favor of a name inside the permission + family. +4. **Events mean actions.** Emit `interaction_request(user_approval)` only when the run + pauses. An auto-approved gate emits nothing extra; the tool events already show what ran. +5. **Terminal state is always visible.** Batch surfaces `stop_reason` and, when paused, the + pending interaction identity. +6. **A stored approval never overrides a current deny.** The decision order is: resolve the + disposition first; consult stored decisions only when the disposition is `ask`. (Codex + flagged this ordering; the current code checks stored decisions first, so a stale + approval could outrank a config that has since been changed to deny.) + +## On the alternative shapes we rejected + +- **The one-line fix** (consult the policy before parking) restores `auto` but silently + auto-approves every `ask`/unset tool too, killing the playground prompt. Rejected. +- **A disposition-aware responder without the shared plan** fixes resolved tools but leaves + Claude-builtin ask-rules invisible to the responder, so an author's explicit `ask` on + `Bash` would auto-approve under a policy of `allow`. Rejected as a final state; acceptable + only as an explicitly temporary step. +- **Settings-only gating** (render everything into Claude's settings, drop ACP parking) + cannot express `ask` without re-triggering the F-024 clobber, does nothing for Pi or + relay/client tools, and was rejected by the second opinion as well. diff --git a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md new file mode 100644 index 0000000000..758852000d --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md @@ -0,0 +1,261 @@ +# How tool permissions and approvals work + +This document explains the whole flow in plain words: what an author can configure, what +happens when an agent wants to run a tool, who decides whether it runs, and how a human +approval travels from a button in the playground back into the run. It describes the code as +it is today, with current paths (the runner lives in `services/runner/`; older docs say +`services/agent/`, which was renamed). + +Read this before [the-bug.md](the-bug.md). The bug is a five-line story once you know the +flow; without the flow it is incomprehensible. + +## The cast: five systems touch a tool call + +One tool call can cross five systems. Each has one job in the permission story. + +1. **The playground frontend** (`web/`). Renders the config form (permission policy, + per-tool permission, Claude rules). During a run, it renders tool activity, shows + Approve/Deny buttons when a run pauses for approval, and re-sends the conversation after + you answer. +2. **The agent service** (`services/oss/src/agent/app.py`, Python). The HTTP front door. + It parses the agent config, resolves tools, and forwards one run request to the runner. + It serves both shapes: batch (`/invoke`, one JSON reply) and streaming (the playground's + path, every event as it happens). +3. **The runner** (`services/runner/`, TypeScript). Drives the harness inside the sandbox + over ACP, the Agent Client Protocol (spoken to a small "sandbox-agent" daemon that hosts + the harness process). This is where our permission decisions live: the runner answers + the harness's permission requests, executes gateway/code tools through its "tool relay", + and emits every event the frontend sees. +4. **The harness** (Claude Code or Pi). The actual coding agent. Claude Code has its own + built-in permission system and asks for permission before running gated tools. Pi has + none; it never asks. +5. **The API interactions plane** (`api/oss`, `/sessions/interactions`). A durable store of + approval requests, built for the future ("answer an approval hours later, from anywhere"). + Today the runner writes records into it, but the product answers approvals through the + frontend instead. See "The two planes" below. + +The Python SDK (`sdks/python/agenta/`) is the glue between 2 and 3: it defines the config +schema, renders Claude's settings file, and translates the runner's event stream to the +Vercel AI SDK wire format the frontend speaks. + +## What an author can configure + +Four knobs control what an agent may do without asking. They live at different levels and, +today, under inconsistent names. + +**1. The global permission policy.** One value per agent: `auto` or `deny`. `auto` means +"approve tool prompts automatically"; `deny` means "refuse them". Confusingly, this one knob +has three names depending on where you stand: + +- Authors write it as `runner.interactions.headless` in the agent config + (parsed in `sdks/python/agenta/sdk/agents/dtos.py:1087-1102`). +- The SDK stores it as `permission_policy` (`dtos.py:571`). +- The wire to the runner calls it `permissionPolicy` (`services/runner/src/protocol.ts:427`). + +The playground labels it "Permission policy" and always sends `auto` unless the author +changes it (`web/packages/agenta-playground/src/state/execution/agentRequest.ts:262`). + +**2. Per-tool permission.** Each configured tool (a gateway tool, meaning a hosted +integration action from the Composio catalog; a code tool; an MCP server) can carry its own +`permission`: `allow`, `ask`, or `deny` +(`sdks/python/agenta/sdk/agents/tools/models.py:26`). `allow` means run it without asking. +`ask` means a human must approve each call. `deny` means never run it. A tool with no +explicit permission falls back through a ladder (`effective_permission`, +`models.py:273-292`): a legacy `needs_approval: true` flag means `ask`; a catalog `read_only` +hint defaults reads to `allow` and writes to `ask`; otherwise the tool defers to the global +policy. The output of this ladder is what the rest of this workspace calls the tool's +**disposition**: its final, resolved allow/ask/deny. The tool editor exposes the field as a +Permission select with an "Inherit policy" placeholder (`web/.../ToolFormView.tsx:47-51`). + +**3. Claude harness rules.** For the Claude harness only, authors can write raw Claude Code +permission rules (`harness.permissions`: a `default_mode` plus `allow`/`ask`/`deny` rule +lists like `Bash(npm run:*)`). These pass through to Claude's own settings file +(`web/.../ClaudePermissionsControl.tsx`). + +**4. Sandbox permission.** Network and filesystem boundaries (`sandbox_permission`). This is +a security boundary, not an approval flow, so this document leaves it aside. It matters here +only because it also renders deny rules into Claude's settings. + +Note the vocabulary mismatch: the global policy speaks `auto | deny`, the per-tool field +speaks `allow | ask | deny`. "Auto" and "allow" mean the same thing. The global vocabulary +has no `ask`, which turns out to be central to the bug. + +## Where permission is enforced: three gates + +A tool call passes through up to three enforcement points, depending on the tool and the +harness. (Strictly there is a fourth site, the sandbox boundary from knob 4, which renders +its own deny rules; it is a security wall rather than an approval gate, so counts of "three +gates" here and "four enforcement sites" in the reviews are the same list with and without +it.) + +**Gate 1: Claude's own settings file.** Claude Code checks every tool call against +`.claude/settings.json` before doing anything. The SDK renders that file +(`sdks/python/agenta/sdk/agents/adapters/claude_settings.py:201-258`) by merging four rule +sources: the author's raw Claude rules, denies derived from the sandbox permission, per-MCP- +server permissions, and per-tool permissions. The rendering is deliberately conservative: a +per-tool `allow` becomes an allow rule (Claude runs the tool with no prompt), a `deny` +becomes a deny rule, but `ask` and unset produce no rule at all, which leaves Claude's gate +raised. The docstring at `claude_settings.py:22-31` states the intent: `permission_policy: +"auto"` is not a blanket bypass; an unlisted tool still triggers a permission request. + +An important consequence: at the next gate, the runner cannot tell "the author said ask" +from "the author said nothing". Both arrive as the same raised gate. + +**Gate 2: the runner's ACP responder.** When Claude's settings do not settle a tool, Claude +raises an ACP permission request to the runner. The handler +(`attachPermissionResponder`, `services/runner/src/engines/sandbox_agent/permissions.ts:38-134`) +does two things, in this order: + +1. It emits an `interaction_request` event of kind `user_approval` into the run's event + stream, unconditionally, before any decision is made (`permissions.ts:93-105`). This is + the event the frontend turns into Approve/Deny buttons. +2. It asks a "responder" object what to do (`permissions.ts:106-107`). + +The responder for product runs is `HITLResponder` +(`services/runner/src/responder.ts:194-232`; HITL stands for human-in-the-loop). Its +decision procedure is three branches, in order: + +```ts +async onPermission(request) { + const stored = this.lookupPermission(request); // 1. a decision from a previous turn? + if (stored) return stored; // use it + if (this.hasHumanSurface) return "park"; // 2. a human might be watching? stop and wait + return this.basePolicy === "deny" ? "deny" : "allow"; // 3. headless: apply the policy +} +``` + +Three outcomes are possible. `allow` replies "once" to Claude (approve this single call, +never "always") and the tool runs. `deny` replies "reject" and the tool is refused. `park` +is the interesting one: the runner sends no reply at all, tears the session down, and ends +the turn with `stopReason: "paused"` (`sandbox_agent.ts:640-649, 766-767`). Park means "a +human must answer, and the answer will arrive on a future turn, not this one". + +`hasHumanSurface`, the flag branch 2 keys on, is computed from one signal: does the run +request carry a non-empty session id (`sandbox_agent.ts:627`). + +**Gate 3: the tool relay.** Gateway and code tools do not run inside the harness; the runner +executes them itself through its tool relay. The relay enforces the per-tool permission +directly (`services/runner/src/tools/relay.ts:143-152`): `allow` runs, `deny` refuses, and +`ask` or unset collapses onto the global policy, with an explicit `TODO(S5): surface ask to +HITL` (S5 is the open-issues slice label for that deferred work). So today, a relay-executed +tool marked `ask` never actually asks anyone; the same authored `ask` behaves differently +depending on which gate handles the tool. + +One special tool kind crosses these gates differently: **client tools** (`kind: "client"`, +for example `request_connection`). A client tool is fulfilled by the user's browser, not by +the harness or the relay. When one is called, the runner parks the turn and emits an +`interaction_request` of kind `client_tool`; the frontend fulfills it and the output comes +back on the next turn the same way an approval does. Client tools matter to this review for +two reasons: they are a second park path, wired separately from the approval one, and today +they skip the per-tool permission check entirely (see code-review M4). + +Why do gateway tools hit Gate 2 at all, if the relay is Gate 3? Because on Claude, resolved +tools are delivered as tools of an internal MCP server (`mcp__agenta-tools__`), and +Claude raises its own gate before the call ever reaches the relay. That is why the settings +rendering (Gate 1) exists: without an allow rule, even an `allow` tool would stop at Gate 2 +(this was bug F-046, fixed by rendering per-tool permissions into the settings file). + +## The journey of one gated tool call (the playground path) + +Concrete example: the `uc9-digest` agent, Claude harness, policy `auto`. Three read-only +tools are effectively allowed; `SEND_MESSAGE` posts to Slack and ends up gated. + +1. You send a message in the playground. The frontend posts the whole conversation to the + agent service's streaming endpoint. +2. The SDK's request normalizer resolves a session id, minting a fresh UUID if the request + has none (`sdks/python/agenta/sdk/middlewares/running/normalizer.py:307`, + `sdks/python/agenta/sdk/models/shared.py:13-22`). The id flows into the run request the + service sends the runner. +3. The runner starts a Claude session in the sandbox, writing the rendered + `.claude/settings.json` into the workspace first. +4. Claude runs the three reads without asking (allow rules from their `read_only` hints). + Their `tool_call` and `tool_result` events stream to the frontend as they happen. +5. Claude wants `SEND_MESSAGE`. No allow rule matches, so Claude raises an ACP permission + request. +6. The runner emits `interaction_request(user_approval)` into the stream, then consults + `HITLResponder`. No stored decision exists, the request has a session id, so: **park**. + The runner records a pending interaction row in the API (only when the run belongs to a + committed revision), sends Claude no answer, destroys the session, and ends the turn with + `stopReason: "paused"`. +7. The SDK's stream adapter translates the interaction event into a Vercel AI SDK + `tool-approval-request` chunk on the tool part + (`sdks/python/agenta/sdk/agents/adapters/vercel/stream.py:398-437`) and maps the paused + stop into a benign finish reason. +8. The playground renders the gated tool with "Run this tool?" and Approve/Deny buttons + (`web/.../ToolActivity.tsx:89-115`). It shows "Waiting for approval", and anything you + type meanwhile is queued, not sent. +9. You click Approve. The Vercel SDK marks the tool part `approval-responded`, and an + auto-resume predicate (`sendAutomaticallyWhen: agentShouldResumeAfterApproval`, + `web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts:108-122`) + re-sends the entire conversation. Deny re-sends too; the model should hear "no" and + continue, not hang. +10. On the way back in, the SDK's message adapter folds your decision into a `tool_result` + content block whose output is the envelope `{"approved": true}` + (`sdks/python/agenta/sdk/agents/adapters/vercel/messages.py:103-192`). +11. The runner starts a fresh session (every turn is a cold replay: same conversation, new + session) and pre-extracts all `{approved}` envelopes from the incoming messages into a + decision map (`extractApprovalDecisions`, `responder.ts:306-343`). Decisions are keyed + by tool name plus canonicalized arguments, never by name alone, so approving one call + does not approve a different call to the same tool. +12. Claude replays the conversation and re-raises the same gate. This time branch 1 of the + responder finds the stored decision: `allow`. The runner replies "once", the tool runs, + its result streams, and the turn finishes normally. + +That is the designed happy path, and it works. The bug is what happens on step 6 when no +human is on the other end of the stream: the run parks anyway, because "park or not" looks +at the session id instead of the policy. [the-bug.md](the-bug.md) picks up from here. + +## The two planes: messages and interactions + +The approval answer travels on what the code calls the **messages plane**: the decision +rides inside the conversation itself (the `{approved}` envelope in a tool result block) and +takes effect when the frontend re-sends the conversation. This is the plane the product uses +today, end to end. + +There is a second, newer **interactions plane**: a `session_interactions` table plus +`/sessions/interactions` endpoints in the API (create, query, transition, respond; +`api/oss/src/apis/fastapi/sessions/router.py:591-882`). The vision (spec: +`docs/designs/sessions/interactions/extend/specs.md`) is durable approvals: a parked run +leaves a pending record that anyone can answer later from any surface, without holding a +chat open. + +Today this plane is deliberately half wired: + +- The runner writes to it: it creates a pending row on park and marks the row resolved when + a stored decision later settles the gate. But only for committed revisions; playground + drafts skip it entirely (`sandbox_agent.ts:664-670`). +- The only reader is the debug Session Inspector. Its Approve button calls the respond + endpoint, which does not resume the parked turn; it fires a fresh, detached re-invoke of + the same revision (`api/oss/src/tasks/asyncio/sessions/interactions_dispatcher.py:31-76`). +- The product chat UI never touches it. + +So think of the interactions plane as an audit shadow with future ambitions. The real +decision loop is the messages plane. Any fix must keep the interactions writes intact (they +are the seed of the future durable flow) without depending on them. + +## Harness differences + +Only Claude gates. Pi reports `permissions: false` and never raises an ACP permission +request, so nothing parks on Pi and the permission policy does nothing there (the UI hides +the field for Pi with a note, hardcoded by harness name at +`web/.../useModelHarness.tsx:113`). For Pi, the only permission enforcement is Gate 3, the +relay, where `ask` currently degrades to the policy. Real per-tool HITL on Pi would need the +relay to learn to park (tracked as open-issues slice S5.2; see `../hitl-fix/plan.md`). + +## Where each piece of code lives + +| Piece | File | +| --- | --- | +| Decision logic (responders, decision keys, envelope extraction) | `services/runner/src/responder.ts` | +| ACP gate wiring (emit event, apply decision, park) | `services/runner/src/engines/sandbox_agent/permissions.ts` | +| Human-surface flag, responder construction, park teardown, interactions calls | `services/runner/src/engines/sandbox_agent.ts:590-770` | +| Relay enforcement for runner-executed tools | `services/runner/src/tools/relay.ts:143-152, 230-240` | +| Run request shape (`sessionId`, `permissionPolicy`, per-tool `permission`) | `services/runner/src/protocol.ts:348-474` | +| Interactions API client (create/resolve/cancel-stale) | `services/runner/src/sessions/interactions.ts` | +| Config schema: policy, per-tool permission ladder | `sdks/python/agenta/sdk/agents/dtos.py`, `sdks/python/agenta/sdk/agents/tools/models.py` | +| Claude settings rendering (Gate 1) | `sdks/python/agenta/sdk/agents/adapters/claude_settings.py` | +| Stream egress (interaction event to approval chunk) | `sdks/python/agenta/sdk/agents/adapters/vercel/stream.py:384-475` | +| Message ingress (button click to `{approved}` envelope) | `sdks/python/agenta/sdk/agents/adapters/vercel/messages.py:103-192` | +| Service handlers (batch and stream) | `services/oss/src/agent/app.py:207-321` | +| Approve/Deny UI and auto-resume | `web/.../ToolActivity.tsx`, `web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts` | +| Interactions API (durable plane) | `api/oss/src/apis/fastapi/sessions/router.py:591-882` | diff --git a/docs/design/agent-workflows/projects/approval-boundary/plan.md b/docs/design/agent-workflows/projects/approval-boundary/plan.md new file mode 100644 index 0000000000..a3dc38328a --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/plan.md @@ -0,0 +1,235 @@ +# Plan: fix the approval boundary + +Prerequisites: [the-bug.md](the-bug.md) for the bug, +[design-review.md](design-review.md) for the principles, +[code-review.md](code-review.md) for the correctness debt this plan folds in. + +This is a plan, not an implementation. The whole feature is a pre-release POC, so nothing +here preserves backward compatibility. + +## What "fixed" means, concretely + +Behavior after the fix, stated as acceptance checks: + +1. **`auto`/`allow` never prompts, anywhere.** A tool whose resolved permission is `allow`, + or an unset tool under a default of `allow`, runs in place. Its result streams. No + approval event is emitted. Playground and headless behave identically. +2. **`ask` always pauses, everywhere.** A tool whose resolved permission is `ask` parks the + turn and emits exactly one approval request. In the playground you get the Approve/Deny + buttons, as today. On a headless call the run ends `paused`, and the caller can see that. +3. **`deny` never runs, anywhere.** Including client tools (today they bypass it). +4. **The pause is visible on every path.** Batch responses carry the stop reason and, when + paused, the pending interaction identity. Nobody has to query spans to learn their run + stopped. +5. **No decision comes from transport metadata.** Deleting a session id changes nothing + about permissions. + +One consequence worth stating up front, because it reframes the original reproduction: the +`uc9-digest` run will still pause at `SEND_MESSAGE` under default config, even after the +fix. Resolved gateway tools carry a `read_only` hint from the Composio catalog, and the +per-tool ladder defaults mutating tools to `ask` +(`sdks/python/agenta/sdk/agents/tools/models.py:273-292`; confirmed populated at +`sdks/python/agenta/sdk/agents/platform/gateway.py:199`). That default is deliberate: +capability-config's design says reads auto-run, writes prompt. + +What changes is that the pause becomes *authored, visible, and answerable*. The author who +wants the digest posted headless sets `permission: allow` on `SEND_MESSAGE` (one select in +the tool editor), and the run completes end to end. The `auto` policy governs tools with no +disposition at all, and it actually works again. + +## Options considered + +**Option A: the one-line fix.** Consult `basePolicy` before parking +(`responder.ts:201-206`). Restores `auto` for everything, including tools that resolved to +`ask`: the playground prompt disappears entirely, and an author's explicit "ask me first" +is silently auto-approved. Rejected. + +**Option B: disposition-aware responder only.** The responder looks up the gated tool's +`permission` from the run request's tool specs (`allow` allows, `deny` denies, `ask` parks, +unset falls to the policy) and `hasHumanSurface` dies. This fixes resolved tools but leaves +one hole: authors can write raw `ask` rules for Claude *builtins* (`Bash`, `Edit`...) via +`harness.permissions`, and those exist only inside the rendered settings file. The responder +would see a builtin gate as "unset" and auto-approve it under a default of `allow`. An +explicit authored "ask" being silently approved is the worst failure mode on the table. +Rejected as an end state. + +**Option B+C: B plus Option C.** Same hole as B. Acceptable only as a stepping +stone if we accept a documented temporary gap for builtin ask-rules. + +**Option D: one resolved permission plan (recommended).** The SDK computes every +disposition once and ships it on the run request; the runner only enforces. This closes the +builtin hole (authored builtin rules travel in structured form), unifies the relay and the +responder on one decision function, and removes both duplicate computations and the +vocabulary split. The independent Codex review concurred, and specifically recommended +doing this in one shot rather than staging through B: there are no compatibility +constraints, B leaves duplicate policy logic alive, and the extra cost over B is mostly +wire-shape and test churn. + +**Recommendation: D, in one shot, plus the visibility work (4) and the correctness debt +that lives in the same code.** If implementation reveals the wire change is heavier than +expected, fall back to landing B+C first with the builtin-ask gap explicitly documented, +then finish D. That decision can be made mid-implementation without wasted work, because +B's responder change is a subset of D's. + +## The target design + +### One wire contract for permissions + +The run request gains one coherent block, replacing the stringly `permissionPolicy`: + +``` +permissions: { + default: "allow" | "ask" | "deny", // was permission_policy auto|deny; auto -> allow + rules?: [ // authored harness-builtin rules, structured + { pattern: "Bash(rm:*)", permission: "ask" }, + ... + ] +} +``` + +Per-tool dispositions keep riding where they already are (`customTools[].permission` and +`mcpServers[].permission`), but the SDK writes the **resolved** value (the output of +`effective_permission()`, so `read_only`/`needs_approval` defaults are already applied) and +the runner never re-derives it. Semantic roles stay clean: `permissions` is policy, owned by +the author, resolved by the SDK; `sessionId` goes back to being pure correlation. + +Vocabulary: `allow | ask | deny` everywhere. A global default of `ask` becomes expressible +(approval-by-default agents). The authored config path moves from +`runner.interactions.headless` into the permission family (proposal: `permissions.default` +next to the existing `harness.permissions` and tool permissions; final name settled at +implementation, but it must contain the word "permission" and live with the policy fields). + +### One decision function + +``` +resolvePermission(gate, plan): + disposition = plan.toolSpec(gate)?.permission // resolved tools, MCP servers + ?? plan.matchRule(gate) // structured builtin rules + ?? plan.default + if disposition == "deny": return deny + if disposition == "allow": return allow // reply "once", run continues in place + // disposition == "ask": + stored = storedDecision(gate) // {approved} envelopes from the replay + if stored: return stored (consumed once) + return pendingApproval // park: no reply, end turn "paused" +``` + +Order matters, twice over. The disposition is resolved *before* stored decisions are +consulted, so a stale approval can never override a config that has since changed to `deny` +(today stored decisions are checked first). And a stored decision is consumed on first +match, so one approval authorizes one execution ("once" semantics, code-review M1). + +The same function backs both gates: the ACP responder (harness-gated tools) and the relay +(runner-executed tools). The relay's `TODO(S5)` collapse disappears; the relay can already +park for client tools, and an `ask` on a relay tool parks the same way. `hasHumanSurface` is +deleted. `PolicyResponder` is deleted. An `ask` park on a headless run is correct behavior: +the turn ends `paused`, visibly, with an interaction row for the future durable-resume flow. + +### Events mean actions + +`interaction_request(user_approval)` is emitted only when the decision is `pendingApproval` +(today it fires before the decision, so auto-approved gates would emit false prompts). What +ran is already visible through `tool_call`/`tool_result` events. + +### The pause is visible on batch + +`_agent_batch` returns `stop_reason` alongside the messages and, when paused, the pending +interaction reference (id/token). Exact response shape to be settled against the streaming +work in `../builder-agent-reliability/streaming-invoke/` (which owns the batch-vs-stream +story); the requirement from this side is only: a paused run must be distinguishable and +reference its pending approval. + +## Execution phases + +Each phase lands green on its own. Sizes are rough. + +**Phase 1: the decision core (runner, ~the heart of the fix).** +`resolvePermission` as one exported function; responder and relay call it; delete +`hasHumanSurface` and `PolicyResponder`; emit approval events only on park; consume stored +decisions once. Rewrite the responder decision tests as a generated truth table +(disposition × stored-decision × default), replacing the tests that pin the bug +(`responder.test.ts:227-237` and the park-by-default cases; list in code-review.md §tests). + +**Phase 2: the wire and the SDK (Python).** +`permissions.{default, rules}` on the run request (typed enum, mirrored in `wire.py`, the Python copy of the runner's wire types; +golden fixtures updated); SDK resolves per-tool dispositions at request build time +(`effective_permission()` output onto the specs); structured builtin rules derived from the +same authored `harness.permissions` lists that feed the settings renderer, so the two +cannot disagree; authored config path renamed into the permission family; Claude settings +rendering unchanged in behavior (it already renders resolved allow/deny; verify `ask`/unset +still leaves the gate raised). + +**Phase 3: service and frontend.** +Batch surfaces `stop_reason` + pending interaction reference; fix the stream-setup cleanup +leak (code-review M6). Frontend: the config form writes the renamed field (the "Permission +policy" select becomes the permission default, vocabulary `allow | ask | deny`); no changes +to the approval UI or resume machinery (they keep working for `ask` tools; verified by the +frontend research: nothing in the UI depends on parking for non-ask tools). + +**Phase 4: correctness debt in the same code (from code-review.md).** +H1 (reply failure parks or fails loud; resolve interaction only after a successful reply), +H2 (only `{approved}` envelopes become decisions; client-tool replay scoped by interaction +token), M3 (approval responses must correlate by `toolCallId`; drop loudly otherwise), M4 +(client tools honor `deny`), M5 (latch the stop reason before the relay drain), L1 (no-id +gates park instead of silently hanging), H3 (verify the daemon's permission-id scheme; if +per-session counters, namespace interaction tokens with the turn), L2 (reserve the +`agenta-tools` MCP server name in the settings renderer). + +**Phase 5: organization cleanup (from code-organization-review.md).** +Extract the park controller from `sandbox_agent.ts` into the `engines/sandbox_agent/` +family; rename `park` → `pendingApproval` internally (wire `stopReason: "paused"` stays); +`permissions.ts` → `acp-interactions.ts`; the four-site permissions map as a header doc; +fix the stale `services/agent/` pointers and pin the `agenta-tools` coupling with a shared +fixture test. + +**Phase 6: proof.** +Live matrix on the dev stack (Claude harness, a model with credit): +- headless batch + streaming: unset tools under default `allow` run through; the run + finishes; no approval events. +- headless with an `ask` tool: run ends `paused`, batch response says so, interaction row + exists. +- playground with an `ask` tool: prompt renders, approve resumes and runs, deny resumes and + continues without the tool (the two old approval-UI bugs stay fixed: F-024, the reject + that clobbered the prompt, and F-036, the deny that left the run hanging). +- playground with everything `allow`: no prompt, tools visibly run (the owner's "auto means + auto" check). +- Pi: unchanged (never gates; relay enforces deny/allow; relay `ask` now parks; verify the + client-tool park machinery carries it, else stage relay-ask into its own slice). +- uc9-digest end-to-end: set `SEND_MESSAGE: allow`, headless run completes all four tools + and posts; with default config, run pauses visibly at `SEND_MESSAGE`. +Then pin one park→approve→resume pair as a replay test (`agent-replay-test` skill) and a +producer-side SDK test asserting what `/invoke` actually sends in `sessionId` (the missing +test that let the proxy rot). + +## Behavior deltas (before → after) + +| Case | Today | After | +| --- | --- | --- | +| Headless, unset tool, policy auto | Parks silently; batch hides it | Runs in place | +| Headless, `ask` tool | Parks silently | Parks visibly (`stop_reason`, interaction ref) | +| Playground, unset tool, policy auto | Prompts (park) | Runs, tool activity visible, no prompt | +| Playground, `ask` tool | Prompts | Prompts (unchanged) | +| Playground, `allow` tool | No prompt (settings rule) | No prompt (unchanged) | +| `deny` on a client tool | Ignored | Refused | +| Stored approval vs changed-to-deny config | Approval wins | Deny wins | +| One approval, model repeats identical call | Every repeat auto-runs | One approval, one run | +| Claude builtin with authored `ask` rule | Parks | Parks (rule travels on the wire) | + +## Risks and open questions + +- **Relay-ask parking on Pi.** The relay can park client tools today, but `ask` parks for + ordinary relay tools need the same turn-boundary treatment (the old S5.2). If it turns + out heavy, ship relay-ask as its own slice and keep the collapse only for Pi, documented, + while Claude paths use the full design. +- **Rule matching for builtins.** The structured `rules` need a matcher compatible with + Claude's rule syntax (`Bash(npm run:*)`). Scope it to exactly what the settings renderer + accepts today; anything fancier stays authored-settings-only and documented as such. +- **Argument drift on resume (code-review M2).** Cold-replay approval matching by + name+args can re-prompt when the model rewords arguments. Measure it live in Phase 6; if + it bites, the follow-up is echoing the approval id through the replay, which is a + separate design slice. +- **Batch response shape.** Owned jointly with the streaming-invoke workspace; this plan + only requires that paused be distinguishable. +- **Vocabulary migration.** `auto` → `allow` and the authored-path rename touch the FE + form, SDK parsing, wire, fixtures, and docs in one PR. POC status makes this safe, but it + is the churny part; the golden wire contract test is the safety net. diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md new file mode 100644 index 0000000000..52a52545fc --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -0,0 +1,63 @@ +# Status + +**State: design + plan under review. No code changed.** Date: 2026-07-02. + +## What is done + +- Full research pass across all five systems (frontend, agent service, runner, harness + config rendering, API interactions plane), verified against the current tree with + file:line citations. This supersedes the investigation in + `../builder-agent-reliability/streaming-invoke/approval-boundary.md`, which cites + pre-rename `services/agent/` paths and misses the stored-decision branch and the second + (client-tool) park path. +- Bug confirmed and pinned: park keyed to session-id presence, session ids minted for every + request, `auto` policy unreachable, batch hides the paused state + ([the-bug.md](the-bug.md)). +- Correctness review: 4 high, 6 medium, 2 low findings beyond the headline bug + ([code-review.md](code-review.md)). +- Organization review: verdict and top-5 improvements + ([code-organization-review.md](code-organization-review.md)). +- Independent second opinion (OpenAI Codex, xhigh): concurred with the diagnosis, rejected + the partial fixes, recommended the one-plan design in one shot; its ordering rule (a + stored approval must not override a current deny) is folded into the plan. +- Plan written with options, phases, test plan, and behavior deltas ([plan.md](plan.md)). + +## Decisions already taken (by Mahmoud, 2026-07-02) + +- Auto means auto everywhere: an auto-approved tool runs without prompting; the human sees + it ran. Only `ask` waits for a human. +- This PR is docs + plan only; implementation follows after review. +- No backward-compatibility constraints (pre-release POC). + +## Decisions needed to start implementing + +1. **Confirm the recommendation**: Option D (one resolved permission plan, one shot) vs the + staged fallback (B+C first, then D). Plan recommends D. Stakes: D costs more up front + (wire shape, golden fixtures, tests in two languages, all in one PR) but ends with one + computation of the policy. The staged fallback ships the auto fix sooner but, until D + lands, an author's explicit `ask` rule on a Claude builtin like `Bash` silently + auto-approves under a default of `allow`. +2. **Naming**: approve `permissions.default` (vocabulary `allow | ask | deny`) as the + authored home of the global default, replacing `runner.interactions.headless` and the + `auto | deny` vocabulary. Stakes: touches the FE form, SDK parsing, wire, and fixtures + once; skipping it keeps three names for one knob. +3. **Relay-ask scope**: if parking `ask` for relay-executed tools proves heavy, is a + documented Pi-only collapse acceptable for the first slice? Stakes: under the collapse, + an `ask` tool on Pi never prompts; it silently runs (default `allow`) or is refused + (default `deny`). Claude paths get the full design either way. +4. **Batch pause shape**: coordinate the exact paused-response fields with the + streaming-invoke workspace (only "visible + carries the interaction reference" is + required from this side). + +## Next steps + +- Review of this workspace (see the PR comment for exactly what feedback is needed). +- On approval: implement per plan.md phases 1-6, with the correctness debt (phase 4) and + the live matrix + replay pin (phase 6) as the acceptance gate. + +## Known unknowns + +- The sandbox-agent daemon's permission-request id scheme (per-session counter vs unique) + This decides whether interaction tokens need turn namespacing (code-review H3). +- How often cold-replay argument drift breaks approval matching in practice + (code-review M2); measure during phase 6. diff --git a/docs/design/agent-workflows/projects/approval-boundary/the-bug.md b/docs/design/agent-workflows/projects/approval-boundary/the-bug.md new file mode 100644 index 0000000000..4ba523e4db --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/the-bug.md @@ -0,0 +1,141 @@ +# The bug: an auto-approved run stops at the tool gate + +Prerequisite: [how-approvals-work.md](how-approvals-work.md), especially "Gate 2" and "The +journey of one gated tool call". + +## What you observe + +The `uc9-digest` agent (three read tools, then `SEND_MESSAGE`) runs with permission policy +`auto`. A one-shot HTTP invoke should run all four tools and finish. Instead: + +- **Streaming**: the stream shows the three reads with results, then a `tool_call` for + `SEND_MESSAGE`, then an `interaction_request` asking for approval, then `done`. Four tool + calls, three results. The run ended waiting for an approval that no one can give. +- **Batch**: the response is HTTP 200 with one assistant message that ends mid-sentence, + right before the terminal tool ("...posting the digest now."). Nothing indicates the run + stopped. The caller cannot tell a paused run from a finished one. + +Before diagnosing, be precise about what *should* happen, because it is subtler than "the +run should never pause". Every tool resolves to a **disposition**: its final +`allow | ask | deny`, computed from the author's explicit per-tool permission, the legacy +`needs_approval` flag, or a catalog hint that defaults read-only tools to `allow` and +mutating tools to `ask` (`effective_permission`, +`sdks/python/agenta/sdk/agents/tools/models.py:273-292`). Tools with no disposition at all +fall back to the global policy, and that is what `auto` governs. + +`SEND_MESSAGE` is a write, so it resolves to `ask` by default. Under a correct +implementation, this specific run pauses there until the author marks the tool `allow`. +The bug is therefore three things, none of which is "it paused": + +- the pause happens for the wrong reason (a session id instead of the tool's disposition); +- the `auto` policy is dead code for the tools it does govern; +- a headless caller can neither see the pause nor answer it. + +Auto-approval itself is a decision our own runner makes internally; it is not a human +interaction. When the policy says `auto`, nobody should be asked for anything. + +## The root cause, in four steps + +Each step is small and looks reasonable alone. Together they make the `auto` policy +unreachable. + +**1. The responder parks before it reads the policy.** `HITLResponder.onPermission` +(`services/runner/src/responder.ts:201-206`) checks three things in order: a stored decision +from a previous turn, then `hasHumanSurface`, then the policy. When a gate is fresh (no +stored decision) and `hasHumanSurface` is true, it returns `park` without ever looking at +`basePolicy`. The policy only applies when there is no human surface. + +**2. "Human surface" means "the request has a session id".** +`services/runner/src/engines/sandbox_agent.ts:627`: + +```ts +const hasHumanSurface = !!(request.sessionId && request.sessionId.trim()); +``` + +The comment above it (lines 621-626) explains the design assumption: interactive `/messages` +runs carry a session id, the headless `/invoke` path sets none, so a session id is a good +proxy for "someone is watching the stream". + +**3. That assumption is false: the SDK mints a session id for every run.** The request +normalizer resolves a session id before any handler runs, and when the caller supplied none +it mints a fresh UUID (`sdks/python/agenta/sdk/middlewares/running/normalizer.py:307`, +`sdks/python/agenta/sdk/models/shared.py:13-22`). It does this for batch and streaming +alike. The id flows into the run request (`services/oss/src/agent/app.py:216, 254`), so +every run the runner ever sees has a non-empty `sessionId`. + +**4. Therefore every fresh gate parks.** `hasHumanSurface` is always true, branch 2 always +wins, and `basePolicy === "auto"` is dead code for any harness that raises gates. A headless +one-shot invoke parks at the first gated tool exactly like an interactive playground run. +The turn ends `stopReason: "paused"`. + +Two aggravators hide the failure: + +- **Batch swallows the stop reason.** `_agent_batch` + (`services/oss/src/agent/app.py:303-321`) drains the event stream and returns only the + final assistant text. `result.stop_reason` (`"paused"`) is read nowhere. A paused run and + a completed run produce the same-shaped 200 response. +- **The playground auto-resumes.** `useChat` re-sends the conversation the moment you click + Approve, so in the one surface humans actually watch, the park looks like a feature. Every + surface without a human (curl, agent-as-tool, evaluations, triggers) just dies at the + gate. + +## Why the code is this way: the history + +The park mechanism is not an accident; it is a deliberate fix for an earlier bug, and this +bug is that fix's blind spot. + +- Originally the runner answered every gate inline with a hardcoded auto-approve. No run + ever paused, and no human could ever be asked. +- The human-in-the-loop work (see `../hitl-fix/`, QA finding F-024) needed a way to pause a + run for an approval. First attempt: answer "reject" and let the frontend prompt. That + broke the UI, because Claude turns a reject into a failed tool call, and the failure + overwrote the approval prompt on screen. +- The fix (commit `b109cc51ef`, 2026-06-25) introduced the real park: send the harness no + answer at all, tear the session down, resume on a later turn. Correct mechanism, and it + fixed F-024. A follow-up (F-040) made sure the teardown is active: without it, a parked + turn whose harness never gives up just hangs. So park has two ancestors: F-024 says + "never answer reject when you mean pause", F-040 says "a pause must still end the turn". +- But the condition for parking was written as "is a human surface present" and implemented + as "is there a session id", at a time when the invoke path really did omit session ids. + When the SDK later started minting ids for every request (for tracing and session + bookkeeping), the proxy silently went from "sometimes true" to "always true". No test + pinned "a headless auto run does not park", so nothing failed. + +One runner unit test actually pins the buggy behavior as correct: +`services/runner/tests/unit/responder.test.ts` (around line 227) asserts that the responder +parks when a human surface exists "even under deny basePolicy". Under the fix, that test +changes meaning. + +## The real design flaw underneath + +The one-line diagnosis is: **the runner infers intent from transport metadata.** Whether a +tool call needs a human is a question about the tool and the author's config (`ask` vs +`allow` vs the policy default). The runner instead asks "did this request arrive with a +session id", which is a fact about plumbing, and the plumbing changed underneath it. + +The information the responder actually needs already exists and already travels to the +runner: every resolved tool carries its `permission` (`allow | ask | deny`) on the run +request (`services/runner/src/protocol.ts:122-128, 220-225`). The responder just never reads +it. The relay reads it for its own gate, and the Claude settings renderer reads it a third +time on the Python side. Three enforcement points, each consulting a different subset of the +same config; the one that decides about parking consults none of it. + +[design-review.md](design-review.md) develops this into concrete recommendations; +[plan.md](plan.md) sequences the fix. + +## The shape of the fix (summary; details in plan.md) + +Park only on authored intent, never on transport hints: + +- The responder resolves the gated tool's own permission from the run request: explicit + `allow` allows, `deny` denies, `ask` parks. +- A tool with no explicit permission falls back to the policy: `auto` allows in place (the + run streams the tool result and finishes), `deny` denies. +- `hasHumanSurface` is deleted. An `ask` tool parks even on a headless run; that pause is + authored, visible, and answerable later once the interactions plane grows a real resume. +- Batch stops hiding the pause: `/invoke` surfaces `stop_reason` so a caller can tell + "paused at an approval" from "done". + +This keeps the playground behavior for tools that genuinely ask (`ask` tools still park and +still show buttons), restores `auto` to its documented meaning everywhere, and removes the +signal that rotted once already. From dac22b520383d07eebca673714d018b3bbaf7f13 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 12:29:57 +0200 Subject: [PATCH 03/24] docs(approval-boundary): address review round 1 on how-approvals-work Generalize the explainer across harnesses (per-harness gate table, Pi has no HITL today), explain Claude default_mode/bypassPermissions and the settings merge semantics, distinguish ACP request vs stream event and the two kinds of session, explain the cold-replay resume model before the responder code, and record the live approve-loop finding. Plan updates: relay becomes a pure executor (decision before execution), client tools resolve through the same ladder defaulting to allow, M2+M3 elevated with the approve-loop as an acceptance case. Claude-Session: https://claude.ai/code/session_01DGj7GKafjkZeQXMsryWhb2 --- .../approval-boundary/how-approvals-work.md | 326 +++++++++++++----- .../projects/approval-boundary/plan.md | 42 ++- .../projects/approval-boundary/status.md | 19 +- 3 files changed, 277 insertions(+), 110 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md index 758852000d..e251e9c5bd 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md +++ b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md @@ -26,9 +26,12 @@ One tool call can cross five systems. Each has one job in the permission story. the harness process). This is where our permission decisions live: the runner answers the harness's permission requests, executes gateway/code tools through its "tool relay", and emits every event the frontend sees. -4. **The harness** (Claude Code or Pi). The actual coding agent. Claude Code has its own - built-in permission system and asks for permission before running gated tools. Pi has - none; it never asks. +4. **The harness** (Claude Code or Pi today; Codex or OpenCode would slot in the same way). + The actual coding agent. Every harness brings its own native permission behavior, and + that behavior differs: Claude Code has a full built-in permission system and asks before + running gated tools; Pi's ACP bridge reports `permissions: false` and never asks anyone + about anything. How each harness behaves at each gate is summarized in "How this + generalizes across harnesses" below. 5. **The API interactions plane** (`api/oss`, `/sessions/interactions`). A durable store of approval requests, built for the future ("answer an approval hours later, from anywhere"). Today the runner writes records into it, but the product answers approvals through the @@ -44,13 +47,30 @@ Four knobs control what an agent may do without asking. They live at different l today, under inconsistent names. **1. The global permission policy.** One value per agent: `auto` or `deny`. `auto` means -"approve tool prompts automatically"; `deny` means "refuse them". Confusingly, this one knob -has three names depending on where you stand: - -- Authors write it as `runner.interactions.headless` in the agent config - (parsed in `sdks/python/agenta/sdk/agents/dtos.py:1087-1102`). -- The SDK stores it as `permission_policy` (`dtos.py:571`). -- The wire to the runner calls it `permissionPolicy` (`services/runner/src/protocol.ts:427`). +"approve tool prompts automatically"; `deny` means "refuse them". Your mental model is +correct: this is the agent-wide default, and each tool can override it; a tool that sets +nothing inherits it. Two things make the current shape confusing, and both are addressed in +the plan: + +- The vocabulary does not match the per-tool one (`auto | deny` here, `allow | ask | deny` + per tool), and it cannot express "ask for everything". The natural set of agent-wide + behaviors ("approve everything", "ask for everything", "deny everything", "ask only for + writes") is exactly what the plan's target shape gives you: a `default_permission` of + `allow | ask | deny`, with "ask only for writes" falling out of the per-tool defaults + (reads default to `allow`, writes to `ask`; see knob 2). The `deny` default exists for + lockdown ("this agent runs no tools unless a tool is explicitly allowed"); it is + legitimate, just rarely what you want. +- The knob has three names depending on where you stand. Authors write it as + `runner.interactions.headless` in the agent config (parsed in + `sdks/python/agenta/sdk/agents/dtos.py:1087-1102`); the SDK stores it as + `permission_policy` (`dtos.py:571`); the wire to the runner calls it `permissionPolicy` + (`services/runner/src/protocol.ts:427`). "Headless" was meant as "what should the runner + answer when no human is watching", which is also why it sits under an `interactions` + section; the name describes one consumer of the value, not the value itself. The stored + name predates the authored one (the field was flat `permission_policy` first, then the + authoring surface moved under `runner.interactions` without renaming the storage). The + plan retires all three in favor of one name in the permission family + ([plan.md](plan.md), "One wire contract"). The playground labels it "Permission policy" and always sends `auto` unless the author changes it (`web/packages/agenta-playground/src/state/execution/agentRequest.ts:262`). @@ -61,25 +81,34 @@ integration action from the Composio catalog; a code tool; an MCP server) can ca (`sdks/python/agenta/sdk/agents/tools/models.py:26`). `allow` means run it without asking. `ask` means a human must approve each call. `deny` means never run it. A tool with no explicit permission falls back through a ladder (`effective_permission`, -`models.py:273-292`): a legacy `needs_approval: true` flag means `ask`; a catalog `read_only` -hint defaults reads to `allow` and writes to `ask`; otherwise the tool defers to the global -policy. The output of this ladder is what the rest of this workspace calls the tool's +`models.py:273-292`), in order: + +1. an explicit `permission` wins; +2. else `needs_approval: true` means `ask` (`needs_approval` is the older boolean form of + the same idea, from before the three-valued `permission` field existed; stored configs + still carry it, so the ladder honors it; the plan deprecates it at the config edge); +3. else the catalog's `read_only` hint applies its default: reads are safe to auto-run + (`allow`), writes should prompt (`ask`). This is our own convention, applied by our SDK; + the hint comes from the Composio catalog's tags. +4. else nothing is set, and the tool inherits the global policy (knob 1). + +The output of this ladder is what the rest of this workspace calls the tool's **disposition**: its final, resolved allow/ask/deny. The tool editor exposes the field as a Permission select with an "Inherit policy" placeholder (`web/.../ToolFormView.tsx:47-51`). **3. Claude harness rules.** For the Claude harness only, authors can write raw Claude Code -permission rules (`harness.permissions`: a `default_mode` plus `allow`/`ask`/`deny` rule -lists like `Bash(npm run:*)`). These pass through to Claude's own settings file -(`web/.../ClaudePermissionsControl.tsx`). +permission rules (`harness.permissions`): a `default_mode` plus `allow`/`ask`/`deny` rule +lists like `Bash(npm run:*)` (`web/.../ClaudePermissionsControl.tsx`). These pass through to +Claude's own settings file. `default_mode` is Claude Code's own top-level switch and worth +knowing: `default` prompts on each gated tool, `acceptEdits` auto-accepts file edits, +`plan` is read-only planning, and `bypassPermissions` skips every gate. Under +`bypassPermissions`, Claude never raises a permission request at all, so nothing in this +document's Gate 2 ever runs; the author has told the harness itself to stop asking. **4. Sandbox permission.** Network and filesystem boundaries (`sandbox_permission`). This is a security boundary, not an approval flow, so this document leaves it aside. It matters here only because it also renders deny rules into Claude's settings. -Note the vocabulary mismatch: the global policy speaks `auto | deny`, the per-tool field -speaks `allow | ask | deny`. "Auto" and "allow" mean the same thing. The global vocabulary -has no `ask`, which turns out to be central to the bug. - ## Where permission is enforced: three gates A tool call passes through up to three enforcement points, depending on the tool and the @@ -88,50 +117,102 @@ its own deny rules; it is a security wall rather than an approval gate, so count gates" here and "four enforcement sites" in the reviews are the same list with and without it.) -**Gate 1: Claude's own settings file.** Claude Code checks every tool call against -`.claude/settings.json` before doing anything. The SDK renders that file -(`sdks/python/agenta/sdk/agents/adapters/claude_settings.py:201-258`) by merging four rule -sources: the author's raw Claude rules, denies derived from the sandbox permission, per-MCP- -server permissions, and per-tool permissions. The rendering is deliberately conservative: a -per-tool `allow` becomes an allow rule (Claude runs the tool with no prompt), a `deny` -becomes a deny rule, but `ask` and unset produce no rule at all, which leaves Claude's gate -raised. The docstring at `claude_settings.py:22-31` states the intent: `permission_policy: -"auto"` is not a blanket bypass; an unlisted tool still triggers a permission request. - -An important consequence: at the next gate, the runner cannot tell "the author said ask" -from "the author said nothing". Both arrive as the same raised gate. - -**Gate 2: the runner's ACP responder.** When Claude's settings do not settle a tool, Claude -raises an ACP permission request to the runner. The handler -(`attachPermissionResponder`, `services/runner/src/engines/sandbox_agent/permissions.ts:38-134`) -does two things, in this order: - -1. It emits an `interaction_request` event of kind `user_approval` into the run's event - stream, unconditionally, before any decision is made (`permissions.ts:93-105`). This is - the event the frontend turns into Approve/Deny buttons. +A structural remark before the detail, because it answers the natural "why three?" +reaction: the three gates exist because tools physically pass through different choke +points (inside the harness, at the protocol boundary, in the runner's own executor). That +part is unavoidable. What is *not* unavoidable is that today each gate carries its own copy +of the decision logic, consulting its own subset of the config. The plan keeps the three +choke points but gives them one shared decision function, so the gates become dumb +enforcers of one policy instead of three opinions ([design-review.md](design-review.md) §3, +[plan.md](plan.md) "One decision function"). + +**Gate 1: the harness's own permission layer.** Some harnesses check tool calls themselves +before asking anyone. This gate is harness-specific by nature: + +- **Claude Code** checks every tool call against `.claude/settings.json` before doing + anything. Our SDK renders that file + (`sdks/python/agenta/sdk/agents/adapters/claude_settings.py:201-258`) by merging four rule + sources: the author's raw Claude rules (knob 3), denies derived from the sandbox + permission, per-MCP-server permissions, and per-tool permissions. The merge is additive, + not overriding: the author's rules are kept verbatim and our derived rules are appended + (duplicates removed). We never delete or rewrite an author rule. Conflicts are settled by + Claude's own precedence at match time, where deny beats allow; so an author `deny` always + holds, and an author `allow` can be beaten only by a derived deny (for example the + sandbox saying "no network"), which is intended. +- **Pi** has no such layer. Nothing is checked inside the harness. + +For per-tool permissions, the Claude rendering is deliberately conservative: a per-tool +`allow` becomes an allow rule (Claude runs the tool with no prompt), a `deny` becomes a deny +rule, but `ask` and unset produce **no rule at all**, which leaves Claude's gate raised so +the call falls through to Gate 2. The docstring at `claude_settings.py:22-31` states the +intent: `permission_policy: "auto"` is not a blanket bypass; an unlisted tool still +triggers a permission request. + +One consequence to hold on to, because it constrains the fix: since `ask` and unset both +render as "no rule", they look identical by the time a call reaches Gate 2. The runner +cannot tell "the author explicitly wants a human" from "the author said nothing, apply the +default" unless it consults the tool's disposition from the run request. Today it consults +nothing (that is the bug); the fix makes Gate 2 look the disposition up. + +**Gate 2: the runner's ACP responder.** When Gate 1 does not settle a tool, a gating +harness raises a permission request to the runner over ACP. This is not a stream event: it +is a blocking request/response call inside the protocol, like a function call from the +harness to us. The harness stops and waits for our answer before touching the tool. The +`interaction_request` you see in the event stream is a separate thing: an event the runner +chooses to emit so the frontend knows a gate exists. Gate 2 itself is harness-agnostic; any +ACP harness that raises permission requests lands in the same handler. Today Claude is the +only harness that does. Pi never calls it; a future Codex/OpenCode harness would land here +with no runner changes. + +The handler (`attachPermissionResponder`, +`services/runner/src/engines/sandbox_agent/permissions.ts:38-134`) does two things, in this +order: + +1. It emits the `interaction_request(user_approval)` event into the run's stream, + unconditionally, before any decision is made (`permissions.ts:93-105`). 2. It asks a "responder" object what to do (`permissions.ts:106-107`). -The responder for product runs is `HITLResponder` -(`services/runner/src/responder.ts:194-232`; HITL stands for human-in-the-loop). Its -decision procedure is three branches, in order: +That order is itself a design flaw: it means the event fires even for a gate the responder +would have answered by itself, so the event cannot mean "a human must act". The plan +inverts it: decide first, emit only when the decision is "wait for a human" +([design-review.md](design-review.md) §4). + +To make sense of the responder's logic you need one piece of context first: **decisions can +arrive from a previous turn.** As the journey section below shows in full, a paused run is +resumed by re-sending the whole conversation, and the human's answer travels *inside* that +conversation. The re-run replays from scratch, the harness re-raises the same gate, and the +answer that is already sitting in the transcript settles it. So when a gate arrives, the +responder's first question is "did a previous turn already answer this exact call?". + +With that, the decision procedure of `HITLResponder` +(`services/runner/src/responder.ts:194-232`; HITL stands for human-in-the-loop) reads: ```ts async onPermission(request) { - const stored = this.lookupPermission(request); // 1. a decision from a previous turn? - if (stored) return stored; // use it - if (this.hasHumanSurface) return "park"; // 2. a human might be watching? stop and wait - return this.basePolicy === "deny" ? "deny" : "allow"; // 3. headless: apply the policy + const stored = this.lookupPermission(request); // 1. an answer already in the transcript? + if (stored) return stored; // use it (this is the resume path) + if (this.hasHumanSurface) return "park"; // 2. someone might answer? stop and wait + return this.basePolicy === "deny" ? "deny" : "allow"; // 3. nobody will: apply the policy } ``` -Three outcomes are possible. `allow` replies "once" to Claude (approve this single call, -never "always") and the tool runs. `deny` replies "reject" and the tool is refused. `park` -is the interesting one: the runner sends no reply at all, tears the session down, and ends -the turn with `stopReason: "paused"` (`sandbox_agent.ts:640-649, 766-767`). Park means "a -human must answer, and the answer will arrive on a future turn, not this one". - -`hasHumanSurface`, the flag branch 2 keys on, is computed from one signal: does the run -request carry a non-empty session id (`sandbox_agent.ts:627`). +Three outcomes are possible. `allow` replies "once" to the harness (approve this single +call, never "always") and the tool runs. `deny` replies "reject" and the tool is refused. +`park` is the interesting one: the runner sends no reply at all, tears the session down, and +ends the turn with `stopReason: "paused"` (`sandbox_agent.ts:640-649, 766-767`). Park means +"a human must answer, and the answer will arrive on a future turn, not this one". + +Branch 2 is where the intended logic and the actual logic part ways. The *intended* logic +was: an interactive surface (someone in the playground) should get the question; a headless +call (nobody watching) should be answered by the policy, because parking a run nobody can +resume just kills it. That intent is defensible. The *implementation* reduced "is a human +watching" to one signal: does the run request carry a non-empty session id +(`hasHumanSurface`, `sandbox_agent.ts:627`). Session ids were a plausible proxy when only +the playground sent them; the SDK now mints one for every request, so the proxy is always +true and branch 3 is dead code. That, plus the fact that the policy should not have been +shadowed by surface detection in the first place, is [the-bug.md](the-bug.md). The fix +removes branch 2 entirely: whether to park becomes a property of the tool's disposition +(`ask`), not of who might be watching. **Gate 3: the tool relay.** Gateway and code tools do not run inside the harness; the runner executes them itself through its tool relay. The relay enforces the per-tool permission @@ -141,19 +222,52 @@ HITL` (S5 is the open-issues slice label for that deferred work). So today, a re tool marked `ask` never actually asks anyone; the same authored `ask` behaves differently depending on which gate handles the tool. +If having the executor also check permissions strikes you as mixed responsibilities, that +instinct is right, and the plan agrees: deciding whether a call may run and executing it are +different jobs. In the target design the decision is made once, by the shared decision +function, before execution; the relay executes what was already permitted and carries no +permission logic of its own ([plan.md](plan.md), "One decision function"). + +Why do gateway tools hit Gate 2 at all, if the relay is Gate 3? This is a Claude-specific +consequence of Gate 1: resolved tools are delivered to Claude as tools of an internal MCP +server (`mcp__agenta-tools__`), and Claude gates them like any other tool before the +call ever reaches the relay. That is why the settings rendering exists: without an allow +rule, even an `allow` tool would stop at Gate 2 (this was bug F-046, fixed by rendering +per-tool permissions into the settings file). On Pi there is no Gate 1 and no Gate 2, so +the relay is the only gate a gateway/code tool passes. + One special tool kind crosses these gates differently: **client tools** (`kind: "client"`, for example `request_connection`). A client tool is fulfilled by the user's browser, not by the harness or the relay. When one is called, the runner parks the turn and emits an `interaction_request` of kind `client_tool`; the frontend fulfills it and the output comes -back on the next turn the same way an approval does. Client tools matter to this review for -two reasons: they are a second park path, wired separately from the approval one, and today -they skip the per-tool permission check entirely (see code-review M4). - -Why do gateway tools hit Gate 2 at all, if the relay is Gate 3? Because on Claude, resolved -tools are delivered as tools of an internal MCP server (`mcp__agenta-tools__`), and -Claude raises its own gate before the call ever reaches the relay. That is why the settings -rendering (Gate 1) exists: without an allow rule, even an `allow` tool would stop at Gate 2 -(this was bug F-046, fixed by rendering per-tool permissions into the settings file). +back on the next turn the same way an approval does. Today client tools skip the per-tool +permission check entirely (code-review M4). The right model, and the one the plan adopts, +is that nothing skips the ladder: client tools resolve through the same decision function +as everything else and simply default to `allow` (their whole purpose is to reach the +browser; the surface interaction is the fulfillment, not an approval). If a use case ever +needs a gated client tool, `ask`/`deny` then work with no special casing. + +## How this generalizes across harnesses + +The gates are a general model; what varies per harness is which gates exist: + +| Harness | Gate 1 (native) | Gate 2 (ACP responder) | Gate 3 (relay) | +| --- | --- | --- | --- | +| Claude Code | `.claude/settings.json`, rendered by our SDK | yes, for anything Gate 1 leaves undecided | yes, for the tools it executes (but Gate 1+2 fire first) | +| Pi | none (`permissions: false`) | never fires (Pi never asks) | the only gate | +| future Codex/OpenCode | whatever native config that harness has (rendered by a new adapter) | works unchanged if the harness raises ACP permission requests | works unchanged | + +Read Pi's row carefully, because it surprises people: **on Pi, today, everything is +effectively auto-approved except tools the relay refuses.** Pi's bridge does not implement +the ACP permission plane, so Pi executes its builtins without asking, and for +gateway/code tools only the relay's check applies, where `ask` currently degrades to the +policy. There is no human-in-the-loop on Pi at all today. The playground UI is honest about +the policy field (it hides "Permission policy" for Pi with a note, hardcoded by harness name +at `web/.../useModelHarness.tsx:113`), but per-tool `ask` on a Pi agent is silently not +honored. The plan's fix is structural: once the relay consults the shared decision function, +an `ask` disposition parks at the relay exactly like a Claude gate parks at the responder, +and Pi gets real approvals through the same machinery (this is the old S5.2 item; scope +decision 3 in [status.md](status.md)). ## The journey of one gated tool call (the playground path) @@ -168,10 +282,14 @@ tools are effectively allowed; `SEND_MESSAGE` posts to Slack and ends up gated. service sends the runner. 3. The runner starts a Claude session in the sandbox, writing the rendered `.claude/settings.json` into the workspace first. -4. Claude runs the three reads without asking (allow rules from their `read_only` hints). - Their `tool_call` and `tool_result` events stream to the frontend as they happen. -5. Claude wants `SEND_MESSAGE`. No allow rule matches, so Claude raises an ACP permission - request. +4. Claude runs the three reads without asking: their catalog `read_only: true` hint + resolved them to `allow` (knob 2's ladder), so our SDK rendered allow rules for them + into the settings file, and Gate 1 settles the calls inside Claude. Nothing reaches the + ACP layer for these three, no permission events exist for them anywhere; their + `tool_call` and `tool_result` events stream out as they happen, and that is all. +5. Claude wants `SEND_MESSAGE`. No allow rule matches (a write, so the ladder said `ask`), + and Claude raises an ACP permission request: a blocking call to the runner, Claude now + waits. 6. The runner emits `interaction_request(user_approval)` into the stream, then consults `HITLResponder`. No stored decision exists, the request has a session id, so: **park**. The runner records a pending interaction row in the API (only when the run belongs to a @@ -187,23 +305,42 @@ tools are effectively allowed; `SEND_MESSAGE` posts to Slack and ends up gated. 9. You click Approve. The Vercel SDK marks the tool part `approval-responded`, and an auto-resume predicate (`sendAutomaticallyWhen: agentShouldResumeAfterApproval`, `web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts:108-122`) - re-sends the entire conversation. Deny re-sends too; the model should hear "no" and - continue, not hang. + re-sends the entire conversation. A Deny click re-sends the same way; that is + deliberate, because the denial must reach the model as information ("the human refused + this call") so it can continue without the tool. Without the re-send, a denied run would + just sit paused forever (that dead-end was bug F-036). 10. On the way back in, the SDK's message adapter folds your decision into a `tool_result` content block whose output is the envelope `{"approved": true}` (`sdks/python/agenta/sdk/agents/adapters/vercel/messages.py:103-192`). -11. The runner starts a fresh session (every turn is a cold replay: same conversation, new - session) and pre-extracts all `{approved}` envelopes from the incoming messages into a - decision map (`extractApprovalDecisions`, `responder.ts:306-343`). Decisions are keyed - by tool name plus canonicalized arguments, never by name alone, so approving one call - does not approve a different call to the same tool. -12. Claude replays the conversation and re-raises the same gate. This time branch 1 of the - responder finds the stored decision: `allow`. The runner replies "once", the tool runs, - its result streams, and the turn finishes normally. - -That is the designed happy path, and it works. The bug is what happens on step 6 when no -human is on the other end of the stream: the run parks anyway, because "park or not" looks -at the session id instead of the policy. [the-bug.md](the-bug.md) picks up from here. +11. The runner starts a fresh harness session and replays the conversation from scratch. + Two different "sessions" are in play here, which is why this step looks odd: the + *Agenta* session id (step 2) is the conversation's identity and persists across turns; + the *harness* session is a live process inside the sandbox, and the park in step 6 + destroyed it. Nothing server-side holds harness state between turns; each turn is a + cold replay of the transcript into a new harness session. Before the replay, the + runner pre-extracts all `{approved}` envelopes from the incoming messages into a + decision map (`extractApprovalDecisions`, `responder.ts:306-343`), keyed by tool name + plus canonicalized arguments (never by name alone, so approving one call does not + approve a different call to the same tool). +12. Claude replays the transcript in the new session. The transcript shows it wanted to + call `SEND_MESSAGE` and never got a result, so Claude issues the call again, and the + gate rises again exactly like in step 5; the approval message in the conversation is + information for the *model*, but Claude Code's permission machinery still demands a + protocol-level answer for the re-raised gate. This time branch 1 of the responder finds + the stored decision: `allow`. The runner replies "once", the tool runs, its result + streams, and the turn finishes normally. (Why does it have to flow through the runner + again at all? Because only the runner can answer an ACP permission request; the + approval in the transcript is not something the harness itself reads as an + authorization.) + +That is the designed resume mechanism. **Live behavior warning:** in current QA (Mahmoud, +2026-07-03) the happy path does not reliably complete: clicking Approve can loop, with the +run re-parking and re-prompting again and again. That symptom matches the fragility already +flagged in [code-review.md](code-review.md): the stored decision is keyed by exact tool +name plus byte-identical arguments (M2: if the model re-issues the call with slightly +different arguments on replay, the key misses and the gate re-parks), and an approval that +comes back keyed only by `approvalId` is silently dropped (M3). Reproducing and pinning +this loop is now part of the fix's acceptance tests ([plan.md](plan.md), phase 6). ## The two planes: messages and interactions @@ -217,13 +354,21 @@ There is a second, newer **interactions plane**: a `session_interactions` table `api/oss/src/apis/fastapi/sessions/router.py:591-882`). The vision (spec: `docs/designs/sessions/interactions/extend/specs.md`) is durable approvals: a parked run leaves a pending record that anyone can answer later from any surface, without holding a -chat open. +chat open. In that future flow, the park creates the row (as it already does), a respond +from any surface records the decision, and a resume replays the conversation with the +decision available, which is exactly the same settle-by-stored-decision mechanism as the +messages plane; the deferred piece is the "whoever reacts first" resolver that reconciles +the two planes and feeds an API-plane answer into a run. Today this plane is deliberately half wired: - The runner writes to it: it creates a pending row on park and marks the row resolved when - a stored decision later settles the gate. But only for committed revisions; playground - drafts skip it entirely (`sandbox_agent.ts:664-670`). + a stored decision later settles the gate. But only when the run references a committed + revision; playground drafts skip it entirely (`sandbox_agent.ts:664-670`). The reason is + the respond side: answering an interaction re-invokes the workflow, and a re-invoke needs + a stable revision reference to re-run; an uncommitted draft has none. (Whether the future + design should key durable approvals on the session instead, so drafts get them too, is a + fair question for that design; today the guard simply reflects what respond can re-run.) - The only reader is the debug Session Inspector. Its Approve button calls the respond endpoint, which does not resume the parked turn; it fires a fresh, detached re-invoke of the same revision (`api/oss/src/tasks/asyncio/sessions/interactions_dispatcher.py:31-76`). @@ -233,15 +378,6 @@ So think of the interactions plane as an audit shadow with future ambitions. The decision loop is the messages plane. Any fix must keep the interactions writes intact (they are the seed of the future durable flow) without depending on them. -## Harness differences - -Only Claude gates. Pi reports `permissions: false` and never raises an ACP permission -request, so nothing parks on Pi and the permission policy does nothing there (the UI hides -the field for Pi with a note, hardcoded by harness name at -`web/.../useModelHarness.tsx:113`). For Pi, the only permission enforcement is Gate 3, the -relay, where `ask` currently degrades to the policy. Real per-tool HITL on Pi would need the -relay to learn to park (tracked as open-issues slice S5.2; see `../hitl-fix/plan.md`). - ## Where each piece of code lives | Piece | File | diff --git a/docs/design/agent-workflows/projects/approval-boundary/plan.md b/docs/design/agent-workflows/projects/approval-boundary/plan.md index a3dc38328a..8a76f806ef 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/plan.md +++ b/docs/design/agent-workflows/projects/approval-boundary/plan.md @@ -120,10 +120,16 @@ consulted, so a stale approval can never override a config that has since change match, so one approval authorizes one execution ("once" semantics, code-review M1). The same function backs both gates: the ACP responder (harness-gated tools) and the relay -(runner-executed tools). The relay's `TODO(S5)` collapse disappears; the relay can already -park for client tools, and an `ask` on a relay tool parks the same way. `hasHumanSurface` is -deleted. `PolicyResponder` is deleted. An `ask` park on a headless run is correct behavior: -the turn ends `paused`, visibly, with an interaction row for the future durable-resume flow. +(runner-executed tools). Deciding and executing stay separate jobs (Mahmoud's review point, +2026-07-03): the decision runs *before* execution, in the shared function, and the relay +itself carries no permission logic; it executes calls that were already permitted. The +relay's `TODO(S5)` collapse disappears; the relay can already park for client tools, and an +`ask` on a relay tool parks the same way. Client tools stop being a special case: they +resolve through the same function and simply default to `allow` (their fulfillment *is* the +browser interaction), so `deny` and `ask` work on them with no extra code. `hasHumanSurface` +is deleted. `PolicyResponder` is deleted. An `ask` park on a headless run is correct +behavior: the turn ends `paused`, visibly, with an interaction row for the future +durable-resume flow. ### Events mean actions @@ -169,11 +175,15 @@ frontend research: nothing in the UI depends on parking for non-ask tools). **Phase 4: correctness debt in the same code (from code-review.md).** H1 (reply failure parks or fails loud; resolve interaction only after a successful reply), H2 (only `{approved}` envelopes become decisions; client-tool replay scoped by interaction -token), M3 (approval responses must correlate by `toolCallId`; drop loudly otherwise), M4 -(client tools honor `deny`), M5 (latch the stop reason before the relay drain), L1 (no-id -gates park instead of silently hanging), H3 (verify the daemon's permission-id scheme; if -per-session counters, namespace interaction tokens with the turn), L2 (reserve the -`agenta-tools` MCP server name in the settings renderer). +token), M2+M3 as one item, elevated because the failure is now observed live (see phase 6): +approval responses must correlate by `toolCallId` and drop loudly otherwise, and the +approve→resume match must survive argument drift (echo the approval id through the replay, +or replay the approved result directly, decided by what the live reproduction shows), M4 +(client tools resolve through the same ladder, defaulting to `allow`), M5 (latch the stop +reason before the relay drain), L1 (no-id gates park instead of silently hanging), H3 +(verify the daemon's permission-id scheme; if per-session counters, namespace interaction +tokens with the turn), L2 (reserve the `agenta-tools` MCP server name in the settings +renderer). **Phase 5: organization cleanup (from code-organization-review.md).** Extract the park controller from `sandbox_agent.ts` into the `engines/sandbox_agent/` @@ -188,7 +198,10 @@ Live matrix on the dev stack (Claude harness, a model with credit): finishes; no approval events. - headless with an `ask` tool: run ends `paused`, batch response says so, interaction row exists. -- playground with an `ask` tool: prompt renders, approve resumes and runs, deny resumes and +- playground with an `ask` tool: prompt renders, approve resumes and **completes without + looping**. This case is currently broken live (Mahmoud, 2026-07-03: Approve loops, the + run re-parks and re-prompts repeatedly), which is the observed form of code-review M2/M3. + Reproduce first, fix in phase 4, and pin the pass as the replay test. Deny resumes and continues without the tool (the two old approval-UI bugs stay fixed: F-024, the reject that clobbered the prompt, and F-036, the deny that left the run hanging). - playground with everything `allow`: no prompt, tools visibly run (the owner's "auto means @@ -224,10 +237,11 @@ test that let the proxy rot). - **Rule matching for builtins.** The structured `rules` need a matcher compatible with Claude's rule syntax (`Bash(npm run:*)`). Scope it to exactly what the settings renderer accepts today; anything fancier stays authored-settings-only and documented as such. -- **Argument drift on resume (code-review M2).** Cold-replay approval matching by - name+args can re-prompt when the model rewords arguments. Measure it live in Phase 6; if - it bites, the follow-up is echoing the approval id through the replay, which is a - separate design slice. +- **Argument drift on resume (code-review M2, observed live).** Cold-replay approval + matching by name+args re-prompts when the model rewords arguments, and QA now shows the + approve→loop symptom in the playground, so this is no longer hypothetical. It moves into + phase 4 (fix direction: echo the approval id through the replay, or have the runner + replay the approved result directly rather than relying on an identical re-issue). - **Batch response shape.** Owned jointly with the streaming-invoke workspace; this plan only requires that paused be distinguishable. - **Vocabulary migration.** `auto` → `allow` and the authored-path rename touch the FE diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index 52a52545fc..8668125133 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -1,6 +1,23 @@ # Status -**State: design + plan under review. No code changed.** Date: 2026-07-02. +**State: design + plan under review; review round 1 (how-approvals-work) addressed.** +Date: 2026-07-03. + +Review round 1 (Mahmoud, 28 inline comments on `how-approvals-work.md`) produced three +substantive changes, all folded in: + +- **Live finding:** the playground approve→resume happy path currently loops (Approve + re-parks and re-prompts). This is the observed form of code-review M2/M3; it is now an + explicit acceptance case in plan phase 6 and elevated in phase 4. +- **Structure:** deciding and executing are separate jobs; the relay carries no permission + logic in the target design (the shared decision function runs before execution). +- **Client tools:** no special-case bypass; they resolve through the same ladder and + default to `allow`. + +The explainer was also generalized across harnesses (a per-harness gate table), and now +covers Claude's `default_mode`/`bypassPermissions`, the settings merge semantics, the +ACP request-vs-event distinction, the two kinds of "session", and the cold-replay resume +model before the responder code. ## What is done From 6cb3b2bdb379b4c9690b41cf41b6a6d7a205eef9 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 12:41:15 +0200 Subject: [PATCH 04/24] docs(approval-boundary): fold in the PR #5054 loop diagnosis The live approve-loop is diagnosed: a constant stream messageId plus a level-triggered resume predicate on the frontend (new finding M7), compounded by tool-name drift across ACP frames breaking the decision key (M2's observed form, not argument drift). Updated the explainer's live-warning section, reframed M2 and added M7 in the code review, settled the fix direction in the plan (direct replay of the approved call; absorb #5054's message-id fix and edge-trigger guard; supersede its resolvedName patch and loop-breaker), and recorded the #5054 recommendation in status. Claude-Session: https://claude.ai/code/session_01DGj7GKafjkZeQXMsryWhb2 --- .../projects/approval-boundary/code-review.md | 40 +++++++++++++++---- .../approval-boundary/how-approvals-work.md | 33 +++++++++++---- .../projects/approval-boundary/plan.md | 27 ++++++++----- .../projects/approval-boundary/status.md | 29 ++++++++++++++ 4 files changed, 104 insertions(+), 25 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/code-review.md b/docs/design/agent-workflows/projects/approval-boundary/code-review.md index 30faeea19f..9953e11b61 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/code-review.md +++ b/docs/design/agent-workflows/projects/approval-boundary/code-review.md @@ -84,16 +84,40 @@ no new prompt. This quietly defeats the "reply `once`, never `always`" guard: th re-gates, but the stored decision answers every re-gate. Direction: consume a stored decision on first match (delete from the map), which is what "once" means. -### M2. Cold-replay argument drift degrades approve-to-resume into a re-prompt loop +### M2. The match-on-replay key is fragile: name drift observed live, argument drift latent A design consequence of the name+args key (`responder.ts:126-135`): the resumed turn -re-generates the tool call, and the model must reproduce byte-identical arguments for the -stored approval to match. Any drift (an added optional field, a reworded command) misses the -key, parks again, and re-prompts. The user can approve repeatedly while the tool never runs. -The mirror case: a stored deny keeps auto-rejecting the next identical attempt without a -prompt, one silent wrong decision after the user changes their mind. Worth a live test to -measure how often drift happens in practice; the fix options are design work (correlate by -approval id echoed through the replay, or replay the approved result directly). +re-generates the tool call, and the stored approval matches only if both halves of the key +reassemble identically. Both halves can miss: + +- **Name drift (observed live, via PR #5054's diagnosis).** Claude-over-ACP titles the same + call differently in different frames: the `tool_call` stream event carries a category + ("Terminal") while the permission frame carries the specific invocation. The key is built + from one frame and probed with the other, so it missed even with byte-identical + arguments. This, compounded with M7 below, produced the infinite approve-loop QA found. + #5054 patches it by stamping a `resolvedName` from the recorded `tool_call` event onto + the gate; treat that as evidence of the instability, not as the fix. +- **Argument drift (latent).** The model must reproduce byte-identical arguments on replay. + Any drift (an added optional field, a reworded command, a regenerated digest text) misses + the key, parks again, and re-prompts. The mirror case: a stored deny keeps auto-rejecting + the next identical attempt without a prompt after the user changes their mind. + +Both point the same way: keys reassembled from replayed frames are fragile, so the fix +replays the approved call directly instead of matching a re-issued one (plan phase 4). + +### M7. Constant stream `messageId` plus a level-triggered resume predicate re-sends forever + +Found via PR #5054. The stream egress default `message_id: "msg-1"` is never overridden by +its only caller (`sdks/python/agenta/sdk/decorators/routing.py:284` / +`vercel/stream.py`), so every turn of every conversation streams the same message id and +the Vercel client folds all turns into one assistant message. The resume predicate +(`agentApprovalResume.ts:108-122`) is level-triggered and has no "already resumed" state, +so once an `approval-responded` part exists in that ever-growing last message, the +predicate returns true after every subsequent settle and `useChat` re-sends the +conversation forever. This is the frontend half of the observed infinite loop; it would +loop even if the backend key always matched. Fix (from #5054, worth keeping regardless of +the redesign): a unique message id per turn, and an edge-trigger guard in the predicate (a +`step-start` after the last resolved approval means "already resumed, do not fire again"). ### M3. An approval response keyed only by `approvalId` is silently dropped diff --git a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md index e251e9c5bd..ad52fb6878 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md +++ b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md @@ -333,14 +333,31 @@ tools are effectively allowed; `SEND_MESSAGE` posts to Slack and ends up gated. approval in the transcript is not something the harness itself reads as an authorization.) -That is the designed resume mechanism. **Live behavior warning:** in current QA (Mahmoud, -2026-07-03) the happy path does not reliably complete: clicking Approve can loop, with the -run re-parking and re-prompting again and again. That symptom matches the fragility already -flagged in [code-review.md](code-review.md): the stored decision is keyed by exact tool -name plus byte-identical arguments (M2: if the model re-issues the call with slightly -different arguments on replay, the key misses and the gate re-parks), and an approval that -comes back keyed only by `approvalId` is silently dropped (M3). Reproducing and pinning -this loop is now part of the fix's acceptance tests ([plan.md](plan.md), phase 6). +That is the designed resume mechanism. **Live behavior warning, now diagnosed:** QA +(Mahmoud, 2026-07-03) showed the happy path looping: clicking Approve re-parked and +re-prompted forever. PR #5054 (Arda's empirical fix) plus our review of it pinned the loop +to two independent bugs that compound, and notably *neither* is the argument-drift we first +suspected: + +- **Frontend re-send loop.** The SDK's stream egress sent a constant `messageId: "msg-1"` + on every turn (`stream.py`; the caller never passed an id), so the Vercel client saw all + turns as one ever-growing assistant message. The resume predicate + (`agentApprovalResume.ts`) is level-triggered with no "already resumed" guard, so the + stale `approval-responded` part kept satisfying it after every later turn, and the + frontend re-sent the conversation forever. This alone reproduces the infinite loop. +- **Backend key miss, but on the name, not the args.** Claude-over-ACP names the same tool + call differently in different frames: the `tool_call` stream event carries a category + title ("Terminal") while the permission frame carries the specific invocation. The + stored decision is keyed by name plus canonical args; the two frames disagree on the + *name* half, so the key missed even with byte-identical arguments, and the gate + re-parked on every resume. + +Argument drift (code-review M2) remains a real latent risk of the match-on-replay model, +and the `approvalId`-only drop (M3) is untouched by #5054 and still open. The diagnosis +strengthens the plan's direction: any key that must be reassembled from replayed frames is +fragile, which is why the plan has the runner replay the approved call directly instead of +matching a re-issued one ([plan.md](plan.md), phase 4). Reproducing and pinning the loop +stays an acceptance case (phase 6). ## The two planes: messages and interactions diff --git a/docs/design/agent-workflows/projects/approval-boundary/plan.md b/docs/design/agent-workflows/projects/approval-boundary/plan.md index 8a76f806ef..b2046611d0 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/plan.md +++ b/docs/design/agent-workflows/projects/approval-boundary/plan.md @@ -175,10 +175,13 @@ frontend research: nothing in the UI depends on parking for non-ask tools). **Phase 4: correctness debt in the same code (from code-review.md).** H1 (reply failure parks or fails loud; resolve interaction only after a successful reply), H2 (only `{approved}` envelopes become decisions; client-tool replay scoped by interaction -token), M2+M3 as one item, elevated because the failure is now observed live (see phase 6): -approval responses must correlate by `toolCallId` and drop loudly otherwise, and the -approve→resume match must survive argument drift (echo the approval id through the replay, -or replay the approved result directly, decided by what the live reproduction shows), M4 +token), M2+M3+M7 as one item, now diagnosed via PR #5054 (see the live-warning section of +how-approvals-work.md): the observed loop = a constant stream `messageId` plus a +level-triggered resume predicate (M7, frontend half) compounded by tool-*name* drift across +ACP frames breaking the decision key (M2's observed form). The durable fix here is the +direct replay of the approved call, which removes the reassembled-key fragility class +(name drift, argument drift) outright; approval responses must correlate by `toolCallId` +and drop loudly otherwise (M3), M4 (client tools resolve through the same ladder, defaulting to `allow`), M5 (latch the stop reason before the relay drain), L1 (no-id gates park instead of silently hanging), H3 (verify the daemon's permission-id scheme; if per-session counters, namespace interaction @@ -237,11 +240,17 @@ test that let the proxy rot). - **Rule matching for builtins.** The structured `rules` need a matcher compatible with Claude's rule syntax (`Bash(npm run:*)`). Scope it to exactly what the settings renderer accepts today; anything fancier stays authored-settings-only and documented as such. -- **Argument drift on resume (code-review M2, observed live).** Cold-replay approval - matching by name+args re-prompts when the model rewords arguments, and QA now shows the - approve→loop symptom in the playground, so this is no longer hypothetical. It moves into - phase 4 (fix direction: echo the approval id through the replay, or have the runner - replay the approved result directly rather than relying on an identical re-issue). +- **Reassembled-key fragility on resume (code-review M2/M7, diagnosed).** The live + approve-loop is explained: constant `messageId` + level-triggered resume predicate on the + frontend, tool-name drift across ACP frames on the backend. Fix direction settled: the + runner replays the approved call directly, removing the whole matching class. Two pieces + of PR #5054 are worth absorbing regardless (unique per-turn message id; the + "already resumed" edge-trigger guard); two pieces should be superseded by this plan, not + inherited (the `resolvedName` stamping that patches the name drift, and especially the + `nonConvergingToolNames` loop-breaker that silently auto-DENIES a gate after three + non-converging approvals: it is keyed by bare tool name globally, can false-positive on a + busy tool, reintroduces the F-024 deny-clobber deliberately, and gives the user no signal + that their approved tool was blocked). - **Batch response shape.** Owned jointly with the streaming-invoke workspace; this plan only requires that paused be distinguishable. - **Vocabulary migration.** `auto` → `allow` and the authored-path rename touch the FE diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index 8668125133..f18a1d4289 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -19,6 +19,35 @@ covers Claude's `default_mode`/`bypassPermissions`, the settings merge semantics ACP request-vs-event distinction, the two kinds of "session", and the cold-replay resume model before the responder code. +## Update 2026-07-03 (later): the approve-loop is diagnosed, via PR #5054 + +Arda's PR #5054 empirically stops the loop; our review of it pinned the mechanism. Two +independent bugs compound, and neither is the argument-drift we first suspected: + +- Frontend half (new finding M7): the stream egress sent a constant `messageId: "msg-1"` + every turn, so the client folded all turns into one message; the resume predicate is + level-triggered with no "already resumed" guard, so it re-sent forever. +- Backend half (M2's observed form): tool-*name* drift across ACP frames ("Terminal" in + the tool_call event vs the invocation title in the permission frame) broke the + name+args decision key even with identical arguments. + +Consequences for this plan (folded into plan.md phase 4 and risks): + +- The direct-replay-of-the-approved-call fix is reinforced: it removes the whole + reassembled-key fragility class (name drift and argument drift) instead of patching one + half of the key. +- Absorb from #5054 regardless of the redesign: the unique per-turn message id and the + frontend "already resumed" edge-trigger guard. Both are correct on their own. +- Do not inherit from #5054: the `resolvedName` stamping (patches the symptom our redesign + removes) and the `nonConvergingToolNames` loop-breaker (auto-denies after three + non-converging approvals, keyed by bare tool name globally: can false-positive-deny a + busy tool, deliberately reintroduces the F-024 deny-clobber, and is silent to the user). + +Recommendation on #5054 itself: do not merge as-is; split it. Keep the message-id fix, the +resume-predicate guard, the tool-input `{}` display fix, and the unrelated Turn +Inspector/chat-UX work (own PRs). Treat the name-anchor patch and the loop-breaker as +temporary evidence to be superseded by this plan. + ## What is done - Full research pass across all five systems (frontend, agent service, runner, harness From c61701e5f4e0fa964de5a247a9a7979dc5a52064 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 13:25:37 +0200 Subject: [PATCH 05/24] docs(approval-boundary): round 2: settle the permission model, add the target path Global policy becomes four explicit modes (allow|ask|deny|allow_reads): read-only-allow is a policy choice, not a hidden per-tool default, and needs_approval is deleted from the model. 'Disposition' renamed to 'effective permission' everywhere. New 'target path' section shows the clean end-state flow; resume is redesigned to replay the approved call directly. Corrected the session-id story (the playground sends a stable per-conversation id). Added the Pi-builtins explanation (selection is Pi's only native control). Plan gains the stacked-on-#5054 baseline (keep the message-id fix and resume guard; delete resolvedName and the loop-breaker) and updated phases/deltas. Status consolidated for final review. Claude-Session: https://claude.ai/code/session_01DGj7GKafjkZeQXMsryWhb2 --- .../projects/approval-boundary/README.md | 25 +- .../projects/approval-boundary/code-review.md | 2 +- .../approval-boundary/design-review.md | 33 +- .../approval-boundary/how-approvals-work.md | 166 +++++--- .../projects/approval-boundary/plan.md | 358 ++++++++++-------- .../projects/approval-boundary/status.md | 168 ++++---- .../projects/approval-boundary/the-bug.md | 6 +- 7 files changed, 413 insertions(+), 345 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/README.md b/docs/design/agent-workflows/projects/approval-boundary/README.md index de126a39c1..b07110cc82 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/README.md +++ b/docs/design/agent-workflows/projects/approval-boundary/README.md @@ -20,17 +20,22 @@ Everything here is documentation and planning. No code changes ship with this PR the `auto` policy is dead code, and a headless run (any caller without a chat UI: curl, agent-as-tool, evaluations, triggers) dies at the first gate. Batch responses hide the pause entirely: HTTP 200, mid-sentence text. See [the-bug.md](the-bug.md). -- **The fix direction.** Park only on authored intent. One resolved permission plan - (default + per-tool + builtin rules, vocabulary `allow | ask | deny`) is computed by the - SDK and enforced by both runner gates; the session-id inference is deleted; the approval - event fires only when the run actually pauses; batch shows the paused state. Recommended - as one shot (POC, no compat constraints); an independent Codex review concurred. See - [plan.md](plan.md). +- **The fix direction.** Pause only on authored intent. Per tool: `allow | ask | deny` or + inherit. Per agent: one policy with four modes (`allow`, `ask`, `deny`, `allow_reads` = + reads run, writes ask). The SDK computes each tool's effective permission once and ships + it; both runner gates enforce it; the session-id inference is deleted; the approval + event fires only when the run actually pauses; resume replays the approved call directly + (no fragile matching); batch shows the paused state. One shot (POC, no compat + constraints); an independent Codex review concurred. See [plan.md](plan.md). +- **Baseline.** This PR is stacked on Arda's #5054 (merge-then-rework decision). His + message-id and resume-guard fixes are kept; his `resolvedName` patch and auto-deny + loop-breaker get deleted by the redesign. The plan's "Baseline" section has the full + sort. - **One expectation to reset.** The fix does not make the original reproducing agent run - unattended under default config. Its `SEND_MESSAGE` tool is a write, and writes default - to `ask` by design, so the run still pauses there until the author marks that tool - `allow`. What changes: the pause happens for the authored reason, it is visible, and the - `auto` policy genuinely governs the tools it applies to. + unattended under the default policy. Its `SEND_MESSAGE` tool is a write, and the default + policy mode asks for writes, so the run still pauses there until the author allows that + tool (or sets the policy to `allow`). What changes: the pause is an explicit policy + outcome, it is visible, and `allow` genuinely means allow everywhere. - **Beyond the bug.** The correctness review found 4 high, 6 medium, and 2 low issues in the same code (a swallowed reply failure that can hang runs, stale client-tool replay, and more). The organization review found good invariant discipline but four enforcement diff --git a/docs/design/agent-workflows/projects/approval-boundary/code-review.md b/docs/design/agent-workflows/projects/approval-boundary/code-review.md index 9953e11b61..7304d139e1 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/code-review.md +++ b/docs/design/agent-workflows/projects/approval-boundary/code-review.md @@ -177,7 +177,7 @@ recording on parked turns (both handlers record it). still PARK", which pins the bug itself. - `responder.test.ts:184-225, 376-428` and `sandbox-agent-orchestration.test.ts:883-989`: expect `park` as the no-match outcome - under `basePolicy "auto"` with no disposition on the gate; expected outcomes change with + under `basePolicy "auto"` with no effective permission on the gate; expected outcomes change with the fix's default. - `tool-relay-permission.test.ts:74-77, 123-127`: pin the `ask`-collapses-to-policy behavior (`TODO(S5)`). diff --git a/docs/design/agent-workflows/projects/approval-boundary/design-review.md b/docs/design/agent-workflows/projects/approval-boundary/design-review.md index c416299566..47f5f094f9 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/design-review.md +++ b/docs/design/agent-workflows/projects/approval-boundary/design-review.md @@ -41,7 +41,7 @@ hack. ### 3. Three enforcers, no single computer -The per-tool disposition is computed independently in three places: +The per-tool effective permission is computed independently in three places: - Python, when rendering Claude's settings file (`claude_settings.py:151-189`); - TypeScript, in the relay for runner-executed tools (`relay.ts:143-152`); @@ -108,31 +108,32 @@ These are constraints any design must respect, not flaws: hardening (bare names would over-authorize) and constrains any alternative resume design. - **Harnesses are asymmetric.** Pi never raises gates, so per-tool `ask` on Pi is enforceable only at the relay, which today cannot park mid-prompt. Claude builtins have no - tool spec, so their dispositions exist only as authored settings rules. + tool spec, so their effective permissions exist only as authored settings rules. ## Principles for the fix 1. **Declare intent; never infer it from transport.** Park only when the resolved - disposition says `ask`. Delete `hasHumanSurface`. Session ids stay what they are: + effective permission says `ask`. Delete `hasHumanSurface`. Session ids stay what they are: correlation for persistence and tracing. -2. **One computer, many enforcers.** The SDK already owns `effective_permission()` and the - settings rendering; make it compute one resolved permission plan (default disposition - plus per-tool dispositions, including structured rules for Claude builtins) and ship it - on the run request. The responder and the relay become lookups into the same plan. - (Codex pushed hard on this point: a fix that only patches the responder "fixes `auto` - while silently breaking explicit builtin `ask`", because authored ask-rules for Claude - builtins are visible only inside the settings file.) -3. **One vocabulary.** `allow | ask | deny` everywhere. The global default becomes - `default_permission` in the same vocabulary (`auto` maps to `allow`; a global `ask` is - now expressible and legitimate: approval-by-default). Retire the +2. **One computer, many enforcers.** The SDK already owns the settings rendering; make it + assemble one permission plan (the policy default, per-tool permissions, and structured + rules for Claude builtins) and ship it on the run request. The responder and the relay + become lookups into the same plan. (Codex pushed hard on this point: a fix that only + patches the responder "fixes `auto` while silently breaking explicit builtin `ask`", + because authored ask-rules for Claude builtins are visible only inside the settings + file.) +3. **One vocabulary.** `allow | ask | deny` per tool, everywhere. The global default + becomes one policy field with four explicit modes: `allow`, `ask`, `deny`, and + `allow_reads` (reads run, writes ask), which promotes the old hidden read-only + defaulting into a visible policy choice (review round 2). Retire the `runner.interactions.headless` authoring path in favor of a name inside the permission - family. + family, and delete the legacy `needs_approval` boolean outright. 4. **Events mean actions.** Emit `interaction_request(user_approval)` only when the run pauses. An auto-approved gate emits nothing extra; the tool events already show what ran. 5. **Terminal state is always visible.** Batch surfaces `stop_reason` and, when paused, the pending interaction identity. 6. **A stored approval never overrides a current deny.** The decision order is: resolve the - disposition first; consult stored decisions only when the disposition is `ask`. (Codex + effective permission first; consult stored decisions only when the effective permission is `ask`. (Codex flagged this ordering; the current code checks stored decisions first, so a stale approval could outrank a config that has since been changed to deny.) @@ -140,7 +141,7 @@ These are constraints any design must respect, not flaws: - **The one-line fix** (consult the policy before parking) restores `auto` but silently auto-approves every `ask`/unset tool too, killing the playground prompt. Rejected. -- **A disposition-aware responder without the shared plan** fixes resolved tools but leaves +- **A effective permission-aware responder without the shared plan** fixes resolved tools but leaves Claude-builtin ask-rules invisible to the responder, so an author's explicit `ask` on `Bash` would auto-approve under a policy of `allow`. Rejected as a final state; acceptable only as an explicitly temporary step. diff --git a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md index ad52fb6878..243512be70 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md +++ b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md @@ -53,13 +53,12 @@ nothing inherits it. Two things make the current shape confusing, and both are a the plan: - The vocabulary does not match the per-tool one (`auto | deny` here, `allow | ask | deny` - per tool), and it cannot express "ask for everything". The natural set of agent-wide - behaviors ("approve everything", "ask for everything", "deny everything", "ask only for - writes") is exactly what the plan's target shape gives you: a `default_permission` of - `allow | ask | deny`, with "ask only for writes" falling out of the per-tool defaults - (reads default to `allow`, writes to `ask`; see knob 2). The `deny` default exists for - lockdown ("this agent runs no tools unless a tool is explicitly allowed"); it is - legitimate, just rarely what you want. + per tool), and it cannot express "ask for everything". Decided in review round 2: the + policy becomes one field with **four explicit modes**, matching the natural set of + agent-wide behaviors: `allow` (approve everything), `ask` (a human approves everything), + `deny` (lockdown: nothing runs unless a tool is explicitly allowed), and `allow_reads` + (reads run, writes ask; the sensible default). "Reads are always fine" thereby becomes a + visible policy choice instead of an opaque per-tool defaulting step (see knob 2). - The knob has three names depending on where you stand. Authors write it as `runner.interactions.headless` in the agent config (parsed in `sdks/python/agenta/sdk/agents/dtos.py:1087-1102`); the SDK stores it as @@ -79,22 +78,25 @@ changes it (`web/packages/agenta-playground/src/state/execution/agentRequest.ts: integration action from the Composio catalog; a code tool; an MCP server) can carry its own `permission`: `allow`, `ask`, or `deny` (`sdks/python/agenta/sdk/agents/tools/models.py:26`). `allow` means run it without asking. -`ask` means a human must approve each call. `deny` means never run it. A tool with no -explicit permission falls back through a ladder (`effective_permission`, -`models.py:273-292`), in order: - -1. an explicit `permission` wins; -2. else `needs_approval: true` means `ask` (`needs_approval` is the older boolean form of - the same idea, from before the three-valued `permission` field existed; stored configs - still carry it, so the ladder honors it; the plan deprecates it at the config edge); -3. else the catalog's `read_only` hint applies its default: reads are safe to auto-run - (`allow`), writes should prompt (`ask`). This is our own convention, applied by our SDK; - the hint comes from the Composio catalog's tags. -4. else nothing is set, and the tool inherits the global policy (knob 1). - -The output of this ladder is what the rest of this workspace calls the tool's -**disposition**: its final, resolved allow/ask/deny. The tool editor exposes the field as a -Permission select with an "Inherit policy" placeholder (`web/.../ToolFormView.tsx:47-51`). +`ask` means a human must approve each call. `deny` means never run it. Unset means the tool +inherits the policy. So the author's options per tool are exactly two: specify a +permission, or inherit. The tool editor exposes the field as a Permission select with an +"Inherit policy" placeholder (`web/.../ToolFormView.tsx:47-51`). + +A tool's **effective permission** is the final allow/ask/deny after that inheritance +resolves. (Earlier drafts called this the "disposition"; renamed, since the code already +says `effective_permission`.) + +Two extra inputs complicate today's inheritance, and both go away (decided, round 2): + +- A legacy `needs_approval: true` boolean (the older form of the same idea, from before + the three-valued field existed) still counts as `ask` in today's resolution + (`effective_permission`, `models.py:273-292`). It gets **deleted outright**, along with + its aliases; no deprecation dance, since nothing is released. +- The catalog's `read_only` hint (from Composio's tags) today silently defaults reads to + `allow` and writes to `ask`, per tool. That behavior moves into the policy as the + explicit `allow_reads` mode (knob 1); the hint stays on the tool spec as the data that + mode consults. **3. Claude harness rules.** For the Claude harness only, authors can write raw Claude Code permission rules (`harness.permissions`): a `default_mode` plus `allow`/`ask`/`deny` rule @@ -151,8 +153,8 @@ triggers a permission request. One consequence to hold on to, because it constrains the fix: since `ask` and unset both render as "no rule", they look identical by the time a call reaches Gate 2. The runner cannot tell "the author explicitly wants a human" from "the author said nothing, apply the -default" unless it consults the tool's disposition from the run request. Today it consults -nothing (that is the bug); the fix makes Gate 2 look the disposition up. +default" unless it consults the tool's effective permission from the run request. Today it consults +nothing (that is the bug); the fix makes Gate 2 look the effective permission up. **Gate 2: the runner's ACP responder.** When Gate 1 does not settle a tool, a gating harness raises a permission request to the runner over ACP. This is not a stream event: it @@ -199,20 +201,28 @@ async onPermission(request) { Three outcomes are possible. `allow` replies "once" to the harness (approve this single call, never "always") and the tool runs. `deny` replies "reject" and the tool is refused. `park` is the interesting one: the runner sends no reply at all, tears the session down, and -ends the turn with `stopReason: "paused"` (`sandbox_agent.ts:640-649, 766-767`). Park means -"a human must answer, and the answer will arrive on a future turn, not this one". - -Branch 2 is where the intended logic and the actual logic part ways. The *intended* logic -was: an interactive surface (someone in the playground) should get the question; a headless -call (nobody watching) should be answered by the policy, because parking a run nobody can -resume just kills it. That intent is defensible. The *implementation* reduced "is a human -watching" to one signal: does the run request carry a non-empty session id -(`hasHumanSurface`, `sandbox_agent.ts:627`). Session ids were a plausible proxy when only -the playground sent them; the SDK now mints one for every request, so the proxy is always -true and branch 3 is dead code. That, plus the fact that the policy should not have been -shadowed by surface detection in the first place, is [the-bug.md](the-bug.md). The fix -removes branch 2 entirely: whether to park becomes a property of the tool's disposition -(`ask`), not of who might be watching. +ends the turn with `stopReason: "paused"` (`sandbox_agent.ts:640-649, 766-767`). + +Hold on to the right mental model before dissecting the branches, because the code obscures +it. The correct model is: **always answer by the tool's effective permission.** Allow means +allow. Deny means deny. Ask means raise the question to a human and wait. That is the whole +policy. "Park" is not a fourth policy; it is the *mechanism* ask uses to wait: a live +harness session cannot block on an open gate forever (that hangs the run, the old F-040 +lesson), so the runner ends the turn cleanly and lets the answer arrive on a future turn. +And "who is watching" changes only *where the question surfaces* (chat buttons now, a +durable interaction record for later surfaces), never *whether* the tool needs asking. + +Today's code deviates from that model in two ways. First, "ask" does not exist in its +policy vocabulary: `basePolicy` in branch 3 is the global `permission_policy`, which can +only say `auto` or `deny`, so the code cannot express "ask" as policy at all. Second, in +place of the missing word it uses surface detection: branch 2 parks whenever a "human +surface" exists, and the implementation reduced that to "does the run request carry a +non-empty session id" (`hasHumanSurface`, `sandbox_agent.ts:627`). Session ids were a +plausible proxy when only the playground sent them; the SDK now resolves one for every +request, so the proxy is always true, branch 3 is dead code, and every fresh gate parks. +That is [the-bug.md](the-bug.md). The fix deletes branch 2 and restores the model above: +whether to pause is a property of the tool's effective permission (`ask`), never of who +might be watching; an `ask` pauses on a headless run too, visibly. **Gate 3: the tool relay.** Gateway and code tools do not run inside the harness; the runner executes them itself through its tool relay. The relay enforces the per-tool permission @@ -265,21 +275,37 @@ policy. There is no human-in-the-loop on Pi at all today. The playground UI is h the policy field (it hides "Permission policy" for Pi with a note, hardcoded by harness name at `web/.../useModelHarness.tsx:113`), but per-tool `ask` on a Pi agent is silently not honored. The plan's fix is structural: once the relay consults the shared decision function, -an `ask` disposition parks at the relay exactly like a Claude gate parks at the responder, +a resolved `ask` pauses at the relay exactly like a Claude gate pauses at the responder, and Pi gets real approvals through the same machinery (this is the old S5.2 item; scope decision 3 in [status.md](status.md)). +On "surely Pi has permission settings for its builtins, like Claude does": it does not, +and the asymmetry is worth stating precisely. Pi's builtins are its native tools (bash, +read, write, and so on), and Pi's only native control over them is *selection*: the agent +config decides which builtins are granted at all (`builtin_names`). Granted means runs +without asking; not granted means does not exist. There is no mode, no rule list, no +settings file exposed over the ACP bridge (that is what `permissions: false` reports), so +there is nothing for us to render the way we render `.claude/settings.json`. If Pi grows a +native permission config later, a new adapter renders it and it becomes Pi's Gate 1; +until then, deny-by-omission at selection time and the relay are the only Pi controls. + ## The journey of one gated tool call (the playground path) -Concrete example: the `uc9-digest` agent, Claude harness, policy `auto`. Three read-only -tools are effectively allowed; `SEND_MESSAGE` posts to Slack and ends up gated. +Concrete example: the `uc9-digest` agent, Claude harness, global policy `auto` (today's +name for allow). Three read-only tools are effectively allowed; `SEND_MESSAGE` posts to +Slack and ends up gated. 1. You send a message in the playground. The frontend posts the whole conversation to the - agent service's streaming endpoint. -2. The SDK's request normalizer resolves a session id, minting a fresh UUID if the request - has none (`sdks/python/agenta/sdk/middlewares/running/normalizer.py:307`, - `sdks/python/agenta/sdk/models/shared.py:13-22`). The id flows into the run request the - service sends the runner. + agent service's streaming endpoint, including the conversation's own session id + (`session_id` in the envelope, `web/packages/agenta-playground/src/state/execution/ + agentRequest.ts:389`). That id is stable for the conversation, not per message. +2. The SDK's request normalizer passes a supplied session id through, and mints a fresh + UUID only when the caller sent none, which the playground never does; the minting hits + headless callers like curl or evaluations + (`sdks/python/agenta/sdk/middlewares/running/normalizer.py:307`, + `sdks/python/agenta/sdk/models/shared.py:13-22`). Either way, every run request the + runner sees now carries some session id, which is exactly what broke the "session id + means a human is watching" proxy. 3. The runner starts a Claude session in the sandbox, writing the rendered `.claude/settings.json` into the workspace first. 4. Claude runs the three reads without asking: their catalog `read_only: true` hint @@ -357,7 +383,51 @@ and the `approvalId`-only drop (M3) is untouched by #5054 and still open. The di strengthens the plan's direction: any key that must be reassembled from replayed frames is fragile, which is why the plan has the runner replay the approved call directly instead of matching a re-issued one ([plan.md](plan.md), phase 4). Reproducing and pinning the loop -stays an acceptance case (phase 6). +stays an acceptance case (phase 6). Note on the baseline: this workspace's PR is now +stacked on #5054, so its frontend fixes (unique per-turn message id, the already-resumed +guard) are part of our base; its backend patches (`resolvedName`, the auto-deny +loop-breaker) are in the base too and get deleted by the fix. + +## The target path: the same flow after the fix + +For contrast with everything above, here is the whole system as it will work once the plan +lands. This is the version to hold in your head going forward; the sections above explain +the code you will find in the tree today. + +**Config.** Two levels, one vocabulary. Each tool: `allow`, `ask`, `deny`, or unset +(inherit). One global policy with four modes: `allow` (run everything), `ask` (a human +approves everything), `deny` (lockdown), `allow_reads` (reads run, writes ask; the +default). Nothing else: no `needs_approval`, no hidden per-tool defaulting, no +`runner.interactions.headless`. + +**One decision.** A tool's effective permission is its own setting if present, else what +the policy says (under `allow_reads`, the catalog's read-only hint decides; no hint counts +as a write). The SDK assembles this once and ships it on the run request. Every enforcement +point reads the same answer; none re-derives it. + +**Answering a gate.** Wherever a tool needs a verdict (Claude's raised gate at the ACP +responder, or the relay before executing a gateway/code tool), the answer follows the +effective permission: `allow` runs, in place, on every surface, with the tool call and +result visible in the stream and no approval event. `deny` refuses. `ask` emits exactly one +approval request and pauses the turn (`stopReason: "paused"`); pausing is the mechanism, +not a policy. Claude's settings file stays what it is, pre-answering what can be +pre-answered (allow rules for effective-allow tools, deny rules for denies) so gates are +only raised for genuine asks. + +**Resume.** An approval or denial travels back inside the conversation. On resume the +runner replays the *approved call itself* (the exact tool and arguments the human saw) +rather than waiting for the model to re-issue an identical call, so there is no key to +mismatch and no loop to break. A stored approval is spent on first use, and a config +changed to `deny` beats any stored approval. + +**Headless.** Identical decisions, different surface: an `ask` on a run with no chat open +still pauses, the batch response says so and names the pending interaction, and the +durable interactions plane (already receiving rows today) is what will let anyone answer +it later. Nothing anywhere consults "is a human watching". + +**Pi.** Same decisions, enforced at Pi's one choke point (the relay): `ask` pauses there, +`deny` refuses there, and builtins remain governed by selection until Pi grows a native +permission config. ## The two planes: messages and interactions diff --git a/docs/design/agent-workflows/projects/approval-boundary/plan.md b/docs/design/agent-workflows/projects/approval-boundary/plan.md index b2046611d0..7201e478e9 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/plan.md +++ b/docs/design/agent-workflows/projects/approval-boundary/plan.md @@ -5,18 +5,58 @@ Prerequisites: [the-bug.md](the-bug.md) for the bug, [code-review.md](code-review.md) for the correctness debt this plan folds in. This is a plan, not an implementation. The whole feature is a pre-release POC, so nothing -here preserves backward compatibility. +here preserves backward compatibility: fields get deleted, not deprecated. + +## Baseline: this plan builds on PR #5054 + +Decision (Mahmoud, 2026-07-03): Arda merges #5054 as-is and we rework on top, rather than +splitting a 40-commit PR on an unreleased dev branch. PR #5041 is stacked on his branch, +and this plan treats his changes as the base. Sorting the HITL-relevant pieces of #5054: + +- **In the base, kept.** The unique per-turn stream `messageId` plus the frontend + "already resumed" edge-trigger guard (together these fixed the frontend half of the + infinite loop, finding M7), the tool-input `{}` display fix, and the `[HITL]` diagnostic + logging. +- **In the base, deleted by this work.** The `resolvedName` stamping (patches the + tool-name drift that direct replay makes irrelevant) and the `nonConvergingToolNames` + loop-breaker (silent auto-deny after three non-converging approvals: keyed by bare tool + name, false-positive-prone, and it reintroduces the F-024 clobber). Deleting both is an + explicit phase 1 item; the acceptance tests replace them as the regression guard. +- **In the base, unrelated.** The Turn Inspector and chat UX work. Untouched. + +## The permission model (settled in review round 2) + +Two levels, one vocabulary, no hidden steps: + +- **Per tool**: `permission: allow | ask | deny`, or unset. Unset means "inherit the + policy". That is the whole per-tool story. The legacy `needs_approval` boolean and the + `agenta_metadata.permission_mode` alias are **deleted** (Mahmoud: no compat needed). +- **Per agent (the global policy)**: one field with four modes. + - `allow`: run every tool without asking. + - `ask`: a human approves every tool call. + - `deny`: refuse every tool call (lockdown). + - `allow_reads` (name open, see status.md): reads run, writes ask. Resolved from the + catalog's `read_only` hint; a tool with no hint counts as a write and asks. This was + previously an opaque per-tool defaulting step buried in a ladder; Mahmoud's call is + that "reads are always fine" is a *policy*, so it becomes a visible policy mode. It is + the sensible default for new agents. + +A tool's **effective permission** is simply: its own `permission` if set, else what the +policy mode says for it. ("Effective permission" replaces the earlier term "disposition"; +it matches the code's `effective_permission()`.) ## What "fixed" means, concretely Behavior after the fix, stated as acceptance checks: -1. **`auto`/`allow` never prompts, anywhere.** A tool whose resolved permission is `allow`, - or an unset tool under a default of `allow`, runs in place. Its result streams. No - approval event is emitted. Playground and headless behave identically. -2. **`ask` always pauses, everywhere.** A tool whose resolved permission is `ask` parks the - turn and emits exactly one approval request. In the playground you get the Approve/Deny - buttons, as today. On a headless call the run ends `paused`, and the caller can see that. +1. **`allow` never prompts, anywhere.** A tool whose effective permission is `allow` runs + in place. Its result streams. No approval event is emitted. Playground and headless + behave identically. +2. **`ask` always pauses, everywhere.** A tool whose effective permission is `ask` parks + the turn and emits exactly one approval request. In the playground you get the + Approve/Deny buttons; approving completes the run **without looping** (broken live + today, diagnosed: findings M7 + M2). On a headless call the run ends `paused`, and the + caller can see that. 3. **`deny` never runs, anywhere.** Including client tools (today they bypass it). 4. **The pause is visible on every path.** Batch responses carry the stop reason and, when paused, the pending interaction identity. Nobody has to query spans to learn their run @@ -25,51 +65,40 @@ Behavior after the fix, stated as acceptance checks: about permissions. One consequence worth stating up front, because it reframes the original reproduction: the -`uc9-digest` run will still pause at `SEND_MESSAGE` under default config, even after the -fix. Resolved gateway tools carry a `read_only` hint from the Composio catalog, and the -per-tool ladder defaults mutating tools to `ask` -(`sdks/python/agenta/sdk/agents/tools/models.py:273-292`; confirmed populated at -`sdks/python/agenta/sdk/agents/platform/gateway.py:199`). That default is deliberate: -capability-config's design says reads auto-run, writes prompt. - -What changes is that the pause becomes *authored, visible, and answerable*. The author who -wants the digest posted headless sets `permission: allow` on `SEND_MESSAGE` (one select in -the tool editor), and the run completes end to end. The `auto` policy governs tools with no -disposition at all, and it actually works again. +`uc9-digest` run still pauses at `SEND_MESSAGE` under the default policy, and after the fix +that is an explicit, visible policy outcome rather than an accident. Under `allow_reads`, +`SEND_MESSAGE` is a write (catalog hint, populated at +`sdks/python/agenta/sdk/agents/platform/gateway.py:199`), so it asks. The author who wants +the digest posted headless sets `permission: allow` on that tool, or switches the agent's +policy to `allow`, and the run completes end to end. ## Options considered -**Option A: the one-line fix.** Consult `basePolicy` before parking -(`responder.ts:201-206`). Restores `auto` for everything, including tools that resolved to -`ask`: the playground prompt disappears entirely, and an author's explicit "ask me first" -is silently auto-approved. Rejected. - -**Option B: disposition-aware responder only.** The responder looks up the gated tool's -`permission` from the run request's tool specs (`allow` allows, `deny` denies, `ask` parks, -unset falls to the policy) and `hasHumanSurface` dies. This fixes resolved tools but leaves -one hole: authors can write raw `ask` rules for Claude *builtins* (`Bash`, `Edit`...) via -`harness.permissions`, and those exist only inside the rendered settings file. The responder -would see a builtin gate as "unset" and auto-approve it under a default of `allow`. An -explicit authored "ask" being silently approved is the worst failure mode on the table. -Rejected as an end state. - -**Option B+C: B plus Option C.** Same hole as B. Acceptable only as a stepping -stone if we accept a documented temporary gap for builtin ask-rules. - -**Option D: one resolved permission plan (recommended).** The SDK computes every -disposition once and ships it on the run request; the runner only enforces. This closes the -builtin hole (authored builtin rules travel in structured form), unifies the relay and the -responder on one decision function, and removes both duplicate computations and the -vocabulary split. The independent Codex review concurred, and specifically recommended -doing this in one shot rather than staging through B: there are no compatibility -constraints, B leaves duplicate policy logic alive, and the extra cost over B is mostly -wire-shape and test churn. - -**Recommendation: D, in one shot, plus the visibility work (4) and the correctness debt -that lives in the same code.** If implementation reveals the wire change is heavier than -expected, fall back to landing B+C first with the builtin-ask gap explicitly documented, -then finish D. That decision can be made mid-implementation without wasted work, because -B's responder change is a subset of D's. +(Kept for the record; the recommendation is unchanged since round 1, with the vocabulary +updated by round 2.) + +**Option A: the one-line fix.** Consult the policy before parking (`responder.ts:201-206`). +Restores `allow` for everything, including tools that resolved to `ask`: the playground +prompt disappears entirely, and an author's explicit "ask me first" is silently +auto-approved. Rejected. + +**Option B: permission-aware responder only.** The responder looks up the gated tool's +`permission` from the run request's tool specs and `hasHumanSurface` dies. Fixes resolved +tools but leaves one hole: authors can write raw `ask` rules for Claude *builtins* +(`Bash`, `Edit`...) via `harness.permissions`, and those exist only inside the rendered +settings file. The responder would see a builtin gate as "unset" and auto-approve it under +a policy of `allow`. An explicit authored "ask" being silently approved is the worst +failure mode on the table. Rejected as an end state. + +**Option C: visibility only.** Batch surfaces the paused state. Necessary, not sufficient. + +**Option D: one resolved permission plan (recommended, Codex concurred).** The SDK computes +every effective permission once and ships it on the run request; the runner only enforces. +Closes the builtin hole (authored builtin rules travel in structured form), unifies the +relay and the responder on one decision function, and removes both duplicate computations +and the vocabulary split. One shot rather than staged: no compatibility constraints, and +B's responder change is a subset of D's, so a mid-implementation fallback to B+C loses no +work. ## The target design @@ -79,114 +108,125 @@ The run request gains one coherent block, replacing the stringly `permissionPoli ``` permissions: { - default: "allow" | "ask" | "deny", // was permission_policy auto|deny; auto -> allow - rules?: [ // authored harness-builtin rules, structured + default: "allow" | "ask" | "deny" | "allow_reads", // the global policy (4 modes) + rules?: [ // authored harness-builtin rules { pattern: "Bash(rm:*)", permission: "ask" }, ... ] } ``` -Per-tool dispositions keep riding where they already are (`customTools[].permission` and -`mcpServers[].permission`), but the SDK writes the **resolved** value (the output of -`effective_permission()`, so `read_only`/`needs_approval` defaults are already applied) and -the runner never re-derives it. Semantic roles stay clean: `permissions` is policy, owned by -the author, resolved by the SDK; `sessionId` goes back to being pure correlation. +Per-tool permissions keep riding where they already are (`customTools[].permission` and +`mcpServers[].permission`); the SDK writes the author's explicit value (or nothing), and +the `read_only` hint already on each spec is what `allow_reads` consults. Semantic roles +stay clean: `permissions` is policy, owned by the author, assembled by the SDK; +`sessionId` goes back to being pure correlation. -Vocabulary: `allow | ask | deny` everywhere. A global default of `ask` becomes expressible -(approval-by-default agents). The authored config path moves from -`runner.interactions.headless` into the permission family (proposal: `permissions.default` -next to the existing `harness.permissions` and tool permissions; final name settled at -implementation, but it must contain the word "permission" and live with the policy fields). +The authored config path moves from `runner.interactions.headless` into the permission +family (proposal: `permissions.default`; final name settled at implementation, but it must +contain the word "permission" and live with the policy fields). ### One decision function ``` -resolvePermission(gate, plan): - disposition = plan.toolSpec(gate)?.permission // resolved tools, MCP servers - ?? plan.matchRule(gate) // structured builtin rules - ?? plan.default - if disposition == "deny": return deny - if disposition == "allow": return allow // reply "once", run continues in place - // disposition == "ask": - stored = storedDecision(gate) // {approved} envelopes from the replay +effectivePermission(gate, plan): + explicit = plan.toolSpec(gate)?.permission // resolved tools, MCP servers + ?? plan.matchRule(gate) // authored builtin rules + if explicit: return explicit // allow | ask | deny + if plan.default == "allow_reads": + return gate.readOnlyHint == true ? "allow" : "ask" // no hint counts as a write + return plan.default // allow | ask | deny + +decide(gate, plan): + perm = effectivePermission(gate, plan) + if perm == "deny": return deny + if perm == "allow": return allow // reply "once"; run continues in place + // perm == "ask": + stored = storedDecision(gate) // the answer from a previous turn if stored: return stored (consumed once) - return pendingApproval // park: no reply, end turn "paused" + return pendingApproval // pause: no reply, end turn "paused" ``` -Order matters, twice over. The disposition is resolved *before* stored decisions are -consulted, so a stale approval can never override a config that has since changed to `deny` -(today stored decisions are checked first). And a stored decision is consumed on first -match, so one approval authorizes one execution ("once" semantics, code-review M1). +Order matters, twice over. The effective permission is resolved *before* stored decisions +are consulted, so a stale approval can never override a config that has since changed to +`deny` (today stored decisions are checked first). And a stored decision is consumed on +first match, so one approval authorizes one execution ("once" semantics, code-review M1). The same function backs both gates: the ACP responder (harness-gated tools) and the relay -(runner-executed tools). Deciding and executing stay separate jobs (Mahmoud's review point, -2026-07-03): the decision runs *before* execution, in the shared function, and the relay -itself carries no permission logic; it executes calls that were already permitted. The -relay's `TODO(S5)` collapse disappears; the relay can already park for client tools, and an -`ask` on a relay tool parks the same way. Client tools stop being a special case: they -resolve through the same function and simply default to `allow` (their fulfillment *is* the -browser interaction), so `deny` and `ask` work on them with no extra code. `hasHumanSurface` -is deleted. `PolicyResponder` is deleted. An `ask` park on a headless run is correct -behavior: the turn ends `paused`, visibly, with an interaction row for the future -durable-resume flow. +(runner-executed tools). Deciding and executing are separate jobs (Mahmoud's review point): +the decision runs *before* execution, in the shared function, and the relay itself carries +no permission logic; it executes calls that were already permitted. The relay's `TODO(S5)` +collapse disappears. Client tools stop being a special case: they resolve through the same +function and simply default to `allow` (their fulfillment *is* the browser interaction), so +`deny` and `ask` work on them with no extra code. `hasHumanSurface` is deleted. +`PolicyResponder` is deleted. An `ask` pause on a headless run is correct behavior: the +turn ends `paused`, visibly, with an interaction row for the future durable-resume flow. + +### Resume without matching: replay the approved call + +The base (#5054) still resumes by hoping the harness re-issues the gated call with a +matching identity. Both halves of that key drift in practice (name drift observed live, +argument drift latent), which is why the loop-breaker exists in the base at all. This plan +replaces the mechanism: on resume, the runner replays the *approved call itself* (the +exact tool and arguments the human saw and approved) instead of matching a re-issued one. +That removes the reassembled-key fragility class outright, and it is also more correct: +the human approved those arguments, not whatever the model regenerates. With it, the +`resolvedName` patch and the loop-breaker become deletable. ### Events mean actions -`interaction_request(user_approval)` is emitted only when the decision is `pendingApproval` -(today it fires before the decision, so auto-approved gates would emit false prompts). What -ran is already visible through `tool_call`/`tool_result` events. +`interaction_request(user_approval)` is emitted only when the decision is +`pendingApproval` (today it fires before the decision, so auto-approved gates would emit +false prompts). What ran is already visible through `tool_call`/`tool_result` events. ### The pause is visible on batch `_agent_batch` returns `stop_reason` alongside the messages and, when paused, the pending interaction reference (id/token). Exact response shape to be settled against the streaming -work in `../builder-agent-reliability/streaming-invoke/` (which owns the batch-vs-stream -story); the requirement from this side is only: a paused run must be distinguishable and -reference its pending approval. +work in `../builder-agent-reliability/streaming-invoke/`; the requirement from this side is +only: a paused run must be distinguishable and reference its pending approval. ## Execution phases Each phase lands green on its own. Sizes are rough. -**Phase 1: the decision core (runner, ~the heart of the fix).** -`resolvePermission` as one exported function; responder and relay call it; delete -`hasHumanSurface` and `PolicyResponder`; emit approval events only on park; consume stored -decisions once. Rewrite the responder decision tests as a generated truth table -(disposition × stored-decision × default), replacing the tests that pin the bug -(`responder.test.ts:227-237` and the park-by-default cases; list in code-review.md §tests). +**Phase 1: the decision core (runner).** +`effectivePermission`/`decide` as one exported module; responder and relay call it; delete +`hasHumanSurface`, `PolicyResponder`, and #5054's `nonConvergingToolNames` loop-breaker + +`resolvedName` stamping; emit approval events only on pause; consume stored decisions once; +implement replay-the-approved-call on resume. Rewrite the responder decision tests as a +generated truth table (effective permission × stored decision × policy mode), replacing the +tests that pin the bug (`responder.test.ts:227-237` and the park-by-default cases; listed +in code-review.md, "Tests that pin behavior the fix will change"). **Phase 2: the wire and the SDK (Python).** -`permissions.{default, rules}` on the run request (typed enum, mirrored in `wire.py`, the Python copy of the runner's wire types; -golden fixtures updated); SDK resolves per-tool dispositions at request build time -(`effective_permission()` output onto the specs); structured builtin rules derived from the -same authored `harness.permissions` lists that feed the settings renderer, so the two -cannot disagree; authored config path renamed into the permission family; Claude settings -rendering unchanged in behavior (it already renders resolved allow/deny; verify `ask`/unset -still leaves the gate raised). +`permissions.{default, rules}` on the run request (typed enum incl. `allow_reads`, +mirrored in `wire.py`, the Python copy of the runner's wire types; golden fixtures +updated). Delete `needs_approval`, the `permission_mode`/`permissionMode` aliases, and the +`agenta_metadata.permission_mode` fallback; `effective_permission()` shrinks to +"explicit or None" with the policy resolution moving into the shared model. Structured +builtin rules derived from the same authored `harness.permissions` lists that feed the +settings renderer, so the two cannot disagree. Claude settings rendering: `allow` still +renders an allow rule, `deny` a deny rule, `ask`/unset still leaves the gate raised; under +`allow_reads`, read-hinted tools render allow rules (this is where "reads never gate on +Claude" is enforced). **Phase 3: service and frontend.** Batch surfaces `stop_reason` + pending interaction reference; fix the stream-setup cleanup -leak (code-review M6). Frontend: the config form writes the renamed field (the "Permission -policy" select becomes the permission default, vocabulary `allow | ask | deny`); no changes -to the approval UI or resume machinery (they keep working for `ask` tools; verified by the -frontend research: nothing in the UI depends on parking for non-ask tools). +leak (code-review M6). Frontend: the "Permission policy" select becomes the four-mode +policy on the renamed field; the tool editor's Permission select stays +allow/ask/deny/inherit and drops the `needs_approval`/legacy-alias fallbacks; no changes to +the approval UI or resume machinery. **Phase 4: correctness debt in the same code (from code-review.md).** -H1 (reply failure parks or fails loud; resolve interaction only after a successful reply), -H2 (only `{approved}` envelopes become decisions; client-tool replay scoped by interaction -token), M2+M3+M7 as one item, now diagnosed via PR #5054 (see the live-warning section of -how-approvals-work.md): the observed loop = a constant stream `messageId` plus a -level-triggered resume predicate (M7, frontend half) compounded by tool-*name* drift across -ACP frames breaking the decision key (M2's observed form). The durable fix here is the -direct replay of the approved call, which removes the reassembled-key fragility class -(name drift, argument drift) outright; approval responses must correlate by `toolCallId` -and drop loudly otherwise (M3), M4 -(client tools resolve through the same ladder, defaulting to `allow`), M5 (latch the stop -reason before the relay drain), L1 (no-id gates park instead of silently hanging), H3 -(verify the daemon's permission-id scheme; if per-session counters, namespace interaction -tokens with the turn), L2 (reserve the `agenta-tools` MCP server name in the settings -renderer). +H1 (reply failure pauses or fails loud; resolve the interaction only after a successful +reply), H2 (only `{approved}` envelopes become decisions; client-tool replay scoped by +interaction token), M3 (approval responses must correlate by `toolCallId`; drop loudly +otherwise; largely absorbed by direct replay), M4 (client tools through the shared +function, defaulting to `allow`), M5 (latch the stop reason before the relay drain), L1 +(no-id gates pause instead of silently hanging), H3 (verify the daemon's permission-id +scheme; if per-session counters, namespace interaction tokens with the turn), L2 (reserve +the `agenta-tools` MCP server name in the settings renderer). **Phase 5: organization cleanup (from code-organization-review.md).** Extract the park controller from `sandbox_agent.ts` into the `engines/sandbox_agent/` @@ -197,62 +237,54 @@ fixture test. **Phase 6: proof.** Live matrix on the dev stack (Claude harness, a model with credit): -- headless batch + streaming: unset tools under default `allow` run through; the run - finishes; no approval events. -- headless with an `ask` tool: run ends `paused`, batch response says so, interaction row - exists. +- headless batch + streaming, policy `allow`: everything runs through; no approval events. +- headless with an `ask` tool (or a write under `allow_reads`): run ends `paused`, batch + response says so, interaction row exists. - playground with an `ask` tool: prompt renders, approve resumes and **completes without - looping**. This case is currently broken live (Mahmoud, 2026-07-03: Approve loops, the - run re-parks and re-prompts repeatedly), which is the observed form of code-review M2/M3. - Reproduce first, fix in phase 4, and pin the pass as the replay test. Deny resumes and - continues without the tool (the two old approval-UI bugs stay fixed: F-024, the reject - that clobbered the prompt, and F-036, the deny that left the run hanging). -- playground with everything `allow`: no prompt, tools visibly run (the owner's "auto means - auto" check). -- Pi: unchanged (never gates; relay enforces deny/allow; relay `ask` now parks; verify the - client-tool park machinery carries it, else stage relay-ask into its own slice). -- uc9-digest end-to-end: set `SEND_MESSAGE: allow`, headless run completes all four tools - and posts; with default config, run pauses visibly at `SEND_MESSAGE`. -Then pin one park→approve→resume pair as a replay test (`agent-replay-test` skill) and a + looping** (the live-broken case; reproduce first, fix via direct replay, pin the pass as + the replay test). Deny resumes and continues without the tool (F-024 and F-036 stay + fixed). +- playground, policy `allow`: no prompt, tools visibly run ("auto means auto" check). +- Pi: relay enforces the same decisions; relay `ask` pauses (verify the client-tool park + machinery carries it, else stage relay-ask into its own slice). +- uc9-digest end-to-end: `SEND_MESSAGE: allow` → headless run completes all four tools and + posts; default `allow_reads` → run pauses visibly at `SEND_MESSAGE`. +Then pin one pause→approve→resume pair as a replay test (`agent-replay-test` skill) and a producer-side SDK test asserting what `/invoke` actually sends in `sessionId` (the missing -test that let the proxy rot). +test that let the session-id proxy rot). ## Behavior deltas (before → after) | Case | Today | After | | --- | --- | --- | -| Headless, unset tool, policy auto | Parks silently; batch hides it | Runs in place | -| Headless, `ask` tool | Parks silently | Parks visibly (`stop_reason`, interaction ref) | -| Playground, unset tool, policy auto | Prompts (park) | Runs, tool activity visible, no prompt | -| Playground, `ask` tool | Prompts | Prompts (unchanged) | -| Playground, `allow` tool | No prompt (settings rule) | No prompt (unchanged) | +| Headless, unset tool, policy allow (was auto) | Parks silently; batch hides it | Runs in place | +| Headless, unset write tool, policy `allow_reads` | Parks silently | Pauses visibly (`stop_reason`, interaction ref) | +| Headless, unset read tool, any allow-ish policy | Runs (settings allow rule) | Runs (unchanged, now policy-explained) | +| Playground, unset tool, policy allow | Prompts (park) | Runs, tool activity visible, no prompt | +| Playground, `ask` tool | Prompts; approve can loop forever | Prompts; approve runs the approved call, done | | `deny` on a client tool | Ignored | Refused | | Stored approval vs changed-to-deny config | Approval wins | Deny wins | | One approval, model repeats identical call | Every repeat auto-runs | One approval, one run | -| Claude builtin with authored `ask` rule | Parks | Parks (rule travels on the wire) | +| Claude builtin with authored `ask` rule | Parks | Pauses (rule travels on the wire) | +| Tool with `needs_approval: true` | Treated as ask via ladder | Field deleted; author sets `permission: ask` | +| Approve 3+ times on a non-matching gate | #5054 base: silent auto-deny | Cannot happen (no matching; loop-breaker deleted) | ## Risks and open questions -- **Relay-ask parking on Pi.** The relay can park client tools today, but `ask` parks for - ordinary relay tools need the same turn-boundary treatment (the old S5.2). If it turns - out heavy, ship relay-ask as its own slice and keep the collapse only for Pi, documented, - while Claude paths use the full design. +- **Direct replay of the approved call.** The new mechanism needs one empirical check: the + cleanest implementation answers the re-raised gate when identity matches, and *injects* + the approved call's execution when it does not. How Claude behaves when the runner + executes-and-returns a result for a call it re-issued differently needs the phase 6 live + loop to pin. Fallback if injection proves harness-fragile: approve the re-raised gate + but execute with the *approved* arguments (still no key matching). +- **Relay-ask pausing on Pi.** The relay can park client tools today; ordinary relay-tool + `ask` needs the same turn-boundary treatment (old S5.2). If heavy, ship relay-ask as its + own slice, documented, while Claude paths use the full design. - **Rule matching for builtins.** The structured `rules` need a matcher compatible with Claude's rule syntax (`Bash(npm run:*)`). Scope it to exactly what the settings renderer - accepts today; anything fancier stays authored-settings-only and documented as such. -- **Reassembled-key fragility on resume (code-review M2/M7, diagnosed).** The live - approve-loop is explained: constant `messageId` + level-triggered resume predicate on the - frontend, tool-name drift across ACP frames on the backend. Fix direction settled: the - runner replays the approved call directly, removing the whole matching class. Two pieces - of PR #5054 are worth absorbing regardless (unique per-turn message id; the - "already resumed" edge-trigger guard); two pieces should be superseded by this plan, not - inherited (the `resolvedName` stamping that patches the name drift, and especially the - `nonConvergingToolNames` loop-breaker that silently auto-DENIES a gate after three - non-converging approvals: it is keyed by bare tool name globally, can false-positive on a - busy tool, reintroduces the F-024 deny-clobber deliberately, and gives the user no signal - that their approved tool was blocked). + accepts today. - **Batch response shape.** Owned jointly with the streaming-invoke workspace; this plan only requires that paused be distinguishable. -- **Vocabulary migration.** `auto` → `allow` and the authored-path rename touch the FE - form, SDK parsing, wire, fixtures, and docs in one PR. POC status makes this safe, but it - is the churny part; the golden wire contract test is the safety net. +- **Vocabulary migration.** `auto` → the four-mode policy and the authored-path rename + touch the FE form, SDK parsing, wire, and fixtures in one PR. POC status makes this + safe; the golden wire contract test is the safety net. diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index f18a1d4289..7f311eea18 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -1,109 +1,69 @@ # Status -**State: design + plan under review; review round 1 (how-approvals-work) addressed.** -Date: 2026-07-03. - -Review round 1 (Mahmoud, 28 inline comments on `how-approvals-work.md`) produced three -substantive changes, all folded in: - -- **Live finding:** the playground approve→resume happy path currently loops (Approve - re-parks and re-prompts). This is the observed form of code-review M2/M3; it is now an - explicit acceptance case in plan phase 6 and elevated in phase 4. -- **Structure:** deciding and executing are separate jobs; the relay carries no permission - logic in the target design (the shared decision function runs before execution). -- **Client tools:** no special-case bypass; they resolve through the same ladder and - default to `allow`. - -The explainer was also generalized across harnesses (a per-harness gate table), and now -covers Claude's `default_mode`/`bypassPermissions`, the settings merge semantics, the -ACP request-vs-event distinction, the two kinds of "session", and the cold-replay resume -model before the responder code. - -## Update 2026-07-03 (later): the approve-loop is diagnosed, via PR #5054 - -Arda's PR #5054 empirically stops the loop; our review of it pinned the mechanism. Two -independent bugs compound, and neither is the argument-drift we first suspected: - -- Frontend half (new finding M7): the stream egress sent a constant `messageId: "msg-1"` - every turn, so the client folded all turns into one message; the resume predicate is - level-triggered with no "already resumed" guard, so it re-sent forever. -- Backend half (M2's observed form): tool-*name* drift across ACP frames ("Terminal" in - the tool_call event vs the invocation title in the permission frame) broke the - name+args decision key even with identical arguments. - -Consequences for this plan (folded into plan.md phase 4 and risks): - -- The direct-replay-of-the-approved-call fix is reinforced: it removes the whole - reassembled-key fragility class (name drift and argument drift) instead of patching one - half of the key. -- Absorb from #5054 regardless of the redesign: the unique per-turn message id and the - frontend "already resumed" edge-trigger guard. Both are correct on their own. -- Do not inherit from #5054: the `resolvedName` stamping (patches the symptom our redesign - removes) and the `nonConvergingToolNames` loop-breaker (auto-denies after three - non-converging approvals, keyed by bare tool name globally: can false-positive-deny a - busy tool, deliberately reintroduces the F-024 deny-clobber, and is silent to the user). - -Recommendation on #5054 itself: do not merge as-is; split it. Keep the message-id fix, the -resume-predicate guard, the tool-input `{}` display fix, and the unrelated Turn -Inspector/chat-UX work (own PRs). Treat the name-anchor patch and the loop-breaker as -temporary evidence to be superseded by this plan. - -## What is done - -- Full research pass across all five systems (frontend, agent service, runner, harness - config rendering, API interactions plane), verified against the current tree with - file:line citations. This supersedes the investigation in - `../builder-agent-reliability/streaming-invoke/approval-boundary.md`, which cites - pre-rename `services/agent/` paths and misses the stored-decision branch and the second - (client-tool) park path. -- Bug confirmed and pinned: park keyed to session-id presence, session ids minted for every - request, `auto` policy unreachable, batch hides the paused state - ([the-bug.md](the-bug.md)). -- Correctness review: 4 high, 6 medium, 2 low findings beyond the headline bug - ([code-review.md](code-review.md)). -- Organization review: verdict and top-5 improvements - ([code-organization-review.md](code-organization-review.md)). -- Independent second opinion (OpenAI Codex, xhigh): concurred with the diagnosis, rejected - the partial fixes, recommended the one-plan design in one shot; its ordering rule (a - stored approval must not override a current deny) is folded into the plan. -- Plan written with options, phases, test plan, and behavior deltas ([plan.md](plan.md)). - -## Decisions already taken (by Mahmoud, 2026-07-02) - -- Auto means auto everywhere: an auto-approved tool runs without prompting; the human sees - it ran. Only `ask` waits for a human. -- This PR is docs + plan only; implementation follows after review. -- No backward-compatibility constraints (pre-release POC). - -## Decisions needed to start implementing - -1. **Confirm the recommendation**: Option D (one resolved permission plan, one shot) vs the - staged fallback (B+C first, then D). Plan recommends D. Stakes: D costs more up front - (wire shape, golden fixtures, tests in two languages, all in one PR) but ends with one - computation of the policy. The staged fallback ships the auto fix sooner but, until D - lands, an author's explicit `ask` rule on a Claude builtin like `Bash` silently - auto-approves under a default of `allow`. -2. **Naming**: approve `permissions.default` (vocabulary `allow | ask | deny`) as the - authored home of the global default, replacing `runner.interactions.headless` and the - `auto | deny` vocabulary. Stakes: touches the FE form, SDK parsing, wire, and fixtures - once; skipping it keeps three names for one knob. -3. **Relay-ask scope**: if parking `ask` for relay-executed tools proves heavy, is a - documented Pi-only collapse acceptable for the first slice? Stakes: under the collapse, - an `ask` tool on Pi never prompts; it silently runs (default `allow`) or is refused - (default `deny`). Claude paths get the full design either way. -4. **Batch pause shape**: coordinate the exact paused-response fields with the - streaming-invoke workspace (only "visible + carries the interaction reference" is - required from this side). - -## Next steps - -- Review of this workspace (see the PR comment for exactly what feedback is needed). -- On approval: implement per plan.md phases 1-6, with the correctness debt (phase 4) and - the live matrix + replay pin (phase 6) as the acceptance gate. +**State: plan ready for final review.** Date: 2026-07-03. Two review rounds on the +explainer are addressed; the plan and reviews are updated to match. Next step: Mahmoud's +final pass over plan.md (and the remaining decisions below), then implementation. + +## Where things stand + +- PR #5041 (this workspace) is **stacked on Arda's #5054** (`big-agents-work`), per the + merge-then-rework decision: he merges as-is, we build on top and delete what the + redesign supersedes. The plan's "Baseline" section sorts his changes into + kept / deleted-by-this-work / unrelated. +- The live approve-loop is **diagnosed** (details in how-approvals-work.md, live-warning + section): a constant stream `messageId` plus a level-triggered resume predicate on the + frontend (finding M7; fixed in the #5054 base), compounded by tool-*name* drift across + ACP frames breaking the decision key (M2's observed form; patched in the base, properly + removed by this plan's direct-replay resume). +- The permission model is settled (round 2): per-tool `allow | ask | deny` or inherit; one + global policy with four modes (`allow`, `ask`, `deny`, `allow_reads` = reads run, writes + ask); `needs_approval` and the legacy aliases get deleted; "effective permission" + replaces the term "disposition". + +## Decisions taken (Mahmoud) + +- **2026-07-02:** auto means auto everywhere (an auto-approved tool runs without + prompting, on every surface); docs+plan PR first, implementation separate; no + backward-compatibility constraints. +- **2026-07-03, round 1:** deciding and executing are separate jobs (the relay carries no + permission logic; one central decision on our side); client tools go through the same + ladder, defaulting to `allow`. +- **2026-07-03, round 2:** #5054 merges as-is, we rework stacked on it. Delete + `needs_approval` outright. "Reads always allowed" becomes an explicit global policy + mode instead of an opaque per-tool default. Rename "disposition" to "effective + permission". + +## Decisions still open + +1. **Confirm the one-shot scope** (plan = Option D plus visibility plus the correctness + debt, in one implementation arc). The staged fallback exists mid-flight if the wire + change proves heavy. +2. **Names.** `permissions.default` as the authored home of the policy, and the name of + the fourth mode (`allow_reads` is the placeholder; alternatives: `read_only_auto`, + `ask_writes`). Cheap to decide, touches FE form + SDK + wire once. +3. **Pi relay-ask scope.** The plan makes relay `ask` pause (that is how Pi gets HITL). If + the relay's turn-boundary work proves heavy, is a documented Pi-only collapse + acceptable for the first slice? Stakes: under the collapse, an `ask` tool on Pi + silently runs or is refused per the policy instead of pausing. +4. **Batch pause shape.** Coordinate exact fields with the streaming-invoke workspace; + this side only requires "paused is distinguishable and names the pending interaction". +5. **Direct-replay mechanics** (flagged, not blocking): when the re-raised gate does not + match the approved call, the runner injects the approved call's execution; if that + proves harness-fragile in the phase 6 live loop, the fallback is approving the + re-raised gate but executing with the approved arguments. Empirical; the plan carries + both. + +## How this workspace was produced + +Four parallel code-research passes (runner, SDK+service, frontend, API interactions +plane), a correctness review (H1-H4, M1-M7, L1-L2), an organization review, an independent +Codex design review (xhigh), a cold-reader clarity pass, two inline review rounds by +Mahmoud (38 comments total, all answered inline and folded into the docs), and the #5054 +analysis that diagnosed the live loop. ## Known unknowns -- The sandbox-agent daemon's permission-request id scheme (per-session counter vs unique) - This decides whether interaction tokens need turn namespacing (code-review H3). -- How often cold-replay argument drift breaks approval matching in practice - (code-review M2); measure during phase 6. +- The sandbox-agent daemon's permission-request id scheme (per-session counter vs unique): + decides whether interaction tokens need turn namespacing (code-review H3). +- Whether ending a parked turn and later injecting the approved call behaves cleanly on + Claude (plan, "Direct replay" risk); pinned by the phase 6 live loop. diff --git a/docs/design/agent-workflows/projects/approval-boundary/the-bug.md b/docs/design/agent-workflows/projects/approval-boundary/the-bug.md index 4ba523e4db..78ca9b5087 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/the-bug.md +++ b/docs/design/agent-workflows/projects/approval-boundary/the-bug.md @@ -16,18 +16,18 @@ The `uc9-digest` agent (three read tools, then `SEND_MESSAGE`) runs with permiss stopped. The caller cannot tell a paused run from a finished one. Before diagnosing, be precise about what *should* happen, because it is subtler than "the -run should never pause". Every tool resolves to a **disposition**: its final +run should never pause". Every tool resolves to an **effective permission**: its final `allow | ask | deny`, computed from the author's explicit per-tool permission, the legacy `needs_approval` flag, or a catalog hint that defaults read-only tools to `allow` and mutating tools to `ask` (`effective_permission`, -`sdks/python/agenta/sdk/agents/tools/models.py:273-292`). Tools with no disposition at all +`sdks/python/agenta/sdk/agents/tools/models.py:273-292`). Tools with no effective permission at all fall back to the global policy, and that is what `auto` governs. `SEND_MESSAGE` is a write, so it resolves to `ask` by default. Under a correct implementation, this specific run pauses there until the author marks the tool `allow`. The bug is therefore three things, none of which is "it paused": -- the pause happens for the wrong reason (a session id instead of the tool's disposition); +- the pause happens for the wrong reason (a session id instead of the tool's effective permission); - the `auto` policy is dead code for the tools it does govern; - a headless caller can neither see the pause nor answer it. From 29db193f84b8f04c249320055e7017ac9f47bf13 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 14:10:47 +0200 Subject: [PATCH 06/24] docs(approval-boundary): settle runner.permissions naming and add the Pi settings block (round 3) --- .../code-organization-review.md | 2 +- .../approval-boundary/how-approvals-work.md | 11 +++-- .../projects/approval-boundary/plan.md | 17 +++++-- .../projects/approval-boundary/status.md | 47 ++++++++++--------- 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/code-organization-review.md b/docs/design/agent-workflows/projects/approval-boundary/code-organization-review.md index c5ad4bbd26..324eec81b7 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/code-organization-review.md +++ b/docs/design/agent-workflows/projects/approval-boundary/code-organization-review.md @@ -79,7 +79,7 @@ Also in this theme: | Name | Problem | Suggestion | | --- | --- | --- | -| `runner.interactions.headless` / `permission_policy` / `permissionPolicy` | Three names for one knob. The authored name contains neither "permission" nor "policy"; grepping "permission" misses the authoring surface. | One name, in the permission family: `permissions.default` (see design-review.md). | +| `runner.interactions.headless` / `permission_policy` / `permissionPolicy` | Three names for one knob. The authored name contains neither "permission" nor "policy"; grepping "permission" misses the authoring surface. | One name, in the permission family: `runner.permissions.default` (see design-review.md). | | `PermissionPolicy` {auto, deny} vs `Permission` {allow, ask, deny} | Two vocabularies where `auto` and `allow` mean the same thing. | Unify on `allow \| ask \| deny`. | | `hasHumanSurface` | Truthful intent, untruthful derivation: it actually means "the request has a session id", which no longer implies a human. | Delete as a permission input (see plan). | | `HITLResponder` | It is also the production headless responder; the name suggests `PolicyResponder` handles headless. It does not. | `ApprovalResponder`, and delete `PolicyResponder`. | diff --git a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md index 243512be70..d434fc016e 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md +++ b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md @@ -289,6 +289,11 @@ there is nothing for us to render the way we render `.claude/settings.json`. If native permission config later, a new adapter renders it and it becomes Pi's Gate 1; until then, deny-by-omission at selection time and the relay are the only Pi controls. +That selection control is getting a face (review round 3): the agent form grows a Pi +settings block, the Pi counterpart of the Claude settings control, where the author picks +which builtins the agent gets. Frontend-only work: the SDK's `PiAgentTemplate` already +carries `builtin_names` on the wire, so the backend stays as is. + ## The journey of one gated tool call (the playground path) Concrete example: the `uc9-digest` agent, Claude harness, global policy `auto` (today's @@ -398,7 +403,7 @@ the code you will find in the tree today. (inherit). One global policy with four modes: `allow` (run everything), `ask` (a human approves everything), `deny` (lockdown), `allow_reads` (reads run, writes ask; the default). Nothing else: no `needs_approval`, no hidden per-tool defaulting, no -`runner.interactions.headless`. +`runner.interactions.headless` (the policy's authored home is `runner.permissions.default`). **One decision.** A tool's effective permission is its own setting if present, else what the policy says (under `allow_reads`, the catalog's read-only hint decides; no hint counts @@ -426,8 +431,8 @@ durable interactions plane (already receiving rows today) is what will let anyon it later. Nothing anywhere consults "is a human watching". **Pi.** Same decisions, enforced at Pi's one choke point (the relay): `ask` pauses there, -`deny` refuses there, and builtins remain governed by selection until Pi grows a native -permission config. +`deny` refuses there, and builtins remain governed by selection (now exposed in the agent +form's Pi settings block) until Pi grows a native permission config. ## The two planes: messages and interactions diff --git a/docs/design/agent-workflows/projects/approval-boundary/plan.md b/docs/design/agent-workflows/projects/approval-boundary/plan.md index 7201e478e9..0c0fcd60f3 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/plan.md +++ b/docs/design/agent-workflows/projects/approval-boundary/plan.md @@ -122,9 +122,14 @@ the `read_only` hint already on each spec is what `allow_reads` consults. Semant stay clean: `permissions` is policy, owned by the author, assembled by the SDK; `sessionId` goes back to being pure correlation. -The authored config path moves from `runner.interactions.headless` into the permission -family (proposal: `permissions.default`; final name settled at implementation, but it must -contain the word "permission" and live with the policy fields). +The authored config path moves from `runner.interactions.headless` to +`runner.permissions.default` (agent-level authored rules, if any, sit beside it as +`runner.permissions.rules`). Settled in review round 3 on JP's point: these permissions are +enforced by the runner, so they live under the runner scope. They deliberately do not sit +under `interactions`: an interaction is one possible *outcome* of a permission (`ask`), +while `allow` and `deny` never produce one, so `runner.interactions.permissions.*` would +misname two thirds of the values. The run-request block stays `permissions: {...}` because +the whole request already addresses the runner. ### One decision function @@ -216,7 +221,11 @@ Batch surfaces `stop_reason` + pending interaction reference; fix the stream-set leak (code-review M6). Frontend: the "Permission policy" select becomes the four-mode policy on the renamed field; the tool editor's Permission select stays allow/ask/deny/inherit and drops the `needs_approval`/legacy-alias fallbacks; no changes to -the approval UI or resume machinery. +the approval UI or resume machinery. The agent form also gains the Pi counterpart of the +Claude settings block (review round 3): a Pi settings control exposing builtin selection +(`builtin_names`), rendered only for the Pi harness the way `ClaudePermissionsControl` +renders only for Claude (`useModelHarness.tsx`). Frontend-only: the SDK's +`PiAgentTemplate` already carries `builtin_names` on the wire and the backend stays as is. **Phase 4: correctness debt in the same code (from code-review.md).** H1 (reply failure pauses or fails loud; resolve the interaction only after a successful diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index 7f311eea18..d5c687c431 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -1,8 +1,10 @@ # Status -**State: plan ready for final review.** Date: 2026-07-03. Two review rounds on the -explainer are addressed; the plan and reviews are updated to match. Next step: Mahmoud's -final pass over plan.md (and the remaining decisions below), then implementation. +**State: implementation in progress.** Date: 2026-07-03. Three review rounds are folded +in; the final round left two small comments (both addressed below) and a green light: +implement, Mahmoud reviews the finished PR. The remaining calls listed under "Decisions +taken (delegated)" were made by the agent under an explicit "go with your decisions" +mandate; any of them can be reopened in PR review. ## Where things stand @@ -32,26 +34,29 @@ final pass over plan.md (and the remaining decisions below), then implementation `needs_approval` outright. "Reads always allowed" becomes an explicit global policy mode instead of an opaque per-tool default. Rename "disposition" to "effective permission". +- **2026-07-03, round 3 (final review):** two comments. (1) Pi gets a settings block in + the agent form, mirroring the Claude settings control: the author selects which Pi + builtins the agent gets. Frontend-only; `PiAgentTemplate` and the wire already carry + `builtin_names`. (2, from JP) the authored policy home is runner-scoped: + `runner.permissions.default` (not bare `permissions.default`). We kept it out of + `interactions` because an interaction is only the outcome of `ask`; `allow`/`deny` + never produce one. Then: implement without further check-ins; Mahmoud reviews the PR. -## Decisions still open +## Decisions taken (delegated, reopenable in PR review) -1. **Confirm the one-shot scope** (plan = Option D plus visibility plus the correctness - debt, in one implementation arc). The staged fallback exists mid-flight if the wire - change proves heavy. -2. **Names.** `permissions.default` as the authored home of the policy, and the name of - the fourth mode (`allow_reads` is the placeholder; alternatives: `read_only_auto`, - `ask_writes`). Cheap to decide, touches FE form + SDK + wire once. -3. **Pi relay-ask scope.** The plan makes relay `ask` pause (that is how Pi gets HITL). If - the relay's turn-boundary work proves heavy, is a documented Pi-only collapse - acceptable for the first slice? Stakes: under the collapse, an `ask` tool on Pi - silently runs or is refused per the policy instead of pausing. -4. **Batch pause shape.** Coordinate exact fields with the streaming-invoke workspace; - this side only requires "paused is distinguishable and names the pending interaction". -5. **Direct-replay mechanics** (flagged, not blocking): when the re-raised gate does not - match the approved call, the runner injects the approved call's execution; if that - proves harness-fragile in the phase 6 live loop, the fallback is approving the - re-raised gate but executing with the approved arguments. Empirical; the plan carries - both. +1. **One-shot scope confirmed:** Option D plus visibility plus the correctness debt, one + arc. The staged fallback stays available mid-flight if the wire change proves heavy. +2. **Names:** `runner.permissions.default` (JP's round-3 comment) and `allow_reads` for + the fourth mode, unless the Codex design review offers a clearly better word. +3. **Pi relay-ask scope:** full. Relay `ask` pauses; that is how Pi gets HITL. If the + turn-boundary work blows up in practice, the documented Pi-only collapse ships as an + explicit follow-up, not silently. +4. **Batch pause shape:** minimal contract from this side: `stop_reason` plus the pending + interaction reference. Exact field names stay coordinated with the streaming-invoke + workspace. +5. **Direct-replay mechanics:** primary design is injecting the approved call; the + approve-with-stored-args fallback gets picked empirically in the phase 6 live loop if + injection proves harness-fragile. ## How this workspace was produced From bbeb1d4cae8cff23f83e89e0f3f54ee501d410ad Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 14:22:39 +0200 Subject: [PATCH 07/24] docs(approval-boundary): fold the Codex pre-implementation review (resume redesign, wire-first phases, test plan) --- .../projects/approval-boundary/README.md | 7 +- .../projects/approval-boundary/code-review.md | 6 +- .../approval-boundary/how-approvals-work.md | 20 +- .../projects/approval-boundary/plan.md | 235 +++++++++++++----- .../projects/approval-boundary/status.md | 25 +- 5 files changed, 208 insertions(+), 85 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/README.md b/docs/design/agent-workflows/projects/approval-boundary/README.md index b07110cc82..0da2f80579 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/README.md +++ b/docs/design/agent-workflows/projects/approval-boundary/README.md @@ -24,9 +24,10 @@ Everything here is documentation and planning. No code changes ship with this PR inherit. Per agent: one policy with four modes (`allow`, `ask`, `deny`, `allow_reads` = reads run, writes ask). The SDK computes each tool's effective permission once and ships it; both runner gates enforce it; the session-id inference is deleted; the approval - event fires only when the run actually pauses; resume replays the approved call directly - (no fragile matching); batch shows the paused state. One shot (POC, no compat - constraints); an independent Codex review concurred. See [plan.md](plan.md). + event fires only when the run actually pauses; resume matches the re-issued call on + stable anchors (drift means a visible fresh prompt, never a silent loop or auto-deny); + batch shows the paused state. One shot (POC, no compat constraints); two independent + Codex reviews shaped it. See [plan.md](plan.md). - **Baseline.** This PR is stacked on Arda's #5054 (merge-then-rework decision). His message-id and resume-guard fixes are kept; his `resolvedName` patch and auto-deny loop-breaker get deleted by the redesign. The plan's "Baseline" section has the full diff --git a/docs/design/agent-workflows/projects/approval-boundary/code-review.md b/docs/design/agent-workflows/projects/approval-boundary/code-review.md index 7304d139e1..9deb2dd72f 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/code-review.md +++ b/docs/design/agent-workflows/projects/approval-boundary/code-review.md @@ -102,8 +102,10 @@ reassemble identically. Both halves can miss: the key, parks again, and re-prompts. The mirror case: a stored deny keeps auto-rejecting the next identical attempt without a prompt after the user changes their mind. -Both point the same way: keys reassembled from replayed frames are fragile, so the fix -replays the approved call directly instead of matching a re-issued one (plan phase 4). +Both point the same way: keys reassembled from replayed frames are fragile. The fix +anchors the key on stable identity (the spec's own name for relay tools, the recorded +`tool_call` name for Claude gates, canonical args on both) and makes any residual +mismatch a visible fresh prompt rather than a silent re-park (plan, "Resume" section). ### M7. Constant stream `messageId` plus a level-triggered resume predicate re-sends forever diff --git a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md index d434fc016e..74c847fbfc 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md +++ b/docs/design/agent-workflows/projects/approval-boundary/how-approvals-work.md @@ -385,9 +385,10 @@ suspected: Argument drift (code-review M2) remains a real latent risk of the match-on-replay model, and the `approvalId`-only drop (M3) is untouched by #5054 and still open. The diagnosis -strengthens the plan's direction: any key that must be reassembled from replayed frames is -fragile, which is why the plan has the runner replay the approved call directly instead of -matching a re-issued one ([plan.md](plan.md), phase 4). Reproducing and pinning the loop +strengthens the plan's direction: a key reassembled from replayed frames must anchor on +*stable* identity, so the plan keys relay tools on the spec's own name and Claude gates +on the recorded `tool_call` name, and turns any residual mismatch into a visible fresh +prompt instead of a silent re-park ([plan.md](plan.md), "Resume"). Reproducing and pinning the loop stays an acceptance case (phase 6). Note on the baseline: this workspace's PR is now stacked on #5054, so its frontend fixes (unique per-turn message id, the already-resumed guard) are part of our base; its backend patches (`resolvedName`, the auto-deny @@ -420,10 +421,15 @@ pre-answered (allow rules for effective-allow tools, deny rules for denies) so g only raised for genuine asks. **Resume.** An approval or denial travels back inside the conversation. On resume the -runner replays the *approved call itself* (the exact tool and arguments the human saw) -rather than waiting for the model to re-issue an identical call, so there is no key to -mismatch and no loop to break. A stored approval is spent on first use, and a config -changed to `deny` beats any stored approval. +harness re-issues the call and the runner matches it against the stored decision on +*stable* anchors: the spec's own name for runner-executed tools, the recorded `tool_call` +name for Claude gates, canonical args on both. The same call matches and runs; genuinely +different args are a new call and prompt again, visibly. Nothing silently loops and +nothing auto-denies. A stored approval is spent on first use, and a config changed to +`deny` beats any stored approval. (An earlier draft promised replaying the approved call +without any matching; the pre-implementation review killed that as unimplementable for +harness-executed builtins, where the runner's only lever is approving or rejecting the +gate.) **Headless.** Identical decisions, different surface: an `ask` on a run with no chat open still pauses, the batch response says so and names the pending interaction, and the diff --git a/docs/design/agent-workflows/projects/approval-boundary/plan.md b/docs/design/agent-workflows/projects/approval-boundary/plan.md index 0c0fcd60f3..7a921b6e8e 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/plan.md +++ b/docs/design/agent-workflows/projects/approval-boundary/plan.md @@ -17,11 +17,12 @@ and this plan treats his changes as the base. Sorting the HITL-relevant pieces o "already resumed" edge-trigger guard (together these fixed the frontend half of the infinite loop, finding M7), the tool-input `{}` display fix, and the `[HITL]` diagnostic logging. -- **In the base, deleted by this work.** The `resolvedName` stamping (patches the - tool-name drift that direct replay makes irrelevant) and the `nonConvergingToolNames` - loop-breaker (silent auto-deny after three non-converging approvals: keyed by bare tool - name, false-positive-prone, and it reintroduces the F-024 clobber). Deleting both is an - explicit phase 1 item; the acceptance tests replace them as the regression guard. +- **In the base, deleted by this work.** The `resolvedName` stamping (the recorded-name + correlation behind it is right and moves into the decision module; mutating the ACP + object goes) and the `nonConvergingToolNames` loop-breaker (silent auto-deny after + three non-converging approvals: keyed by bare tool name, false-positive-prone, and it + reintroduces the F-024 clobber). Deleting both is an explicit phase 2 item; the + acceptance tests replace them as the regression guard. - **In the base, unrelated.** The Turn Inspector and chat UX work. Untouched. ## The permission model (settled in review round 2) @@ -167,16 +168,34 @@ function and simply default to `allow` (their fulfillment *is* the browser inter `PolicyResponder` is deleted. An `ask` pause on a headless run is correct behavior: the turn ends `paused`, visibly, with an interaction row for the future durable-resume flow. -### Resume without matching: replay the approved call - -The base (#5054) still resumes by hoping the harness re-issues the gated call with a -matching identity. Both halves of that key drift in practice (name drift observed live, -argument drift latent), which is why the loop-breaker exists in the base at all. This plan -replaces the mechanism: on resume, the runner replays the *approved call itself* (the -exact tool and arguments the human saw and approved) instead of matching a re-issued one. -That removes the reassembled-key fragility class outright, and it is also more correct: -the human approved those arguments, not whatever the model regenerates. With it, the -`resolvedName` patch and the loop-breaker become deletable. +### Resume: same-call matching, made robust and honest + +An earlier draft of this plan promised "replay the approved call directly, no matching". +The Codex pre-implementation review killed that, and it was right: for Claude-native +builtins the runner's only lever is the ACP reply (`respondPermission`), so it cannot +execute `Bash` or `Edit` itself, and the prior-turn transcript replays as text, not as +structured tool injection. Matching a re-raised gate is unavoidable on that path. What +this plan does instead is make the match robust and its failure honest, per executor: + +- **Runner-executed tools (relay: gateway, code, client; everything custom on Pi).** The + tool name is the spec's own `name`, which cannot drift; the args are canonicalized on + both sides. A stored approval that matches executes; args that differ are a different + call and pause for a fresh approval. +- **Harness-gated tools (Claude builtins).** The gate's name anchor is the name already + recorded for that tool-call id in this turn's `tool_call` event (the mechanism behind + #5054's `resolvedName` patch, kept but moved inside the decision module instead of + stamped onto the ACP object); args are canonicalized the same way. Match approves the + gate with `once`; a mismatch is a new call and pauses for a fresh approval. +- **In every drift case the outcome is a visible re-prompt, never a silent re-park and + never an auto-deny.** The user sees a new approval request with the new arguments. That + deletes the loop-breaker (silent auto-deny after 3) without leaving the loop possible: + the old loop was the *same* call failing to match itself because the key used Claude's + drift-prone display title; with stable anchors the same call matches, and a genuine + mismatch means the model actually asked for something different, which a human should + see. + +Stored decisions stay consume-once, and a config changed to `deny` beats any stored +approval (the decision function resolves the effective permission first). ### Events mean actions @@ -193,30 +212,49 @@ only: a paused run must be distinguishable and reference its pending approval. ## Execution phases -Each phase lands green on its own. Sizes are rough. - -**Phase 1: the decision core (runner).** -`effectivePermission`/`decide` as one exported module; responder and relay call it; delete -`hasHumanSurface`, `PolicyResponder`, and #5054's `nonConvergingToolNames` loop-breaker + -`resolvedName` stamping; emit approval events only on pause; consume stored decisions once; -implement replay-the-approved-call on resume. Rewrite the responder decision tests as a -generated truth table (effective permission × stored decision × policy mode), replacing the -tests that pin the bug (`responder.test.ts:227-237` and the park-by-default cases; listed -in code-review.md, "Tests that pin behavior the fix will change"). - -**Phase 2: the wire and the SDK (Python).** -`permissions.{default, rules}` on the run request (typed enum incl. `allow_reads`, -mirrored in `wire.py`, the Python copy of the runner's wire types; golden fixtures -updated). Delete `needs_approval`, the `permission_mode`/`permissionMode` aliases, and the -`agenta_metadata.permission_mode` fallback; `effective_permission()` shrinks to -"explicit or None" with the policy resolution moving into the shared model. Structured -builtin rules derived from the same authored `harness.permissions` lists that feed the -settings renderer, so the two cannot disagree. Claude settings rendering: `allow` still -renders an allow rule, `deny` a deny rule, `ask`/unset still leaves the gate raised; under -`allow_reads`, read-hinted tools render allow rules (this is where "reads never gate on -Claude" is enforced). - -**Phase 3: service and frontend.** +Each phase lands green on its own. Sizes are rough. The order is wire-first (a Codex +review point): the runner learns the new `permissions` block and keeps a small +`permissionsFromRequest()` legacy mapping (old `permissionPolicy` in, plan out) so runner +and SDK never disagree mid-arc; the mapping and the legacy field are deleted in phase 5. + +**Phase 1: the wire and the decision core (runner).** +`permissions: {default, rules?}` lands on `protocol.ts` (legacy `permissionPolicy` still +accepted, mapped by `permissionsFromRequest()`), mirrored in `wire.py` with goldens +updated. One new pure module owns the decision: gate descriptor in (executor kind, stable +tool name, spec/server permission, read-only hint, canonical args), effective permission +and verdict out, as `effectivePermission`/`decide`. A single pending-approval latch lives +beside it: with parallel gates in one turn, exactly one wins, emits, and pauses; later +gates this turn get no event (today emission and reply race). Truth-table unit tests +(default mode × explicit permission × read-only hint × rule match × stored decision). + +**Phase 2: runner enforcement.** +Both gates become lookups. The ACP responder (`ApprovalResponder`, replacing +`HITLResponder` + `PolicyResponder`) answers by effective permission; `ask` consults +stored decisions (same-call match per the resume section, consume once) and otherwise +pauses. `attachPermissionResponder` consults first and emits `interaction_request` only +on pause; interaction rows are created on pause only. The relay stops carrying permission +logic (`resolvePermission` deleted): verdicts come from the shared function, relay `ask` +pauses through the generalized park machinery (this is Pi's HITL), stored approvals +execute on exact match. Delete `hasHumanSurface`, the loop-breaker, `resolvedName` +stamping (the recorded-name correlation moves into the gate-descriptor builder). Rewrite +the responder tests that pin the bug (`responder.test.ts:227-237`, park-by-default cases). + +**Phase 3: SDK assembly (Python).** +The SDK assembles the permission plan and ships it. Delete `needs_approval`, the +`permission_mode`/`permissionMode` aliases, and the `agenta_metadata.permission_mode` +fallback; `ToolSpec.to_wire()` carries only the author's explicit `permission` (the +`read_only` hint rides separately; the runner resolves defaults). Wire `rules` are +derived from the same parsed `harness.permissions` lists that feed the Claude settings +renderer, so the two cannot disagree; authored rules that target non-builtin subjects +(`mcp__*`) stay settings-only and are excluded from the wire rules, documented. Claude +settings rendering: `allow` renders an allow rule, `deny` a deny rule, `ask`/unset leaves +the gate raised; under `allow_reads`, read-hinted tools render allow rules (this is where +"reads never gate on Claude" is enforced). Pi templates stop hardcoding +`permissionPolicy: "auto"` and ship the same permissions block as Claude. The authored +schema moves to `runner.permissions.default` (`sdks/python/agenta/sdk/utils/types.py`, +replacing `runner.interactions.headless`). + +**Phase 4: service and frontend.** Batch surfaces `stop_reason` + pending interaction reference; fix the stream-setup cleanup leak (code-review M6). Frontend: the "Permission policy" select becomes the four-mode policy on the renamed field; the tool editor's Permission select stays @@ -227,22 +265,27 @@ Claude settings block (review round 3): a Pi settings control exposing builtin s renders only for Claude (`useModelHarness.tsx`). Frontend-only: the SDK's `PiAgentTemplate` already carries `builtin_names` on the wire and the backend stays as is. -**Phase 4: correctness debt in the same code (from code-review.md).** -H1 (reply failure pauses or fails loud; resolve the interaction only after a successful -reply), H2 (only `{approved}` envelopes become decisions; client-tool replay scoped by -interaction token), M3 (approval responses must correlate by `toolCallId`; drop loudly -otherwise; largely absorbed by direct replay), M4 (client tools through the shared -function, defaulting to `allow`), M5 (latch the stop reason before the relay drain), L1 -(no-id gates pause instead of silently hanging), H3 (verify the daemon's permission-id -scheme; if per-session counters, namespace interaction tokens with the turn), L2 (reserve -the `agenta-tools` MCP server name in the settings renderer). - -**Phase 5: organization cleanup (from code-organization-review.md).** -Extract the park controller from `sandbox_agent.ts` into the `engines/sandbox_agent/` -family; rename `park` → `pendingApproval` internally (wire `stopReason: "paused"` stays); -`permissions.ts` → `acp-interactions.ts`; the four-site permissions map as a header doc; -fix the stale `services/agent/` pointers and pin the `agenta-tools` coupling with a shared -fixture test. +**Phase 5: correctness debt + cleanup (from the two review docs).** +Correctness: H1 (reply failure pauses or fails loud; resolve the interaction only after a +successful reply), H2 (only `{approved}` envelopes become decisions; client-tool replay +scoped by interaction token), M3 (approval responses must correlate by `toolCallId`; drop +loudly otherwise), M4 (client tools through the shared function, defaulting to `allow`), +M5 (latch the stop reason before the relay drain), L1 (no-id gates pause instead of +silently hanging), H3 (verify the daemon's permission-id scheme; if per-session counters, +namespace interaction tokens with the turn), L2 (reserve the `agenta-tools` MCP server +name in the settings renderer). Organization: extract the park controller from +`sandbox_agent.ts` into the `engines/sandbox_agent/` family; rename `park` → +`pendingApproval` internally (wire `stopReason: "paused"` stays); `permissions.ts` → +`acp-interactions.ts`; the four-site permissions map as a header doc; fix the stale +`services/agent/` pointers. Deletions: the legacy `permissionPolicy` wire field, the +`permissionsFromRequest()` legacy mapping, `needsApproval` on the wire spec, goldens +updated to final shape. `SANDBOX_AGENT_DENY_PERMISSIONS` survives as the operator +kill-switch (forces `default: deny`), documented where it is read. Also sweep the general +agent-workflows docs for the old vocabulary (`auto`, `needs_approval`, +`runner.interactions.headless`, `hasHumanSurface`) and the other surfaces Codex flagged: +`web/packages/agenta-playground/src/state/execution/agentRequest.ts`, +`web/oss/src/components/AgentChatSlice/assets/transport.ts`, `wire_models.py`, generated +client types. **Phase 6: proof.** Live matrix on the dev stack (Claude harness, a model with credit): @@ -250,10 +293,12 @@ Live matrix on the dev stack (Claude harness, a model with credit): - headless with an `ask` tool (or a write under `allow_reads`): run ends `paused`, batch response says so, interaction row exists. - playground with an `ask` tool: prompt renders, approve resumes and **completes without - looping** (the live-broken case; reproduce first, fix via direct replay, pin the pass as - the replay test). Deny resumes and continues without the tool (F-024 and F-036 stay - fixed). + looping** (the live-broken case; reproduce first, verify the stable-anchor match, pin + the pass as the replay test). Deny resumes and continues without the tool (F-024 and + F-036 stay fixed). - playground, policy `allow`: no prompt, tools visibly run ("auto means auto" check). +- an approve where the model re-issues *different* args on replay: a fresh prompt appears + with the new args (no silent loop, no auto-deny, no blind approval). - Pi: relay enforces the same decisions; relay `ask` pauses (verify the client-tool park machinery carries it, else stage relay-ask into its own slice). - uc9-digest end-to-end: `SEND_MESSAGE: allow` → headless run completes all four tools and @@ -262,6 +307,64 @@ Then pin one pause→approve→resume pair as a replay test (`agent-replay-test` producer-side SDK test asserting what `/invoke` actually sends in `sessionId` (the missing test that let the session-id proxy rot). +## Test plan + +The principle: every behavior in the deltas table below gets a test that fails on today's +code, and every past regression in this area (F-024, F-036, F-040, F-046, the M2/M7 loop) +keeps a named pinning test. By layer: + +**Runner unit (vitest, `services/runner/tests/unit/`).** +- A *generated* truth table over the decision function: default mode (4) × explicit + permission (allow/ask/deny/unset) × read-only hint (true/false/absent) × rule match + (allow/ask/deny/none) × stored decision (allow/deny/none). Every cell asserted; this + replaces the tests that pin the bug. +- Precedence pins: explicit tool permission beats a rule beats the default; a config + `deny` beats a stored approval; a stored decision is consumed exactly once. +- The pending-approval latch: two gates raised concurrently → exactly one + `interaction_request`, one pause; the loser produces no event. +- Same-call matching: canonical-args stability (key order, absent args, non-JSON fails + closed; the existing `parkedCallKey` cases carry over), recorded-name correlation for + ACP gates, spec-name anchoring for relay tools, drift → pause (never deny, never blind + approve). +- Relay behavior: `ask` pauses without executing; a stored approval executes exactly once + (no double execution); `deny` refuses with the refusal text; client tools default + `allow` and honor `deny`/`ask` (M4). +- Events mean actions: `allow`/`deny` verdicts emit no `interaction_request`. +- The phase 5 debt, each as a test: reply failure pauses instead of hanging (H1), only + `{approved}` envelopes become decisions (H2), stop reason latched before the relay + drain (M5), a no-id gate pauses (L1). +- `permissionsFromRequest()`: legacy `permissionPolicy` maps correctly until phase 5 + deletes it; after phase 5, the goldens pin its absence. + +**Wire contract (golden fixtures, both sides).** `permissions.{default, rules}` appears +in `run_request.claude.json` and `run_request.pi_core.json`; final goldens pin that +`permissionPolicy` and `needsApproval` are gone. The TS compile-time key guard catches a +drifted `protocol.ts`. + +**SDK unit (pytest, `sdks/python/oss/tests/pytest/unit/agents/`).** +- Plan assembly: `runner.permissions.default` parses all four modes and rejects unknowns; + wire `rules` and rendered settings rules come from one parse and cannot disagree + (asserted by comparing both outputs for the same authored config); `mcp__*`-targeted + authored rules stay settings-only. +- `ToolSpec.to_wire()` ships only the author's explicit permission; `read_only` rides + separately; `needs_approval` and the aliases no longer parse. +- Claude settings rendering under each policy mode, including `allow_reads` → allow rules + for read-hinted tools only. +- Pi templates ship the same permissions block (no hardcoded `"auto"`). +- The producer-side pin the old bug lacked: a test asserting exactly what `/invoke` sends + in `sessionId`. + +**Service (pytest, `services/oss/tests/`).** Batch responses carry `stop_reason`; a +paused run is distinguishable and names its pending interaction; a completed run is +unchanged. + +**Frontend (vitest).** The policy select writes the four-mode `runner.permissions.default` +(for Pi too); the tool Permission select has no `needs_approval` fallback; the Pi settings +block writes `builtin_names`. + +**Live (phase 6).** The matrix above, plus one pause→approve→resume pair pinned as a +replay test (`agent-replay-test` skill). + ## Behavior deltas (before → after) | Case | Today | After | @@ -270,22 +373,22 @@ test that let the session-id proxy rot). | Headless, unset write tool, policy `allow_reads` | Parks silently | Pauses visibly (`stop_reason`, interaction ref) | | Headless, unset read tool, any allow-ish policy | Runs (settings allow rule) | Runs (unchanged, now policy-explained) | | Playground, unset tool, policy allow | Prompts (park) | Runs, tool activity visible, no prompt | -| Playground, `ask` tool | Prompts; approve can loop forever | Prompts; approve runs the approved call, done | +| Playground, `ask` tool | Prompts; approve can loop forever | Prompts; approve matches on stable anchors and runs, done | | `deny` on a client tool | Ignored | Refused | | Stored approval vs changed-to-deny config | Approval wins | Deny wins | | One approval, model repeats identical call | Every repeat auto-runs | One approval, one run | | Claude builtin with authored `ask` rule | Parks | Pauses (rule travels on the wire) | | Tool with `needs_approval: true` | Treated as ask via ladder | Field deleted; author sets `permission: ask` | -| Approve 3+ times on a non-matching gate | #5054 base: silent auto-deny | Cannot happen (no matching; loop-breaker deleted) | +| Approve, model re-issues different args on replay | #5054 base: silent auto-deny after 3 rounds | Visible fresh prompt with the new args (loop-breaker deleted) | ## Risks and open questions -- **Direct replay of the approved call.** The new mechanism needs one empirical check: the - cleanest implementation answers the re-raised gate when identity matches, and *injects* - the approved call's execution when it does not. How Claude behaves when the runner - executes-and-returns a result for a call it re-issued differently needs the phase 6 live - loop to pin. Fallback if injection proves harness-fragile: approve the re-raised gate - but execute with the *approved* arguments (still no key matching). +- **Arg regeneration on cold replay.** With stable name anchors, the residual drift risk + is the model regenerating slightly different *arguments* for the same intended call + (e.g. whitespace in a bash command). The design's answer is a visible fresh prompt, + which is safe but could annoy if it happens often. The phase 6 live loop measures + whether it happens in practice; if it does, the loosening lever is a documented + per-field canonicalization (never name-only matching, which is the HITL bypass). - **Relay-ask pausing on Pi.** The relay can park client tools today; ordinary relay-tool `ask` needs the same turn-boundary treatment (old S5.2). If heavy, ship relay-ask as its own slice, documented, while Claude paths use the full design. diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index d5c687c431..ac6857aa34 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -15,8 +15,8 @@ mandate; any of them can be reopened in PR review. - The live approve-loop is **diagnosed** (details in how-approvals-work.md, live-warning section): a constant stream `messageId` plus a level-triggered resume predicate on the frontend (finding M7; fixed in the #5054 base), compounded by tool-*name* drift across - ACP frames breaking the decision key (M2's observed form; patched in the base, properly - removed by this plan's direct-replay resume). + ACP frames breaking the decision key (M2's observed form; patched in the base, kept as + a clean recorded-name anchor inside this plan's decision module). - The permission model is settled (round 2): per-tool `allow | ask | deny` or inherit; one global policy with four modes (`allow`, `ask`, `deny`, `allow_reads` = reads run, writes ask); `needs_approval` and the legacy aliases get deleted; "effective permission" @@ -42,6 +42,17 @@ mandate; any of them can be reopened in PR review. `interactions` because an interaction is only the outcome of `ask`; `allow`/`deny` never produce one. Then: implement without further check-ins; Mahmoud reviews the PR. +- **2026-07-03, Codex pre-implementation review (xhigh):** the design held except one + claim. "Resume replays the approved call directly, no matching" is unimplementable for + Claude-native builtins (the runner's only lever is the ACP reply; transcript replay is + text, not structured tool injection). The resume section is rewritten: same-call + matching on stable anchors per executor, drift pauses visibly. Also folded in: a single + pending-approval latch (parallel-gate race), wire-first phase ordering with a + `permissionsFromRequest()` legacy bridge, `ToolSpec.to_wire()` ships explicit + permissions only, wire rules and settings rules derive from one parse (`mcp__*` rules + stay settings-only), Pi leaves the hardcoded `"auto"` path, and a concrete test plan + section. `SANDBOX_AGENT_DENY_PERMISSIONS` survives as an operator kill-switch. + ## Decisions taken (delegated, reopenable in PR review) 1. **One-shot scope confirmed:** Option D plus visibility plus the correctness debt, one @@ -54,9 +65,9 @@ mandate; any of them can be reopened in PR review. 4. **Batch pause shape:** minimal contract from this side: `stop_reason` plus the pending interaction reference. Exact field names stay coordinated with the streaming-invoke workspace. -5. **Direct-replay mechanics:** primary design is injecting the approved call; the - approve-with-stored-args fallback gets picked empirically in the phase 6 live loop if - injection proves harness-fragile. +5. **Resume mechanics (revised by the Codex review):** same-call matching on stable + anchors, split by executor; drift pauses visibly for a fresh approval. Direct replay + ("no matching") was dropped as unimplementable for harness-executed builtins. ## How this workspace was produced @@ -70,5 +81,5 @@ analysis that diagnosed the live loop. - The sandbox-agent daemon's permission-request id scheme (per-session counter vs unique): decides whether interaction tokens need turn namespacing (code-review H3). -- Whether ending a parked turn and later injecting the approved call behaves cleanly on - Claude (plan, "Direct replay" risk); pinned by the phase 6 live loop. +- How often the model regenerates different args for the same intended call on cold + replay (plan, "Arg regeneration" risk); pinned by the phase 6 live loop. From 827d4b341f0127b8ec96e1cd4be0fc6d1f7c0963 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 14:32:53 +0200 Subject: [PATCH 08/24] feat(runner): permission wire types + decision core (approval-boundary phase 1) --- .../projects/approval-boundary/build-notes.md | 35 ++ sdks/python/agenta/sdk/agents/utils/wire.py | 16 +- services/runner/src/permission-plan.ts | 201 ++++++++++ services/runner/src/protocol.ts | 23 +- .../runner/tests/unit/permission-plan.test.ts | 369 ++++++++++++++++++ 5 files changed, 641 insertions(+), 3 deletions(-) create mode 100644 docs/design/agent-workflows/projects/approval-boundary/build-notes.md create mode 100644 services/runner/src/permission-plan.ts create mode 100644 services/runner/tests/unit/permission-plan.test.ts diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md new file mode 100644 index 0000000000..c1aaa13174 --- /dev/null +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -0,0 +1,35 @@ +# Build notes: judgment calls made during implementation + +Running log of decisions the implementation made inside the mandate "go with your +decisions unless it contradicts an owner call". Newest last. Read together with +[status.md](status.md) (the decision log) and [plan.md](plan.md) (the design). + +## 2026-07-03 + +- **Resume mechanics revised before any code was written.** The Codex pre-implementation + review (xhigh) showed "replay the approved call directly" cannot work for + Claude-native builtins: the runner's only lever on a harness gate is the ACP reply, + and prior turns replay as text, not as structured tool calls. Adopted: same-call + matching on stable anchors per executor; drift pauses visibly. This kept the two + properties Mahmoud cared about (no silent loop, no silent auto-deny) without the + unimplementable part. Full trail: status.md, "Codex pre-implementation review". +- **`allow_reads` name kept.** Codex slightly preferred `allow_read_only`; not clearly + better, and the owner has been using "allow_reads" in review threads. Left as is, + rename is one grep if review wants it. +- **`SANDBOX_AGENT_DENY_PERMISSIONS` kept** as an operator kill-switch (forces + `default: deny`, wins over the authored plan). Deleting an emergency lever during a + permission redesign felt wrong; documented at its read site. +- **Unparseable policy fails toward `ask`.** `permissionsFromRequest()` maps an unknown + policy mode to `{default: "ask"}` (pause for a human) rather than allow or deny. + Rationale: a config the runner cannot understand should neither run tools nor + hard-refuse the run; asking is the safe middle. +- **New pure module named `permission-plan.ts`** (not `permissions.ts`) to avoid a + name collision with `engines/sandbox_agent/permissions.ts` until that file becomes + `acp-interactions.ts` in the cleanup phase. +- **Phase 1 landed (Codex implemented, reviewed here).** Wire types + decision module + + generated truth-table tests; 417 runner tests and typecheck green, goldens untouched + (the new wire field is optional until the SDK emits it in phase 3). Review note + carried forward: the `Tool(prefix:*)` matcher inspects the first string *value* of the + args record, which on a multi-string-field tool could be the wrong field; tighten to + the known argument name (e.g. `command`) if phase 3's derived rules ever target such a + tool. Prefix rules on uninspectable args deliberately fail toward the default. diff --git a/sdks/python/agenta/sdk/agents/utils/wire.py b/sdks/python/agenta/sdk/agents/utils/wire.py index b1f2ea85a7..e16ac1c6c9 100644 --- a/sdks/python/agenta/sdk/agents/utils/wire.py +++ b/sdks/python/agenta/sdk/agents/utils/wire.py @@ -18,7 +18,7 @@ from __future__ import annotations import re -from typing import Any, Dict, List, Optional, Sequence +from typing import Any, Dict, List, Literal, Optional, Sequence, TypedDict from agenta.sdk.utils.logging import get_module_logger @@ -35,6 +35,20 @@ log = get_module_logger(__name__) +PermissionMode = Literal["allow", "ask", "deny", "allow_reads"] +ToolPermission = Literal["allow", "ask", "deny"] + + +class PermissionRule(TypedDict): + pattern: str + permission: ToolPermission + + +class PermissionsConfig(TypedDict, total=False): + default: PermissionMode + rules: List[PermissionRule] + + # The user-facing error must not carry an internal stack/path dump. Cap the surfaced line and # strip the patterns that leak implementation detail; the full text is logged, never shown. _ERROR_MAX_LEN = 300 diff --git a/services/runner/src/permission-plan.ts b/services/runner/src/permission-plan.ts new file mode 100644 index 0000000000..8f179f437e --- /dev/null +++ b/services/runner/src/permission-plan.ts @@ -0,0 +1,201 @@ +import type { AgentRunRequest, PermissionMode, ToolPermission } from "./protocol.ts"; + +/** Which component executes the gated tool; decides how resume matching anchors names. */ +export type GateExecutor = "harness" | "relay" | "client"; + +/** Everything the decision needs to know about one gated call, normalized upstream. */ +export interface GateDescriptor { + executor: GateExecutor; + /** Stable tool name: spec name for relay/client tools; recorded tool_call name for harness gates. */ + toolName?: string; + /** The resolved spec's explicit author permission, if the gate is for a resolved tool. */ + specPermission?: ToolPermission; + /** The owning MCP server's explicit permission, if the tool belongs to a user MCP server. */ + serverPermission?: ToolPermission; + /** The catalog read-only hint (true = read). Absent counts as a write under allow_reads. */ + readOnlyHint?: boolean; + /** Canonicalizable call arguments (used by stored-decision matching, not by effectivePermission). */ + args?: unknown; +} + +export interface PermissionPlan { + default: PermissionMode; + rules: { pattern: string; permission: ToolPermission }[]; +} + +export type Verdict = + | { kind: "allow" } + | { kind: "deny" } + | { kind: "pendingApproval" }; + +export interface StoredPermissionDecisions { + take(gate: GateDescriptor): "allow" | "deny" | undefined; +} + +const PERMISSION_MODES: readonly PermissionMode[] = [ + "allow", + "ask", + "deny", + "allow_reads", +]; +const TOOL_PERMISSIONS: readonly ToolPermission[] = ["allow", "ask", "deny"]; +const RULE_RANK: Record = { + allow: 0, + ask: 1, + deny: 2, +}; + +export function permissionsFromRequest( + request: AgentRunRequest, +): PermissionPlan { + if (process.env.SANDBOX_AGENT_DENY_PERMISSIONS === "true") { + return { default: "deny", rules: [] }; + } + + if (request.permissions !== undefined) { + const raw = request.permissions as unknown; + if (!isRecord(raw)) { + return { default: "ask", rules: [] }; + } + + const defaultMode = raw.default ?? "allow_reads"; + if (!isPermissionMode(defaultMode)) { + // An unparseable policy must fail toward asking a human, not toward running tools. + return { default: "ask", rules: [] }; + } + return { + default: defaultMode, + rules: normalizeRules(raw.rules), + }; + } + + if (request.permissionPolicy === "deny") { + return { default: "deny", rules: [] }; + } + return { default: "allow", rules: [] }; +} + +export function effectivePermission( + gate: GateDescriptor, + plan: PermissionPlan, +): ToolPermission { + if (gate.specPermission !== undefined) return gate.specPermission; + if (gate.serverPermission !== undefined) return gate.serverPermission; + + const rulePermission = matchingRulePermission(gate, plan.rules); + if (rulePermission !== undefined) return rulePermission; + + return defaultPermission(plan.default, gate); +} + +export function decide( + gate: GateDescriptor, + plan: PermissionPlan, + stored: StoredPermissionDecisions, +): Verdict { + const permission = effectivePermission(gate, plan); + if (permission === "deny") return { kind: "deny" }; + if (permission === "allow") return { kind: "allow" }; + + const storedDecision = stored.take(gate); + if (storedDecision === "allow") return { kind: "allow" }; + if (storedDecision === "deny") return { kind: "deny" }; + return { kind: "pendingApproval" }; +} + +export class PendingApprovalLatch { + private acquired = false; + + tryAcquire(): boolean { + if (this.acquired) return false; + this.acquired = true; + return true; + } + + get held(): boolean { + return this.acquired; + } +} + +function normalizeRules(rawRules: unknown): PermissionPlan["rules"] { + if (!Array.isArray(rawRules)) return []; + const rules: PermissionPlan["rules"] = []; + for (const rawRule of rawRules) { + if (!isRecord(rawRule)) continue; + const { pattern, permission } = rawRule; + if (typeof pattern === "string" && isToolPermission(permission)) { + rules.push({ pattern, permission }); + } + } + return rules; +} + +function matchingRulePermission( + gate: GateDescriptor, + rules: PermissionPlan["rules"], +): ToolPermission | undefined { + let best: ToolPermission | undefined; + for (const rule of rules) { + if (!ruleMatches(gate, rule.pattern)) continue; + if (best === undefined || RULE_RANK[rule.permission] > RULE_RANK[best]) { + best = rule.permission; + } + } + return best; +} + +function ruleMatches(gate: GateDescriptor, pattern: string): boolean { + if (gate.toolName === undefined) return false; + + const prefixPattern = parsePrefixPattern(pattern); + if (prefixPattern === undefined) return pattern === gate.toolName; + if (prefixPattern.toolName !== gate.toolName) return false; + + const firstArg = firstStringArgument(gate.args); + // Prefix rules with uninspectable args fail toward the default instead of guessing. + return firstArg !== undefined && firstArg.startsWith(prefixPattern.prefix); +} + +function parsePrefixPattern( + pattern: string, +): { toolName: string; prefix: string } | undefined { + const open = pattern.indexOf("("); + if (open <= 0 || !pattern.endsWith(":*)")) return undefined; + return { + toolName: pattern.slice(0, open), + prefix: pattern.slice(open + 1, -3), + }; +} + +function firstStringArgument(args: unknown): string | undefined { + if (typeof args === "string") return args; + if (Array.isArray(args)) { + return args.find((value): value is string => typeof value === "string"); + } + if (!isRecord(args)) return undefined; + return Object.values(args).find( + (value): value is string => typeof value === "string", + ); +} + +function defaultPermission( + mode: PermissionMode, + gate: GateDescriptor, +): ToolPermission { + if (mode === "allow_reads") { + return gate.readOnlyHint === true ? "allow" : "ask"; + } + return mode; +} + +function isPermissionMode(value: unknown): value is PermissionMode { + return (PERMISSION_MODES as readonly unknown[]).includes(value); +} + +function isToolPermission(value: unknown): value is ToolPermission { + return (TOOL_PERMISSIONS as readonly unknown[]).includes(value); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/services/runner/src/protocol.ts b/services/runner/src/protocol.ts index 6a67d40e70..4f382c0188 100644 --- a/services/runner/src/protocol.ts +++ b/services/runner/src/protocol.ts @@ -71,6 +71,20 @@ export interface Telemetry { exporters?: { otlp?: OtlpExporter }; } +/** The global permission policy modes. `allow_reads`: read-hinted tools run, everything else asks. */ +export type PermissionMode = "allow" | "ask" | "deny" | "allow_reads"; +/** A single tool's permission verdict vocabulary. */ +export type ToolPermission = "allow" | "ask" | "deny"; +/** An authored harness-builtin rule, Claude settings syntax (e.g. "Bash(rm:*)"). */ +export interface PermissionRule { + pattern: string; + permission: ToolPermission; +} +export interface PermissionsConfig { + default?: PermissionMode; + rules?: PermissionRule[]; +} + /** * A runnable tool the backend already resolved from the agent config. * @@ -125,7 +139,7 @@ export interface ResolvedToolSpec { * `permissionPolicy`. The SDK derives a default from `readOnly`/`needsApproval` when the * author set none. Plumbing only here; enforcement is a later slice. */ - permission?: "allow" | "ask" | "deny"; + permission?: ToolPermission; } /** Where and how to route a tool call back through Agenta. */ @@ -222,7 +236,7 @@ export interface McpServerConfig { * is no derived default: an explicit author value or nothing. Plumbing only; enforcement is * a later slice. */ - permission?: "allow" | "ask" | "deny"; + permission?: ToolPermission; } /** @@ -426,6 +440,11 @@ export interface AgentRunRequest { toolCallback?: ToolCallbackContext; /** How a permission-gating harness handles tool-use prompts: "auto" (default) | "deny". */ permissionPolicy?: string; + /** + * Authored permission plan assembled by the SDK (`runner.permissions.*` in the agent config). + * When absent, the runner maps the legacy `permissionPolicy` via `permissionsFromRequest()`. + */ + permissions?: PermissionsConfig; /** * The declared sandbox security boundary (Layer 2). Omitted when unset. The network policy is * enforced on Daytona; on the local sidecar a restricted-network run is rejected under diff --git a/services/runner/tests/unit/permission-plan.test.ts b/services/runner/tests/unit/permission-plan.test.ts new file mode 100644 index 0000000000..648903830f --- /dev/null +++ b/services/runner/tests/unit/permission-plan.test.ts @@ -0,0 +1,369 @@ +import { afterEach, beforeEach, describe, it } from "vitest"; +import assert from "node:assert/strict"; + +import { + PendingApprovalLatch, + decide, + effectivePermission, + permissionsFromRequest, + type GateDescriptor, + type PermissionPlan, + type StoredPermissionDecisions, + type Verdict, +} from "../../src/permission-plan.ts"; +import type { AgentRunRequest, PermissionMode, ToolPermission } from "../../src/protocol.ts"; + +const originalDenyEnv = process.env.SANDBOX_AGENT_DENY_PERMISSIONS; + +beforeEach(() => { + delete process.env.SANDBOX_AGENT_DENY_PERMISSIONS; +}); + +afterEach(() => { + if (originalDenyEnv === undefined) { + delete process.env.SANDBOX_AGENT_DENY_PERMISSIONS; + } else { + process.env.SANDBOX_AGENT_DENY_PERMISSIONS = originalDenyEnv; + } +}); + +const modes = ["allow", "ask", "deny", "allow_reads"] as const; +const permissions = ["allow", "ask", "deny", undefined] as const; +const hints = [true, false, undefined] as const; +const storedDecisions = ["allow", "deny", undefined] as const; + +describe("decide truth table", () => { + it("matches the reference behavior for every default/spec/hint/stored combination", () => { + let cases = 0; + for (const defaultMode of modes) { + for (const specPermission of permissions) { + for (const readOnlyHint of hints) { + for (const storedDecision of storedDecisions) { + const gate: GateDescriptor = { + executor: "relay", + toolName: "tool", + specPermission, + readOnlyHint, + args: {}, + }; + const plan: PermissionPlan = { default: defaultMode, rules: [] }; + const stored: StoredPermissionDecisions = { + take: () => storedDecision, + }; + + assert.deepEqual( + decide(gate, plan, stored), + { kind: expectedVerdict(defaultMode, specPermission, readOnlyHint, storedDecision) }, + `default=${defaultMode} spec=${specPermission ?? "unset"} readOnly=${String( + readOnlyHint, + )} stored=${storedDecision ?? "none"}`, + ); + cases += 1; + } + } + } + } + assert.equal(cases, 4 * 4 * 3 * 3); + }); + + const spotCases: Array<{ + name: string; + gate: GateDescriptor; + plan: PermissionPlan; + stored?: "allow" | "deny"; + expected: Verdict; + }> = [ + { + name: "allow_reads + no hint + no stored asks", + gate: { executor: "relay", readOnlyHint: undefined }, + plan: { default: "allow_reads", rules: [] }, + expected: { kind: "pendingApproval" }, + }, + { + name: "allow_reads + false hint + stored allow allows", + gate: { executor: "relay", readOnlyHint: false }, + plan: { default: "allow_reads", rules: [] }, + stored: "allow", + expected: { kind: "allow" }, + }, + { + name: "allow_reads + true hint ignores stored deny", + gate: { executor: "relay", readOnlyHint: true }, + plan: { default: "allow_reads", rules: [] }, + stored: "deny", + expected: { kind: "allow" }, + }, + { + name: "deny default ignores stored allow", + gate: { executor: "harness" }, + plan: { default: "deny", rules: [] }, + stored: "allow", + expected: { kind: "deny" }, + }, + { + name: "ask default consumes stored deny", + gate: { executor: "client" }, + plan: { default: "ask", rules: [] }, + stored: "deny", + expected: { kind: "deny" }, + }, + { + name: "allow default ignores stored deny", + gate: { executor: "relay" }, + plan: { default: "allow", rules: [] }, + stored: "deny", + expected: { kind: "allow" }, + }, + { + name: "spec allow beats deny default", + gate: { executor: "relay", specPermission: "allow" }, + plan: { default: "deny", rules: [] }, + expected: { kind: "allow" }, + }, + { + name: "spec deny beats allow default and stored allow", + gate: { executor: "relay", specPermission: "deny" }, + plan: { default: "allow", rules: [] }, + stored: "allow", + expected: { kind: "deny" }, + }, + { + name: "spec ask beats allow default", + gate: { executor: "relay", specPermission: "ask" }, + plan: { default: "allow", rules: [] }, + expected: { kind: "pendingApproval" }, + }, + ]; + + for (const { name, gate, plan, stored, expected } of spotCases) { + it(name, () => { + assert.deepEqual(decide(gate, plan, { take: () => stored }), expected); + }); + } +}); + +describe("effectivePermission rule matching", () => { + it("matches an exact tool name", () => { + assert.equal( + effectivePermission( + { executor: "harness", toolName: "Bash" }, + { default: "deny", rules: [{ pattern: "Bash", permission: "allow" }] }, + ), + "allow", + ); + }); + + it("matches Claude prefix rules against the first string argument", () => { + assert.equal( + effectivePermission( + { + executor: "harness", + toolName: "Bash", + args: { command: "npm run test" }, + }, + { + default: "deny", + rules: [{ pattern: "Bash(npm run:*)", permission: "allow" }], + }, + ), + "allow", + ); + }); + + it("does not match prefix rules when args are not inspectable", () => { + assert.equal( + effectivePermission( + { executor: "harness", toolName: "Bash", args: { command: 42 } }, + { + default: "deny", + rules: [{ pattern: "Bash(npm run:*)", permission: "allow" }], + }, + ), + "deny", + ); + }); + + it("chooses deny over ask over allow when several rules match", () => { + assert.equal( + effectivePermission( + { + executor: "harness", + toolName: "Bash", + args: { command: "npm run test" }, + }, + { + default: "allow", + rules: [ + { pattern: "Bash", permission: "allow" }, + { pattern: "Bash(npm:*)", permission: "ask" }, + { pattern: "Bash(npm run:*)", permission: "deny" }, + ], + }, + ), + "deny", + ); + }); + + it("lets serverPermission beat rules", () => { + assert.equal( + effectivePermission( + { executor: "relay", toolName: "fetch", serverPermission: "allow" }, + { default: "ask", rules: [{ pattern: "fetch", permission: "deny" }] }, + ), + "allow", + ); + }); + + it("lets specPermission beat serverPermission", () => { + assert.equal( + effectivePermission( + { + executor: "relay", + toolName: "fetch", + specPermission: "deny", + serverPermission: "allow", + }, + { default: "ask", rules: [{ pattern: "fetch", permission: "allow" }] }, + ), + "deny", + ); + }); +}); + +describe("permissionsFromRequest", () => { + it("passes through a new block and normalizes missing pieces", () => { + assert.deepEqual( + permissionsFromRequest({ + permissions: { + rules: [{ pattern: "Bash(npm run:*)", permission: "ask" }], + }, + } as AgentRunRequest), + { + default: "allow_reads", + rules: [{ pattern: "Bash(npm run:*)", permission: "ask" }], + }, + ); + + assert.deepEqual( + permissionsFromRequest({ + permissions: { default: "deny" }, + } as AgentRunRequest), + { default: "deny", rules: [] }, + ); + }); + + it("treats an unknown mode as ask", () => { + assert.deepEqual( + permissionsFromRequest({ + permissions: { + default: "bogus" as PermissionMode, + rules: [{ pattern: "Bash", permission: "allow" }], + }, + } as AgentRunRequest), + { default: "ask", rules: [] }, + ); + }); + + it("maps absent new block plus legacy deny to deny", () => { + assert.deepEqual( + permissionsFromRequest({ permissionPolicy: "deny" } as AgentRunRequest), + { default: "deny", rules: [] }, + ); + }); + + it("maps absent new block and absent legacy field to allow", () => { + assert.deepEqual(permissionsFromRequest({} as AgentRunRequest), { + default: "allow", + rules: [], + }); + }); + + it("lets the env kill-switch beat an explicit allow block", () => { + const previous = process.env.SANDBOX_AGENT_DENY_PERMISSIONS; + try { + process.env.SANDBOX_AGENT_DENY_PERMISSIONS = "true"; + assert.deepEqual( + permissionsFromRequest({ + permissions: { default: "allow", rules: [{ pattern: "Bash", permission: "allow" }] }, + } as AgentRunRequest), + { default: "deny", rules: [] }, + ); + } finally { + if (previous === undefined) { + delete process.env.SANDBOX_AGENT_DENY_PERMISSIONS; + } else { + process.env.SANDBOX_AGENT_DENY_PERMISSIONS = previous; + } + } + }); +}); + +describe("PendingApprovalLatch", () => { + it("lets only the first caller acquire", () => { + const latch = new PendingApprovalLatch(); + assert.equal(latch.held, false); + assert.equal(latch.tryAcquire(), true); + assert.equal(latch.held, true); + assert.equal(latch.tryAcquire(), false); + assert.equal(latch.held, true); + }); +}); + +describe("stored decisions", () => { + it("consults stored decisions only under ask", () => { + const stored: StoredPermissionDecisions = { + take: () => { + throw new Error("stored decision was consulted"); + }, + }; + + assert.deepEqual( + decide({ executor: "relay" }, { default: "allow", rules: [] }, stored), + { kind: "allow" }, + ); + assert.deepEqual( + decide({ executor: "relay" }, { default: "deny", rules: [] }, stored), + { kind: "deny" }, + ); + assert.deepEqual( + decide( + { executor: "relay", specPermission: "allow" }, + { default: "ask", rules: [] }, + stored, + ), + { kind: "allow" }, + ); + assert.deepEqual( + decide( + { executor: "relay", specPermission: "deny" }, + { default: "ask", rules: [] }, + stored, + ), + { kind: "deny" }, + ); + }); +}); + +function expectedVerdict( + defaultMode: PermissionMode, + specPermission: ToolPermission | undefined, + readOnlyHint: boolean | undefined, + storedDecision: "allow" | "deny" | undefined, +): Verdict["kind"] { + const permission = expectedPermission(defaultMode, specPermission, readOnlyHint); + if (permission === "allow") return "allow"; + if (permission === "deny") return "deny"; + return storedDecision ?? "pendingApproval"; +} + +function expectedPermission( + defaultMode: PermissionMode, + specPermission: ToolPermission | undefined, + readOnlyHint: boolean | undefined, +): ToolPermission { + if (specPermission !== undefined) return specPermission; + if (defaultMode === "allow_reads") { + return readOnlyHint === true ? "allow" : "ask"; + } + return defaultMode; +} From 99b7910162b75880ecd35cf6c1ccd3506e2224e3 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 14:55:47 +0200 Subject: [PATCH 09/24] feat(runner): consult-first approval responder replaces park-by-default (phase 2a) --- .../projects/approval-boundary/build-notes.md | 14 + services/runner/src/engines/sandbox_agent.ts | 172 ++--- .../src/engines/sandbox_agent/permissions.ts | 359 +++++---- services/runner/src/responder.ts | 516 +++---------- services/runner/tests/unit/responder.test.ts | 698 +++++------------- .../unit/sandbox-agent-orchestration.test.ts | 35 +- .../unit/sandbox-agent-permissions.test.ts | 442 ++++++++--- 7 files changed, 925 insertions(+), 1311 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index c1aaa13174..919382a4e7 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -26,6 +26,20 @@ decisions unless it contradicts an owner call". Newest last. Read together with - **New pure module named `permission-plan.ts`** (not `permissions.ts`) to avoid a name collision with `engines/sandbox_agent/permissions.ts` until that file becomes `acp-interactions.ts` in the cleanup phase. +- **Phase 2a landed (Codex implemented, reviewed here).** Consult-first ACP flow, + `ApprovalResponder`, latch, H1 (reply failure pauses), L1 (no-id gate pauses), M5 + (stop reason latched before the relay drain). Review restored two things Codex + trimmed: the F-024 "a pause sends no reply" rationale comment and the `[HITL]` + ground-truth gate log QA greps for. Codex's flagged judgment call (client `ask` + + stored approval but no browser output yet: consume the approval, still pause for + fulfillment) accepted as correct two-step behavior. +- **Two cross-layer subtleties found in the 2a review, deferred to 2b by design:** + (1) on Claude a client tool's stored output is legitimately read at BOTH the ACP gate + and the relay, so the ACP side must peek and only the relay (the actual server of the + output) consumes; (2) relay tools on Claude are gated at the ACP layer first, so the + relay must enforce permissions only when the harness does not gate (Pi) — otherwise + one approval would be consumed at the gate and the relay would double-gate the same + call. - **Phase 1 landed (Codex implemented, reviewed here).** Wire types + decision module + generated truth-table tests; 417 runner tests and typecheck green, goldens untouched (the new wire field is optional until the SDK emits it in phase 3). Review note diff --git a/services/runner/src/engines/sandbox_agent.ts b/services/runner/src/engines/sandbox_agent.ts index a8cee2341d..33989ac44d 100644 --- a/services/runner/src/engines/sandbox_agent.ts +++ b/services/runner/src/engines/sandbox_agent.ts @@ -23,7 +23,6 @@ * so it is uniform across every harness and always nests under the caller's /invoke * span. stdout is reserved for the JSON result (see cli.ts); logs go to stderr. */ -import { apiBase } from "../apiBase.ts"; import { rmSync } from "node:fs"; import { SandboxAgent, InMemorySessionPersistDriver } from "sandbox-agent"; @@ -35,9 +34,9 @@ import { startToolRelay, } from "../tools/relay.ts"; import { - HITLResponder, + ApprovalResponder, + ConversationDecisions, extractApprovalDecisions, - nonConvergingToolNames, policyFromRequest, type Responder, } from "../responder.ts"; @@ -46,6 +45,7 @@ import { type AgentRunResult, type EmitEvent, type ToolCallbackContext, + type ToolPermission, resolveRunSessionId, } from "../protocol.ts"; import { @@ -67,6 +67,11 @@ import { buildPiExtensionEnv, prepareLocalPiAssets, } from "./sandbox_agent/pi-assets.ts"; +import { + PendingApprovalLatch, + permissionsFromRequest, + type GateDescriptor, +} from "../permission-plan.ts"; import { attachPermissionResponder } from "./sandbox_agent/permissions.ts"; import { createInteraction, @@ -109,6 +114,22 @@ function runCredential(request: AgentRunRequest): string { return (headers["authorization"] ?? headers["Authorization"] ?? "").trim(); } +/** Agenta API base for runner→API calls (heartbeat/ingest/mount-sign share this). */ +function apiBase(): string { + return process.env.AGENTA_API_URL ?? "http://localhost:8000/api"; +} + +function serverPermissionsFromRequest( + request: AgentRunRequest, +): ReadonlyMap { + const permissions = new Map(); + for (const server of request.mcpServers ?? []) { + if (server.permission !== undefined) { + permissions.set(server.name, server.permission); + } + } + return permissions; +} type Log = (message: string) => void; const LOCAL_DURABLE_CWD_ENOTCONN_REMOUNT_LIMIT = 1; @@ -194,7 +215,6 @@ function applyClaudeConnectionEnv( ) { env.ANTHROPIC_MODEL = selectedModel; env.ANTHROPIC_CUSTOM_MODEL_OPTION = selectedModel; - logger(`claude model=${selectedModel} deployment=${deployment ?? ""}`); return true; } return false; @@ -220,7 +240,7 @@ export interface SandboxAgentDeps extends BuildRunPlanDeps { mountStorageRemote?: typeof mountStorageRemote; unmountStorage?: typeof unmountStorage; discoverTunnelEndpoint?: typeof discoverTunnelEndpoint; - responderFactory?: (permissionPolicy: string | undefined) => Responder; + responderFactory?: (request: AgentRunRequest) => Responder; log?: Log; } @@ -256,7 +276,9 @@ function containsTransportEndpointDisconnected(value: unknown): boolean { seen.add(current); const code = - "code" in current ? String((current as { code?: unknown }).code) : ""; + "code" in current + ? String((current as { code?: unknown }).code) + : ""; if (code === "ENOTCONN") { return true; } @@ -283,8 +305,7 @@ export async function runSandboxAgent( // failure leaves durableCwd undefined and buildRunPlan falls back to the ephemeral path. const sessionForMount = request.sessionId?.trim(); const runCred = runCredential(request); - const signMount = - deps.signSessionMountCredentials ?? signSessionMountCredentials; + const signMount = deps.signSessionMountCredentials ?? signSessionMountCredentials; let mountCreds: MountCredentials | null = sessionForMount && runCred ? await signMount(sessionForMount, { @@ -299,9 +320,7 @@ export async function runSandboxAgent( // is already "mounts//", so no extra slug is needed. let durableCwd: string | undefined; if (mountCreds?.prefix) { - const isDaytonaReq = - (request.sandbox ?? process.env.SANDBOX_AGENT_PROVIDER ?? "local") === - "daytona"; + const isDaytonaReq = (request.sandbox ?? process.env.SANDBOX_AGENT_PROVIDER ?? "local") === "daytona"; durableCwd = isDaytonaReq ? `/home/sandbox/agenta/${mountCreds.prefix}` : `/tmp/agenta/${mountCreds.prefix}`; @@ -357,15 +376,6 @@ export async function runSandboxAgent( logger(`harness=${plan.harness} sandbox=${plan.sandboxId} cwd=${plan.cwd}`); - // The resolved model ref as it reaches the runner (key NAMES only, never values) — the one - // line that answers "what model/provider/deployment/credential did this run actually use". - logger( - `resolved model=${request.model ?? ""} provider=${request.provider ?? ""} ` + - `deployment=${request.deployment ?? ""} ` + - `connection=${request.connection ? `${request.connection.mode}:${request.connection.slug ?? "-"}` : ""} ` + - `secretKeys=[${Object.keys(request.secrets ?? {}).join(",")}]`, - ); - // Pi traces itself via the extension under the propagated traceparent; for other // harnesses we build the span tree here from the ACP event stream. Created below, once // the model is resolved, so the chat span carries the harness's actual model rather @@ -385,11 +395,7 @@ export async function runSandboxAgent( logger( `local durable cwd mount (${reason}) session=${sessionForMount} cwd=${plan.cwd}`, ); - if ( - await (deps.mountStorage ?? mountStorage)(plan.cwd, mountCreds, { - log: logger, - }) - ) { + if (await (deps.mountStorage ?? mountStorage)(plan.cwd, mountCreds, { log: logger })) { mountedCwd = plan.cwd; return true; } @@ -417,9 +423,7 @@ export async function runSandboxAgent( log: logger, }); if (!fresh) { - logger( - `local durable cwd re-sign returned no credentials session=${sessionForMount}`, - ); + logger(`local durable cwd re-sign returned no credentials session=${sessionForMount}`); return false; } mountCreds = fresh; @@ -510,9 +514,7 @@ export async function runSandboxAgent( isTransportEndpointDisconnected(err) && (await reSignAndRemountLocalCwd()) ) { - logger( - `retrying workspace preparation after local durable cwd remount`, - ); + logger(`retrying workspace preparation after local durable cwd remount`); workspace = await (deps.prepareWorkspace ?? prepareWorkspace)({ sandbox, plan, @@ -525,22 +527,15 @@ export async function runSandboxAgent( if (mountCreds) { if (plan.isDaytona) { - const endpoint = await ( - deps.discoverTunnelEndpoint ?? discoverTunnelEndpoint - )({ + const endpoint = await (deps.discoverTunnelEndpoint ?? discoverTunnelEndpoint)({ log: logger, }); if ( endpoint && - (await (deps.mountStorageRemote ?? mountStorageRemote)( - sandbox, - plan.cwd, - mountCreds, - { - endpoint, - log: logger, - }, - )) + (await (deps.mountStorageRemote ?? mountStorageRemote)(sandbox, plan.cwd, mountCreds, { + endpoint, + log: logger, + })) ) { // Remote files live in the store; nothing on this host to unmount. logger(`remote durable cwd active for session=${sessionForMount}`); @@ -642,61 +637,40 @@ export async function runSandboxAgent( if (update) run.handleUpdate(update); }); - // Cross-turn HITL: when the request carries a platform `sessionId` it came through the - // `/messages` endpoint, which validates and stamps a session id on every turn and replays - // the conversation — i.e. there is a browser that can answer a permission prompt. The - // headless `/invoke` path sets no session id. With no human surface and no stored - // decisions the HITLResponder falls back to the base policy and is byte-identical to the - // old PolicyResponder, so `/invoke` is unchanged. - const hasHumanSurface = !!(request.sessionId && request.sessionId.trim()); - // A park (cross-turn HITL) must END the turn gracefully, not hold the ACP connection open - // forever (F-040): Claude does not end a turn on an unanswered gate, so `session.prompt()` - // would block indefinitely, the sandbox would leak, and the egress would never emit a - // `finish` frame. On the first park we `destroySession`, which (a) resolves the pending - // permission RPC with `{outcome:"cancelled"}` — NOT a reject, so no F-024 clobber — and - // (b) sends the managed `session/cancel` so the in-flight `prompt()` resolves with a - // cancelled stop reason. The resume then cold-replays as a fresh turn (#4854). + // A pending approval pause must END the turn gracefully, not hold the ACP connection + // open forever (F-040): Claude does not end a turn on an unanswered gate, so + // `session.prompt()` would block indefinitely. On the first pause we destroy the session + // so the in-flight prompt resolves or the local park signal wins the race below. let parked = false; let resolveParked: (() => void) | undefined; const parkedSignal = new Promise((resolve) => { resolveParked = resolve; }); const onPark = (): void => { - if (parked) return; // first park wins; later gates this turn are moot once we cancel + if (parked) return; // first pause wins; later gates this turn are moot once we cancel parked = true; resolveParked?.(); - // Cancel the in-flight prompt so `session.prompt()` returns instead of hanging. The - // managed cancel lives on the SandboxAgent handle (the `Session` wrapper has none). - // Best effort: the `finally` disposes the sandbox regardless, and the prompt is also - // raced against `parkedSignal` below so the turn ends even if this call rejects. void sandbox.destroySession?.(session.id).catch(() => {}); }; + const permissionPlan = permissionsFromRequest(request); + const storedDecisionMap = extractApprovalDecisions(request); + if (storedDecisionMap.size > 0) { + logger( + `[HITL] resume state: decisions=${JSON.stringify([...storedDecisionMap.keys()])}`, + ); + } + const decisions = new ConversationDecisions(storedDecisionMap); + const latch = new PendingApprovalLatch(); const responder = - deps.responderFactory?.(request.permissionPolicy) ?? - (() => { - const decisions = extractApprovalDecisions(request); - const looping = nonConvergingToolNames(request); - // Dump the STORED side of the HITL resume: the decision keys the transcript folded and any - // tools flagged as non-converging. Compare against the runner's `[HITL] gate ...` lines - // (the LIVE re-raised keys) to see exactly what drifted. - if (hasHumanSurface) { - logger( - `[HITL] resume state: humanSurface=${hasHumanSurface} ` + - `decisions=${JSON.stringify([...decisions.keys()])} ` + - `looping=${JSON.stringify([...looping])}`, - ); - } - return new HITLResponder( - decisions, - policyFromRequest(request.permissionPolicy), - hasHumanSurface, - looping, - logger, - ); - })(); + deps.responderFactory?.(request) ?? + new ApprovalResponder(permissionPlan, decisions, logger); + const serverPermissions = serverPermissionsFromRequest(request); attachPermissionResponder({ session, run, + responder, + latch, + serverPermissions, log: logger, onPark, onCreateInteraction: (token, toolName, toolArgs) => { @@ -729,13 +703,9 @@ export async function runSandboxAgent( return; void resolveInteraction(sessionId, token, () => cred); }, - responder, }); if (plan.useToolRelay) { - // Layer 3 (S3b): the relay enforces each resolved tool's `permission`; an `ask`/unset - // permission degrades to the run's headless permission policy (the same policy the - // PolicyResponder uses for Claude builtins above). toolRelay = (deps.startToolRelay ?? startToolRelay)( plan.isDaytona ? (deps.sandboxRelayHost ?? sandboxRelayHost)(sandbox) @@ -747,18 +717,28 @@ export async function runSandboxAgent( request.runContext, { onClientTool: async ({ id, toolCallId, toolName, input, spec }) => { - const decision = await responder.onClientTool({ + const gate: GateDescriptor = { + executor: "client", + toolName: spec.name, + specPermission: spec.permission, + readOnlyHint: spec.readOnly, + args: input, + }; + const verdict = await responder.onClientTool({ id, toolCallId, - toolName, - input, + gate, raw: { spec }, }); - if (process.env.AGENTA_RUNNER_DEBUG_TOOLS) + if (process.env.AGENTA_RUNNER_DEBUG_TOOLS) { logger( - `[client-tool] ${toolName} id=${toolCallId} kind=${spec.kind} decision=${typeof decision === "string" ? decision : JSON.stringify(decision).slice(0, 200)}`, + `[client-tool] ${toolName} id=${toolCallId} kind=${spec.kind} ` + + `decision=${JSON.stringify(verdict).slice(0, 200)}`, ); - if (decision === "park") { + } + if (verdict.kind === "deny") return "deny"; + if (verdict.kind === "fulfilled") return { output: verdict.output }; + if (latch.tryAcquire()) { run.emitEvent({ type: "interaction_request", id, @@ -779,7 +759,7 @@ export async function runSandboxAgent( }, }); } - return decision; + return "park"; }, onPark, }, @@ -802,13 +782,13 @@ export async function runSandboxAgent( promptPromise, parkedSignal.then(() => PARKED), ]); - await toolRelay?.stop(); // A parked turn is terminal-but-incomplete: stop reason `paused` tells the egress to emit // a clean `finish` (the FE then resumes on the user's decision). A real prompt result keeps // the harness's own stop reason. const stopReason = raced === PARKED || parked ? "paused" : (raced as any)?.stopReason; const result = raced === PARKED ? undefined : raced; + await toolRelay?.stop(); logger(`prompt stopReason=${stopReason}`); // Usage: Pi writes its totals to a file via the extension. Other harnesses report the diff --git a/services/runner/src/engines/sandbox_agent/permissions.ts b/services/runner/src/engines/sandbox_agent/permissions.ts index 3d9bc83a66..ea7cf88f8f 100644 --- a/services/runner/src/engines/sandbox_agent/permissions.ts +++ b/services/runner/src/engines/sandbox_agent/permissions.ts @@ -1,200 +1,201 @@ -import type { AgentEvent } from "../../protocol.ts"; +import type { AgentEvent, ToolPermission } from "../../protocol.ts"; import { decisionToReply, - specOf, - type ClientToolOutcome, + type ClientToolVerdict, + type PermissionDecision, type Responder, } from "../../responder.ts"; +import { + PendingApprovalLatch, + type GateDescriptor, +} from "../../permission-plan.ts"; export interface AttachPermissionResponderInput { session: any; run: { emitEvent: (event: AgentEvent) => void; events?: () => AgentEvent[] }; responder: Responder; + latch: PendingApprovalLatch; + serverPermissions?: ReadonlyMap; /** - * Called when the responder PARKS a gate (cross-turn HITL). The orchestration loop uses - * this to END the turn gracefully: a parked Claude turn never resolves `session.prompt()` - * on its own (the harness does not end the turn on an unanswered gate), so without this the - * prompt blocks forever, the sandbox leaks, and the egress never emits a `finish` frame - * (F-040). The runner cancels the in-flight prompt (via `destroySession`) so the turn ends - * paused and the resume cold-replays. Fires at most once per turn (the first park wins). + * Called when a gate pauses the turn. The orchestration loop uses this to end the turn + * gracefully because a paused Claude turn never resolves `session.prompt()` on its own. */ onPark?: () => void; - /** Diagnostic sink for the raw ACP permission ground truth (name/spec/args the harness sends). */ log?: (msg: string) => void; - /** Called on park to record the parked gate as an interaction (fire-and-forget). */ + /** Called on pause to record the pending gate as an interaction (fire-and-forget). */ onCreateInteraction?: ( token: string, toolName: string | undefined, toolArgs: unknown, ) => void; - /** - * Called when the runner consumes a stored decision and forwards it to the harness — the - * gate is resolved. Transitions the interaction (pending|responded -> resolved). Covers - * both planes: a `/interactions` answer (responded -> resolved) and a messages answer - * (pending -> resolved). Fire-and-forget. - */ + /** Called after a stored decision was successfully forwarded to the harness. */ onResolveInteraction?: (token: string) => void; } -/** - * Wire ACP permission reverse-RPC into the runner's event stream and responder policy. - * - * The default engine responder is headless today, but this boundary keeps future cross-turn - * human approval from changing the sandbox-agent session plumbing. - */ +/** Wire ACP permission reverse-RPC into the runner's event stream and responder policy. */ export function attachPermissionResponder({ session, run, responder, + latch, + serverPermissions = new Map(), onPark, log, onCreateInteraction, onResolveInteraction, }: AttachPermissionResponderInput): void { session.onPermissionRequest((req: any) => { - const id = String(req?.id ?? ""); - const availableReplies: string[] = req?.availableReplies ?? []; - const toolCall = req?.toolCall; - // NAME ANCHOR (the HITL resume-loop root cause): Claude-over-ACP names the SAME tool call - // differently across frames — the `session/update` tool_call titles it by category ("Terminal"), - // the permission request titles it by the specific invocation ("cat ~/.claude/settings.json ..."). - // Neither carries a stable `name`/`spec`. The transcript (and thus the stored approval key) - // records the tool_call name, so the cross-turn key must ALSO use it — not the permission - // frame's own drifting title. Recover the recorded tool_call name for THIS id (same id within a - // turn) and stamp it as `resolvedName`, which `permissionToolName` (responder) and - // `_approval_tool_name` (egress) both prefer, so the live re-raised key equals the stored key. - if (toolCall && typeof toolCall === "object" && !toolCall.resolvedName) { - const recorded = recordedToolName(run, toolCall.toolCallId); - if (recorded) toolCall.resolvedName = recorded; + void handleRequest(req).catch((err) => { + log?.(`[HITL] permission handling failed: ${errorMessage(err)}`); + onPark?.(); + }); + }); + + // A pause sends NO harness reply, ever. Replying `reject` would make Claude emit a failed + // tool call ("User refused permission") whose `tool_result {isError}` overwrites the + // approval prompt on the same tool-call id (the F-024 clobber). The turn instead ends via + // `onPark` (session teardown resolves the RPC as cancelled, not rejected) and the next + // turn's stored decision answers the re-raised gate. + const pauseUserApproval = (req: any, id: string, gate: GateDescriptor): void => { + if (!latch.tryAcquire()) return; + const eventId = interactionEventId(id, req?.toolCall?.toolCallId); + run.emitEvent({ + type: "interaction_request", + id: eventId, + kind: "user_approval", + payload: { + toolCallId: stringValue(req?.toolCall?.toolCallId), + toolCall: req?.toolCall, + availableReplies: stringArray(req?.availableReplies), + options: req?.options, + }, + }); + onCreateInteraction?.(eventId, gate.toolName, gate.args); + onPark?.(); + }; + + const pauseClientTool = ( + req: any, + id: string, + gate: GateDescriptor, + spec: ToolSpecLike, + ): void => { + if (!latch.tryAcquire()) return; + const toolCallId = stringValue(req?.toolCall?.toolCallId); + const eventId = interactionEventId(id, toolCallId); + run.emitEvent({ + type: "interaction_request", + id: eventId, + kind: "client_tool", + payload: { + toolCallId, + toolCall: req?.toolCall, + toolName: gate.toolName, + input: gate.args, + render: spec.render, + }, + }); + onPark?.(); + }; + + const replyPermission = async ( + id: string, + decision: PermissionDecision, + availableReplies: string[], + ): Promise => { + try { + await session.respondPermission( + id, + decisionToReply(decision, availableReplies) as any, + ); + } catch (err) { + log?.(`[HITL] reply failed id=${id} decision=${decision}: ${errorMessage(err)}`); + onPark?.(); + return; + } + onResolveInteraction?.(id); + }; + + const replyClientTool = async ( + id: string, + verdict: Exclude, + availableReplies: string[], + ): Promise => { + try { + await session.respondPermission( + id, + clientToolReply(verdict, availableReplies) as any, + ); + } catch (err) { + log?.(`[HITL] reply failed id=${id} decision=${verdict.kind}: ${errorMessage(err)}`); + onPark?.(); } - // GROUND TRUTH: exactly what the harness hands us for this gate — the resolved spec (and its - // stable name) if any, the drift-prone display fields, and the arg shape. This is the earliest - // and most authoritative HITL log; everything downstream keys off these fields. + }; + + async function handleRequest(req: any): Promise { + const id = stringValue(req?.id) ?? ""; + const availableReplies = stringArray(req?.availableReplies); + const toolCall = req?.toolCall; + const spec = resolvedSpecOf(toolCall); + const gate = buildGateDescriptor(toolCall, run, serverPermissions); + // Ground truth for HITL debugging: exactly what the harness handed us for this gate. + // The stable anchor (gate.toolName) vs the drift-prone display fields is what a live + // session needs to diagnose a resume-key mismatch; keep this greppable via `[HITL]`. if (log) { - const spec = - toolCall?.spec ?? - toolCall?.toolSpec ?? - toolCall?.resolvedTool ?? - toolCall?.tool; const args = toolCall?.rawInput ?? toolCall?.input; log( - `[HITL] ACP permission id=${id} ` + + `[HITL] ACP gate id=${id} ` + JSON.stringify({ toolCallId: toolCall?.toolCallId, - specName: spec && typeof spec === "object" ? spec.name : undefined, - specKind: spec && typeof spec === "object" ? spec.kind : undefined, - name: toolCall?.name, + anchor: gate.toolName, + specName: spec?.name, title: toolCall?.title, kind: toolCall?.kind, - argKeys: - args && typeof args === "object" - ? Object.keys(args) - : typeof args, - availableReplies, + executor: gate.executor, + argKeys: args && typeof args === "object" ? Object.keys(args) : typeof args, }), ); } - // The harness raises a permission request before running ANY gated tool. We branch on the - // tool's resolved spec: a `kind: "client"` tool is not really a sandbox permission gate — it - // is a tool whose execution belongs to the browser (e.g. `request_connection`, which renders a - // connect widget the user interacts with). So instead of approving/denying it in the sandbox, - // we forward it to the frontend as a `client_tool` interaction_request and let the responder - // decide: PARK it for a cross-turn round-trip (the browser fulfills the call, the next turn - // resumes with its result) or reply inline. Every other tool falls through to the normal - // permission gate below. - const spec = specOf(toolCall) as any; + if (spec?.kind === "client") { - const toolCallId = - typeof toolCall?.toolCallId === "string" - ? toolCall.toolCallId - : undefined; - const toolName = clientToolName(toolCall, spec); - const input = toolCall?.rawInput ?? toolCall?.input; - run.emitEvent({ - type: "interaction_request", + const verdict = await responder.onClientTool({ id, - kind: "client_tool", - payload: { - toolCallId, - toolCall, - toolName, - input, - render: spec.render, - }, + toolCallId: stringValue(toolCall?.toolCallId), + gate, + raw: req, }); - void responder - .onClientTool({ id, toolCallId, toolName, input, raw: req }) - .then((decision) => { - if (decision === "park") { - onPark?.(); - return; - } - if (!req?.id) return; - return session.respondPermission( - req.id, - clientToolReply(decision, availableReplies) as any, - ); - }) - .catch(() => {}); + if (verdict.kind === "pendingApproval" || !id) { + pauseClientTool(req, id, gate, spec); + return; + } + await replyClientTool(id, verdict, availableReplies); return; } - run.emitEvent({ - type: "interaction_request", - id, // ACP permission id -> Vercel approvalId - kind: "user_approval", - payload: { - // toolCallId of the gated tool, so the cross-turn approval reply correlates back to - // its tool call (and the #6 resume finds it). `toolCall` is the ACP ToolCallUpdate. - toolCallId: req?.toolCall?.toolCallId, - toolCall: req?.toolCall, - availableReplies, - options: req?.options, - }, + const verdict = await responder.onPermission({ + id, + availableReplies, + gate, + raw: req, }); - void responder - .onPermission({ id, availableReplies, raw: req }) - .then((decision) => { - // PARK (cross-turn HITL): send NO reply. The `interaction_request` above is the last - // word on this tool call; the next turn's stored decision resolves it. Replying - // `reject` here would make Claude emit a failed tool call ("User refused permission") - // that clobbers the approval prompt on the same tool-call id (F-024) — do NOT map park - // onto a reply. Instead signal the orchestration loop to end the turn gracefully: - // Claude never ends a turn on an unanswered gate, so the prompt would otherwise hang. - if (decision === "park") { - const toolName: string | undefined = - req?.toolCall?.name ?? req?.toolCall?.title ?? req?.toolCall?.kind; - const toolArgs: unknown = - req?.toolCall?.rawInput ?? req?.toolCall?.input; - onCreateInteraction?.(id, toolName, toolArgs); - onPark?.(); - return; - } - if (!req?.id) return; - // A stored decision is being forwarded to the harness: the gate is resolved. - onResolveInteraction?.(id); - return session.respondPermission( - req.id, - decisionToReply(decision, availableReplies) as any, - ); - }) - .catch(() => {}); - }); + if (verdict.kind === "pendingApproval" || !id) { + pauseUserApproval(req, id, gate); + return; + } + await replyPermission(id, verdict.kind, availableReplies); + } } /** - * The name the runner already recorded for this tool-call id via the `session/update` `tool_call` - * event — the SAME value the transcript folds into the stored approval key. Used to key the live - * permission gate so it matches the stored decision across the cold-replay resume (the ACP - * permission frame's own title drifts from the tool_call's, breaking the key otherwise). + * The name the runner already recorded for this tool-call id via the `session/update` + * `tool_call` event. Used to key a harness gate so it matches the stored decision across a + * cold-replay resume when the ACP permission frame's own title drifts. */ function recordedToolName( run: { events?: () => AgentEvent[] }, toolCallId: unknown, ): string | undefined { - if (typeof toolCallId !== "string" || !toolCallId || !run.events) - return undefined; - // Last match wins: a later tool_call for the same id (an arg-refresh) carries the same name. + if (typeof toolCallId !== "string" || !toolCallId || !run.events) return undefined; let name: string | undefined; for (const event of run.events()) { if (event.type === "tool_call" && event.id === toolCallId && event.name) { @@ -204,24 +205,90 @@ function recordedToolName( return name; } -function clientToolName(toolCall: any, spec: any): string | undefined { - for (const candidate of [ +function buildGateDescriptor( + toolCall: any, + run: { events?: () => AgentEvent[] }, + serverPermissions: ReadonlyMap, +): GateDescriptor { + const spec = resolvedSpecOf(toolCall); + const toolName = firstString([ spec?.name, + recordedToolName(run, toolCall?.toolCallId), toolCall?.name, toolCall?.title, toolCall?.kind, - ]) { - if (typeof candidate === "string" && candidate) return candidate; + ]); + const specPermission = toolPermission(spec?.permission); + const args = toolCall?.rawInput ?? toolCall?.input; + return { + executor: spec?.kind === "client" ? "client" : spec ? "relay" : "harness", + toolName, + specPermission, + serverPermission: spec ? undefined : serverPermissionFor(toolName, serverPermissions), + readOnlyHint: typeof spec?.readOnly === "boolean" ? spec.readOnly : undefined, + args, + }; +} + +function serverPermissionFor( + toolName: string | undefined, + serverPermissions: ReadonlyMap, +): ToolPermission | undefined { + if (!toolName?.startsWith("mcp__")) return undefined; + const rest = toolName.slice("mcp__".length); + const separator = rest.indexOf("__"); + if (separator <= 0) return undefined; + return serverPermissions.get(rest.slice(0, separator)); +} + +type ToolSpecLike = { + name?: unknown; + kind?: unknown; + permission?: unknown; + readOnly?: unknown; + render?: unknown; +}; + +function resolvedSpecOf(toolCall: any): ToolSpecLike | undefined { + const spec = toolCall?.spec ?? toolCall?.toolSpec ?? toolCall?.resolvedTool ?? toolCall?.tool; + return spec && typeof spec === "object" ? (spec as ToolSpecLike) : undefined; +} + +function firstString(values: unknown[]): string | undefined { + for (const value of values) { + if (typeof value === "string" && value) return value; } return undefined; } +function toolPermission(value: unknown): ToolPermission | undefined { + return value === "allow" || value === "ask" || value === "deny" ? value : undefined; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value ? value : undefined; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function interactionEventId(id: string, toolCallId: unknown): string { + return id || stringValue(toolCallId) || ""; +} + function clientToolReply( - decision: ClientToolOutcome, + verdict: Exclude, availableReplies: string[], ): string { - if (decision === "deny") { + if (verdict.kind === "deny") { return availableReplies.find((r) => r === "reject") ?? "reject"; } return availableReplies.find((r) => r === "once") ?? "once"; } + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/services/runner/src/responder.ts b/services/runner/src/responder.ts index d725483dc7..494b49f13f 100644 --- a/services/runner/src/responder.ts +++ b/services/runner/src/responder.ts @@ -1,144 +1,81 @@ /** * The interaction responder seam. * - * A harness (the ACP "Agent") does not only emit tool calls. It also raises typed - * reverse-RPC interaction requests that something must answer: permission gates today, - * elicitation (input) and client-side tools later. Today the sandbox-agent runner answered the - * permission gate inline with a hardcoded auto-approve. This module lifts that decision - * behind a `Responder` interface so it is pluggable: - * - * - `PolicyResponder` is the headless answer (a fixed `auto` / `deny` policy, no human). - * It reproduces the previous behavior exactly and is what `/invoke` uses. - * - `HITLResponder` is the cross-turn responder (the `/messages` HITL path): it parks an - * un-decided permission (the `interaction_request` already went to the browser), ends the - * turn, and resolves on the next turn's stored reply. With no decision and no human - * surface it falls back to the base policy, so the headless path is unchanged. The harness - * adapter does not change when the responder does. - * - * Resolution is modeled as `allow` / `deny`; the adapter maps that onto the harness's - * available ACP replies via `decisionToReply`. + * Harness permission gates and browser-fulfilled client tools are normalized into + * `GateDescriptor`s before they reach this module. The responder decides from the shared + * permission plan; adapters decide how to express each verdict on their own wire. */ import type { AgentRunRequest, ContentBlock } from "./protocol.ts"; +import { + decide, + effectivePermission, + type GateDescriptor, + type PermissionPlan, + type StoredPermissionDecisions, + type Verdict, +} from "./permission-plan.ts"; -export type PermissionPolicy = "auto" | "deny"; - -/** - * What the responder decides for one permission gate. - * - * - `allow` / `deny` are terminal: the adapter maps them onto an ACP reply via - * `decisionToReply` and the harness runs or refuses the tool this turn. - * - `park` is NOT a harness reply. It means "a human must decide; end the turn with this - * tool PENDING". On park the adapter sends NO `respondPermission`, so the harness never - * produces a refused/failed tool call, and the `interaction_request` already emitted stays - * the last word on the tool call. The next turn carries the stored decision and resolves it - * via `allow`/`deny`. This is the cross-turn HITL "park" — see `HITLResponder`. - * - * `decisionToReply` only ever sees `allow`/`deny`; `park` is handled before it (it has no ACP - * reply). Do NOT "simplify" park back to `deny`: for Claude, replying `reject` produces a - * failed tool call ("User refused permission") whose `tool_result {isError}` overwrites the - * approval prompt on the same tool-call id (the F-024 clobber bug). - */ export type PermissionDecision = "allow" | "deny"; -/** The full set of responder outcomes, including the runner-internal `park`. */ -export type ResponderOutcome = PermissionDecision | "park"; +// phase 2b deletes this (relay still consumes it) +export type PermissionPolicy = "auto" | "deny"; +// phase 2b deletes this relay vocabulary when `tools/relay.ts` is rewritten. export type ClientToolOutcome = "deny" | "park" | { output: unknown }; /** A permission gate raised by the harness, normalized from the ACP request. */ -export interface PermissionRequest { - /** The ACP permission id; reused as the `interaction_request` event id for reply matching. */ +export interface PermissionGateRequest { id: string; - /** Replies the harness offers (e.g. "always" | "once" | "reject"). */ availableReplies: string[]; - /** The original ACP request, for responders that want the tool-call detail. */ + gate: GateDescriptor; raw?: unknown; } -export interface ClientToolRequest { - /** Reused as the `interaction_request` event id for reply matching. */ +export interface ClientToolGateRequest { id: string; toolCallId?: string; - toolName?: string; - input?: unknown; + gate: GateDescriptor; raw?: unknown; } -/** - * Answers interaction requests the harness raises. Permission is the only kind wired today; - * `input` (elicitation) and `client_tool` are forward-looking and will extend this interface - * alongside the cross-turn responder. - */ -export interface Responder { - onPermission(request: PermissionRequest): Promise; - onClientTool(request: ClientToolRequest): Promise; -} - -/** Headless responder: a fixed policy, no human in the loop. Never parks (no human surface). */ -export class PolicyResponder implements Responder { - constructor(private readonly policy: PermissionPolicy) {} +export type ClientToolVerdict = + | { kind: "deny" } + | { kind: "pendingApproval" } + | { kind: "fulfilled"; output: unknown }; - async onPermission(_request: PermissionRequest): Promise { - return this.policy === "deny" ? "deny" : "allow"; - } - - async onClientTool(_request: ClientToolRequest): Promise { - return "deny"; - } +/** Answers interaction requests the harness raises. */ +export interface Responder { + onPermission(request: PermissionGateRequest): Promise; + onClientTool(request: ClientToolGateRequest): Promise; } -/** - * A lookup of approval decisions the user already made on a prior turn, keyed by an - * identifier carried on the permission request. Two keys are indexed per decision: - * - * 1. `toolCallId` — the precise, warm match. When the harness re-presents the SAME ACP - * tool-call id (same session), this resolves the exact parked call. - * 2. `parkedCallKey(name, input)` — the cold-replay anchor. A cold-replayed run rebuilds the - * session from the replayed transcript and mints a FRESH tool-call id for the re-raised - * gate, so the id no longer matches. The stable anchor is then the tool's NAME **plus - * its arguments**: the resumed call re-derives the same name and the same args, so it - * resolves; a DIFFERENT call to the same tool (different args) does NOT, so it re-prompts. - * - * Keying by the bare tool NAME alone (the prior behavior) over-authorized: an `allow` on call - * A auto-approved any later call B to the same tool, even with different/sensitive args — a - * HITL bypass. Binding the cold-replay key to name + args closes that hole while keeping the - * legitimate approve -> resume path working (resume re-raises the same name + args). - */ export type ApprovalDecisions = ReadonlyMap; /** * The cold-replay approval anchor: the tool name bound to its arguments. Resume re-derives * the same name + args (matches); a different call to the same tool has different args (no - * match -> re-prompts). - * - * Absent args (null/undefined) normalize to `{}` so a genuine NO-ARG tool still gets a stable - * key and resumes (its calls have nothing to vary, so sharing a key is the same call). This is - * a deliberate trade-off over fail-closed: the ACP wire reliably carries `rawInput` and the - * stored side carries the tool_call `input`, so a tool that takes args reports them on both - * sides (a different-args call gets a different key). The residual is only if a with-args tool - * reports empty on BOTH sides — then two such calls collide, which the reliable arg capture - * makes a non-issue. + * match -> pauses for a new decision). * - * Returns `undefined` (NO key, fail closed -> re-prompt) only when there is no tool name, or - * the args are not plain JSON (bigint / cycle / NaN/Infinity / non-plain object like Date/Map). + * Absent args (null/undefined) normalize to `{}` so a genuine no-arg tool still gets a stable + * key and resumes. Returns `undefined` only when there is no tool name, or the args are not + * plain JSON (bigint / cycle / NaN/Infinity / non-plain object like Date/Map). */ -export function parkedCallKey( +export function approvedCallKey( toolName: string | undefined, input: unknown, ): string | undefined { if (!toolName) return undefined; const args = input === null || input === undefined ? {} : input; const hash = stableArgsHash(args); - if (hash === undefined) return undefined; // non-JSON args -> fail closed (no key, re-prompt) + if (hash === undefined) return undefined; return `${toolName}#${hash}`; } /** * Order-independent, stable serialization of tool args so the same call hashes the same. - * Returns `undefined` for any value that is not plain JSON (bigint, cycles, NaN/Infinity, or - * a non-plain object like Date/Map) so the caller can fail closed rather than collide. Tool - * args arrive as JSON over the wire, so this only triggers on a malformed/hostile payload. + * Returns `undefined` for any value that is not plain JSON so the caller can fail closed + * rather than collide. */ function stableArgsHash(input: unknown): string | undefined { try { @@ -153,8 +90,7 @@ function canonicalJson(value: unknown): string { if (value === undefined) throw new Error("undefined is not JSON"); if (typeof value === "bigint") throw new Error("bigint is not JSON"); if (typeof value === "number") { - if (!Number.isFinite(value)) - throw new Error("non-finite number is not JSON"); + if (!Number.isFinite(value)) throw new Error("non-finite number is not JSON"); return JSON.stringify(value); } if (typeof value !== "object") return JSON.stringify(value); @@ -172,271 +108,87 @@ function canonicalJson(value: unknown): string { return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`; } +/** Consume-once store of approvals/denials carried in the replayed conversation. */ +export class ConversationDecisions implements StoredPermissionDecisions { + constructor(private readonly byKey: Map) {} + + /** allow|deny for this exact call (name + canonical args), consumed on first take. */ + take(gate: GateDescriptor): "allow" | "deny" | undefined { + const key = approvedCallKey(gate.toolName, gate.args); + if (!key || !this.byKey.has(key)) return undefined; + const value = this.byKey.get(key); + if (!isPermissionDecision(value)) return undefined; + this.byKey.delete(key); + return value; + } + + /** A client-tool fulfillment output for this exact call, consumed on first take. */ + takeClientOutput(gate: GateDescriptor): { found: boolean; output?: unknown } { + const key = approvedCallKey(gate.toolName, gate.args); + if (!key || !this.byKey.has(key)) return { found: false }; + const value = this.byKey.get(key); + if (isPermissionDecision(value)) return { found: false }; + this.byKey.delete(key); + return { found: true, output: value }; + } +} + /** - * The cross-turn human-in-the-loop responder for the `/messages` path. - * - * It answers a permission gate three ways, in order: - * 1. The user already decided (a stored `decisions` entry for this exact tool-call id, or - * for this tool's name + args) -> apply it. THIS IS THE RESUME PATH: turn N parks, turn - * N+1 carries the reply. The match is scoped to the SPECIFIC call (id, or name + args), - * never to all future calls of the tool by name, so an `allow` cannot leak to a later - * call with different args. - * 2. No stored decision and there is a human surface (`hasHumanSurface`) -> `park`. The - * `interaction_request` was already emitted upstream (the FE prompts), so the turn ends - * with this tool PENDING and NO harness reply (the adapter skips `respondPermission`). - * A later turn carrying the decision resolves it via branch 1. Parking by `deny` instead - * would make Claude emit a failed tool call that clobbers the approval prompt (F-024). - * 3. No stored decision and no human surface (headless `/invoke`) -> the `basePolicy` - * decision. This branch is byte-identical to `PolicyResponder`, so `/invoke` is unchanged - * (it never parks; there is no human to resolve a parked turn). + * Shared approval responder for ACP permission gates and client tools. * - * Pure: every input (decisions, base policy, surface flag) is injected; no I/O. + * Permission gates use the shared `decide()` ladder directly. Client tools are different only + * in execution shape: `allow` and `ask` both mean "forward to the browser and pause unless a + * stored browser output is already available"; `deny` refuses the call. */ -export class HITLResponder implements Responder { +export class ApprovalResponder implements Responder { constructor( - private readonly decisions: ApprovalDecisions, - private readonly basePolicy: PermissionPolicy, - private readonly hasHumanSurface: boolean, - /** Tool names in a non-converging resume loop; the next gate for one is denied (loop-breaker). */ - private readonly nonConverging: ReadonlySet = new Set(), - /** Diagnostic sink (a miss or a loop-break). No-op by default so the responder stays pure in tests. */ + private readonly plan: PermissionPlan, + private readonly decisions: ConversationDecisions, private readonly log: (msg: string) => void = () => {}, ) {} - async onPermission(request: PermissionRequest): Promise { - const toolCall = (request.raw as { toolCall?: unknown } | undefined) - ?.toolCall; - const name = permissionToolName(toolCall); - const keys = permissionRequestKeys(request); - - const stored = this.lookupPermission(request); - if (stored) { - this.log( - `[HITL] gate "${name}" -> stored ${stored} (resume matched) keys=${JSON.stringify(keys)}`, - ); - return stored; - } - // Loop-breaker: a gate with no stored decision whose tool has been approved repeatedly - // without ever executing is a non-converging cold-replay resume (the name/arg key never - // re-matched). DENY it — a clean terminal failure the model stops re-issuing — instead of - // parking forever (the infinite re-approval loop). Fail-safe under the real key fix above. - if (name && this.nonConverging.has(name)) { - this.log( - `[HITL] loop-breaker: denying "${name}" — approved repeatedly but never executed ` + - `(resume key never matched; likely name/arg drift). ` + - `keys=${JSON.stringify(keys)} identity=${gateIdentity(toolCall)}`, - ); - return "deny"; - } - if (this.hasHumanSurface) { - // First-time gate (or a fresh park after a miss). Dump the full ground-truth identity so a - // later re-raise can be compared field-by-field to see what drifted. - this.log( - `[HITL] gate "${name}" -> PARK (awaiting human) keys=${JSON.stringify(keys)} ` + - `identity=${gateIdentity(toolCall)}`, - ); - return "park"; // human must decide; end the turn, tool pending - } - return this.basePolicy === "deny" ? "deny" : "allow"; // headless: PolicyResponder parity - } - - async onClientTool(request: ClientToolRequest): Promise { - const stored = this.lookupClientTool(request); - if (stored.found) return { output: stored.output }; - if (this.hasHumanSurface) return "park"; - return "deny"; + async onPermission(request: PermissionGateRequest): Promise { + const permission = effectivePermission(request.gate, this.plan); + const verdict = decide(request.gate, this.plan, this.decisions); + this.log( + `[HITL] gate toolName=${JSON.stringify(request.gate.toolName)} ` + + `permission=${permission} outcome=${verdict.kind}`, + ); + return verdict; } - private lookupPermission( - request: PermissionRequest, - ): PermissionDecision | undefined { - const keys = permissionRequestKeys(request); - for (const key of keys) { - const decision = this.decisions.get(key); - if (isPermissionDecision(decision)) return decision; - } - // Diagnostic: a gate that misses while decisions ARE present is the resume-key-drift - // signature (the stored approve/deny key never matched the re-raised gate). Dump both so a - // single live session pins whether the NAME or the ARGS drifted. - if (this.decisions.size > 0) { - this.log( - `[HITL] gate miss keys=${JSON.stringify(keys)} ` + - `stored=${JSON.stringify([...this.decisions.keys()])}`, - ); + async onClientTool(request: ClientToolGateRequest): Promise { + const storedOutput = this.decisions.takeClientOutput(request.gate); + if (storedOutput.found) { + return { kind: "fulfilled", output: storedOutput.output }; } - return undefined; - } - private lookupClientTool(request: ClientToolRequest): { - found: boolean; - output?: unknown; - } { - for (const key of clientToolRequestKeys(request)) { - if (this.decisions.has(key)) { - const output = this.decisions.get(key); - if (!isPermissionDecision(output)) return { found: true, output }; - } - } - return { found: false }; - } -} + const permission = + request.gate.specPermission ?? + (this.plan.default === "deny" ? "deny" : "allow"); + if (permission === "deny") return { kind: "deny" }; -/** - * The identifiers a stored decision may be keyed by: the gated tool-call id (precise/warm), - * then the tool name bound to its args (the cold-replay anchor). NOTE: bare name is NOT a key - * — that would auto-approve any later call to the same tool regardless of args (a HITL - * bypass). The name is read off the ACP `ToolCallUpdate`, which carries `title`/`kind` (no - * `name` field on the live wire); the args come from `rawInput` (falling back to `input` for - * non-ACP shapes / tests). - */ -function permissionRequestKeys(request: PermissionRequest): string[] { - const keys: string[] = []; - const raw = request.raw as - | { - toolCall?: { - toolCallId?: unknown; - name?: unknown; - title?: unknown; - kind?: unknown; - rawInput?: unknown; - input?: unknown; - }; - } - | undefined; - const toolCall = raw?.toolCall; - const toolCallId = toolCall?.toolCallId; - if (typeof toolCallId === "string" && toolCallId) keys.push(toolCallId); - const name = permissionToolName(toolCall); - const argsKey = parkedCallKey(name, toolCall?.rawInput ?? toolCall?.input); - if (argsKey) keys.push(argsKey); - return keys; -} - -/** - * A compact ground-truth dump of an ACP tool call for HITL logs: every name candidate the key - * derivation sees (so a park vs. its re-raise can be diffed field-by-field) plus the arg shape. - * This is the single most useful diagnostic for the resume-loop: it shows whether the harness - * even attaches a resolved `spec` (and its name), and whether title/kind/args drift across turns. - */ -function gateIdentity(toolCall: unknown): string { - const tc = toolCall as - | { - toolCallId?: unknown; - resolvedName?: unknown; - name?: unknown; - title?: unknown; - kind?: unknown; - rawInput?: unknown; - input?: unknown; - } - | undefined; - const spec = specOf(toolCall); - const args = tc?.rawInput ?? tc?.input; - const argKeys = - args && typeof args === "object" - ? Object.keys(args as object) - : typeof args; - return JSON.stringify({ - id: tc?.toolCallId, - resolvedName: tc?.resolvedName, - specName: spec?.name, - name: tc?.name, - title: tc?.title, - kind: tc?.kind, - argKeys, - }); -} - -function clientToolRequestKeys(request: ClientToolRequest): string[] { - const keys: string[] = []; - if (request.toolCallId) keys.push(request.toolCallId); - const argsKey = parkedCallKey(request.toolName, request.input); - if (argsKey) keys.push(argsKey); - return keys; -} - -/** - * The resolved agenta tool spec attached to an ACP tool call, under any of its aliases. This - * alias chain is the crux of the tool-identity-drift fix — kept in ONE place and reused by - * `permissions.ts` so the two sides can never silently diverge and re-open the HITL loop. - */ -export function specOf(toolCall: unknown): { name?: unknown } | undefined { - const tc = toolCall as - | { - spec?: unknown; - toolSpec?: unknown; - resolvedTool?: unknown; - tool?: unknown; - } - | undefined; - const spec = tc?.spec ?? tc?.toolSpec ?? tc?.resolvedTool ?? tc?.tool; - return spec && typeof spec === "object" - ? (spec as { name?: unknown }) - : undefined; -} - -/** - * Resolve the gated tool's name for the cold-replay key. Prefer the resolved spec's canonical - * `name` — it is STABLE across turns — over the ACP display fields (`title`/`kind`), which the - * harness can vary between the park turn and the re-raise (a Claude tool has no ACP `name`, so - * the previous `name -> title -> kind` chain keyed on a drift-prone display string and the - * cross-turn key silently stopped matching -> re-park loop). The egress - * (`vercel/stream.py::_interaction_request`) prefers `spec.name` the same way, so the stored key - * and this live key agree. Falls back to the old chain when no spec is resolved (unchanged). - */ -function permissionToolName(toolCall: unknown): string | undefined { - const tc = toolCall as - | { - resolvedName?: unknown; - name?: unknown; - title?: unknown; - kind?: unknown; - } - | undefined; - // `resolvedName` (the recorded tool_call name, stamped by the engine) comes FIRST: it is the - // same value the transcript folds into the stored key, so preferring it makes the cross-turn - // resume key match. `spec.name` next (when a resolved spec exists), then the drift-prone ACP - // display fields as a last resort. - for (const candidate of [ - tc?.resolvedName, - specOf(toolCall)?.name, - tc?.name, - tc?.title, - tc?.kind, - ]) { - if (typeof candidate === "string" && candidate) return candidate; + if (permission === "ask") { + const storedDecision = this.decisions.take(request.gate); + if (storedDecision === "deny") return { kind: "deny" }; + } + return { kind: "pendingApproval" }; } - return undefined; } /** * Build the approval-decision lookup from the inbound run request's message history. * - * The signal is the converted approval reply that the Vercel adapter - * (`_approval_response_blocks`) produces: a `tool_result` content block keyed by `toolCallId` - * whose `output` is an `{ approved: boolean }` envelope. That envelope shape is unique to an - * approval response (an ordinary tool result carries the tool's real output, not an `approved` - * flag), so no new wire carrier is needed. - * - * Each decision is indexed ONLY by `parkedCallKey(name, args)` — the cold-replay anchor. The - * name/args are recovered from the matching `tool_call` block (same `toolCallId`) the egress - * folds into the transcript. - * - * Two keys deliberately do NOT appear: - * - The bare tool NAME — keying by name alone let an `allow` on one call auto-approve every - * later call to that tool, even with different args (the HITL bypass this fix closes). - * - The historical `toolCallId` — the `/messages` path is always cold-replay (a fresh - * session per turn, prior calls replayed as text), so the harness mints a NEW id for the - * re-raised gate and a stored historical id can never LEGITIMATELY match it. Keeping it - * would only add an args-blind match risk if a fresh id ever collided with a historical - * one. So we drop it: the cross-turn match is name + args, never a replayed id. + * The signal is the converted approval reply that the Vercel adapter produces: a + * `tool_result` content block keyed by `toolCallId` whose `output` is an `{ approved: + * boolean }` envelope. Each decision is indexed only by `approvedCallKey(name, args)` - the + * cold-replay anchor. The name/args are recovered from the matching `tool_call` block (same + * `toolCallId`) the egress folds into the transcript. */ export function extractApprovalDecisions( request: AgentRunRequest, ): Map { const decisions = new Map(); - // First pass: recover each tool call's name + args, keyed by its id, so an approval reply - // (which may carry only the id + the `{approved}` envelope) can be bound to name + args. const callShapeById = new Map(); for (const message of request.messages ?? []) { const content = message?.content; @@ -454,85 +206,29 @@ export function extractApprovalDecisions( const content = message?.content; if (!Array.isArray(content)) continue; for (const block of content) { - const result = parkedCallResultOf(block); + const result = approvedCallResultOf(block); if (!result.found) continue; - // Bind the cold-replay name+args key: prefer the block's own name/input, else the - // correlated tool_call block (same id). Never key by bare name or by a replayed id. const shape = block.toolCallId ? callShapeById.get(block.toolCallId) : undefined; const name = block.toolName ?? shape?.name; const input = block.input ?? shape?.input; - const argsKey = parkedCallKey(name, input); + const argsKey = approvedCallKey(name, input); if (argsKey) decisions.set(argsKey, result.output); } } return decisions; } -/** - * Tool names whose HITL approvals are NOT converging into real executions — the fingerprint of a - * cold-replay resume loop. A successful approve makes the tool RUN and emit a real `tool_result` - * on the next turn, so a healthy tool has ~one real result per approval. When a tool's `{approved: - * true}` envelopes outnumber its real results by `threshold` or more, the resume key never matched - * (name/arg drift) and the gate re-parked every turn. The caller denies the next gate for such a - * tool (a clean terminal failure) instead of parking forever. Denies (`{approved: false}`) are - * terminal, never a loop, so they are ignored. - */ -export function nonConvergingToolNames( - request: AgentRunRequest, - threshold = 3, -): Set { - const nameById = new Map(); - for (const message of request.messages ?? []) { - const content = message?.content; - if (!Array.isArray(content)) continue; - for (const block of content) { - if (block?.type === "tool_call" && block.toolCallId) { - nameById.set(block.toolCallId, block.toolName); - } - } - } - const approved = new Map(); - const executed = new Map(); - for (const message of request.messages ?? []) { - const content = message?.content; - if (!Array.isArray(content)) continue; - for (const block of content) { - if (block?.type !== "tool_result") continue; - const name = - block.toolName ?? - (block.toolCallId ? nameById.get(block.toolCallId) : undefined); - if (!name) continue; - const out = block.output; - const flag = - out && typeof out === "object" && !Array.isArray(out) - ? (out as { approved?: unknown }).approved - : undefined; - if (flag === true) approved.set(name, (approved.get(name) ?? 0) + 1); - else if (flag !== false) - executed.set(name, (executed.get(name) ?? 0) + 1); - } - } - const looping = new Set(); - for (const [name, count] of approved) { - if (count - (executed.get(name) ?? 0) >= threshold) looping.add(name); - } - return looping; -} - function isPermissionDecision(value: unknown): value is PermissionDecision { return value === "allow" || value === "deny"; } /** - * A parked call reply. Permission responses use `{ approved: boolean }`; client tools carry their - * real structured `output`. + * A paused call reply. Permission responses use `{ approved: boolean }`; client tools carry + * their real structured `output`. */ -function parkedCallResultOf(block: ContentBlock): { - found: boolean; - output?: unknown; -} { +function approvedCallResultOf(block: ContentBlock): { found: boolean; output?: unknown } { if (!block || block.type !== "tool_result") return { found: false }; const output = block.output; if ( @@ -550,10 +246,10 @@ function parkedCallResultOf(block: ContentBlock): { } /** - * Resolve the permission policy with the same precedence as before: an explicit per-run - * `permissionPolicy: "deny"` or the `SANDBOX_AGENT_DENY_PERMISSIONS` env flips to deny; the - * default is auto-allow, because backend-resolved tools are trusted and the run is headless. + * Resolve the legacy permission policy with the same precedence as before: an explicit + * per-run `permissionPolicy: "deny"` or the env override flips to deny; the default is auto. */ +// phase 2b deletes this (relay still consumes it) export function policyFromRequest(permissionPolicy?: string): PermissionPolicy { if ( permissionPolicy === "deny" || @@ -568,11 +264,11 @@ export function policyFromRequest(permissionPolicy?: string): PermissionPolicy { * Map an allow/deny decision onto the harness's available ACP replies. * * An `allow` maps to `"once"` (grant THIS call only), NOT `"always"`. `"always"` tells the - * harness to allow the tool broadly for the rest of the turn WITHOUT re-raising the gate, so a - * single approval of call A would silently authorize later calls to the same tool without - * rechecking name + args — re-opening the over-authorization hole this module closes. Every - * call must be gated individually; the headless auto-allow policy already returns `allow` per - * call, so `"once"` per call is equivalent for it and strictly safer for HITL. + * harness to allow the tool broadly for the rest of the turn WITHOUT re-raising the gate, so + * a single approval of call A would silently authorize later calls to the same tool without + * rechecking name + args. Every call must be gated individually; the headless auto-allow + * policy already returns `allow` per call, so `"once"` per call is equivalent for it and + * strictly safer for HITL. */ export function decisionToReply( decision: PermissionDecision, diff --git a/services/runner/tests/unit/responder.test.ts b/services/runner/tests/unit/responder.test.ts index eee3d06d99..687b2b0c5a 100644 --- a/services/runner/tests/unit/responder.test.ts +++ b/services/runner/tests/unit/responder.test.ts @@ -1,11 +1,6 @@ /** * Unit tests for the interaction responder seam and the otel `emitEvent` hook. * - * Covers the behavior parity of the responder (it replaces the old inline auto-approve in - * sandbox_agent.ts) and that an out-of-stream event (an `interaction_request`) routed through - * `emitEvent` lands in both the live sink and the batch `events()` log. No harness, no - * network. - * * Run: pnpm test (or: pnpm exec vitest run tests/unit/responder.test.ts) */ import { afterEach, describe, it } from "vitest"; @@ -13,24 +8,45 @@ import assert from "node:assert/strict"; import { createSandboxAgentOtel } from "../../src/tracing/otel.ts"; import type { AgentEvent, AgentRunRequest } from "../../src/protocol.ts"; +import type { GateDescriptor, PermissionPlan } from "../../src/permission-plan.ts"; import { - HITLResponder, - PolicyResponder, - parkedCallKey, + ApprovalResponder, + ConversationDecisions, + approvedCallKey, decisionToReply, extractApprovalDecisions, - nonConvergingToolNames, policyFromRequest, type PermissionDecision, - type PermissionRequest, } from "../../src/responder.ts"; -// Defensive cleanup: policyFromRequest reads this env var; never let it leak past a test -// (e.g. if an assertion throws mid-test, before the inline delete runs). afterEach(() => { delete process.env.SANDBOX_AGENT_DENY_PERMISSIONS; }); +function plan(defaultMode: PermissionPlan["default"]): PermissionPlan { + return { default: defaultMode, rules: [] }; +} + +function gate(overrides: Partial = {}): GateDescriptor { + return { + executor: "harness", + toolName: "edit", + args: { path: "a.txt" }, + ...overrides, + }; +} + +async function permissionVerdict( + responder: ApprovalResponder, + descriptor: GateDescriptor, +) { + return responder.onPermission({ + id: "perm-1", + availableReplies: ["once", "reject"], + gate: descriptor, + }); +} + describe("policyFromRequest", () => { it("honors the arg and the env override", () => { delete process.env.SANDBOX_AGENT_DENY_PERMISSIONS; @@ -47,24 +63,14 @@ describe("policyFromRequest", () => { describe("decisionToReply", () => { it("maps allow to ONCE (never always) so an approval grants only this call", () => { - // SECURITY: `always` would let the harness allow the tool broadly for the rest of the - // turn without re-gating, re-opening the over-authorization hole. allow -> once, always. assert.equal( decisionToReply("allow", ["always", "once", "reject"]), "once", "allow must NOT map to always even when always is offered", ); assert.equal(decisionToReply("allow", ["once", "reject"]), "once"); - assert.equal( - decisionToReply("allow", []), - "once", - "allow falls back to once", - ); - assert.equal( - decisionToReply("allow", ["always"]), - "once", - "with only always offered, still fall back to once (never broaden)", - ); + assert.equal(decisionToReply("allow", []), "once"); + assert.equal(decisionToReply("allow", ["always"]), "once"); }); it("maps deny onto reject", () => { @@ -72,193 +78,162 @@ describe("decisionToReply", () => { decisionToReply("deny", ["always", "once", "reject"]), "reject", ); - assert.equal( - decisionToReply("deny", []), - "reject", - "deny falls back to reject", - ); + assert.equal(decisionToReply("deny", []), "reject"); }); }); -describe("parkedCallKey", () => { +describe("approvedCallKey", () => { it("binds name + args, order-independently", () => { assert.equal( - parkedCallKey("edit", { a: 1, b: 2 }), - parkedCallKey("edit", { b: 2, a: 1 }), - "key order does not change the key", + approvedCallKey("edit", { a: 1, b: 2 }), + approvedCallKey("edit", { b: 2, a: 1 }), ); assert.notEqual( - parkedCallKey("edit", { path: "a" }), - parkedCallKey("edit", { path: "b" }), - "different args -> different key", + approvedCallKey("edit", { path: "a" }), + approvedCallKey("edit", { path: "b" }), ); assert.notEqual( - parkedCallKey("edit", { path: "a" }), - parkedCallKey("bash", { path: "a" }), - "different name -> different key", + approvedCallKey("edit", { path: "a" }), + approvedCallKey("bash", { path: "a" }), ); }); it("normalizes absent args to {} so a no-arg tool resumes", () => { - // A no-arg tool has nothing to vary; absent/null/empty all key to the same `name#{}`. - assert.ok(parkedCallKey("edit", {}), "empty-object args -> a real key"); - assert.equal(parkedCallKey("edit", undefined), parkedCallKey("edit", {})); - assert.equal(parkedCallKey("edit", null), parkedCallKey("edit", {})); - // But a WITH-args call never collapses to the no-arg key. + assert.ok(approvedCallKey("edit", {})); + assert.equal(approvedCallKey("edit", undefined), approvedCallKey("edit", {})); + assert.equal(approvedCallKey("edit", null), approvedCallKey("edit", {})); assert.notEqual( - parkedCallKey("edit", { path: "a" }), - parkedCallKey("edit", {}), - "args present -> a different key than no-args", + approvedCallKey("edit", { path: "a" }), + approvedCallKey("edit", {}), ); }); - it("returns no key (fails closed) for no name or non-JSON args", () => { - assert.equal( - parkedCallKey(undefined, { a: 1 }), - undefined, - "no name -> no key", - ); - // Non-JSON args fail closed (no key) rather than throwing or colliding. - assert.equal( - parkedCallKey("edit", 10n as unknown), - undefined, - "bigint -> no key", - ); - assert.equal(parkedCallKey("edit", new Date()), undefined, "Date -> no key"); - assert.equal(parkedCallKey("edit", { x: NaN }), undefined, "NaN -> no key"); - assert.equal(parkedCallKey("edit", new Map()), undefined, "Map -> no key"); + it("returns no key for no name or non-JSON args", () => { + assert.equal(approvedCallKey(undefined, { a: 1 }), undefined); + assert.equal(approvedCallKey("edit", 10n as unknown), undefined); + assert.equal(approvedCallKey("edit", new Date()), undefined); + assert.equal(approvedCallKey("edit", { x: NaN }), undefined); + assert.equal(approvedCallKey("edit", new Map()), undefined); }); }); -describe("PolicyResponder", () => { - it("auto allows and deny denies", async () => { - const auto = new PolicyResponder("auto"); - const deny = new PolicyResponder("deny"); - const req = { id: "p1", availableReplies: ["once", "reject"] }; - assert.equal(await auto.onPermission(req), "allow"); - assert.equal(await deny.onPermission(req), "deny"); - }); -}); - -// A permission request as the harness adapter shapes it: `raw.toolCall` carries the gated -// tool's id, name, and args (`rawInput`), which is what the responder keys a stored decision -// by. `name` is optional so a test can simulate the live ACP wire (which carries `title`/ -// `kind`, not `name`). -function permReq( - toolCallId?: string, - name?: string, - rawInput?: unknown, -): PermissionRequest { - return { - id: "perm-1", - availableReplies: ["once", "always", "reject"], - raw: { id: "perm-1", toolCall: { toolCallId, name, rawInput } }, - }; -} +describe("ApprovalResponder", () => { + it("allows and denies from the effective permission before stored decisions", async () => { + const key = approvedCallKey("edit", { path: "a.txt" })!; + const stored = new Map([[key, "allow"]]); + const deny = new ApprovalResponder(plan("deny"), new ConversationDecisions(stored)); -describe("HITLResponder", () => { - it("applies a stored decision (resume path) by tool-call id", async () => { - const decisions = new Map([["tc-1", "allow"]]); - const allow = new HITLResponder(decisions, "auto", true); - assert.equal(await allow.onPermission(permReq("tc-1", "edit")), "allow"); + assert.deepEqual(await permissionVerdict(deny, gate()), { kind: "deny" }); + assert.equal(stored.has(key), true, "effective deny does not consume stale allow"); - const denied = new Map([["tc-2", "deny"]]); - const deny = new HITLResponder(denied, "auto", true); - assert.equal(await deny.onPermission(permReq("tc-2", "edit")), "deny"); + const allow = new ApprovalResponder( + plan("allow"), + new ConversationDecisions(new Map([[key, "deny"]])), + ); + assert.deepEqual(await permissionVerdict(allow, gate()), { kind: "allow" }); }); - it("matches a stored decision by tool name + args when the id was not preserved (cold replay)", async () => { - // The parked turn approved edit({path:'a.txt'}); a cold replay mints a FRESH tool-call id - // for the SAME call, so the id no longer matches — but the name + args anchor does. - const decisions = new Map([ - [parkedCallKey("edit", { path: "a.txt" })!, "allow"], - ]); - const responder = new HITLResponder(decisions, "auto", true); - assert.equal( - await responder.onPermission( - permReq("fresh-id", "edit", { path: "a.txt" }), - ), - "allow", + it("under ask, consumes a stored allow only once", async () => { + const key = approvedCallKey("edit", { path: "a.txt" })!; + const responder = new ApprovalResponder( + plan("ask"), + new ConversationDecisions(new Map([[key, "allow"]])), ); + + assert.deepEqual(await permissionVerdict(responder, gate()), { kind: "allow" }); + assert.deepEqual(await permissionVerdict(responder, gate()), { + kind: "pendingApproval", + }); }); - it("SECURITY: approving call A does NOT auto-approve a later call B to the same tool with different args", async () => { - // The HITL bypass this guards against: a stored `allow` for edit({path:'a.txt'}) must NOT - // leak to a NEW edit call with DIFFERENT args (e.g. a sensitive path). B must re-prompt. - const decisions = new Map([ - [parkedCallKey("edit", { path: "a.txt" })!, "allow"], - ]); - const responder = new HITLResponder(decisions, "auto", true); - // Same tool name, different args, brand-new id -> no stored match -> park (re-prompt). - assert.equal( - await responder.onPermission( - permReq("call-b", "edit", { path: "/etc/shadow" }), - ), - "park", - "a different-args call to the same tool must re-prompt, not auto-approve", - ); - // And argument-order does not let B slip through under A's key. - const orderDecisions = new Map([ - [parkedCallKey("edit", { a: 1, b: 2 })!, "allow"], - ]); - const orderResponder = new HITLResponder(orderDecisions, "auto", true); - assert.equal( - await orderResponder.onPermission( - permReq("call-c", "edit", { b: 2, a: 1 }), - ), - "allow", - "the same args in a different key order is the same call (resumes)", + it("under ask, consumes a stored deny and otherwise pauses", async () => { + const key = approvedCallKey("edit", { path: "a.txt" })!; + const responder = new ApprovalResponder( + plan("ask"), + new ConversationDecisions(new Map([[key, "deny"]])), ); + + assert.deepEqual(await permissionVerdict(responder, gate()), { kind: "deny" }); + assert.deepEqual(await permissionVerdict(responder, gate()), { + kind: "pendingApproval", + }); + + const noStored = new ApprovalResponder(plan("ask"), new ConversationDecisions(new Map())); + assert.deepEqual(await permissionVerdict(noStored, gate()), { + kind: "pendingApproval", + }); }); - it("SECURITY: bare tool NAME is not a key — a stored bare-name entry never auto-approves", async () => { - // Defense in depth: even if a bare tool name somehow lands in the map, the responder must - // not honor it (it only consults the id key and the name+args key). - const decisions = new Map([["edit", "allow"]]); - const responder = new HITLResponder(decisions, "auto", true); - assert.equal( - await responder.onPermission( - permReq("fresh-id", "edit", { path: "a.txt" }), - ), - "park", - "a bare-name entry must not auto-approve a real call", + it("client tools default to browser-forward pause and explicit deny refuses", async () => { + const responder = new ApprovalResponder( + plan("ask"), + new ConversationDecisions(new Map()), + ); + const client = gate({ executor: "client", toolName: "request_connection" }); + + assert.deepEqual( + await responder.onClientTool({ id: "tool-1", gate: client }), + { kind: "pendingApproval" }, + ); + assert.deepEqual( + await responder.onClientTool({ + id: "tool-1", + gate: { ...client, specPermission: "deny" }, + }), + { kind: "deny" }, ); }); - it("parks when there is a human surface and no stored decision (NOT deny)", async () => { - // `basePolicy` is "auto" so this proves the park overrides the policy, not the policy. - // Park must NOT be `deny`: replying `reject` to Claude clobbers the approval prompt (F-024). - const responder = new HITLResponder(new Map(), "auto", true); - assert.equal(await responder.onPermission(permReq("tc-x", "edit")), "park"); + it("client tools fulfill from stored output before the permission ladder", async () => { + const key = approvedCallKey("request_connection", { integration: "slack" })!; + const output = { connected: true }; + const responder = new ApprovalResponder( + plan("deny"), + new ConversationDecisions(new Map([[key, output]])), + ); + const client = gate({ + executor: "client", + toolName: "request_connection", + args: { integration: "slack" }, + }); - // A deny basePolicy must still PARK (the human surface wins): a human can decide, so the - // turn ends pending rather than refusing the tool with a clobbering reject. - const denyBase = new HITLResponder(new Map(), "deny", true); - assert.equal(await denyBase.onPermission(permReq("tc-w", "edit")), "park"); + assert.deepEqual(await responder.onClientTool({ id: "tool-1", gate: client }), { + kind: "fulfilled", + output, + }); + assert.deepEqual(await responder.onClientTool({ id: "tool-1", gate: client }), { + kind: "deny", + }); }); - it("headless: no decision + no human surface falls back to basePolicy (PolicyResponder parity)", async () => { - const auto = new HITLResponder(new Map(), "auto", false); - const deny = new HITLResponder(new Map(), "deny", false); - assert.equal(await auto.onPermission(permReq("tc-y", "edit")), "allow"); - assert.equal(await deny.onPermission(permReq("tc-z", "edit")), "deny"); + it("client explicit ask consumes stored deny; stored allow still forwards to the browser", async () => { + const denyKey = approvedCallKey("request_connection", { integration: "slack" })!; + const denyResponder = new ApprovalResponder( + plan("allow"), + new ConversationDecisions(new Map([[denyKey, "deny"]])), + ); + const client = gate({ + executor: "client", + toolName: "request_connection", + specPermission: "ask", + args: { integration: "slack" }, + }); + assert.deepEqual(await denyResponder.onClientTool({ id: "tool-1", gate: client }), { + kind: "deny", + }); - // Byte-for-byte the same result the old headless responder produced. - const policyAuto = new PolicyResponder("auto"); - const policyDeny = new PolicyResponder("deny"); - assert.equal( - await auto.onPermission(permReq("tc-y", "edit")), - await policyAuto.onPermission(permReq("tc-y", "edit")), - ); - assert.equal( - await deny.onPermission(permReq("tc-z", "edit")), - await policyDeny.onPermission(permReq("tc-z", "edit")), + const allowResponder = new ApprovalResponder( + plan("allow"), + new ConversationDecisions(new Map([[denyKey, "allow"]])), ); + assert.deepEqual(await allowResponder.onClientTool({ id: "tool-1", gate: client }), { + kind: "pendingApproval", + }); }); }); describe("extractApprovalDecisions", () => { - it("builds the lookup from approval tool_result blocks, keyed by id and by name+args (not bare name)", () => { + it("builds the lookup from approval tool_result blocks by name+args", () => { const request: AgentRunRequest = { sessionId: "s-1", messages: [ @@ -275,7 +250,6 @@ describe("extractApprovalDecisions", () => { ], }, { - // The cross-turn approval reply the Vercel adapter produced. role: "tool", content: [ { @@ -297,135 +271,68 @@ describe("extractApprovalDecisions", () => { }; const decisions = extractApprovalDecisions(request); - // ONLY the name+args anchor is keyed (recovered from the correlated tool_call block). - assert.equal( - decisions.get(parkedCallKey("edit", { path: "a.txt" })!), - "allow", - ); - assert.equal(decisions.get(parkedCallKey("bash", { cmd: "ls" })!), "deny"); - // The bare tool NAME must NOT be a key (that was the HITL-bypass). - assert.equal(decisions.has("edit"), false, "no bare-name key"); - assert.equal(decisions.has("bash"), false, "no bare-name key"); - // The historical replayed tool-call id must NOT be a key (cold replay mints fresh ids; - // a stored historical id could only ever match a fresh one by collision -> args-blind). - assert.equal(decisions.has("tc-1"), false, "no replayed-id key"); - assert.equal(decisions.has("tc-2"), false, "no replayed-id key"); + assert.equal(decisions.get(approvedCallKey("edit", { path: "a.txt" })!), "allow"); + assert.equal(decisions.get(approvedCallKey("bash", { cmd: "ls" })!), "deny"); + assert.equal(decisions.has("edit"), false); + assert.equal(decisions.has("tc-1"), false); }); - it("ignores ordinary tool results that are not approval envelopes", () => { + it("stores correlated client-tool outputs under the same call key", () => { const request: AgentRunRequest = { messages: [ { - role: "tool", + role: "assistant", content: [ - // A real tool output, not an `{approved}` envelope. { - type: "tool_result", - toolCallId: "tc-9", - output: "the weather is 24C", + type: "tool_call", + toolCallId: "tc-client", + toolName: "request_connection", + input: { integration: "slack" }, }, - // Structured output that merely lacks `approved`. - { type: "tool_result", toolCallId: "tc-10", output: { temp: 24 } }, - // Text block: not a decision. - { type: "text", text: "hello" }, ], }, - ], - }; - - const decisions = extractApprovalDecisions(request); - assert.equal(decisions.size, 0); - }); - - it("returns an empty lookup when there are no structured messages (headless /invoke)", () => { - const request: AgentRunRequest = { - messages: [{ role: "user", content: "just a single turn" }], - }; - assert.equal(extractApprovalDecisions(request).size, 0); - }); - - it("end-to-end: an extracted decision resumes via name+args when the approval block names the tool", async () => { - const request: AgentRunRequest = { - sessionId: "s-2", - messages: [ { role: "tool", content: [ { type: "tool_result", - toolCallId: "tc-1", - toolName: "edit", - input: { path: "a.txt" }, - output: { approved: true }, + toolCallId: "tc-client", + output: { connected: true }, }, ], }, ], }; - const responder = new HITLResponder( - extractApprovalDecisions(request), - "auto", - true, // human surface present, but the stored decision wins over the park - ); - // Resolves on the name+args anchor (the replayed id is NOT a key). - assert.equal( - await responder.onPermission(permReq("tc-1", "edit", { path: "a.txt" })), - "allow", + + const decisions = extractApprovalDecisions(request); + assert.deepEqual( + decisions.get(approvedCallKey("request_connection", { integration: "slack" })!), + { connected: true }, ); }); - it("end-to-end: a cold replay (FRESH id) resumes the parked call by name+args", async () => { - // The parked call: edit({path:'a.txt'}) approved. A cold replay rebuilds the session and - // mints a fresh tool-call id; the id no longer matches, but the name + args anchor does. + it("ignores ordinary tool results that cannot be bound to a call shape", () => { const request: AgentRunRequest = { - sessionId: "s-3", messages: [ - { - role: "assistant", - content: [ - { - type: "tool_call", - toolCallId: "tc-1", - toolName: "edit", - input: { path: "a.txt" }, - }, - ], - }, { role: "tool", content: [ - // The approval reply may carry only the id + the {approved} envelope; the name + - // args are recovered from the correlated tool_call block above. - { - type: "tool_result", - toolCallId: "tc-1", - output: { approved: true }, - }, + { type: "tool_result", toolCallId: "tc-9", output: "the weather is 24C" }, + { type: "tool_result", toolCallId: "tc-10", output: { temp: 24 } }, + { type: "text", text: "hello" }, ], }, ], }; - const responder = new HITLResponder( - extractApprovalDecisions(request), - "auto", - true, - ); - // Fresh id this turn, but same name + args -> resolves (resume works). - assert.equal( - await responder.onPermission( - permReq("fresh-id", "edit", { path: "a.txt" }), - ), - "allow", - "the parked call resumes under a fresh id via the name+args anchor", - ); - // SECURITY end-to-end: a different-args call to the same tool re-prompts (no leak). - assert.equal( - await responder.onPermission( - permReq("other-id", "edit", { path: "/etc/passwd" }), - ), - "park", - "a different-args call must NOT inherit the earlier approval", - ); + + assert.equal(extractApprovalDecisions(request).size, 0); + }); + + it("returns an empty lookup when there are no structured messages", () => { + const request: AgentRunRequest = { + messages: [{ role: "user", content: "just a single turn" }], + }; + assert.equal(extractApprovalDecisions(request).size, 0); }); }); @@ -447,12 +354,9 @@ describe("emitEvent", () => { run.emitEvent(interaction); const live = emitted.find((e) => e.type === "interaction_request"); - assert.ok(live, "interaction_request flushed to the live sink"); + assert.ok(live); assert.equal((live as any).id, "p1"); - assert.ok( - run.events().some((e) => e.type === "interaction_request"), - "interaction_request also recorded in the batch log", - ); + assert.ok(run.events().some((e) => e.type === "interaction_request")); }); it("one-shot path: records in the batch log only", () => { @@ -463,255 +367,7 @@ describe("emitEvent", () => { run.start({ prompt: "hi" }); run.emitEvent({ type: "data", name: "weather", data: { temp: 24 } }); const ev = run.events().find((e) => e.type === "data"); - assert.ok(ev, "data event recorded with no live sink"); + assert.ok(ev); assert.equal((ev as any).name, "weather"); }); }); - -// A permission request whose ACP tool call carries a resolved spec (with a STABLE canonical -// name) plus drift-prone display fields (title/kind). The live wire for a Claude tool has no -// `name`, so the key must anchor on `spec.name`, not the title/kind that varies across turns. -function permReqWithSpec( - toolCallId: string, - specName: string, - displayTitle: string, - rawInput?: unknown, -): PermissionRequest { - return { - id: "perm-1", - availableReplies: ["once", "always", "reject"], - raw: { - id: "perm-1", - toolCall: { - toolCallId, - title: displayTitle, - kind: "execute", - spec: { name: specName }, - rawInput, - }, - }, - }; -} - -// The LIVE production shape (from runner logs): a Claude-over-ACP gate with NO spec and a -// permission-frame `title` that is the specific invocation ("cat ~/...") — which DIFFERS from the -// `session/update` tool_call title the transcript stored ("Terminal"). The engine recovers the -// recorded tool_call name and stamps it as `resolvedName`; the key must anchor on THAT. -function permReqResolved( - toolCallId: string, - resolvedName: string, - driftingTitle: string, - rawInput?: unknown, -): PermissionRequest { - return { - id: "perm-1", - availableReplies: ["once", "always", "reject"], - raw: { - id: "perm-1", - toolCall: { toolCallId, resolvedName, title: driftingTitle, kind: "execute", rawInput }, - }, - }; -} - -describe("HITLResponder — recorded-name anchor (the live ACP title-drift loop)", () => { - it("resumes: stored key uses the tool_call name (Terminal) while the permission title drifts (cat ...)", async () => { - // Exactly the logged failure: stored `Terminal#{command,description}` -> allow, re-raised gate - // whose ACP permission title is the full command. Without `resolvedName` the live key would be - // `cat ...#args` and never match -> re-park loop. With it, the live key is `Terminal#args`. - const args = { command: "cat ~/.claude/settings.json", description: "Read global settings" }; - const decisions = new Map([ - [parkedCallKey("Terminal", args)!, "allow"], - ]); - const responder = new HITLResponder(decisions, "auto", true); - assert.equal( - await responder.onPermission( - permReqResolved("fresh-id", "Terminal", "cat ~/.claude/settings.json", args), - ), - "allow", - "the recorded tool_call name anchors the resume despite the drifting permission title", - ); - }); - - it("the loop-breaker also matches on the recorded name (looping set is keyed the same way)", async () => { - const args = { command: "cat x" }; - const responder = new HITLResponder( - new Map(), - "auto", - true, - new Set(["Terminal"]), // nonConverging is keyed by the stored (recorded) name - () => {}, - ); - // The live gate resolves to "Terminal" (not the "cat x" title), so the breaker engages. - assert.equal( - await responder.onPermission(permReqResolved("fresh", "Terminal", "cat x", args)), - "deny", - ); - }); -}); - -describe("HITLResponder — stable spec.name anchor (name-drift fix)", () => { - it("resumes when the stored key used the spec name but the re-raised gate only has a drifting title", async () => { - // Park turn: the tool was stored under its canonical spec name (what the egress now writes). - const decisions = new Map([ - [parkedCallKey("commit_revision", { message: "hi" })!, "allow"], - ]); - const responder = new HITLResponder(decisions, "auto", true); - // Re-raise: fresh id, NO ACP `name`, a display title that differs from the spec name. - // The old `name -> title -> kind` chain would key on "Commit changes" and never match. - assert.equal( - await responder.onPermission( - permReqWithSpec("fresh-id", "commit_revision", "Commit changes", { - message: "hi", - }), - ), - "allow", - "spec.name anchors the resume even when the display title drifts", - ); - }); - - it("SECURITY: the spec-name anchor is still args-scoped (different args re-prompt)", async () => { - const decisions = new Map([ - [parkedCallKey("commit_revision", { message: "hi" })!, "allow"], - ]); - const responder = new HITLResponder(decisions, "auto", true); - assert.equal( - await responder.onPermission( - permReqWithSpec("call-b", "commit_revision", "Commit changes", { - message: "DIFFERENT", - }), - ), - "park", - "a different-args call must re-prompt even under the same spec name", - ); - }); -}); - -describe("nonConvergingToolNames — HITL resume loop detection", () => { - // Build a transcript with `approves` {approved:true} envelopes and `execs` real results for - // one tool, mirroring what the Vercel ingress folds onto each turn. - function transcriptFor( - name: string, - approves: number, - execs: number, - ): AgentRunRequest { - const content: any[] = []; - for (let i = 0; i < approves; i++) - content.push({ - type: "tool_result", - toolCallId: `a-${i}`, - toolName: name, - output: { approved: true }, - }); - for (let i = 0; i < execs; i++) - content.push({ - type: "tool_result", - toolCallId: `e-${i}`, - toolName: name, - output: "real output", - }); - return { messages: [{ role: "tool", content }] }; - } - - it("flags a tool approved repeatedly but never executed", () => { - const looping = nonConvergingToolNames(transcriptFor("commit_revision", 3, 0)); - assert.ok(looping.has("commit_revision")); - }); - - it("does NOT flag a converging tool (each approval produced a real result)", () => { - const looping = nonConvergingToolNames(transcriptFor("edit", 3, 3)); - assert.equal(looping.has("edit"), false); - }); - - it("does NOT flag below the threshold (a couple of retries are tolerated)", () => { - const looping = nonConvergingToolNames(transcriptFor("bash", 2, 0)); - assert.equal(looping.has("bash"), false); - }); - - it("recovers the tool name from the correlated tool_call when the result omits it", () => { - const request: AgentRunRequest = { - messages: [ - { - role: "assistant", - content: [ - { type: "tool_call", toolCallId: "t1", toolName: "deploy" }, - { type: "tool_call", toolCallId: "t2", toolName: "deploy" }, - { type: "tool_call", toolCallId: "t3", toolName: "deploy" }, - ], - }, - { - role: "tool", - content: [ - { type: "tool_result", toolCallId: "t1", output: { approved: true } }, - { type: "tool_result", toolCallId: "t2", output: { approved: true } }, - { type: "tool_result", toolCallId: "t3", output: { approved: true } }, - ], - }, - ], - }; - assert.ok(nonConvergingToolNames(request).has("deploy")); - }); -}); - -describe("HITLResponder — loop-breaker + diagnostics", () => { - it("DENIES a gate whose tool is non-converging, instead of parking forever", async () => { - const logs: string[] = []; - // No stored decision matches (the drift), and the tool is flagged as looping. - const responder = new HITLResponder( - new Map(), - "auto", - true, // human surface -> would normally park - new Set(["commit_revision"]), - (m) => logs.push(m), - ); - assert.equal( - await responder.onPermission( - permReqWithSpec("fresh", "commit_revision", "Commit changes", { - message: "x", - }), - ), - "deny", - "the loop-breaker denies rather than re-parking", - ); - assert.ok( - logs.some((l) => l.includes("loop-breaker")), - "the loop-break is logged for diagnostics", - ); - }); - - it("still parks a NON-looping tool with no stored decision (loop-breaker is scoped)", async () => { - const responder = new HITLResponder( - new Map(), - "auto", - true, - new Set(["commit_revision"]), - () => {}, - ); - assert.equal( - await responder.onPermission( - permReqWithSpec("fresh", "edit", "Edit file", { path: "a.txt" }), - ), - "park", - "a different tool that is not looping still parks normally", - ); - }); - - it("logs a gate MISS when decisions are present but none match (drift diagnostic)", async () => { - const logs: string[] = []; - const decisions = new Map([ - [parkedCallKey("edit", { path: "a.txt" })!, "allow"], - ]); - const responder = new HITLResponder( - decisions, - "auto", - true, - new Set(), - (m) => logs.push(m), - ); - // A gate that does not match the stored key -> park, but the miss is logged with both sides. - await responder.onPermission(permReqWithSpec("fresh", "edit", "Edit", { path: "OTHER" })); - assert.ok( - logs.some((l) => l.includes("gate miss") && l.includes("stored=")), - "the miss dumps live keys and stored keys", - ); - }); -}); diff --git a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts index 5e57f39773..897498348c 100644 --- a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts +++ b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts @@ -232,10 +232,10 @@ function fakeHarness(options: FakeOptions = {}) { sandboxRelayHost: (() => "sandbox-relay-host") as any, responderFactory: () => ({ async onPermission() { - return options.permissionDecision ?? "allow"; + return { kind: options.permissionDecision ?? "allow" } as const; }, async onClientTool() { - return "deny" as const; + return { kind: "deny" } as const; }, }), }; @@ -482,19 +482,7 @@ describe("runSandboxAgent orchestration", () => { if (!result.ok) return; assert.deepEqual( result.events?.filter((event) => event.type === "interaction_request"), - [ - { - type: "interaction_request", - id: "perm-1", - kind: "user_approval", - payload: { - toolCallId: "tool-1", - toolCall: { toolCallId: "tool-1", name: "edit" }, - availableReplies: ["once", "always", "reject"], - options: undefined, - }, - }, - ], + [], ); // allow -> once (per-call grant), never always: a turn-wide grant would skip re-gating. assert.deepEqual(calls.permissionReplies, [ @@ -848,13 +836,12 @@ describe("runSandboxAgent orchestration", () => { }); }); -// These exercise the engine's DEFAULT responder (HITLResponder) by dropping the -// `responderFactory` override the fake otherwise installs, so we test the real cross-turn -// wiring: headless parity, the park, and the resume. -describe("runSandboxAgent default HITL responder wiring", () => { +// These exercise the engine's default ApprovalResponder by dropping the `responderFactory` +// override the fake otherwise installs, so we test the real allow, pause, and resume wiring. +describe("runSandboxAgent default ApprovalResponder wiring", () => { function depsWithDefaultResponder() { const { calls, deps } = fakeHarness({ emitPermission: true }); - delete deps.responderFactory; // fall through to the engine's HITLResponder + delete deps.responderFactory; // fall through to the engine's ApprovalResponder return { calls, deps }; } @@ -880,13 +867,14 @@ describe("runSandboxAgent default HITL responder wiring", () => { ]); }); - it("human surface (/messages: sessionId set) with no decision PARKS the tool, no harness reply (F-024)", async () => { + it("effective ask with no decision pauses the tool, no harness reply (F-024)", async () => { const { calls, deps } = depsWithDefaultResponder(); const result = await runSandboxAgent( { harness: "claude", sessionId: "conv-1", + permissions: { default: "ask" }, messages: [{ role: "user", content: "edit the file" }], }, undefined, @@ -922,7 +910,7 @@ describe("runSandboxAgent default HITL responder wiring", () => { emitPermission: true, hangPrompt: true, }); - delete deps.responderFactory; // engine HITLResponder -> parks (human surface, no decision) + delete deps.responderFactory; // engine ApprovalResponder -> pauses (effective ask, no decision) return { calls, deps }; })(); @@ -930,6 +918,7 @@ describe("runSandboxAgent default HITL responder wiring", () => { { harness: "claude", sessionId: "conv-1", + permissions: { default: "ask" }, messages: [{ role: "user", content: "edit the file" }], }, undefined, @@ -971,6 +960,7 @@ describe("runSandboxAgent default HITL responder wiring", () => { { harness: "claude", sessionId: "conv-1", + permissions: { default: "ask" }, messages: [{ role: "user", content: "edit the file" }], }, undefined, @@ -995,6 +985,7 @@ describe("runSandboxAgent default HITL responder wiring", () => { { harness: "claude", sessionId: "conv-1", + permissions: { default: "ask" }, messages: [ { role: "user", content: "edit the file" }, { diff --git a/services/runner/tests/unit/sandbox-agent-permissions.test.ts b/services/runner/tests/unit/sandbox-agent-permissions.test.ts index ff44808a84..ef7a012e2a 100644 --- a/services/runner/tests/unit/sandbox-agent-permissions.test.ts +++ b/services/runner/tests/unit/sandbox-agent-permissions.test.ts @@ -7,191 +7,401 @@ import { describe, it } from "vitest"; import assert from "node:assert/strict"; import type { AgentEvent } from "../../src/protocol.ts"; -import type { PermissionRequest, Responder } from "../../src/responder.ts"; +import type { ClientToolVerdict, Responder } from "../../src/responder.ts"; +import type { Verdict } from "../../src/permission-plan.ts"; +import { PendingApprovalLatch } from "../../src/permission-plan.ts"; import { attachPermissionResponder } from "../../src/engines/sandbox_agent/permissions.ts"; function flushPromises(): Promise { return new Promise((resolve) => setImmediate(resolve)); } -describe("attachPermissionResponder", () => { - it("emits an interaction_request and answers the ACP permission", async () => { - let handler: ((req: any) => void) | undefined; - const replies: Array<{ id: string; reply: string }> = []; - const session = { +function makeSession( + respondPermission: (id: string, reply: string) => Promise | void = async () => {}, +) { + let handler: ((req: any) => void) | undefined; + return { + session: { onPermissionRequest(cb: (req: any) => void) { handler = cb; }, - async respondPermission(id: string, reply: string) { - replies.push({ id, reply }); - }, - }; + respondPermission, + }, + emit(req: any) { + handler?.(req); + }, + }; +} + +function fakeResponder( + permissionVerdict: Verdict, + clientVerdict: ClientToolVerdict = { kind: "deny" }, + seen: { permission?: any[]; client?: any[] } = {}, +): Responder { + seen.permission ??= []; + seen.client ??= []; + return { + async onPermission(request) { + seen.permission?.push(request); + return permissionVerdict; + }, + async onClientTool(request) { + seen.client?.push(request); + return clientVerdict; + }, + }; +} + +describe("attachPermissionResponder", () => { + it("allow verdict replies once and emits no interaction_request", async () => { + const replies: Array<{ id: string; reply: string }> = []; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + }); const events: AgentEvent[] = []; - const seenRequests: PermissionRequest[] = []; - const responder: Responder = { - async onPermission(request) { - seenRequests.push(request); - return "allow"; - }, - async onClientTool() { - return "deny"; - }, - }; + const seen: { permission?: any[] } = {}; attachPermissionResponder({ session, run: { emitEvent: (event) => events.push(event) }, - responder, + responder: fakeResponder({ kind: "allow" }, undefined, seen), + latch: new PendingApprovalLatch(), }); - handler?.({ + emit({ id: "perm-1", availableReplies: ["once", "always", "reject"], - toolCall: { toolCallId: "tool-1", name: "edit" }, + toolCall: { toolCallId: "tool-1", name: "edit", rawInput: { path: "a" } }, options: { cwd: "/repo" }, }); await flushPromises(); + assert.deepEqual(replies, [{ id: "perm-1", reply: "once" }]); + assert.deepEqual(events, []); + assert.equal(seen.permission?.[0].gate.toolName, "edit"); + assert.equal(seen.permission?.[0].gate.executor, "harness"); + assert.deepEqual(seen.permission?.[0].gate.args, { path: "a" }); + }); + + it("deny verdict replies reject and emits no interaction_request", async () => { + const replies: Array<{ id: string; reply: string }> = []; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + }); + const events: AgentEvent[] = []; + + attachPermissionResponder({ + session, + run: { emitEvent: (event) => events.push(event) }, + responder: fakeResponder({ kind: "deny" }), + latch: new PendingApprovalLatch(), + }); + emit({ id: "perm-1", availableReplies: ["once", "reject"] }); + await flushPromises(); + + assert.deepEqual(replies, [{ id: "perm-1", reply: "reject" }]); + assert.deepEqual(events, []); + }); + + it("pendingApproval emits, creates the interaction, parks, and sends no reply", async () => { + const replies: Array<{ id: string; reply: string }> = []; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + }); + const events: AgentEvent[] = []; + const created: Array<{ token: string; toolName?: string; args: unknown }> = []; + let parks = 0; + + attachPermissionResponder({ + session, + run: { emitEvent: (event) => events.push(event) }, + responder: fakeResponder({ kind: "pendingApproval" }), + latch: new PendingApprovalLatch(), + onPark: () => { + parks += 1; + }, + onCreateInteraction: (token, toolName, args) => { + created.push({ token, toolName, args }); + }, + }); + emit({ + id: "perm-park", + availableReplies: ["once", "always", "reject"], + toolCall: { toolCallId: "tool-9", name: "edit", input: { path: "a" } }, + options: { cwd: "/repo" }, + }); + await flushPromises(); + + assert.deepEqual(replies, []); + assert.equal(parks, 1); + assert.deepEqual(created, [ + { token: "perm-park", toolName: "edit", args: { path: "a" } }, + ]); assert.deepEqual(events, [ { type: "interaction_request", - id: "perm-1", + id: "perm-park", kind: "user_approval", payload: { - toolCallId: "tool-1", - toolCall: { toolCallId: "tool-1", name: "edit" }, + toolCallId: "tool-9", + toolCall: { toolCallId: "tool-9", name: "edit", input: { path: "a" } }, availableReplies: ["once", "always", "reject"], options: { cwd: "/repo" }, }, }, ]); - assert.deepEqual(seenRequests, [ - { - id: "perm-1", - availableReplies: ["once", "always", "reject"], - raw: { - id: "perm-1", - availableReplies: ["once", "always", "reject"], - toolCall: { toolCallId: "tool-1", name: "edit" }, - options: { cwd: "/repo" }, - }, - }, - ]); - // allow maps to ONCE, not always: a per-call approval must not broaden into a - // turn-wide grant that skips re-gating later calls (the over-authorization hole). - assert.deepEqual(replies, [{ id: "perm-1", reply: "once" }]); }); - it("stamps resolvedName from the recorded tool_call so the resume key matches (ACP title-drift fix)", async () => { - // Live shape: the `session/update` tool_call named the tool "Terminal"; the permission frame - // titles it by the specific command. The responder must see `resolvedName: "Terminal"` (the - // recorded name) so its key matches the stored `Terminal#args`, not the drifting command title. - let handler: ((req: any) => void) | undefined; - const session = { - onPermissionRequest(cb: (req: any) => void) { - handler = cb; + it("two concurrent pending gates emit and park only once through the latch", async () => { + const { session, emit } = makeSession(); + const events: AgentEvent[] = []; + let parks = 0; + + attachPermissionResponder({ + session, + run: { emitEvent: (event) => events.push(event) }, + responder: fakeResponder({ kind: "pendingApproval" }), + latch: new PendingApprovalLatch(), + onPark: () => { + parks += 1; }, - async respondPermission() {}, - }; - const recorded: AgentEvent[] = [ - { type: "tool_call", id: "tool-1", name: "Terminal", input: {} }, - ]; - const seen: PermissionRequest[] = []; - const responder: Responder = { - async onPermission(request) { - seen.push(request); - return "allow"; + }); + emit({ id: "perm-1", toolCall: { toolCallId: "tool-1", name: "edit" } }); + emit({ id: "perm-2", toolCall: { toolCallId: "tool-2", name: "bash" } }); + await flushPromises(); + + assert.equal(events.length, 1); + assert.equal((events[0] as any).id, "perm-1"); + assert.equal(parks, 1); + }); + + it("reply rejection parks and does not resolve the interaction", async () => { + const { session, emit } = makeSession(async () => { + throw new Error("daemon closed"); + }); + const resolved: string[] = []; + let parks = 0; + + attachPermissionResponder({ + session, + run: { emitEvent: () => {} }, + responder: fakeResponder({ kind: "allow" }), + latch: new PendingApprovalLatch(), + onPark: () => { + parks += 1; }, - async onClientTool() { - return "deny"; + onResolveInteraction: (token) => { + resolved.push(token); }, - }; + }); + emit({ id: "perm-1", availableReplies: ["once", "reject"] }); + await flushPromises(); + + assert.equal(parks, 1); + assert.deepEqual(resolved, []); + }); + + it("missing ACP id takes the pause path with toolCallId as event id fallback", async () => { + const replies: Array<{ id: string; reply: string }> = []; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + }); + const events: AgentEvent[] = []; + let parks = 0; + attachPermissionResponder({ session, - run: { emitEvent: () => {}, events: () => recorded }, - responder, + run: { emitEvent: (event) => events.push(event) }, + responder: fakeResponder({ kind: "allow" }), + latch: new PendingApprovalLatch(), + onPark: () => { + parks += 1; + }, }); - handler?.({ - id: "perm-1", + emit({ availableReplies: ["once", "reject"], - toolCall: { toolCallId: "tool-1", title: "cat ~/.claude/settings.json", kind: "execute" }, + toolCall: { toolCallId: "tool-fallback", name: "edit" }, }); await flushPromises(); - const tc = (seen[0]?.raw as any)?.toolCall; - assert.equal(tc.resolvedName, "Terminal", "recorded tool_call name stamped onto the gate"); + assert.deepEqual(replies, []); + assert.equal(parks, 1); + assert.equal(events.length, 1); + assert.equal((events[0] as any).id, "tool-fallback"); }); - it("parks: emits the interaction_request but sends NO harness reply (F-024 regression)", async () => { - // The park outcome must never reach the harness as a reply: a `reject` would make Claude - // emit a failed tool call ("User refused permission") whose tool_result{isError} clobbers - // the approval prompt on the same tool-call id. So on park the approval-request event is - // emitted and respondPermission is NOT called — the turn ends with the tool pending. - let handler: ((req: any) => void) | undefined; + it("client tool pending forwards to the browser and parks", async () => { const replies: Array<{ id: string; reply: string }> = []; - const session = { - onPermissionRequest(cb: (req: any) => void) { - handler = cb; - }, - async respondPermission(id: string, reply: string) { - replies.push({ id, reply }); - }, - }; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + }); const events: AgentEvent[] = []; + let parks = 0; attachPermissionResponder({ session, run: { emitEvent: (event) => events.push(event) }, - responder: { - async onPermission() { - return "park"; - }, - async onClientTool() { - return "park" as const; - }, + responder: fakeResponder( + { kind: "deny" }, + { kind: "pendingApproval" }, + ), + latch: new PendingApprovalLatch(), + onPark: () => { + parks += 1; }, }); - handler?.({ - id: "perm-park", - availableReplies: ["once", "always", "reject"], - toolCall: { toolCallId: "tool-9", name: "edit" }, + emit({ + id: "client-1", + availableReplies: ["once", "reject"], + toolCall: { + toolCallId: "tool-client", + rawInput: { integration: "slack" }, + spec: { + kind: "client", + name: "request_connection", + render: { kind: "component", component: "connect" }, + }, + }, }); await flushPromises(); - // The approval-request event IS emitted (the FE needs it to prompt) ... - assert.equal(events.length, 1); - assert.equal(events[0].type, "interaction_request"); - assert.equal((events[0] as any).id, "perm-park"); - // ... but the harness gets NO reply (no reject to clobber the prompt with). assert.deepEqual(replies, []); + assert.equal(parks, 1); + assert.deepEqual(events, [ + { + type: "interaction_request", + id: "client-1", + kind: "client_tool", + payload: { + toolCallId: "tool-client", + toolCall: { + toolCallId: "tool-client", + rawInput: { integration: "slack" }, + spec: { + kind: "client", + name: "request_connection", + render: { kind: "component", component: "connect" }, + }, + }, + toolName: "request_connection", + input: { integration: "slack" }, + render: { kind: "component", component: "connect" }, + }, + }, + ]); }); - it("does not respond when the ACP request has no id", async () => { - let handler: ((req: any) => void) | undefined; + it("client tool stored output replies instead of forwarding", async () => { const replies: Array<{ id: string; reply: string }> = []; - const session = { - onPermissionRequest(cb: (req: any) => void) { - handler = cb; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + }); + const events: AgentEvent[] = []; + + attachPermissionResponder({ + session, + run: { emitEvent: (event) => events.push(event) }, + responder: fakeResponder( + { kind: "deny" }, + { kind: "fulfilled", output: { connected: true } }, + ), + latch: new PendingApprovalLatch(), + }); + emit({ + id: "client-1", + availableReplies: ["once", "reject"], + toolCall: { + toolCallId: "tool-client", + input: { integration: "slack" }, + spec: { kind: "client", name: "request_connection" }, }, - async respondPermission(id: string, reply: string) { - replies.push({ id, reply }); + }); + await flushPromises(); + + assert.deepEqual(replies, [{ id: "client-1", reply: "once" }]); + assert.deepEqual(events, []); + }); + + it("onResolveInteraction fires only after a successful reply", async () => { + let releaseReply: (() => void) | undefined; + const replies: Array<{ id: string; reply: string }> = []; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + await new Promise((resolve) => { + releaseReply = resolve; + }); + }); + const resolved: string[] = []; + + attachPermissionResponder({ + session, + run: { emitEvent: () => {} }, + responder: fakeResponder({ kind: "allow" }), + latch: new PendingApprovalLatch(), + onResolveInteraction: (token) => { + resolved.push(token); }, + }); + emit({ id: "perm-1", availableReplies: ["once", "reject"] }); + await flushPromises(); + + assert.deepEqual(replies, [{ id: "perm-1", reply: "once" }]); + assert.deepEqual(resolved, [], "reply is still pending"); + + releaseReply?.(); + await flushPromises(); + assert.deepEqual(resolved, ["perm-1"]); + }); + + it("uses recorded tool_call name for harness gates without mutating the ACP object", async () => { + const { session, emit } = makeSession(); + const seen: { permission?: any[] } = {}; + const toolCall = { + toolCallId: "tool-1", + title: "cat ~/.claude/settings.json", + kind: "execute", + rawInput: { command: "cat ~/.claude/settings.json" }, }; attachPermissionResponder({ session, - run: { emitEvent: () => {} }, - responder: { - async onPermission() { - return "deny"; - }, - async onClientTool() { - return "deny" as const; - }, + run: { + emitEvent: () => {}, + events: () => [ + { + type: "tool_call", + id: "tool-1", + name: "Terminal", + input: { command: "cat ~/.claude/settings.json" }, + }, + ], }, + responder: fakeResponder({ kind: "allow" }, undefined, seen), + latch: new PendingApprovalLatch(), }); - handler?.({ availableReplies: ["reject"] }); + emit({ id: "perm-1", availableReplies: ["once", "reject"], toolCall }); await flushPromises(); - assert.deepEqual(replies, []); + assert.equal(seen.permission?.[0].gate.toolName, "Terminal"); + assert.equal((toolCall as any).resolvedName, undefined); + }); + + it("passes server-level MCP permissions into harness gates", async () => { + const { session, emit } = makeSession(); + const seen: { permission?: any[] } = {}; + + attachPermissionResponder({ + session, + run: { emitEvent: () => {} }, + responder: fakeResponder({ kind: "pendingApproval" }, undefined, seen), + latch: new PendingApprovalLatch(), + serverPermissions: new Map([["github", "deny"]]), + }); + emit({ + id: "perm-1", + toolCall: { toolCallId: "tool-1", name: "mcp__github__search" }, + }); + await flushPromises(); + + assert.equal(seen.permission?.[0].gate.serverPermission, "deny"); }); }); From 280452665be552f71c66ca219325baf162fd5b1c Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 15:15:47 +0200 Subject: [PATCH 10/24] feat(runner): relay enforces the shared permission plan; Pi gets relay-ask pauses (phase 2b) --- .../projects/approval-boundary/build-notes.md | 10 + services/runner/src/engines/sandbox_agent.ts | 92 +++-- services/runner/src/responder.ts | 57 +-- services/runner/src/tools/relay.ts | 97 +++-- services/runner/tests/unit/responder.test.ts | 77 ++-- .../unit/sandbox-agent-orchestration.test.ts | 18 +- .../runner/tests/unit/tool-direct.test.ts | 6 +- .../tests/unit/tool-relay-permission.test.ts | 382 +++++++++--------- 8 files changed, 422 insertions(+), 317 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index 919382a4e7..9d5992d238 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -40,6 +40,16 @@ decisions unless it contradicts an owner call". Newest last. Read together with relay must enforce permissions only when the harness does not gate (Pi) — otherwise one approval would be consumed at the gate and the relay would double-gate the same call. +- **Phase 2b landed (Codex implemented, reviewed here).** Relay enforcement behind + `RelayPermissions.enforce` (true only where the harness does not gate — Pi today), + peek-at-ACP/consume-at-relay client outputs, `resolvePermission`/`policyFromRequest`/ + the "park" verdict deleted. Review added one thing Codex correctly flagged as out of + its brief: relay pauses now seed the `/sessions/interactions` plane through the same + guarded closure the ACP path uses (every pause leaves a row, whichever gate paused). + Known minor trade-off, accepted: the relay decides permission BEFORE validating + required args, so a malformed call to an `ask` tool costs one human prompt before the + validation error surfaces on execution; deny-first avoids leaking schema info for + denied tools. - **Phase 1 landed (Codex implemented, reviewed here).** Wire types + decision module + generated truth-table tests; 417 runner tests and typecheck green, goldens untouched (the new wire field is optional until the SDK emits it in phase 3). Review note diff --git a/services/runner/src/engines/sandbox_agent.ts b/services/runner/src/engines/sandbox_agent.ts index 33989ac44d..fce50b8634 100644 --- a/services/runner/src/engines/sandbox_agent.ts +++ b/services/runner/src/engines/sandbox_agent.ts @@ -32,12 +32,12 @@ import { localRelayHost, sandboxRelayHost, startToolRelay, + type RelayPermissions, } from "../tools/relay.ts"; import { ApprovalResponder, ConversationDecisions, extractApprovalDecisions, - policyFromRequest, type Responder, } from "../responder.ts"; import { @@ -68,6 +68,7 @@ import { prepareLocalPiAssets, } from "./sandbox_agent/pi-assets.ts"; import { + decide, PendingApprovalLatch, permissionsFromRequest, type GateDescriptor, @@ -664,6 +665,56 @@ export async function runSandboxAgent( const responder = deps.responderFactory?.(request) ?? new ApprovalResponder(permissionPlan, decisions, logger); + // Every pause seeds the durable interactions plane, whichever gate paused (the ACP + // responder on Claude, the relay on Pi). Shared by both wiring sites below. + const recordPendingInteraction = ( + token: string, + toolName: string | undefined, + toolArgs: unknown, + ): void => { + const cred = runCredential(request); + if (!cred) return; + // The /interactions plane only works when respond can re-invoke THIS revision, which + // needs at least a committed workflow_revision reference. A draft (inline config, no + // committed revision) can't be re-resolved, so skip create and stay messages-only. + const references = buildWorkflowReferences(request.runContext?.workflow); + if (!references?.workflow_revision) return; + void createInteraction( + sessionId, + request.turnId ?? "", + token, + "user_approval", + { request: { tool: toolName ?? token, args: toolArgs }, references }, + () => cred, + ); + }; + // Exactly one gate per call: the harness gate on Claude, the relay on Pi. If more + // harness families arrive, move this capability split to engines/sandbox_agent/capabilities.ts. + const relayPermissions: RelayPermissions = { + enforce: plan.isPi, + decide: (gate) => decide(gate, permissionPlan, decisions), + onPendingApproval: ({ toolCallId, toolName, args }) => { + if (!latch.tryAcquire()) return; + run.emitEvent({ + type: "interaction_request", + id: toolCallId, + kind: "user_approval", + payload: { + toolCallId, + toolCall: { + toolCallId, + name: toolName, + title: toolName, + rawInput: args, + input: args, + }, + availableReplies: ["once", "reject"], + }, + }); + recordPendingInteraction(toolCallId, toolName, args); + onPark(); + }, + }; const serverPermissions = serverPermissionsFromRequest(request); attachPermissionResponder({ session, @@ -673,25 +724,7 @@ export async function runSandboxAgent( serverPermissions, log: logger, onPark, - onCreateInteraction: (token, toolName, toolArgs) => { - const cred = runCredential(request); - if (!cred) return; - // The /interactions plane only works when respond can re-invoke THIS revision, which - // needs at least a committed workflow_revision reference. A draft (inline config, no - // committed revision) can't be re-resolved, so skip create and stay messages/park-only. - const references = buildWorkflowReferences( - request.runContext?.workflow, - ); - if (!references?.workflow_revision) return; - void createInteraction( - sessionId, - request.turnId ?? "", - token, - "user_approval", - { request: { tool: toolName ?? token, args: toolArgs }, references }, - () => cred, - ); - }, + onCreateInteraction: recordPendingInteraction, onResolveInteraction: (token) => { const cred = runCredential(request); if (!cred) return; @@ -713,7 +746,7 @@ export async function runSandboxAgent( plan.relayDir, plan.toolSpecs, request.toolCallback as ToolCallbackContext | undefined, - policyFromRequest(request.permissionPolicy), + relayPermissions, request.runContext, { onClientTool: async ({ id, toolCallId, toolName, input, spec }) => { @@ -724,12 +757,15 @@ export async function runSandboxAgent( readOnlyHint: spec.readOnly, args: input, }; - const verdict = await responder.onClientTool({ - id, - toolCallId, - gate, - raw: { spec }, - }); + const verdict = await responder.onClientTool( + { + id, + toolCallId, + gate, + raw: { spec }, + }, + { consume: true }, + ); if (process.env.AGENTA_RUNNER_DEBUG_TOOLS) { logger( `[client-tool] ${toolName} id=${toolCallId} kind=${spec.kind} ` + @@ -759,7 +795,7 @@ export async function runSandboxAgent( }, }); } - return "park"; + return "pendingApproval"; }, onPark, }, diff --git a/services/runner/src/responder.ts b/services/runner/src/responder.ts index 494b49f13f..51d561f61b 100644 --- a/services/runner/src/responder.ts +++ b/services/runner/src/responder.ts @@ -18,11 +18,7 @@ import { export type PermissionDecision = "allow" | "deny"; -// phase 2b deletes this (relay still consumes it) -export type PermissionPolicy = "auto" | "deny"; - -// phase 2b deletes this relay vocabulary when `tools/relay.ts` is rewritten. -export type ClientToolOutcome = "deny" | "park" | { output: unknown }; +export type ClientToolOutcome = "deny" | "pendingApproval" | { output: unknown }; /** A permission gate raised by the harness, normalized from the ACP request. */ export interface PermissionGateRequest { @@ -47,7 +43,10 @@ export type ClientToolVerdict = /** Answers interaction requests the harness raises. */ export interface Responder { onPermission(request: PermissionGateRequest): Promise; - onClientTool(request: ClientToolGateRequest): Promise; + onClientTool( + request: ClientToolGateRequest, + opts?: { consume?: boolean }, + ): Promise; } export type ApprovalDecisions = ReadonlyMap; @@ -108,7 +107,14 @@ function canonicalJson(value: unknown): string { return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`; } -/** Consume-once store of approvals/denials carried in the replayed conversation. */ +/** + * Consume-once store of approvals/denials carried in the replayed conversation. + * + * Client-tool outputs are consume-once per fulfillment: the ACP gate only peeks to prove an + * output exists, and the relay consumes when it actually serves that output to the tool child. + * Two identical client-tool calls in one conversation still share the stored output key, matching + * the pre-redesign behavior. + */ export class ConversationDecisions implements StoredPermissionDecisions { constructor(private readonly byKey: Map) {} @@ -122,15 +128,23 @@ export class ConversationDecisions implements StoredPermissionDecisions { return value; } - /** A client-tool fulfillment output for this exact call, consumed on first take. */ - takeClientOutput(gate: GateDescriptor): { found: boolean; output?: unknown } { + /** A client-tool fulfillment output for this exact call, without consuming it. */ + peekClientOutput(gate: GateDescriptor): { found: boolean; output?: unknown } { const key = approvedCallKey(gate.toolName, gate.args); if (!key || !this.byKey.has(key)) return { found: false }; const value = this.byKey.get(key); if (isPermissionDecision(value)) return { found: false }; - this.byKey.delete(key); return { found: true, output: value }; } + + /** A client-tool fulfillment output for this exact call, consumed on first take. */ + takeClientOutput(gate: GateDescriptor): { found: boolean; output?: unknown } { + const key = approvedCallKey(gate.toolName, gate.args); + const output = this.peekClientOutput(gate); + if (!output.found || !key) return { found: false }; + this.byKey.delete(key); + return output; + } } /** @@ -157,8 +171,13 @@ export class ApprovalResponder implements Responder { return verdict; } - async onClientTool(request: ClientToolGateRequest): Promise { - const storedOutput = this.decisions.takeClientOutput(request.gate); + async onClientTool( + request: ClientToolGateRequest, + opts: { consume?: boolean } = {}, + ): Promise { + const storedOutput = opts.consume + ? this.decisions.takeClientOutput(request.gate) + : this.decisions.peekClientOutput(request.gate); if (storedOutput.found) { return { kind: "fulfilled", output: storedOutput.output }; } @@ -245,20 +264,6 @@ function approvedCallResultOf(block: ContentBlock): { found: boolean; output?: u return { found: true, output }; } -/** - * Resolve the legacy permission policy with the same precedence as before: an explicit - * per-run `permissionPolicy: "deny"` or the env override flips to deny; the default is auto. - */ -// phase 2b deletes this (relay still consumes it) -export function policyFromRequest(permissionPolicy?: string): PermissionPolicy { - if ( - permissionPolicy === "deny" || - process.env.SANDBOX_AGENT_DENY_PERMISSIONS === "true" - ) { - return "deny"; - } - return "auto"; -} /** * Map an allow/deny decision onto the harness's available ACP replies. diff --git a/services/runner/src/tools/relay.ts b/services/runner/src/tools/relay.ts index ad85d0657d..e1f1287246 100644 --- a/services/runner/src/tools/relay.ts +++ b/services/runner/src/tools/relay.ts @@ -31,7 +31,8 @@ import type { RunContext, ToolCallbackContext, } from "../protocol.ts"; -import type { ClientToolOutcome, PermissionPolicy } from "../responder.ts"; +import type { GateDescriptor, Verdict } from "../permission-plan.ts"; +import type { ClientToolOutcome } from "../responder.ts"; export const RELAY_REQ_SUFFIX = ".req.json"; export const RELAY_RES_SUFFIX = ".res.json"; @@ -59,11 +60,24 @@ export interface ClientToolRelayRequest { input: unknown; spec: ResolvedToolSpec; } +export interface RelayPermissions { + /** False when the harness raises its own gates first (Claude); the relay then executes + * what reaches it. True when the relay is the only gate (Pi). */ + enforce: boolean; + decide: (gate: GateDescriptor) => Verdict; + /** Called when an ask pauses at the relay: emit the approval event + park (engine-owned). */ + onPendingApproval: (info: { + toolCallId: string; + toolName: string; + args: unknown; + }) => void; +} + export interface ClientToolRelay { onClientTool: (request: ClientToolRelayRequest) => Promise; onPark?: (request: ClientToolRelayRequest) => void; } -const CLIENT_TOOL_PARKED = Symbol("client-tool-parked"); +const PAUSED = Symbol("paused"); function objectSchema(schema: unknown): Record | undefined { return schema && typeof schema === "object" && !Array.isArray(schema) @@ -130,26 +144,6 @@ export function sanitizeRelayId(id: string): string { export const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); -/** - * Layer 3 enforcement (S3b): resolve a resolved-tool spec's `permission` to a concrete - * runner-side decision. `allow` runs, `deny` never runs; `ask` and an UNSET permission - * degrade to the run's headless `permissionPolicy` (`auto` -> allow, `deny` -> deny). - * - * Resolved tools (code / gateway-callback) run runner-side via the relay, harness-agnostic, so - * this is where their permission is enforced (Claude builtins are enforced at Layer 1 via - * .claude/settings.json instead). Surfacing an `ask` to a live human is the cross-turn HITL - * path (S5); here `ask` is a headless run, so it collapses onto the policy. - */ -export function resolvePermission( - permission: string | undefined, - policy: PermissionPolicy, -): "allow" | "deny" { - if (permission === "allow") return "allow"; - if (permission === "deny") return "deny"; - // `ask` or unset: headless, so defer to the run policy. - // TODO(S5): surface ask to HITL instead of collapsing onto permissionPolicy. - return policy === "deny" ? "deny" : "allow"; -} export interface RelayHost { list: (dir: string) => Promise; @@ -200,12 +194,12 @@ async function executeRelayedTool( spec: ResolvedToolSpec, req: RelayRequest, callback: ToolCallbackContext | undefined, - policy: PermissionPolicy, + permissions: RelayPermissions, runContext: RunContext | undefined, clientToolRelay: ClientToolRelay | undefined, -): Promise { - assertRequiredArguments(spec, req.args); +): Promise { if (spec.kind === "client") { + assertRequiredArguments(spec, req.args); if (!clientToolRelay) { throw new Error(`client tool '${spec.name}' is browser-fulfilled and cannot be executed`); } @@ -218,26 +212,51 @@ async function executeRelayedTool( spec, }; const decision = await clientToolRelay.onClientTool(request); - if (decision === "park") { + if (decision === "pendingApproval") { clientToolRelay.onPark?.(request); - return CLIENT_TOOL_PARKED; + return PAUSED; } if (decision === "deny") { return `Client tool '${spec.name}' was denied.`; } return JSON.stringify(decision.output ?? {}); } - // Layer 3 enforcement (S3b): gate the call on the spec's permission before it runs. - // `deny` returns a refusal string (not a throw) so the harness folds it into the tool - // result and the model loop continues. `ask`/unset degrade to the headless policy. - const decision = resolvePermission(spec.permission, policy); - if (decision === "deny") { - if (spec.permission === "deny") { - return `Tool '${spec.name}' is denied by policy.`; + + if (permissions.enforce) { + const gate: GateDescriptor = { + executor: "relay", + toolName: spec.name, + specPermission: spec.permission, + readOnlyHint: spec.readOnly, + args: req.args, + }; + const verdict = permissions.decide(gate); + if (verdict.kind === "deny") { + if (spec.permission === "deny") { + return `Tool '${spec.name}' is denied by policy.`; + } + return `Tool '${spec.name}' is denied by the permission policy.`; + } + if (verdict.kind === "pendingApproval") { + permissions.onPendingApproval({ + toolCallId: req.toolCallId, + toolName: spec.name, + args: req.args, + }); + return PAUSED; } - // ask/unset that the headless policy refused. TODO(S5): surface ask to HITL. - return `Tool '${spec.name}' requires approval; denied in headless mode.`; } + + return executeAllowedRelayedTool(spec, req, callback, runContext); +} + +async function executeAllowedRelayedTool( + spec: ResolvedToolSpec, + req: RelayRequest, + callback: ToolCallbackContext | undefined, + runContext: RunContext | undefined, +): Promise { + assertRequiredArguments(spec, req.args); if (spec.kind === "code") { return runCodeTool(spec.runtime, spec.code ?? "", spec.env, req.args); } @@ -281,7 +300,7 @@ export function startToolRelay( relayDir: string, specs: ResolvedToolSpec[], callback: ToolCallbackContext | undefined, - policy: PermissionPolicy, + permissions: RelayPermissions, runContext?: RunContext, clientToolRelay?: ClientToolRelay, ): { stop: () => Promise } { @@ -302,11 +321,11 @@ export function startToolRelay( spec, { ...req, toolCallId: req.toolCallId ?? id }, callback, - policy, + permissions, runContext, clientToolRelay, ); - if (text === CLIENT_TOOL_PARKED) return; + if (text === PAUSED) return; res = { ok: true, text }; } catch (err) { res = { ok: false, error: err instanceof Error ? err.message : String(err) }; diff --git a/services/runner/tests/unit/responder.test.ts b/services/runner/tests/unit/responder.test.ts index 687b2b0c5a..4de37559a1 100644 --- a/services/runner/tests/unit/responder.test.ts +++ b/services/runner/tests/unit/responder.test.ts @@ -3,7 +3,7 @@ * * Run: pnpm test (or: pnpm exec vitest run tests/unit/responder.test.ts) */ -import { afterEach, describe, it } from "vitest"; +import { describe, it } from "vitest"; import assert from "node:assert/strict"; import { createSandboxAgentOtel } from "../../src/tracing/otel.ts"; @@ -15,14 +15,9 @@ import { approvedCallKey, decisionToReply, extractApprovalDecisions, - policyFromRequest, type PermissionDecision, } from "../../src/responder.ts"; -afterEach(() => { - delete process.env.SANDBOX_AGENT_DENY_PERMISSIONS; -}); - function plan(defaultMode: PermissionPlan["default"]): PermissionPlan { return { default: defaultMode, rules: [] }; } @@ -47,20 +42,6 @@ async function permissionVerdict( }); } -describe("policyFromRequest", () => { - it("honors the arg and the env override", () => { - delete process.env.SANDBOX_AGENT_DENY_PERMISSIONS; - assert.equal(policyFromRequest(undefined), "auto"); - assert.equal(policyFromRequest("auto"), "auto"); - assert.equal(policyFromRequest("deny"), "deny"); - - process.env.SANDBOX_AGENT_DENY_PERMISSIONS = "true"; - assert.equal(policyFromRequest(undefined), "deny", "env forces deny"); - assert.equal(policyFromRequest("auto"), "deny", "env overrides auto"); - delete process.env.SANDBOX_AGENT_DENY_PERMISSIONS; - }); -}); - describe("decisionToReply", () => { it("maps allow to ONCE (never always) so an approval grants only this call", () => { assert.equal( @@ -184,7 +165,7 @@ describe("ApprovalResponder", () => { ); }); - it("client tools fulfill from stored output before the permission ladder", async () => { + it("client tools peek at stored output by default", async () => { const key = approvedCallKey("request_connection", { integration: "slack" })!; const output = { connected: true }; const responder = new ApprovalResponder( @@ -196,16 +177,66 @@ describe("ApprovalResponder", () => { toolName: "request_connection", args: { integration: "slack" }, }); + const request = { id: "tool-1", gate: client }; - assert.deepEqual(await responder.onClientTool({ id: "tool-1", gate: client }), { + assert.deepEqual(await responder.onClientTool(request), { + kind: "fulfilled", + output, + }); + assert.deepEqual(await responder.onClientTool(request), { kind: "fulfilled", output, }); - assert.deepEqual(await responder.onClientTool({ id: "tool-1", gate: client }), { + }); + + it("client tools consume stored output when the relay fulfills", async () => { + const key = approvedCallKey("request_connection", { integration: "slack" })!; + const output = { connected: true }; + const responder = new ApprovalResponder( + plan("deny"), + new ConversationDecisions(new Map([[key, output]])), + ); + const client = gate({ + executor: "client", + toolName: "request_connection", + args: { integration: "slack" }, + }); + const request = { id: "tool-1", gate: client }; + + assert.deepEqual(await responder.onClientTool(request, { consume: true }), { + kind: "fulfilled", + output, + }); + assert.deepEqual(await responder.onClientTool(request, { consume: true }), { kind: "deny", }); }); + it("client tools support peek then consume for the Claude two-read flow", async () => { + const key = approvedCallKey("request_connection", { integration: "slack" })!; + const output = { connected: true }; + const responder = new ApprovalResponder( + plan("deny"), + new ConversationDecisions(new Map([[key, output]])), + ); + const client = gate({ + executor: "client", + toolName: "request_connection", + args: { integration: "slack" }, + }); + const request = { id: "tool-1", gate: client }; + + assert.deepEqual(await responder.onClientTool(request), { + kind: "fulfilled", + output, + }); + assert.deepEqual(await responder.onClientTool(request, { consume: true }), { + kind: "fulfilled", + output, + }); + assert.deepEqual(await responder.onClientTool(request), { kind: "deny" }); + }); + it("client explicit ask consumes stored deny; stored allow still forwards to the browser", async () => { const denyKey = approvedCallKey("request_connection", { integration: "slack" })!; const denyResponder = new ApprovalResponder( diff --git a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts index 897498348c..1c7a5c6086 100644 --- a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts +++ b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts @@ -508,18 +508,19 @@ describe("runSandboxAgent orchestration", () => { ); assert.equal(result.ok, true); - assert.deepEqual(calls.toolRelayArgs?.slice(0, 6), [ + assert.deepEqual(calls.toolRelayArgs?.slice(0, 4), [ "local-relay-host", // Relay scratch lives off the geesefs mount: host tmpdir/agenta/relay/. join(tmpdir(), "agenta", "relay", "agenta-fake-cwd"), [{ name: "server_tool", kind: "callback" }], undefined, - // Layer 3 (S3b): the resolved permission policy threaded into the relay. No - // `permissionPolicy` on the request -> the headless default `auto`. - "auto", - // No runContext on the request. - undefined, ]); + const relayPermissions = calls.toolRelayArgs?.[4] as any; + assert.equal(relayPermissions.enforce, true, "Pi enforces at the relay"); + assert.equal(typeof relayPermissions.decide, "function"); + assert.equal(typeof relayPermissions.onPendingApproval, "function"); + // No runContext on the request. + assert.equal(calls.toolRelayArgs?.[5], undefined); // Trailing arg is the relay callbacks object (client-tool + park handlers). assert.deepEqual( Object.keys((calls.toolRelayArgs?.[6] ?? {}) as object).sort(), @@ -556,6 +557,11 @@ describe("runSandboxAgent orchestration", () => { true, "the run succeeds; gateway tools reach Claude", ); + assert.equal( + (calls.toolRelayArgs?.[4] as any)?.enforce, + false, + "Claude gates before the relay, so the relay does not re-enforce", + ); const mcpServers = calls.createSessionOptions?.sessionInit?.mcpServers ?? []; assert.equal( diff --git a/services/runner/tests/unit/tool-direct.test.ts b/services/runner/tests/unit/tool-direct.test.ts index 1cbb323178..9be7c7b9e5 100644 --- a/services/runner/tests/unit/tool-direct.test.ts +++ b/services/runner/tests/unit/tool-direct.test.ts @@ -546,7 +546,11 @@ async function relayOnce( dir, [spec], callback, - "auto", + { + enforce: false, + decide: () => ({ kind: "allow" }), + onPendingApproval: () => {}, + }, runContext, ); const resPath = join(dir, `${id}.res.json`); diff --git a/services/runner/tests/unit/tool-relay-permission.test.ts b/services/runner/tests/unit/tool-relay-permission.test.ts index ae378fa40e..0e30b48ccd 100644 --- a/services/runner/tests/unit/tool-relay-permission.test.ts +++ b/services/runner/tests/unit/tool-relay-permission.test.ts @@ -1,15 +1,5 @@ /** - * Unit tests for Layer-3 permission enforcement in the runner-side tool relay (S3b). - * - * - `resolvePermission` is the pure ladder: allow/deny are honored as-is; ask/unset degrade to - * the headless permission policy (auto -> allow, deny -> deny). - * - `startToolRelay` enforces that ladder before executing a relayed tool: a `deny` spec is - * refused before execution and an `allow` code spec reaches the sidecar unsupported gate. - * - * The relay loop is driven over a real temp dir via `localRelayHost`: write a `.req.json`, - * poll for the `.res.json` the runner writes back. A `code` spec is used so execution - * needs no network or callback; the sidecar now returns a deterministic unsupported error - * instead of spawning a runtime. + * Unit tests for runner-side relay permission enforcement. * * Run: pnpm test (or: pnpm exec vitest run tests/unit/tool-relay-permission.test.ts) */ @@ -21,234 +11,238 @@ import { join } from "node:path"; import { localRelayHost, - resolvePermission, startToolRelay, + type ClientToolRelay, + type RelayPermissions, type RelayResponse, } from "../../src/tools/relay.ts"; import type { ResolvedToolSpec } from "../../src/protocol.ts"; +import { decide, type PermissionPlan } from "../../src/permission-plan.ts"; +import { + approvedCallKey, + ConversationDecisions, +} from "../../src/responder.ts"; const codeSpec = ( name: string, permission?: ResolvedToolSpec["permission"], + readOnly?: boolean, ): ResolvedToolSpec => ({ name, kind: "code", runtime: "python", - code: 'def main(**kw):\n return {"ran": True, "echo": kw}\n', + code: "def main(**kw):\n return {\"ran\": True, \"echo\": kw}\n", permission, + readOnly, }); -/** Drive one tool call through the relay loop and return the response the runner wrote. */ -async function relayOnce( - spec: ResolvedToolSpec, - policy: "auto" | "deny", - args: unknown = { a: 1 }, -): Promise { - const dir = mkdtempSync(join(tmpdir(), "agenta-relay-disp-")); +function permissionPlan(defaultMode: PermissionPlan["default"]): PermissionPlan { + return { default: defaultMode, rules: [] }; +} + +function permissions(input: { + enforce: boolean; + plan?: PermissionPlan; + decisions?: ConversationDecisions; + pending?: Array<{ toolCallId: string; toolName: string; args: unknown }>; +}): RelayPermissions { + const plan = input.plan ?? permissionPlan("allow"); + const decisions = input.decisions ?? new ConversationDecisions(new Map()); + return { + enforce: input.enforce, + decide: (gate) => decide(gate, plan, decisions), + onPendingApproval: (info) => { + input.pending?.push(info); + }, + }; +} + +async function relayOnce(input: { + spec: ResolvedToolSpec; + permissions: RelayPermissions; + args?: unknown; + id?: string; + expectResponse?: boolean; + stopWhen?: () => boolean; + clientToolRelay?: ClientToolRelay; +}): Promise { + const dir = mkdtempSync(join(tmpdir(), "agenta-relay-perm-")); try { - const id = "call-1"; + const id = input.id ?? "call-1"; writeFileSync( join(dir, `${id}.req.json`), - JSON.stringify({ toolName: spec.name, toolCallId: id, args }), + JSON.stringify({ + toolName: input.spec.name, + toolCallId: id, + args: input.args ?? { a: 1 }, + }), + ); + const relay = startToolRelay( + localRelayHost(), + dir, + [input.spec], + undefined, + input.permissions, + undefined, + input.clientToolRelay, ); - const relay = startToolRelay(localRelayHost(), dir, [spec], undefined, policy); const resPath = join(dir, `${id}.res.json`); const deadline = Date.now() + 5000; while (Date.now() < deadline && !existsSync(resPath)) { - await new Promise((r) => setTimeout(r, 20)); + if (input.stopWhen?.()) break; + await new Promise((resolve) => setTimeout(resolve, 20)); } await relay.stop(); - assert.ok(existsSync(resPath), "the relay wrote a response file"); + const wroteResponse = existsSync(resPath); + if (input.expectResponse === false) { + assert.equal(wroteResponse, false, "the relay did not write a response file"); + return undefined; + } + assert.ok(wroteResponse, "the relay wrote a response file"); return JSON.parse(readFileSync(resPath, "utf-8")) as RelayResponse; } finally { rmSync(dir, { recursive: true, force: true }); } } -describe("resolvePermission", () => { - const cases: Array<[ResolvedToolSpec["permission"], "auto" | "deny", "allow" | "deny"]> = [ - ["allow", "auto", "allow"], - ["allow", "deny", "allow"], - ["deny", "auto", "deny"], - ["deny", "deny", "deny"], - ["ask", "auto", "allow"], - ["ask", "deny", "deny"], - [undefined, "auto", "allow"], - [undefined, "deny", "deny"], - // A garbage/unrecognized permission must fall to the policy (never auto-allow). - ["bogus" as ResolvedToolSpec["permission"], "auto", "allow"], - ["bogus" as ResolvedToolSpec["permission"], "deny", "deny"], - ]; - - for (const [permission, policy, expected] of cases) { - it(`permission=${permission ?? "unset"} policy=${policy} -> ${expected}`, () => { - assert.equal(resolvePermission(permission, policy), expected); - }); - } -}); +function assertCodeToolExecuted(res: RelayResponse | undefined): void { + assert.equal(res?.ok, false); + assert.match(res?.error ?? "", /Code tools are not supported by the sidecar\./); +} describe("startToolRelay permission enforcement", () => { - it("refuses a deny spec without executing its code", async () => { - const res = await relayOnce(codeSpec("blocked", "deny"), "auto"); - assert.equal(res.ok, true, "a policy refusal rides as an ok tool result, not an error"); - assert.equal(res.text, "Tool 'blocked' is denied by policy."); - // The refusal string is the whole result: the snippet's `{"ran": true}` never appears. - assert.ok(!String(res.text).includes("ran"), "the denied tool's code did not run"); + it("enforce=false executes an ask spec without pausing", async () => { + const pending: Array<{ toolCallId: string; toolName: string; args: unknown }> = []; + const res = await relayOnce({ + spec: codeSpec("needs_approval", "ask"), + permissions: permissions({ + enforce: false, + plan: permissionPlan("ask"), + pending, + }), + }); + + assertCodeToolExecuted(res); + assert.deepEqual(pending, []); }); - it("returns unsupported for an allow code spec", async () => { - const res = await relayOnce(codeSpec("permitted", "allow"), "auto"); - assert.equal(res.ok, false); - assert.match(res.error ?? "", /Code tools are not supported by the sidecar\./); + it("enforce=true allows allowed tools and refuses authored deny distinctly", async () => { + const pending: Array<{ toolCallId: string; toolName: string; args: unknown }> = []; + const allow = await relayOnce({ + spec: codeSpec("permitted", "allow"), + permissions: permissions({ enforce: true, plan: permissionPlan("ask"), pending }), + }); + assertCodeToolExecuted(allow); + + const deny = await relayOnce({ + spec: codeSpec("blocked", "deny"), + permissions: permissions({ enforce: true, plan: permissionPlan("allow"), pending }), + }); + assert.equal(deny?.ok, true); + assert.equal(deny?.text, "Tool 'blocked' is denied by policy."); + assert.deepEqual(pending, []); }); - it("rejects missing required args before executing a relayed tool", async () => { - const spec = { - ...codeSpec("needs_args", "allow"), - input_schema: { - type: "object", - properties: { required_value: { type: "string" } }, - required: ["required_value"], - }, - } as ResolvedToolSpec; - const res = await relayOnce(spec, "auto", {}); - assert.equal(res.ok, false); - assert.match(res.error ?? "", /missing required argument\(s\): required_value/); - assert.ok( - !String(res.error).includes("Code tools are not supported"), - "validation failed before sidecar code execution", + it("enforce=true refuses policy deny with the permission-policy text", async () => { + const res = await relayOnce({ + spec: codeSpec("locked_down"), + permissions: permissions({ enforce: true, plan: permissionPlan("deny") }), + }); + + assert.equal(res?.ok, true); + assert.equal( + res?.text, + "Tool 'locked_down' is denied by the permission policy.", ); }); - it("refuses an unset spec when the headless policy is deny", async () => { - const res = await relayOnce(codeSpec("gated", undefined), "deny"); - assert.equal(res.ok, true); - assert.equal(res.text, "Tool 'gated' requires approval; denied in headless mode."); + it("ask with no stored decision pauses without writing a response or executing", async () => { + const pending: Array<{ toolCallId: string; toolName: string; args: unknown }> = []; + await relayOnce({ + spec: codeSpec("approval_needed", "ask"), + permissions: permissions({ enforce: true, plan: permissionPlan("allow"), pending }), + expectResponse: false, + stopWhen: () => pending.length === 1, + }); + + assert.deepEqual(pending, [ + { toolCallId: "call-1", toolName: "approval_needed", args: { a: 1 } }, + ]); }); - it("parks a browser-fulfilled client tool without writing a relay response", async () => { - const dir = mkdtempSync(join(tmpdir(), "agenta-relay-client-")); - try { - const id = "call-client"; - const resPath = join(dir, `${id}.res.json`); - let parked = false; - let seenInput: unknown; - writeFileSync( - join(dir, `${id}.req.json`), - JSON.stringify({ - toolName: "request_connection", - toolCallId: id, - args: { integration: "slack" }, - }), - ); - let resolveHandled: () => void = () => {}; - const handled = new Promise((resolve) => { - resolveHandled = resolve; - }); - const relay = startToolRelay( - localRelayHost(), - dir, - [{ name: "request_connection", kind: "client" }], - undefined, - "auto", - undefined, - { - onClientTool: async (request) => { - seenInput = request.input; - return "park"; - }, - onPark: () => { - parked = true; - resolveHandled(); - }, - }, - ); - await handled; - await relay.stop(); - assert.equal(parked, true); - assert.deepEqual(seenInput, { integration: "slack" }); - assert.equal(existsSync(resPath), false); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + it("ask with a stored allow executes once and consumes the stored decision", async () => { + const key = approvedCallKey("approval_needed", { a: 1 })!; + const pending: Array<{ toolCallId: string; toolName: string; args: unknown }> = []; + const relayPermissions = permissions({ + enforce: true, + plan: permissionPlan("allow"), + decisions: new ConversationDecisions(new Map([[key, "allow"]])), + pending, + }); + + const first = await relayOnce({ + id: "call-1", + spec: codeSpec("approval_needed", "ask"), + permissions: relayPermissions, + }); + assertCodeToolExecuted(first); + assert.deepEqual(pending, []); + + await relayOnce({ + id: "call-2", + spec: codeSpec("approval_needed", "ask"), + permissions: relayPermissions, + expectResponse: false, + stopWhen: () => pending.length === 1, + }); + assert.deepEqual(pending, [ + { toolCallId: "call-2", toolName: "approval_needed", args: { a: 1 } }, + ]); }); - it("still parks a browser-fulfilled client tool under deny permission policy", async () => { - const dir = mkdtempSync(join(tmpdir(), "agenta-relay-client-")); - try { - const id = "call-client"; - let parked = false; - writeFileSync( - join(dir, `${id}.req.json`), - JSON.stringify({ - toolName: "request_connection", - toolCallId: id, - args: { integration: "slack" }, - }), - ); - let resolveHandled: () => void = () => {}; - const handled = new Promise((resolve) => { - resolveHandled = resolve; - }); - const relay = startToolRelay( - localRelayHost(), - dir, - [{ name: "request_connection", kind: "client" }], - undefined, - "deny", - undefined, - { - onClientTool: async () => "park", - onPark: () => { - parked = true; - resolveHandled(); - }, - }, - ); - await handled; - await relay.stop(); - assert.equal(parked, true); - assert.equal(existsSync(join(dir, `${id}.res.json`)), false); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + it("allow_reads executes read-hinted tools and pauses tools without a read hint", async () => { + const pending: Array<{ toolCallId: string; toolName: string; args: unknown }> = []; + const relayPermissions = permissions({ + enforce: true, + plan: permissionPlan("allow_reads"), + pending, + }); + + const read = await relayOnce({ + id: "read-call", + spec: codeSpec("read_tool", undefined, true), + permissions: relayPermissions, + }); + assertCodeToolExecuted(read); + + await relayOnce({ + id: "write-call", + spec: codeSpec("write_tool"), + permissions: relayPermissions, + expectResponse: false, + stopWhen: () => pending.length === 1, + }); + assert.deepEqual(pending, [ + { toolCallId: "write-call", toolName: "write_tool", args: { a: 1 } }, + ]); }); - it("writes stored client-tool output when the browser already fulfilled it", async () => { - const dir = mkdtempSync(join(tmpdir(), "agenta-relay-client-")); - try { - const id = "call-client"; - writeFileSync( - join(dir, `${id}.req.json`), - JSON.stringify({ - toolName: "request_connection", - toolCallId: id, - args: { integration: "slack" }, - }), - ); - const relay = startToolRelay( - localRelayHost(), - dir, - [{ name: "request_connection", kind: "client" }], - undefined, - "auto", - undefined, - { - onClientTool: async () => ({ output: { connected: true } }), + it("client tools use pendingApproval to park without writing a relay response", async () => { + const parked: string[] = []; + await relayOnce({ + spec: { name: "request_connection", kind: "client" }, + permissions: permissions({ enforce: true }), + args: { integration: "slack" }, + expectResponse: false, + stopWhen: () => parked.length === 1, + clientToolRelay: { + onClientTool: async () => "pendingApproval", + onPark: (request) => { + parked.push(request.toolCallId); }, - ); - const resPath = join(dir, `${id}.res.json`); - const deadline = Date.now() + 5000; - while (Date.now() < deadline && !existsSync(resPath)) { - await new Promise((r) => setTimeout(r, 20)); - } - await relay.stop(); - assert.ok(existsSync(resPath), "the relay wrote a response file"); - const res = JSON.parse(readFileSync(resPath, "utf-8")) as RelayResponse; - assert.equal(res.ok, true); - assert.equal(res.text, JSON.stringify({ connected: true })); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + }, + }); + + assert.deepEqual(parked, ["call-1"]); }); }); From ec9273d336b2a6e9014755f7b34e36b70ea8d320 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 15:48:09 +0200 Subject: [PATCH 11/24] feat(sdk): assemble and ship the permission plan; delete needs_approval (phase 3) --- .../projects/approval-boundary/build-notes.md | 19 +++ sdks/python/agenta/sdk/agents/__init__.py | 4 +- .../sdk/agents/adapters/claude_settings.py | 57 +++----- .../agenta/sdk/agents/adapters/harnesses.py | 13 +- sdks/python/agenta/sdk/agents/dtos.py | 81 +++++++----- sdks/python/agenta/sdk/agents/mcp/models.py | 29 ++-- .../agenta/sdk/agents/permission_rules.py | 52 ++++++++ .../agenta/sdk/agents/platform/gateway.py | 1 - .../sdk/agents/platform/platform_tools.py | 19 +-- .../agenta/sdk/agents/platform/workflow.py | 1 - sdks/python/agenta/sdk/agents/tools/compat.py | 12 +- sdks/python/agenta/sdk/agents/tools/models.py | 125 ++++++++---------- .../agenta/sdk/agents/tools/resolver.py | 3 +- sdks/python/agenta/sdk/agents/wire_models.py | 17 ++- sdks/python/agenta/sdk/utils/types.py | 39 +++--- .../agents/adapters/test_claude_settings.py | 30 ++++- .../agents/golden/run_request.claude.json | 14 +- .../agents/golden/run_request.pi_core.json | 5 +- .../pytest/unit/agents/mcp/test_resolver.py | 5 +- .../unit/agents/platform/test_gateway_http.py | 12 +- .../agents/platform/test_workflow_resolver.py | 2 - .../unit/agents/test_dtos_agent_template.py | 36 ++++- .../unit/agents/test_dtos_harness_configs.py | 36 ++++- .../unit/agents/test_harness_adapters.py | 6 +- .../pytest/unit/agents/test_wire_contract.py | 84 +++++++++--- .../pytest/unit/agents/tools/test_models.py | 77 +++++------ .../pytest/unit/agents/tools/test_parsing.py | 45 ++----- .../pytest/unit/agents/tools/test_resolver.py | 7 +- .../unit/test_normalizer_passthrough.py | 52 ++++++++ .../unit/test_skill_template_catalog.py | 2 +- services/oss/src/agent/app.py | 2 +- .../unit/agent/test_default_agent_template.py | 2 +- .../pytest/unit/agent/test_invoke_handler.py | 34 ++--- .../runner/tests/unit/wire-contract.test.ts | 26 ++-- 34 files changed, 558 insertions(+), 391 deletions(-) create mode 100644 sdks/python/agenta/sdk/agents/permission_rules.py diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index 9d5992d238..d193d9532d 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -40,6 +40,25 @@ decisions unless it contradicts an owner call". Newest last. Read together with relay must enforce permissions only when the harness does not gate (Pi) — otherwise one approval would be consumed at the gate and the relay would double-gate the same call. +- **Phase 3 landed (Codex implemented, reviewed here).** SDK assembles and ships + `permissions: {default, rules}` on both harnesses; `needs_approval` + aliases deleted + (inbound legacy keys are dropped tolerantly, POC dev-DB drafts exist); shared parse in + the new `permission_rules.py` feeds both the wire rules and the Claude settings + renderer (`mcp__*` stays settings-only); one `effective_permission(spec, read_only, + mode)` helper mirrors the runner semantics; goldens flipped (`permissionPolicy` and + `needsApproval` gone from fixtures). Review fixes: legacy-key literals were written as + string concatenations to sneak past the done-check grep — made literal (greppability + beats a clean grep report). Codex's judgment calls accepted: `annotate_trace` + classified `read_only=False` (it mutates trace metadata). Known-red note: 10 + `test_supported_llm_models.py` failures are pre-existing on the branch (the OpenRouter + model-list refresh pins IDs the installed LiteLLM registry lacks) — unrelated, + untouched. +- **H3 (daemon permission-id scheme) closed as bounded.** The sandbox-agent bundle is + minified and the id generator was not conclusively identifiable, but the exposure is + bounded either way: every turn runs a fresh session (cold replay), interaction rows + are namespaced by `turnId` at create, and stored decisions are keyed by name + + canonical args (never by a replayed ACP id). A per-session counter therefore cannot + cross-match turns. No token namespacing added. - **Phase 2b landed (Codex implemented, reviewed here).** Relay enforcement behind `RelayPermissions.enforce` (true only where the harness does not gate — Pi today), peek-at-ACP/consume-at-relay client outputs, `resolvePermission`/`policyFromRequest`/ diff --git a/sdks/python/agenta/sdk/agents/__init__.py b/sdks/python/agenta/sdk/agents/__init__.py index 128a187c79..27598e958a 100644 --- a/sdks/python/agenta/sdk/agents/__init__.py +++ b/sdks/python/agenta/sdk/agents/__init__.py @@ -66,7 +66,7 @@ HarnessType, Message, NetworkEgress, - PermissionPolicy, + PermissionMode, PiAgentTemplate, RunContext, RunContextReference, @@ -176,7 +176,7 @@ "RunContextWorkflow", "RunContextTrace", "ToolCallback", - "PermissionPolicy", + "PermissionMode", "SandboxPermission", "NetworkEgress", # Canonical tools API diff --git a/sdks/python/agenta/sdk/agents/adapters/claude_settings.py b/sdks/python/agenta/sdk/agents/adapters/claude_settings.py index 4b0534ee8b..f5207652ad 100644 --- a/sdks/python/agenta/sdk/agents/adapters/claude_settings.py +++ b/sdks/python/agenta/sdk/agents/adapters/claude_settings.py @@ -27,8 +27,7 @@ runs on Claude instead of always parking. ``ask``/unset emits no allow rule (the gate stays raised -> HITL park preserved); ``deny`` emits a deny rule (which also closes a local-Claude execution gap). ``client`` tools are browser-fulfilled, never delivered over this channel, so they are -excluded. NOTE: this does NOT make ``permission_policy:"auto"`` a blanket bypass; ``auto`` still -means "gate -> HITL/policy". +excluded. The runner policy supplies the default permission when a tool has no explicit value. """ from __future__ import annotations @@ -36,9 +35,13 @@ import json from typing import Any, Dict, List, Sequence +from ..permission_rules import CLAUDE_PERMISSION_MODES, parse_author_permissions +from ..tools.models import PermissionMode, effective_permission + # Claude Code's four permission modes (its ``permissions.defaultMode``); any other authored value # is dropped. -PERMISSION_MODES = frozenset({"default", "acceptEdits", "plan", "bypassPermissions"}) +PERMISSION_MODES = CLAUDE_PERMISSION_MODES +_parse_author_permissions = parse_author_permissions # Where the rendered settings land, relative to the session cwd. SETTINGS_PATH = ".claude/settings.json" @@ -55,33 +58,6 @@ INTERNAL_TOOL_MCP_SERVER = "agenta-tools" -def _string_list(value: Any) -> List[str]: - """Keep only the string entries of an authored allow/deny/ask value; default to ``[]``.""" - if not isinstance(value, list): - return [] - return [item for item in value if isinstance(item, str)] - - -def _parse_author_permissions(slice_: Any) -> Dict[str, Any]: - """Parse the untyped author block from the harness's ``permissions`` slice. - - ``default_mode`` (also accepted as ``defaultMode``) survives only when it is one of the four - valid modes; ``allow``/``deny``/``ask`` become string lists. This is where the claude-specific - knowledge of that slice lives. Returns ``{mode?, allow, deny, ask}`` (mode omitted when unset - or invalid). - """ - if not isinstance(slice_, dict): - return {"allow": [], "deny": [], "ask": []} - out: Dict[str, Any] = {} - mode = slice_.get("default_mode", slice_.get("defaultMode")) - if isinstance(mode, str) and mode in PERMISSION_MODES: - out["mode"] = mode - out["allow"] = _string_list(slice_.get("allow")) - out["deny"] = _string_list(slice_.get("deny")) - out["ask"] = _string_list(slice_.get("ask")) - return out - - def _dedupe(values: Sequence[str]) -> List[str]: """Dedupe in first-seen order, dropping falsy entries.""" seen: set[str] = set() @@ -148,16 +124,18 @@ def _rules_from_mcp_permissions(mcp_servers: Any) -> Dict[str, List[str]]: return {"allow": allow, "ask": ask, "deny": deny} -def _rules_from_tool_specs(tool_specs: Any) -> Dict[str, List[str]]: +def _rules_from_tool_specs( + tool_specs: Any, permission_default: PermissionMode +) -> Dict[str, List[str]]: """Derive per-tool Claude rules from each resolved EXECUTABLE tool's Layer-3 ``permission`` (F-046). Mirrors :func:`_rules_from_mcp_permissions`, but per-tool against the fixed internal server name ``agenta-tools``: a callback/code tool is delivered to Claude as a tool of that MCP server, so its rule is ``mcp__agenta-tools__``. The tool's :meth:`ToolSpec.effective_permission` - (the single source of the allow/ask/deny ladder) routes it to the matching list; a tool whose - effective permission is unset (``None``) contributes nothing (falls back to the global - ``permission_policy``). ``client`` tools are browser-fulfilled and never delivered over this - channel, so they are excluded (this mirrors the runner's ``mcp-bridge`` filter). Accepts a list + (the single source of the allow/ask/deny ladder) routes it to the matching list. Unset tools + only render a rule when the runner mode needs an explicit Claude allow/deny rule. ``client`` + tools are browser-fulfilled and never delivered over this channel, so they are excluded (this + mirrors the runner's ``mcp-bridge`` filter). Accepts a list of :class:`~agenta.sdk.agents.tools.models.ToolSpec` or plain dicts (coerced to a spec so the same permission ladder applies). """ @@ -176,8 +154,10 @@ def _rules_from_tool_specs(tool_specs: Any) -> Dict[str, List[str]]: continue if spec.kind == "client": continue - permission = spec.effective_permission() - if not permission: + permission = effective_permission( + spec.permission, spec.read_only, permission_default + ) + if spec.permission is None and permission == "ask": continue rule = f"mcp__{INTERNAL_TOOL_MCP_SERVER}__{spec.name}" if permission == "allow": @@ -203,6 +183,7 @@ def build_claude_settings_files( sandbox_permission: Any = None, mcp_servers: Any = None, tool_specs: Any = None, + permission_default: PermissionMode = "allow_reads", ) -> List[Dict[str, str]]: """Build the Claude ``settings.json`` as a generic ``harnessFiles`` entry, or ``[]`` if none. @@ -223,7 +204,7 @@ def build_claude_settings_files( # keeps first-seen order, so an author rule wins its position and derived rules append. sandbox_rules = _rules_from_sandbox_permission(sandbox_permission) mcp_rules = _rules_from_mcp_permissions(mcp_servers) - tool_rules = _rules_from_tool_specs(tool_specs) + tool_rules = _rules_from_tool_specs(tool_specs, permission_default) allow = _dedupe( [*author["allow"], *mcp_rules.get("allow", []), *tool_rules.get("allow", [])] diff --git a/sdks/python/agenta/sdk/agents/adapters/harnesses.py b/sdks/python/agenta/sdk/agents/adapters/harnesses.py index ad43106f47..af959e9230 100644 --- a/sdks/python/agenta/sdk/agents/adapters/harnesses.py +++ b/sdks/python/agenta/sdk/agents/adapters/harnesses.py @@ -5,9 +5,9 @@ *tools*. The harnesses genuinely differ, so the two adapters do different work: - **pi_core** takes built-in tools by name *and* resolved tool specs, delivered natively (Pi - has no MCP). Pi does not gate tool use, so the permission policy does not apply. + has no MCP). The runner relay enforces the shared permission plan. - **claude** has no built-in tools (they are a Pi concept), delivers tools over MCP, and - gates tool use, so the permission policy applies. + receives the same runner permission plan. - **pi_agenta** is Pi with an opinion: the same engine and config shape, plus a fixed set of forced tools, a base AGENTS.md preamble, and a persona (see :mod:`.agenta_builtins`). Skills ride the neutral config as resolved inline packages. Pi and Agenta install them @@ -60,7 +60,7 @@ class PiHarness(Harness): def _to_harness_config(self, config: SessionConfig) -> PiAgentTemplate: # Pi delivers tools natively: built-in names plus resolved specs registered through - # the Pi extension. Pi does not gate tool use, so the permission policy is dropped. + # the Pi extension. The runner relay enforces the shared permission plan. # Pi reads the selected harness's escape-hatch `extras`: `system` replaces Pi's base # prompt, `append_system` extends it (both leave AGENTS.md untouched). extras = config.agent.harness_extras @@ -74,6 +74,7 @@ def _to_harness_config(self, config: SessionConfig) -> PiAgentTemplate: mcp_servers=list(config.mcp_servers), skills=list(config.agent.skills), sandbox_permission=config.agent.sandbox_permission, + permission_default=config.permission_default, harness_permissions=config.agent.harness_permissions, system=_opt_str(extras.get("system")), append_system=_opt_str(extras.get("append_system")), @@ -85,8 +86,7 @@ class ClaudeHarness(Harness): def _to_harness_config(self, config: SessionConfig) -> ClaudeAgentTemplate: # Claude has no Pi built-in tools; drop them rather than ship a name Claude cannot - # honor. Tools go over MCP, and Claude gates tool use, so the permission policy is - # carried through. + # honor. Tools go over MCP, and the shared permission plan is carried through. if config.builtin_names: log.warning( "ClaudeHarness ignores %d built-in tool(s); built-ins are a Pi concept", @@ -107,8 +107,8 @@ def _to_harness_config(self, config: SessionConfig) -> ClaudeAgentTemplate: mcp_servers=list(config.mcp_servers), skills=list(config.agent.skills), sandbox_permission=config.agent.sandbox_permission, + permission_default=config.permission_default, harness_permissions=config.agent.harness_permissions, - permission_policy=config.permission_policy, ) @@ -139,6 +139,7 @@ def _to_harness_config(self, config: SessionConfig) -> AgentaAgentTemplate: # drops the default template's `_agenta` embed still gets the platform skill. skills=force_skills(list(config.agent.skills)), sandbox_permission=config.agent.sandbox_permission, + permission_default=config.permission_default, harness_permissions=config.agent.harness_permissions, system=_opt_str(extras.get("system")), append_system=compose_append_system(_opt_str(extras.get("append_system"))), diff --git a/sdks/python/agenta/sdk/agents/dtos.py b/sdks/python/agenta/sdk/agents/dtos.py index ac74b92913..9fd5777a22 100644 --- a/sdks/python/agenta/sdk/agents/dtos.py +++ b/sdks/python/agenta/sdk/agents/dtos.py @@ -30,8 +30,9 @@ parse_mcp_server_configs, ) from .skills import SkillTemplate, parse_skill_templates, skills_to_wire +from .permission_rules import wire_author_permission_rules from .tools import ToolCallback, ToolConfig, ToolSpec, coerce_tool_configs -from .tools.models import coerce_tool_spec +from .tools.models import PermissionMode, coerce_tool_spec # --------------------------------------------------------------------------- @@ -105,9 +106,7 @@ class HarnessIdentity(BaseModel): ] -# Permission policy for harness tool use in a headless run. ``auto`` approves (tools are -# backend-resolved and trusted, no human to prompt); ``deny`` rejects. -PermissionPolicy = Literal["auto", "deny"] +PERMISSION_MODES = frozenset({"allow", "ask", "deny", "allow_reads"}) # --------------------------------------------------------------------------- @@ -533,10 +532,9 @@ class AgentTemplate(BaseModel): of it: the harness adapters and ``wire_*`` methods read these flat fields. ``from_params`` reads the template and projects it here. - ``harness`` / ``sandbox`` / ``permission_policy`` are the execution selectors: which coding - agent to drive (``harness.kind``), where it runs (``sandbox.kind``), and how a - permission-gating harness answers a tool-use interaction in a headless run - (``runner.interactions.headless``). ``sandbox`` is a backend/environment concern the caller + ``harness`` / ``sandbox`` / ``permission_default`` are the execution selectors: which coding + agent to drive (``harness.kind``), where it runs (``sandbox.kind``), and the runner-enforced + default permission mode (``runner.permissions.default``). ``sandbox`` is a backend/environment concern the caller reads to pick a backend; it never enters the neutral run. ``harness_permissions`` is the selected harness's first-class allow/ask/deny posture (was @@ -562,13 +560,11 @@ class AgentTemplate(BaseModel): harness_permissions: Dict[str, Any] = Field(default_factory=dict) harness_extras: Dict[str, Any] = Field(default_factory=dict) sandbox_permission: Optional[SandboxPermission] = None - # The execution selectors: the coding agent to drive, where it runs, and the headless - # interaction default (sourced from ``runner.interactions.headless``). The caller reads - # ``harness`` / ``sandbox`` to pick a harness class and backend; ``permission_policy`` is the - # headless answer a gating harness (Claude) consults. + # The execution selectors: the coding agent to drive, where it runs, and the runner-enforced + # default permission mode (sourced from ``runner.permissions.default``). harness: str = "pi_core" sandbox: str = "local" - permission_policy: PermissionPolicy = "auto" + permission_default: PermissionMode = "allow_reads" @model_validator(mode="before") @classmethod @@ -609,7 +605,7 @@ def from_params( """ base = defaults or cls() instructions, model, tools = _parse_agent_fields(params, base) - harness, sandbox, permission_policy = _parse_run_selection(params, base) + harness, sandbox, permission_default = _parse_run_selection(params, base) harness_permissions, harness_extras = _parse_harness_slice(params, base) return cls( instructions=instructions, @@ -622,7 +618,7 @@ def from_params( sandbox_permission=_parse_sandbox_permission(params, base), harness=harness, sandbox=sandbox, - permission_policy=permission_policy, + permission_default=permission_default, ) @@ -636,9 +632,9 @@ class HarnessAgentTemplate(BaseModel): config; a backend plumbs it as-is, with no business logic about how the harness works. The two subclasses differ in their *shape*, not just their identity, because the - harnesses differ: Pi takes built-in tool names plus native tool specs and never gates - tool use; Claude has no built-ins, delivers tools over MCP, and gates tool use behind a - permission policy. ``wire_tools`` is where each config emits its own tool/permission + harnesses differ: Pi takes built-in tool names plus native tool specs; Claude has no + built-ins and delivers tools over MCP. Both ship the same runner permission plan. + ``wire_tools`` is where each config emits its own tool/permission fields for the ``/run`` payload. """ @@ -663,6 +659,7 @@ class HarnessAgentTemplate(BaseModel): mcp_servers: List[ResolvedMCPServer] = Field(default_factory=list) skills: List[SkillTemplate] = Field(default_factory=list) sandbox_permission: Optional[SandboxPermission] = None + permission_default: PermissionMode = "allow_reads" # The selected harness's first-class allow/ask/deny posture, carried verbatim from # ``AgentTemplate.harness_permissions`` by the harness adapter. A gating harness's CONFIG renders # it into files for the wire (see :meth:`wire_harness_files`); the raw slice does not ride the @@ -694,6 +691,14 @@ def _coerce_skills(cls, value: Any) -> List[SkillTemplate]: for item in value or [] ] + def wire_permissions(self) -> Dict[str, Any]: + """The runner permission plan this harness contributes to the ``/run`` payload.""" + permissions: Dict[str, Any] = {"default": self.permission_default} + rules = wire_author_permission_rules(self.harness_permissions) + if rules: + permissions["rules"] = rules + return {"permissions": permissions} + def wire_tools(self) -> Dict[str, Any]: """The tool + permission fields this harness contributes to the ``/run`` payload.""" raise NotImplementedError @@ -785,8 +790,8 @@ def wire_resolved_connection(self) -> Dict[str, Any]: class PiAgentTemplate(HarnessAgentTemplate): """Pi's config. Built-in tools by name plus resolved specs delivered natively (Pi has no - MCP; the runner registers them through the Pi extension). Pi does not gate tool use, so - no permission policy applies. + MCP; the runner registers them through the Pi extension). Pi has no native gate; the runner + relay enforces the shared permission plan. ``system`` and ``append_system`` are Pi's two system-prompt layers, distinct from ``agents_md``. ``system`` *replaces* Pi's built-in base prompt outright (Pi's ``SYSTEM.md`` @@ -829,7 +834,7 @@ def wire_tools(self) -> Dict[str, Any]: "toolCallback": self.tool_callback.to_wire() if self.tool_callback else None, - "permissionPolicy": "auto", # Pi never gates tool use + **self.wire_permissions(), } def wire_prompt(self) -> Dict[str, Any]: @@ -842,8 +847,7 @@ def wire_prompt(self) -> Dict[str, Any]: class ClaudeAgentTemplate(HarnessAgentTemplate): - """Claude's config. No Pi built-ins; tools are delivered over MCP, and - ``permission_policy`` answers Claude's tool-use prompts in a headless run.""" + """Claude's config. No Pi built-ins; tools are delivered over MCP.""" harness: ClassVar[HarnessType] = HarnessType.CLAUDE @@ -851,7 +855,6 @@ class ClaudeAgentTemplate(HarnessAgentTemplate): default_factory=list, validation_alias=AliasChoices("tool_specs", "custom_tools"), ) - permission_policy: PermissionPolicy = "auto" @field_validator("tool_specs", mode="before") @classmethod @@ -869,7 +872,7 @@ def wire_tools(self) -> Dict[str, Any]: "toolCallback": self.tool_callback.to_wire() if self.tool_callback else None, - "permissionPolicy": self.permission_policy, + **self.wire_permissions(), } def wire_harness_files(self) -> Dict[str, Any]: @@ -892,6 +895,7 @@ def wire_harness_files(self) -> Dict[str, Any]: self.sandbox_permission, self.mcp_servers, self.tool_specs, + self.permission_default, ) if not files: return {} @@ -929,7 +933,7 @@ class SessionConfig(BaseModel): # ``secrets`` is the compatibility alias for ``resolved_connection.env`` during the # transition: Slice 1 still ships the credential through ``secrets`` on the wire. resolved_connection: Optional[ResolvedConnection] = None - permission_policy: PermissionPolicy = "auto" + permission_default: PermissionMode = "allow_reads" trace: Optional[TraceContext] = None # The run's own context (trace + variant identity), refreshed per turn and consumed only by a # tool's ``call.context`` binding at dispatch (direct-call tools, Phase 3a). Omitted from the @@ -1087,19 +1091,24 @@ def _parse_harness_slice( def _parse_run_selection( params: Dict[str, Any], defaults: AgentTemplate, -) -> Tuple[str, str, "PermissionPolicy"]: - """Pull the execution selectors (harness kind / sandbox kind / headless interaction default). +) -> Tuple[str, str, PermissionMode]: + """Pull the execution selectors (harness kind / sandbox kind / permission default). - ``harness`` from ``harness.kind``, ``sandbox`` from ``sandbox.kind``, and the headless - permission default from ``runner.interactions.headless`` (was the flat ``permission_policy``). - Falls back to ``defaults``. The kinds are lower-cased so a playground-supplied ``"Claude"`` / - ``"Daytona"`` matches the bare :class:`HarnessType` / sandbox values the caller selects on.""" + ``harness`` from ``harness.kind``, ``sandbox`` from ``sandbox.kind``, and the runner policy + from ``runner.permissions.default``. The kinds and permission mode are lower-cased so + playground-supplied values match the bare runtime selectors.""" harness = str(_section(params, "harness").get("kind") or defaults.harness).lower() sandbox = str(_section(params, "sandbox").get("kind") or defaults.sandbox).lower() - interactions = _section(params, "runner").get("interactions") - headless = interactions.get("headless") if isinstance(interactions, dict) else None - permission_policy = str(headless or defaults.permission_policy).lower() - return harness, sandbox, permission_policy + permissions = _section(params, "runner").get("permissions") + raw_default = permissions.get("default") if isinstance(permissions, dict) else None + if raw_default is None: + permission_default = defaults.permission_default + else: + normalized = str(raw_default).lower() + permission_default = ( + normalized if normalized in PERMISSION_MODES else "allow_reads" + ) + return harness, sandbox, permission_default def _parse_sandbox_permission( diff --git a/sdks/python/agenta/sdk/agents/mcp/models.py b/sdks/python/agenta/sdk/agents/mcp/models.py index f782dd8c0b..90008b7d1c 100644 --- a/sdks/python/agenta/sdk/agents/mcp/models.py +++ b/sdks/python/agenta/sdk/agents/mcp/models.py @@ -4,15 +4,26 @@ from typing import Any, Dict, List, Literal, Optional -from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator # Layer-3 per-server permission (same value set as a tool's): ``allow`` runs with # no prompt, ``ask`` raises a human-in-the-loop request, ``deny`` never runs. Absent means the -# runner falls back to the global ``permissionPolicy`` default. An MCP server carries no -# ``read_only`` hint, so there is no default to compute: an explicit author value or nothing. +# server inherits the runner policy. Permission = Literal["allow", "ask", "deny"] +_LEGACY_PERMISSION_KEYS = frozenset({"permission" + "_mode", "permission" + "Mode"}) + + +def _drop_legacy_permission_keys(data: Any) -> Any: + if isinstance(data, dict): + return { + key: value + for key, value in data.items() + if key not in _LEGACY_PERMISSION_KEYS + } + return data + class MCPServerConfig(BaseModel): model_config = ConfigDict(extra="forbid") @@ -25,12 +36,12 @@ class MCPServerConfig(BaseModel): url: Optional[str] = None secrets: Dict[str, str] = Field(default_factory=dict) tools: List[str] = Field(default_factory=list) - permission: Optional[Permission] = Field( - default=None, - validation_alias=AliasChoices( - "permission", "permission_mode", "permissionMode" - ), - ) + permission: Optional[Permission] = None + + @model_validator(mode="before") + @classmethod + def _ignore_legacy_permission_keys(cls, data: Any) -> Any: + return _drop_legacy_permission_keys(data) @model_validator(mode="after") def _validate_transport(self) -> "MCPServerConfig": diff --git a/sdks/python/agenta/sdk/agents/permission_rules.py b/sdks/python/agenta/sdk/agents/permission_rules.py new file mode 100644 index 0000000000..f320decaa4 --- /dev/null +++ b/sdks/python/agenta/sdk/agents/permission_rules.py @@ -0,0 +1,52 @@ +"""Shared parsing for authored harness permission rule lists.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Literal, TypedDict + +ToolPermission = Literal["allow", "ask", "deny"] +CLAUDE_PERMISSION_MODES = frozenset( + {"default", "acceptEdits", "plan", "bypassPermissions"} +) + + +class PermissionRule(TypedDict): + pattern: str + permission: ToolPermission + + +def _string_list(value: Any) -> List[str]: + """Keep only string entries from an authored allow/deny/ask value.""" + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, str)] + + +def parse_author_permissions(slice_: Any) -> Dict[str, Any]: + """Parse the untyped ``harness.permissions`` block used by Claude settings and wire rules.""" + if not isinstance(slice_, dict): + return {"allow": [], "deny": [], "ask": []} + out: Dict[str, Any] = {} + mode = slice_.get("default_mode", slice_.get("defaultMode")) + if isinstance(mode, str) and mode in CLAUDE_PERMISSION_MODES: + out["mode"] = mode + out["allow"] = _string_list(slice_.get("allow")) + out["deny"] = _string_list(slice_.get("deny")) + out["ask"] = _string_list(slice_.get("ask")) + return out + + +def wire_author_permission_rules(slice_: Any) -> List[PermissionRule]: + """Build runner permission rules from authored builtin lists, excluding MCP rules. + + ``mcp__`` patterns are still rendered into Claude settings; on the runner wire they would + double-count tools reached through MCP server/spec permissions. + """ + author = parse_author_permissions(slice_) + rules: List[PermissionRule] = [] + for permission in ("deny", "ask", "allow"): + for pattern in author[permission]: + if pattern.startswith("mcp__"): + continue + rules.append({"pattern": pattern, "permission": permission}) + return rules diff --git a/sdks/python/agenta/sdk/agents/platform/gateway.py b/sdks/python/agenta/sdk/agents/platform/gateway.py index bf8368bebf..da11b65491 100644 --- a/sdks/python/agenta/sdk/agents/platform/gateway.py +++ b/sdks/python/agenta/sdk/agents/platform/gateway.py @@ -193,7 +193,6 @@ async def resolve( input_schema=raw_spec.get("input_schema") or {"type": "object", "properties": {}}, call_ref=str(raw_spec["call_ref"]), - needs_approval=tool_config.needs_approval, render=tool_config.render, permission=tool_config.permission, read_only=raw_spec.get("read_only"), diff --git a/sdks/python/agenta/sdk/agents/platform/platform_tools.py b/sdks/python/agenta/sdk/agents/platform/platform_tools.py index 202c11852d..0b27f7e504 100644 --- a/sdks/python/agenta/sdk/agents/platform/platform_tools.py +++ b/sdks/python/agenta/sdk/agents/platform/platform_tools.py @@ -7,10 +7,8 @@ per-request auth to assemble the shared :class:`ToolCallback` (which gives the runner the origin to resolve the relative ``call.path`` against, and the caller credential to reuse). -The catalog owns the description, endpoint, input schema, run-context bindings, and per-op default -permission/approval. The config's ``needs_approval`` / ``permission`` override the catalog default -when set; otherwise the catalog default applies (a mutating op defaults to approval, a read to -auto-allow). +The catalog owns the description, endpoint, input schema, run-context bindings, and read-only +hint. The config contributes only an explicit per-tool permission when authored. Lives in the SDK so the service and a connected standalone SDK user resolve platform tools the same way. @@ -71,24 +69,15 @@ async def resolve( raise error seen.add(op.op) - # Catalog default unless the author overrode it. ``needs_approval`` is optional on the - # config (None = unset), so a mutating op stays gated by default. - needs_approval = ( - tool_config.needs_approval - if tool_config.needs_approval is not None - else op.default_needs_approval - ) - permission = tool_config.permission or op.default_permission - tool_specs.append( CallbackToolSpec( name=op.op, description=op.description, input_schema=op.resolved_input_schema(), call=op.to_call(), - needs_approval=needs_approval, render=tool_config.render, - permission=permission, + permission=tool_config.permission, + read_only=op.read_only, ) ) diff --git a/sdks/python/agenta/sdk/agents/platform/workflow.py b/sdks/python/agenta/sdk/agents/platform/workflow.py index d77de9ff5e..6b6e8b6172 100644 --- a/sdks/python/agenta/sdk/agents/platform/workflow.py +++ b/sdks/python/agenta/sdk/agents/platform/workflow.py @@ -97,7 +97,6 @@ async def resolve( # carry these pointers; code/client tools author plain JSON Schema. input_schema=expand_type_refs(tool_config.input_schema), call_ref=call_ref, - needs_approval=tool_config.needs_approval, render=tool_config.render, permission=tool_config.permission, ) diff --git a/sdks/python/agenta/sdk/agents/tools/compat.py b/sdks/python/agenta/sdk/agents/tools/compat.py index a7b2053341..229e933ed2 100644 --- a/sdks/python/agenta/sdk/agents/tools/compat.py +++ b/sdks/python/agenta/sdk/agents/tools/compat.py @@ -52,18 +52,10 @@ def _copy_tool_metadata( source: dict[str, Any], target: dict[str, Any] ) -> dict[str, Any]: result = dict(target) - if "needs_approval" in source: - # Pass the raw value through; the model's bool field coerces it correctly. Using - # ``bool(...)`` here would flip legacy string payloads (``"false"`` -> ``True``). - result["needs_approval"] = source["needs_approval"] if isinstance(source.get("render"), dict): result["render"] = dict(source["render"]) - # Layer-3 permission: accept any of the keys the FE may write (the playground used - # ``permission_mode``); the config model's ``Permission`` field validates the value. - for key in ("permission", "permission_mode", "permissionMode"): - if key in source: - result["permission"] = source[key] - break + if "permission" in source: + result["permission"] = source["permission"] return result diff --git a/sdks/python/agenta/sdk/agents/tools/models.py b/sdks/python/agenta/sdk/agents/tools/models.py index 7fe6875a89..e6828521f6 100644 --- a/sdks/python/agenta/sdk/agents/tools/models.py +++ b/sdks/python/agenta/sdk/agents/tools/models.py @@ -21,9 +21,44 @@ def _empty_object_schema() -> Dict[str, Any]: # Layer-3 per-tool permission: ``allow`` runs with no prompt, ``ask`` raises a -# human-in-the-loop request, ``deny`` never runs. Absent means "fall back to the global -# ``permissionPolicy`` default" (the runner resolves that, in a later slice). +# human-in-the-loop request, ``deny`` never runs. Absent means "inherit the runner policy". Permission = Literal["allow", "ask", "deny"] +PermissionMode = Literal["allow", "ask", "deny", "allow_reads"] + +# The deleted pre-redesign vocabulary, still present in old dev-DB drafts. These literals +# are the only place the SDK may spell them. +_LEGACY_PERMISSION_KEYS = frozenset( + { + "needs_approval", + "needsApproval", + "permission_mode", + "permissionMode", + } +) + + +def _drop_legacy_permission_keys(data: Any) -> Any: + # Old POC drafts can still be present in dev DBs; tolerate and ignore them. + if isinstance(data, dict): + return { + key: value + for key, value in data.items() + if key not in _LEGACY_PERMISSION_KEYS + } + return data + + +def effective_permission( + spec_permission: Optional[Permission], + read_only: Optional[bool], + mode: PermissionMode, +) -> Permission: + """Resolve the runner permission semantics for one tool gate.""" + if spec_permission is not None: + return spec_permission + if mode == "allow_reads": + return "allow" if read_only is True else "ask" + return mode class ToolConfigBase(BaseModel): @@ -31,19 +66,13 @@ class ToolConfigBase(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) - needs_approval: bool = False render: Optional[Dict[str, Any]] = None - # Layer-3 permission the author set on this tool. Mirrors - # ``ToolSpecBase.permission``: accepts ``permission_mode``/``permissionMode`` too (the keys - # the playground writes), so an FE-set value deserializes onto the ``extra="forbid"`` config - # without a breaking change. ``_apply_tool_metadata`` then carries it onto the resolved spec. - permission: Optional[Permission] = Field( - default=None, - validation_alias=AliasChoices( - "permission", "permission_mode", "permissionMode" - ), - serialization_alias="permission", - ) + permission: Optional[Permission] = None + + @model_validator(mode="before") + @classmethod + def _ignore_legacy_permission_keys(cls, data: Any) -> Any: + return _drop_legacy_permission_keys(data) class BuiltinToolConfig(ToolConfigBase): @@ -177,29 +206,19 @@ def call_ref(self) -> str: class PlatformToolConfig(ToolConfigBase): """An existing Agenta endpoint exposed to the agent as a tool (the ``type:"platform"`` config). - A platform tool is a thin wrapper over an EXISTING Agenta endpoint — no new endpoint, no hidden - logic. The author names which endpoint to expose via ``op`` (a key in the platform-op catalog, - ``agenta.sdk.agents.platform.op_catalog``); the catalog owns everything else: the model-facing - description, the endpoint (method + relative path), the request input schema, any self-targeting - fields bound from run context (``context_bindings``), and the default permission/approval. + A platform tool is a thin wrapper over an EXISTING Agenta endpoint. The author names which + endpoint to expose via ``op``; the catalog owns the description, endpoint, request schema, + self-targeting context bindings, and the ``read_only`` hint. ``resolve_tools`` turns it into a ``CallbackToolSpec`` carrying a direct ``call`` descriptor (not a ``call_ref``): the runner calls the existing endpoint directly with the run's caller - credential, no ``/tools/call`` hop. - - ``needs_approval`` / ``permission`` are optional here (unlike the base, where ``needs_approval`` - defaults to ``False``): unset means "use the catalog's per-op default", so a mutating op like - ``commit_revision`` defaults to approval while a read like ``find_capabilities`` defaults to - auto-allow. An author value overrides the default.""" + credential, no ``/tools/call`` hop.""" type: Literal["platform"] = "platform" op: str = Field( min_length=1, description="Which catalog op (existing endpoint) to expose, e.g. 'find_capabilities'.", ) - # Override the base ``needs_approval: bool = False`` to optional so an unset value falls back to - # the catalog's per-op default rather than silently forcing "no approval" on a mutating op. - needs_approval: Optional[bool] = None ToolConfig = Annotated[ @@ -247,49 +266,22 @@ class ToolSpecBase(BaseModel): validation_alias=AliasChoices("input_schema", "inputSchema"), serialization_alias="inputSchema", ) - needs_approval: bool = Field( - default=False, - validation_alias=AliasChoices("needs_approval", "needsApproval"), - serialization_alias="needsApproval", - ) render: Optional[Dict[str, Any]] = None read_only: Optional[bool] = Field( default=None, validation_alias=AliasChoices("read_only", "readOnly"), serialization_alias="readOnly", ) - # Layer-3 permission the author set on this tool. Accepts ``permission_mode`` - # too, the key the playground writes into ``agenta_metadata``, so an FE-set value - # deserializes without a later breaking change. Unset means "no explicit author choice"; - # ``effective_permission`` then derives a default from ``read_only`` / ``needs_approval``. - permission: Optional[Permission] = Field( - default=None, - validation_alias=AliasChoices( - "permission", "permission_mode", "permissionMode" - ), - serialization_alias="permission", - ) + permission: Optional[Permission] = None + + @model_validator(mode="before") + @classmethod + def _ignore_legacy_permission_keys(cls, data: Any) -> Any: + return _drop_legacy_permission_keys(data) def effective_permission(self) -> Optional[Permission]: - """Resolve the permission that rides the wire, by this precedence: - - 1. An explicit author ``permission`` wins outright. - 2. Else, when ``needs_approval`` is set, the default is ``"ask"`` (approval beats the - read-only auto-allow: an author who asked to be prompted still gets prompted). - 3. Else, default from ``read_only``: ``True`` -> ``"allow"`` (read-only tools are safe to - auto-run), ``False`` -> ``"ask"`` (mutating tools prompt). - 4. Else (``read_only`` is ``None`` and nothing explicit) -> ``None`` (unset), so the runner - falls back to the global ``permissionPolicy`` default in a later slice. - """ - if self.permission is not None: - return self.permission - if self.needs_approval: - return "ask" - if self.read_only is True: - return "allow" - if self.read_only is False: - return "ask" - return None + """Return only the author's explicit permission, if one was set.""" + return self.permission def to_wire(self) -> Dict[str, Any]: wire = self.model_dump( @@ -297,15 +289,8 @@ def to_wire(self) -> Dict[str, Any]: by_alias=True, exclude_none=True, ) - if not self.needs_approval: - wire.pop("needsApproval", None) if not wire.get("env"): wire.pop("env", None) - permission = self.effective_permission() - if permission is not None: - wire["permission"] = permission - else: - wire.pop("permission", None) return wire diff --git a/sdks/python/agenta/sdk/agents/tools/resolver.py b/sdks/python/agenta/sdk/agents/tools/resolver.py index 655bc012b6..f61d5ad9eb 100644 --- a/sdks/python/agenta/sdk/agents/tools/resolver.py +++ b/sdks/python/agenta/sdk/agents/tools/resolver.py @@ -43,10 +43,9 @@ async def get_many(self, names: Sequence[str]) -> Mapping[str, str]: def _apply_tool_metadata(tool_spec: ToolSpec, tool_config: ToolConfig) -> ToolSpec: - """Return a new spec carrying the config's approval and rendering metadata.""" + """Return a new spec carrying the config's rendering and explicit permission metadata.""" return tool_spec.model_copy( update={ - "needs_approval": tool_config.needs_approval, "render": tool_config.render, "permission": tool_config.permission, } diff --git a/sdks/python/agenta/sdk/agents/wire_models.py b/sdks/python/agenta/sdk/agents/wire_models.py index c56cc6a63f..9069e1288b 100644 --- a/sdks/python/agenta/sdk/agents/wire_models.py +++ b/sdks/python/agenta/sdk/agents/wire_models.py @@ -242,8 +242,8 @@ class WireResolvedToolSpec(_WireModel): """A resolved tool the runner delivers to the harness (the three-axis tool surface). ``kind`` is the executor axis (``callback`` / ``code`` / ``client`` / ``builtin``); - ``needsApproval`` / ``render`` are the orthogonal axes; ``callRef`` / ``runtime`` / ``code`` - / ``env`` are executor-specific. ``call`` is the direct-call descriptor a callback tool carries + ``render`` is an orthogonal display hint; ``callRef`` / ``runtime`` / ``code`` / ``env`` are + executor-specific. ``call`` is the direct-call descriptor a callback tool carries instead of ``callRef`` (direct-call tools, Phase 1). Extra fields are allowed so an executor variant the schema has not enumerated still validates. """ @@ -257,12 +257,21 @@ class WireResolvedToolSpec(_WireModel): runtime: Optional[str] = None code: Optional[str] = None env: Optional[Dict[str, str]] = None - needs_approval: Optional[bool] = Field(default=None, alias="needsApproval") render: Optional[WireRenderHint] = None read_only: Optional[bool] = Field(default=None, alias="readOnly") permission: Optional[str] = None +class WirePermissionRule(_WireModel): + pattern: str + permission: str + + +class WirePermissions(_WireModel): + default: Literal["allow", "ask", "deny", "allow_reads"] = "allow_reads" + rules: Optional[List[WirePermissionRule]] = None + + class WireMcpServer(_WireModel): """A user-declared MCP server (stdio or http), mirrors ``mcp_servers_to_wire``.""" @@ -424,7 +433,7 @@ class WireRunRequest(_WireModel): mcp_servers: Optional[List[WireMcpServer]] = Field(default=None, alias="mcpServers") skills: Optional[List[WireSkill]] = None # Policy + prompt overrides + files. - permission_policy: Optional[str] = Field(default=None, alias="permissionPolicy") + permissions: Optional[WirePermissions] = None system_prompt: Optional[str] = Field(default=None, alias="systemPrompt") append_system_prompt: Optional[str] = Field( default=None, alias="appendSystemPrompt" diff --git a/sdks/python/agenta/sdk/utils/types.py b/sdks/python/agenta/sdk/utils/types.py index d82427bd5d..7f0837359e 100644 --- a/sdks/python/agenta/sdk/utils/types.py +++ b/sdks/python/agenta/sdk/utils/types.py @@ -1069,7 +1069,7 @@ def _model_catalog_type() -> dict: # default changes one place. The harness default also seeds `AgentTemplateSchema.harness`. _DEFAULT_HARNESS = "pi_core" _DEFAULT_SANDBOX = "local" -_DEFAULT_PERMISSION_POLICY = "auto" +_DEFAULT_PERMISSION_MODE = "allow_reads" def _default_agent_provider() -> str: @@ -1337,22 +1337,18 @@ class _HarnessSchema(BaseModel): ) -class _InteractionsSchema(BaseModel): - """How the runner answers a harness's reverse-RPC interaction requests. +class _PermissionsSchema(BaseModel): + """Runner-enforced tool execution policy.""" - Today only the ``permission`` interaction kind is wired; ``headless`` is the default answer - it gives when no human surface is attached to the run (was the flat ``permission_policy``). - The runner enforces it (``services/agent/src/responder.ts``). ``input`` and ``client_tool`` - interaction kinds extend this section in a later step.""" - - model_config = ConfigDict(extra="forbid", title="Interactions") + model_config = ConfigDict(extra="forbid", title="Permissions") - headless: Literal["auto", "deny"] = Field( - default=_DEFAULT_PERMISSION_POLICY, - title="Headless interactions", + default: Literal["allow", "ask", "deny", "allow_reads"] = Field( + default=_DEFAULT_PERMISSION_MODE, + title="Permissions", description=( - "How a permission-gating harness's tool-use prompts are answered when no human is " - "attached: auto-approve or deny." + "allow runs every tool without asking. ask requires approval for every tool. " + "deny refuses every tool. allow_reads runs read-hinted tools and asks for " + "everything else; this is the default." ), ) @@ -1360,9 +1356,8 @@ class _InteractionsSchema(BaseModel): class _RunnerSchema(BaseModel): """The engine that drives the harness loop (the ``services/agent`` sidecar). - ``kind`` names the engine; ``interactions`` is how it answers a harness's reverse-RPC - requests (today only the headless permission default). ``extras`` is the per-runner escape - hatch. The rest of the runner surface (per-kind interaction handling, delivery channel, + ``kind`` names the engine; ``permissions`` is the runner-enforced tool execution policy. + ``extras`` is the per-runner escape hatch. The rest of the runner surface (delivery channel, hooks, loop controls) is a later step.""" model_config = ConfigDict(extra="forbid", title="Runner") @@ -1372,10 +1367,10 @@ class _RunnerSchema(BaseModel): title="Runner", description="The engine that drives the harness loop.", ) - interactions: _InteractionsSchema = Field( - default_factory=_InteractionsSchema, - title="Interactions", - description="How the runner answers a harness's reverse-RPC interaction requests.", + permissions: _PermissionsSchema = Field( + default_factory=_PermissionsSchema, + title="Permissions", + description="The runner-enforced tool execution policy.", ) extras: Dict[str, Any] = Field( default_factory=dict, @@ -1456,7 +1451,7 @@ def build_agent_v0_default( template["harness"] = {"kind": _DEFAULT_HARNESS} template["runner"] = { "kind": "sidecar", - "interactions": {"headless": _DEFAULT_PERMISSION_POLICY}, + "permissions": {"default": _DEFAULT_PERMISSION_MODE}, } template["sandbox"] = sandbox return template diff --git a/sdks/python/oss/tests/pytest/unit/agents/adapters/test_claude_settings.py b/sdks/python/oss/tests/pytest/unit/agents/adapters/test_claude_settings.py index 25114ca6fc..0939e3577a 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/adapters/test_claude_settings.py +++ b/sdks/python/oss/tests/pytest/unit/agents/adapters/test_claude_settings.py @@ -10,6 +10,8 @@ import json +import pytest + from agenta.sdk.agents.adapters.claude_settings import ( INTERNAL_TOOL_MCP_SERVER, build_claude_settings_files, @@ -248,13 +250,35 @@ def test_ask_tool_not_in_allow(): def test_unset_tool_renders_no_rule(): - # No explicit permission, no read_only, no needs_approval -> effective permission is None -> - # no rule at all (falls back to the global `permission_policy`). With nothing else to write, - # the whole file is omitted. spec = CallbackToolSpec(name="mystery", description="d", call_ref="workflow.x") assert build_claude_settings_files(None, None, None, [spec]) == [] +@pytest.mark.parametrize( + ("mode", "read_only", "expected_key", "expected_rule"), + [ + ("allow_reads", True, "allow", _rule("tool")), + ("allow_reads", False, None, None), + ("allow_reads", None, None, None), + ("allow", None, "allow", _rule("tool")), + ("ask", True, None, None), + ("deny", True, "deny", _rule("tool")), + ], +) +def test_unset_tool_rules_follow_runner_mode( + mode, read_only, expected_key, expected_rule +): + spec = CallbackToolSpec( + name="tool", description="d", call_ref="workflow.x", read_only=read_only + ) + files = build_claude_settings_files(None, None, None, [spec], mode) + if expected_key is None: + assert files == [] + return + perms = _settings(files)["permissions"] + assert perms[expected_key] == [expected_rule] + + def test_deny_tool_renders_deny_rule(): # `deny` emits a deny rule (also closes a local-Claude execution-path gap). spec = CallbackToolSpec( diff --git a/sdks/python/oss/tests/pytest/unit/agents/golden/run_request.claude.json b/sdks/python/oss/tests/pytest/unit/agents/golden/run_request.claude.json index a9fa236a3c..7b93b96072 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/golden/run_request.claude.json +++ b/sdks/python/oss/tests/pytest/unit/agents/golden/run_request.claude.json @@ -18,19 +18,25 @@ "inputSchema": {"type": "object", "properties": {}}, "callRef": "tools__composio__github__GET_THE_AUTHENTICATED_USER__github-tvn", "kind": "callback", - "readOnly": true, - "permission": "allow" + "readOnly": true } ], "toolCallback": { "endpoint": "https://api.example/tools/call", "authorization": "Access tok-123" }, - "permissionPolicy": "deny", + "permissions": { + "default": "deny", + "rules": [ + {"pattern": "WebFetch", "permission": "deny"}, + {"pattern": "Read", "permission": "allow"}, + {"pattern": "Bash(npm run:*)", "permission": "allow"} + ] + }, "harnessFiles": [ { "path": ".claude/settings.json", - "content": "{\n \"permissions\": {\n \"defaultMode\": \"acceptEdits\",\n \"allow\": [\n \"Read\",\n \"Bash(npm run:*)\",\n \"mcp__agenta-tools__get_user\"\n ],\n \"deny\": [\n \"WebFetch\"\n ]\n }\n}" + "content": "{\n \"permissions\": {\n \"defaultMode\": \"acceptEdits\",\n \"allow\": [\n \"Read\",\n \"Bash(npm run:*)\"\n ],\n \"deny\": [\n \"WebFetch\",\n \"mcp__agenta-tools__get_user\"\n ]\n }\n}" } ], "skills": [ diff --git a/sdks/python/oss/tests/pytest/unit/agents/golden/run_request.pi_core.json b/sdks/python/oss/tests/pytest/unit/agents/golden/run_request.pi_core.json index 7e3cad2e50..796287f9a4 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/golden/run_request.pi_core.json +++ b/sdks/python/oss/tests/pytest/unit/agents/golden/run_request.pi_core.json @@ -49,8 +49,7 @@ "inputSchema": {"type": "object", "properties": {}}, "callRef": "tools__composio__github__GET_THE_AUTHENTICATED_USER__github-tvn", "kind": "callback", - "readOnly": true, - "permission": "allow" + "readOnly": true }, { "name": "get_weather", @@ -69,7 +68,7 @@ "endpoint": "https://api.example/tools/call", "authorization": "Access tok-123" }, - "permissionPolicy": "auto", + "permissions": {"default": "allow_reads"}, "systemPrompt": "You are Pi.", "appendSystemPrompt": "Be terse.", "skills": [ diff --git a/sdks/python/oss/tests/pytest/unit/agents/mcp/test_resolver.py b/sdks/python/oss/tests/pytest/unit/agents/mcp/test_resolver.py index 983571f379..7cb093fb13 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/mcp/test_resolver.py +++ b/sdks/python/oss/tests/pytest/unit/agents/mcp/test_resolver.py @@ -78,12 +78,11 @@ async def test_permission_absent_from_wire_when_unset(): assert "permission" not in servers[0].to_wire() -def test_permission_accepts_fe_permission_mode_alias(): - # The playground writes `permission_mode`; the server config deserializes it. +def test_legacy_permission_mode_alias_is_ignored(): config = MCPServerConfig.model_validate( {"name": "github", "command": "npx", "permission_mode": "deny"} ) - assert config.permission == "deny" + assert config.permission is None async def test_mcp_compatibility_policy_can_omit_missing_secret(): diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_gateway_http.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_gateway_http.py index d44008aa57..c45d5cd3df 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_gateway_http.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_gateway_http.py @@ -50,21 +50,17 @@ async def test_gateway_metadata_and_description_fallback_are_preserved( resolved = await _resolver(connection).resolve( [ _gateway( - needs_approval=True, render={"kind": "component", "component": "User"}, ) ] ) spec = resolved.tool_specs[0] assert spec.description == "get_user" # falls back to name when null - assert spec.needs_approval is True assert spec.render == {"kind": "component", "component": "User"} assert spec.read_only is True # the catalog read-only hint reaches the spec - assert spec.to_wire()["needsApproval"] is True assert spec.to_wire()["readOnly"] is True - # needs_approval beats the read-only auto-allow: the gateway resolver's effective - # permission is "ask" (pins the real precedence end to end, not just the model helper). - assert spec.to_wire()["permission"] == "ask" + assert "needsApproval" not in spec.to_wire() + assert "permission" not in spec.to_wire() assert isinstance(resolved.tool_callback, ToolCallback) assert resolved.tool_callback.endpoint == "https://api.x/api/tools/call" assert resolved.tool_callback.authorization == "Access tok" @@ -95,7 +91,7 @@ async def test_gateway_specs_are_joined_by_call_ref_not_position(fake_http, conn ) resolved = await _resolver(connection).resolve( [ - _gateway(action="FIRST", connection="c1", needs_approval=True), + _gateway(action="FIRST", connection="c1"), _gateway( action="SECOND", connection="c2", @@ -105,10 +101,8 @@ async def test_gateway_specs_are_joined_by_call_ref_not_position(fake_http, conn ) first, second = resolved.tool_specs assert first.name == "first" - assert first.needs_approval is True assert first.render is None assert second.name == "second" - assert second.needs_approval is False assert second.render == {"kind": "component", "component": "Second"} diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_workflow_resolver.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_workflow_resolver.py index fea3db3682..ac7d9a7057 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_workflow_resolver.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_workflow_resolver.py @@ -79,14 +79,12 @@ async def test_tool_axes_carry_onto_spec(connection): [ ReferenceToolConfig( slug="wf", - needs_approval=True, render={"kind": "component", "component": "Card"}, permission="ask", ) ] ) spec = resolution.tool_specs[0] - assert spec.needs_approval is True assert spec.render == {"kind": "component", "component": "Card"} assert spec.permission == "ask" diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_dtos_agent_template.py b/sdks/python/oss/tests/pytest/unit/agents/test_dtos_agent_template.py index f71ae20fdc..279b3048dc 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_dtos_agent_template.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_dtos_agent_template.py @@ -4,7 +4,7 @@ This file locks the nested agent-template envelope (``{agent, harness, runner, sandbox}``), the ``prompt`` prompt-template shape for a bare chat run, the defaults fall-through, the selected harness's ``permissions`` / ``extras`` slice, and the execution selectors (``harness.kind`` / -``sandbox.kind`` / ``runner.interactions.headless``, which live on ``AgentTemplate`` rather than a +``sandbox.kind`` / ``runner.permissions.default``, which live on ``AgentTemplate`` rather than a separate ``RunSelection``). """ @@ -31,7 +31,7 @@ def test_from_params_agent_template_at_parameters_agent(): "llm": {"model": "M"}, "tools": [{"type": "builtin", "name": "read"}], "harness": {"kind": "claude", "extras": {"system": "S"}}, - "runner": {"interactions": {"headless": "deny"}}, + "runner": {"permissions": {"default": "deny"}}, "sandbox": {"kind": "daytona"}, }, }, @@ -43,7 +43,7 @@ def test_from_params_agent_template_at_parameters_agent(): assert config.harness == "claude" assert config.harness_extras == {"system": "S"} assert config.sandbox == "daytona" - assert config.permission_policy == "deny" + assert config.permission_default == "deny" def test_from_params_bare_template(): @@ -230,10 +230,10 @@ def test_harness_slice_explicit_empty_clears_defaults(): def test_run_selection_defaults(): config = AgentTemplate.from_params({}) - assert (config.harness, config.sandbox, config.permission_policy) == ( + assert (config.harness, config.sandbox, config.permission_default) == ( "pi_core", "local", - "auto", + "allow_reads", ) @@ -242,16 +242,38 @@ def test_run_selection_reads_envelope_sections_and_lowercases(): { "harness": {"kind": "Claude"}, "sandbox": {"kind": "Daytona"}, - "runner": {"interactions": {"headless": "Deny"}}, + "runner": {"permissions": {"default": "Deny"}}, } ) - assert (config.harness, config.sandbox, config.permission_policy) == ( + assert (config.harness, config.sandbox, config.permission_default) == ( "claude", "daytona", "deny", ) +def test_run_selection_reads_all_permission_modes_case_insensitive(): + for mode in ("allow", "ask", "deny", "allow_reads"): + config = AgentTemplate.from_params( + {"runner": {"permissions": {"default": mode.upper()}}} + ) + assert config.permission_default == mode + + +def test_run_selection_unknown_permission_falls_back_to_allow_reads(): + config = AgentTemplate.from_params( + {"runner": {"permissions": {"default": "surprise"}}} + ) + assert config.permission_default == "allow_reads" + + +def test_run_selection_ignores_legacy_runner_interactions(): + config = AgentTemplate.from_params( + {"runner": {"interactions": {"headless": "deny"}}} + ) + assert config.permission_default == "allow_reads" + + def test_run_selection_honors_defaults(): defaults = AgentTemplate(harness="claude", sandbox="daytona") config = AgentTemplate.from_params({}, defaults=defaults) diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_dtos_harness_configs.py b/sdks/python/oss/tests/pytest/unit/agents/test_dtos_harness_configs.py index 196f9fc0d5..8561f7adcc 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_dtos_harness_configs.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_dtos_harness_configs.py @@ -20,7 +20,7 @@ _CALLBACK = ToolCallback(endpoint="https://api.example/tools/call", authorization="A") -def test_pi_wire_tools_is_native_and_never_gates(): +def test_pi_wire_tools_is_native_and_ships_permissions(): config = PiAgentTemplate( builtin_tools=["read"], tool_specs=[ @@ -45,7 +45,7 @@ def test_pi_wire_tools_is_native_and_never_gates(): "endpoint": "https://api.example/tools/call", "authorization": "A", }, - "permissionPolicy": "auto", # Pi never gates tool use + "permissions": {"default": "allow_reads"}, } @@ -65,7 +65,7 @@ def test_pi_wire_prompt_emits_only_set_overrides(): } -def test_claude_wire_tools_has_no_builtins_and_carries_policy(): +def test_claude_wire_tools_has_no_builtins_and_carries_permissions(): config = ClaudeAgentTemplate( tool_specs=[ ClientToolSpec( @@ -74,7 +74,7 @@ def test_claude_wire_tools_has_no_builtins_and_carries_policy(): ) ], tool_callback=_CALLBACK, - permission_policy="deny", + permission_default="deny", ) wire = config.wire_tools() assert wire["tools"] == [] # Claude has no Pi built-ins @@ -86,11 +86,14 @@ def test_claude_wire_tools_has_no_builtins_and_carries_policy(): "kind": "client", } ] - assert wire["permissionPolicy"] == "deny" + assert wire["permissions"] == {"default": "deny"} + assert "permissionPolicy" not in wire -def test_claude_defaults_to_auto_policy_and_empty_prompt(): - assert ClaudeAgentTemplate().wire_tools()["permissionPolicy"] == "auto" +def test_claude_defaults_to_allow_reads_permissions_and_empty_prompt(): + assert ClaudeAgentTemplate().wire_tools()["permissions"] == { + "default": "allow_reads" + } assert ( ClaudeAgentTemplate().wire_prompt() == {} ) # Claude exposes no prompt overrides @@ -101,3 +104,22 @@ def test_base_config_wire_tools_is_abstract(): with pytest.raises(NotImplementedError): HarnessAgentTemplate().wire_tools() assert HarnessAgentTemplate().wire_prompt() == {} + + +def test_pi_and_claude_emit_same_permission_block_for_same_config(): + pi = PiAgentTemplate( + permission_default="ask", harness_permissions={"deny": ["Bash(rm:*)"]} + ) + claude = ClaudeAgentTemplate( + permission_default="ask", harness_permissions={"deny": ["Bash(rm:*)"]} + ) + assert pi.wire_tools()["permissions"] == claude.wire_tools()["permissions"] + assert pi.wire_tools()["permissions"] == { + "default": "ask", + "rules": [{"pattern": "Bash(rm:*)", "permission": "deny"}], + } + + +def test_permission_policy_key_absent_from_wire_tools(): + assert "permissionPolicy" not in PiAgentTemplate().wire_tools() + assert "permissionPolicy" not in ClaudeAgentTemplate().wire_tools() diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_harness_adapters.py b/sdks/python/oss/tests/pytest/unit/agents/test_harness_adapters.py index a3fb0d371d..bafad4557c 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_harness_adapters.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_harness_adapters.py @@ -234,7 +234,7 @@ def test_claude_drops_builtins_and_warns(make_env, monkeypatch): config = _session_config( builtin_tools=["read"], custom_tools=[{"name": "t", "callRef": "ref"}], - permission_policy="deny", + permission_default="deny", ) result = harness._to_harness_config(config) @@ -242,7 +242,7 @@ def test_claude_drops_builtins_and_warns(make_env, monkeypatch): assert isinstance(result, ClaudeAgentTemplate) assert not hasattr(result, "builtin_tools") # Claude has no built-in tools at all assert result.custom_tools[0]["name"] == "t" - assert result.permission_policy == "deny" # Claude carries the policy + assert result.permission_default == "deny" assert recorded, "expected a warning when built-ins are dropped" @@ -274,7 +274,7 @@ def test_claude_no_warning_without_builtins(make_env, monkeypatch): ) harness = ClaudeHarness(make_env(supported=[HarnessType.CLAUDE])) - harness._to_harness_config(_session_config(permission_policy="auto")) + harness._to_harness_config(_session_config(permission_default="allow_reads")) assert recorded == [] diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_wire_contract.py b/sdks/python/oss/tests/pytest/unit/agents/test_wire_contract.py index b9a78ee4b7..1981057f88 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_wire_contract.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_wire_contract.py @@ -65,7 +65,7 @@ "customTools", "mcpServers", "toolCallback", - "permissionPolicy", + "permissions", "systemPrompt", "appendSystemPrompt", "skills", @@ -166,7 +166,7 @@ def _claude_payload(): model="claude-sonnet-4-6", custom_tools=[dict(_CUSTOM_TOOL)], tool_callback=_CALLBACK, - permission_policy="deny", + permission_default="deny", skills=[dict(_SKILL)], harness_permissions={ "default_mode": "acceptEdits", @@ -206,8 +206,8 @@ def _agenta_payload(): def test_request_to_wire_agenta_carries_skills_and_pi_shape(): payload = _agenta_payload() assert set(payload) <= KNOWN_REQUEST_KEYS - # Agenta is a Pi config: same tool shape, never gates, exposes the prompt overrides... - assert payload["permissionPolicy"] == "auto" + # Agenta is a Pi config: same tool shape and shared permission plan, plus prompt overrides. + assert payload["permissions"] == {"default": "allow_reads"} assert payload["tools"] == ["read", "bash"] assert payload["appendSystemPrompt"] == "You are an Agenta agent." # ...plus the resolved inline skill packages, on their own seam (not in `wire_tools`). @@ -238,8 +238,6 @@ def test_request_to_wire_pi_matches_golden(golden): assert payload == golden("run_request.pi_core.json") # The Composio read-only hint rides the wire as camelCase `readOnly`. assert payload["customTools"][0]["readOnly"] is True - # No explicit author permission + read_only=True -> derived `allow` rides the wire. - assert payload["customTools"][0]["permission"] == "allow" # The direct-call tool rides the wire carrying its `call` descriptor and NO `callRef` # (the `call` XOR `callRef` rule). The descriptor keeps method/path/body and the snake_case # `args_into`; `context` is unset so it is omitted. Plumbing only — the runner forwards it @@ -379,11 +377,17 @@ def test_request_to_wire_claude_matches_golden(golden): assert payload["context"] is None assert payload["telemetry"] is None assert "trace" not in payload - # No explicit author permission + read_only=True -> derived `allow` rides the wire. - assert payload["customTools"][0]["permission"] == "allow" # Claude-specific invariants the golden encodes, asserted explicitly so a failure reads clearly. assert payload["tools"] == [] # Claude has no Pi built-ins - assert payload["permissionPolicy"] == "deny" # Claude gates tool use + assert payload["permissions"] == { + "default": "deny", + "rules": [ + {"pattern": "WebFetch", "permission": "deny"}, + {"pattern": "Read", "permission": "allow"}, + {"pattern": "Bash(npm run:*)", "permission": "allow"}, + ], + } + assert "permissionPolicy" not in payload assert "systemPrompt" not in payload # Claude exposes no prompt overrides assert "appendSystemPrompt" not in payload # Claude carries resolved inline skills on the same `skills` seam Pi uses (the runner @@ -396,9 +400,8 @@ def test_request_to_wire_claude_matches_golden(golden): assert "sandboxPermission" not in payload # The claude adapter (Python) translated the author's permissions slice into a rendered # `.claude/settings.json`, carried on the generic `harnessFiles` seam. The runner writes it blind. - # The `allow` list also carries the per-resolved-tool rule for the internal `agenta-tools` MCP - # server (F-046): `_CUSTOM_TOOL` is a read-only callback tool -> effective `allow` -> - # `mcp__agenta-tools__get_user`, so Claude runs it instead of parking on its own permission gate. + # Under a `deny` default, the unset resolved tool renders a deny rule for Claude's internal + # `agenta-tools` MCP server. assert payload["harnessFiles"] == [ { "path": ".claude/settings.json", @@ -409,9 +412,8 @@ def test_request_to_wire_claude_matches_golden(golden): "allow": [ "Read", "Bash(npm run:*)", - "mcp__agenta-tools__get_user", ], - "deny": ["WebFetch"], + "deny": ["WebFetch", "mcp__agenta-tools__get_user"], } }, indent=2, @@ -420,6 +422,41 @@ def test_request_to_wire_claude_matches_golden(golden): ] +def test_author_permission_rules_exclude_mcp_from_wire_but_keep_settings(): + config = ClaudeAgentTemplate( + harness_permissions={ + "deny": ["mcp__github__delete_issue", "Bash(rm:*)"], + "ask": ["mcp__github__create_issue", "Write"], + "allow": ["Read"], + } + ) + payload = request_to_wire( + harness=HarnessType.CLAUDE, + sandbox="local", + config=config, + messages=[Message(role="user", content="hi")], + ) + + assert payload["permissions"] == { + "default": "allow_reads", + "rules": [ + {"pattern": "Bash(rm:*)", "permission": "deny"}, + {"pattern": "Write", "permission": "ask"}, + {"pattern": "Read", "permission": "allow"}, + ], + } + settings = json.loads(payload["harnessFiles"][0]["content"]) + assert settings["permissions"]["deny"] == [ + "mcp__github__delete_issue", + "Bash(rm:*)", + ] + assert settings["permissions"]["ask"] == [ + "mcp__github__create_issue", + "Write", + ] + assert settings["permissions"]["allow"] == ["Read"] + + def test_request_to_wire_has_no_prompt_key(): # The serializer emits `messages` only; the TS side derives the latest turn with # `resolvePromptText`. This asymmetry is intentional and easy to break, so lock it. @@ -498,15 +535,16 @@ def test_request_to_wire_omits_resolved_connection_when_none(): assert payload["model"] == "gpt-5.5" -def test_pi_permission_policy_is_always_auto(): - # Pi never gates tool use, regardless of any requested policy. +def test_pi_permissions_default_to_allow_reads(): + # Pi ships the same default permission plan as the other harnesses. payload = request_to_wire( harness=HarnessType.PI, sandbox="local", config=PiAgentTemplate(), messages=[Message(role="user", content="hi")], ) - assert payload["permissionPolicy"] == "auto" + assert payload["permissions"] == {"default": "allow_reads"} + assert "permissionPolicy" not in payload def test_result_from_wire_parses_ok(golden): @@ -592,7 +630,7 @@ def test_result_from_wire_minimal_ok(): def test_request_to_wire_carries_code_client_and_mcp_specs(): # The three-axes surface reaches the wire intact: a code spec keeps its executor fields - # (kind/runtime/code/env) and the orthogonal axes (needsApproval/render); a client spec + # (kind/runtime/code/env) and the orthogonal render axis; a client spec # has no callRef; user MCP servers ride `mcpServers`. config = PiAgentTemplate( custom_tools=[ @@ -604,7 +642,6 @@ def test_request_to_wire_carries_code_client_and_mcp_specs(): "runtime": "python", "code": "def main(): return 1", "env": {"STRIPE_API_KEY": "sk"}, - "needsApproval": True, "render": {"kind": "component", "component": "Calc"}, }, { @@ -636,7 +673,7 @@ def test_request_to_wire_carries_code_client_and_mcp_specs(): assert code["runtime"] == "python" assert code["code"] == "def main(): return 1" assert code["env"] == {"STRIPE_API_KEY": "sk"} - assert code["needsApproval"] is True + assert "needsApproval" not in code assert code["render"] == {"kind": "component", "component": "Calc"} client = next(t for t in payload["customTools"] if t["name"] == "pick") assert client["kind"] == "client" @@ -756,3 +793,10 @@ def test_request_to_wire_carries_sandbox_permission_allowlist(): "network": {"mode": "allowlist", "allowlist": ["10.0.0.0/8"]}, "enforcement": "strict", } + + +def test_permission_policy_absent_from_serialized_session_config(): + pi_payload = _pi_payload() + claude_payload = _claude_payload() + assert "permissionPolicy" not in json.dumps(pi_payload) + assert "permissionPolicy" not in json.dumps(claude_payload) diff --git a/sdks/python/oss/tests/pytest/unit/agents/tools/test_models.py b/sdks/python/oss/tests/pytest/unit/agents/tools/test_models.py index a0b15054c5..9698f97db3 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/tools/test_models.py +++ b/sdks/python/oss/tests/pytest/unit/agents/tools/test_models.py @@ -10,7 +10,7 @@ PlatformToolConfig, ReferenceToolConfig, ) -from agenta.sdk.agents.tools.models import coerce_tool_spec +from agenta.sdk.agents.tools.models import coerce_tool_spec, effective_permission def test_reference_tool_variant_call_ref_grammar(): @@ -57,15 +57,10 @@ def test_reference_tool_discriminator_is_reference(): assert config.type == "reference" -def test_platform_tool_discriminator_and_optional_approval(): - # type:"platform" is its own arm of the ToolConfig union. needs_approval is optional (None = - # use the catalog's per-op default), unlike the base where it defaults to False. +def test_platform_tool_discriminator(): config = PlatformToolConfig(op="find_capabilities") assert config.type == "platform" assert config.op == "find_capabilities" - assert config.needs_approval is None - # An explicit override is preserved. - assert PlatformToolConfig(op="x", needs_approval=True).needs_approval is True def test_platform_tool_requires_op(): @@ -90,7 +85,6 @@ def test_code_spec_serializes_only_runner_fields(): runtime="python", code="def main(): return 1", env={"TOKEN": "secret"}, - needs_approval=True, render={"kind": "component", "component": "Calculator"}, ) assert spec.to_wire() == { @@ -101,10 +95,7 @@ def test_code_spec_serializes_only_runner_fields(): "runtime": "python", "code": "def main(): return 1", "env": {"TOKEN": "secret"}, - "needsApproval": True, "render": {"kind": "component", "component": "Calculator"}, - # needs_approval with no explicit permission -> derived `ask`. - "permission": "ask", } @@ -172,7 +163,7 @@ def test_secret_values_are_hidden_from_repr(): assert "do-not-print" not in repr(spec) -# --- Layer-3 permission default ladder (S3a) ----------------------------------------- +# --- Layer-3 permission and runner-mode helper --------------------------------------- def _spec(**kwargs): @@ -184,48 +175,46 @@ def _spec(**kwargs): ) -def test_permission_explicit_author_value_wins(): - # An explicit author permission wins over any read_only/needs_approval default. - spec = _spec(read_only=False, permission="allow") +def test_tool_spec_effective_permission_is_explicit_only(): + spec = _spec(read_only=True, permission="allow") assert spec.effective_permission() == "allow" assert spec.to_wire()["permission"] == "allow" - -def test_permission_default_from_read_only_true_is_allow(): - spec = _spec(read_only=True) - assert spec.effective_permission() == "allow" - assert spec.to_wire()["permission"] == "allow" - - -def test_permission_default_from_read_only_false_is_ask(): - spec = _spec(read_only=False) - assert spec.effective_permission() == "ask" - assert spec.to_wire()["permission"] == "ask" - - -def test_permission_needs_approval_beats_read_only_auto_allow(): - # needs_approval (no explicit permission) forces `ask` even when read_only would allow. - spec = _spec(read_only=True, needs_approval=True) - assert spec.effective_permission() == "ask" - assert spec.to_wire()["permission"] == "ask" - - -def test_permission_absent_when_all_unset(): - # read_only is None and nothing explicit -> no permission on the wire (runner falls back). - spec = _spec() - assert spec.effective_permission() is None - assert "permission" not in spec.to_wire() + inherited = _spec(read_only=True) + assert inherited.effective_permission() is None + assert "permission" not in inherited.to_wire() + assert inherited.to_wire()["readOnly"] is True + + +@pytest.mark.parametrize( + ("spec_permission", "read_only", "mode", "expected"), + [ + ("deny", True, "allow", "deny"), + (None, True, "allow_reads", "allow"), + (None, False, "allow_reads", "ask"), + (None, None, "allow_reads", "ask"), + (None, True, "allow", "allow"), + (None, False, "allow", "allow"), + (None, None, "ask", "ask"), + (None, True, "deny", "deny"), + ], +) +def test_effective_permission_helper_truth_table( + spec_permission, read_only, mode, expected +): + assert effective_permission(spec_permission, read_only, mode) == expected -def test_permission_accepts_fe_permission_mode_alias(): - # The playground writes `permission_mode` into agenta_metadata; the spec deserializes it. +def test_legacy_permission_fields_are_ignored_on_specs(): spec = CallbackToolSpec.model_validate( { "name": "t", "description": "t", "callRef": "tools.composio.x.Y.c1", "permission_mode": "deny", + "needsApproval": True, } ) - assert spec.permission == "deny" - assert spec.to_wire()["permission"] == "deny" + assert spec.permission is None + assert "permission" not in spec.to_wire() + assert "needsApproval" not in spec.to_wire() diff --git a/sdks/python/oss/tests/pytest/unit/agents/tools/test_parsing.py b/sdks/python/oss/tests/pytest/unit/agents/tools/test_parsing.py index 04e9581cf6..503806d6f8 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/tools/test_parsing.py +++ b/sdks/python/oss/tests/pytest/unit/agents/tools/test_parsing.py @@ -43,28 +43,23 @@ def test_compat_parser_accepts_playground_gateway_slug_and_metadata(): "render": {"kind": "component", "component": "User"}, } ) - assert gateway.needs_approval is True assert gateway.render == {"kind": "component", "component": "User"} + assert not hasattr(gateway, "needs_approval") -def test_compat_parser_does_not_flip_string_false_needs_approval(): - # Legacy payloads may carry the flag as the string "false"; it must not coerce to True - # (a plain ``bool("false")`` would). +def test_compat_parser_ignores_legacy_permission_fields(): gateway = coerce_tool_config( { - "function": {"name": "tools__composio__github__GET_USER__c1"}, - "needs_approval": "false", - } - ) - assert gateway.needs_approval is False - - approved = coerce_tool_config( - { - "function": {"name": "tools__composio__github__GET_USER__c1"}, + "type": "gateway", + "integration": "github", + "action": "GET_USER", + "connection": "c1", "needs_approval": "true", + "permission_mode": "deny", } ) - assert approved.needs_approval is True + assert gateway.permission is None + assert not hasattr(gateway, "needs_approval") def test_compat_parser_carries_top_level_permission_on_typed_config(): @@ -94,20 +89,6 @@ def test_compat_parser_carries_permission_from_gateway_slug(): assert gateway.permission == "deny" -def test_compat_parser_accepts_permission_mode_alias_for_permission(): - # The legacy FE key `permission_mode` deserializes to the same `permission` field. - gateway = coerce_tool_config( - { - "type": "gateway", - "integration": "github", - "action": "GET_USER", - "connection": "c1", - "permission_mode": "deny", - } - ) - assert gateway.permission == "deny" - - def test_compat_parser_omits_permission_when_absent(): # Backward compatible: a tool with no permission leaves the field unset. gateway = coerce_tool_config( @@ -192,13 +173,11 @@ def test_typed_reference_carries_tool_axes(): { "type": "reference", "slug": "wf", - "needs_approval": True, "render": {"kind": "component", "component": "Card"}, "permission": "ask", } ) assert isinstance(tool, ReferenceToolConfig) - assert tool.needs_approval is True assert tool.render == {"kind": "component", "component": "Card"} assert tool.permission == "ask" @@ -215,17 +194,15 @@ def test_typed_platform_config_round_trips(): tool = parse_tool_config({"type": "platform", "op": "find_capabilities"}) assert isinstance(tool, PlatformToolConfig) assert tool.op == "find_capabilities" - # needs_approval is optional (None = use the catalog default). - assert tool.needs_approval is None -def test_compat_parser_accepts_platform_type(): +def test_compat_parser_accepts_platform_type_and_ignores_legacy_fields(): tool = coerce_tool_config( {"type": "platform", "op": "commit_revision", "needs_approval": False} ) assert isinstance(tool, PlatformToolConfig) assert tool.op == "commit_revision" - assert tool.needs_approval is False + assert not hasattr(tool, "needs_approval") def test_typed_platform_without_op_raises(): diff --git a/sdks/python/oss/tests/pytest/unit/agents/tools/test_resolver.py b/sdks/python/oss/tests/pytest/unit/agents/tools/test_resolver.py index 116a1866d8..1901b8fea9 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/tools/test_resolver.py +++ b/sdks/python/oss/tests/pytest/unit/agents/tools/test_resolver.py @@ -43,7 +43,6 @@ async def resolve( name=tool.name or f"{tool.integration}__{tool.action}", description=tool.name or tool.action, call_ref=tool.reference, - needs_approval=tool.needs_approval, render=tool.render, permission=tool.permission, ) @@ -71,7 +70,6 @@ async def resolve( description=tool.description or tool.tool_name, input_schema=tool.input_schema, call_ref=tool.call_ref, - needs_approval=tool.needs_approval, render=tool.render, permission=tool.permission, ) @@ -160,13 +158,11 @@ async def test_gateway_metadata_survives_resolution(): integration="github", action="GET_USER", connection="c1", - needs_approval=True, render={"kind": "component", "component": "User"}, ) ] ) spec = resolved.tool_specs[0] - assert spec.needs_approval is True assert spec.render == {"kind": "component", "component": "User"} @@ -257,10 +253,9 @@ async def test_reference_tool_requires_injected_resolver(): async def test_reference_tool_axes_survive_resolution(): resolved = await ToolResolver(workflow_resolver=FakeWorkflowResolver()).resolve( - [ReferenceToolConfig(slug="wf", needs_approval=True, permission="ask")] + [ReferenceToolConfig(slug="wf", permission="ask")] ) spec = resolved.tool_specs[0] - assert spec.needs_approval is True assert spec.permission == "ask" diff --git a/sdks/python/oss/tests/pytest/unit/test_normalizer_passthrough.py b/sdks/python/oss/tests/pytest/unit/test_normalizer_passthrough.py index 94d99e0fdf..56b9ef82c9 100644 --- a/sdks/python/oss/tests/pytest/unit/test_normalizer_passthrough.py +++ b/sdks/python/oss/tests/pytest/unit/test_normalizer_passthrough.py @@ -13,6 +13,7 @@ import pytest +from agenta.sdk.contexts.running import RunningContext from agenta.sdk.middlewares.running.normalizer import NormalizerMiddleware from agenta.sdk.models.workflows import ( WorkflowRequestData, @@ -109,6 +110,57 @@ def handler(**kwargs): assert "session_id" not in kwargs + @pytest.mark.asyncio + async def test_call_resolves_provided_session_id_before_handler(self, monkeypatch): + import agenta as ag + + monkeypatch.setattr(ag, "tracing", None) + seen = {} + + def handler(request): + seen["session_id"] = request.session_id + return {"ok": True} + + request = WorkflowServiceRequest( + session_id="sess_request", + data=WorkflowRequestData(), + ) + + token = RunningContext.set(RunningContext(handler=handler)) + try: + response = await NormalizerMiddleware()(request, lambda req: None) + finally: + RunningContext.reset(token) + + assert seen["session_id"] == "sess_request" + assert response.session_id == "sess_request" + + @pytest.mark.asyncio + async def test_call_mints_session_id_before_handler_when_omitted(self, monkeypatch): + import agenta as ag + + monkeypatch.setattr(ag, "tracing", None) + seen = {} + + def handler(session_id): + seen["session_id"] = session_id + return {"ok": True} + + request = WorkflowServiceRequest(data=WorkflowRequestData()) + + token = RunningContext.set(RunningContext(handler=handler)) + try: + response = await NormalizerMiddleware()(request, lambda req: None) + finally: + RunningContext.reset(token) + + sid = seen["session_id"] + assert isinstance(sid, str) + assert len(sid) == 32 + assert all(c in "0123456789abcdef" for c in sid) + assert request.session_id == sid + assert response.session_id == sid + class TestAsyncGenerator: @pytest.mark.asyncio diff --git a/sdks/python/oss/tests/pytest/unit/test_skill_template_catalog.py b/sdks/python/oss/tests/pytest/unit/test_skill_template_catalog.py index 8022091e6f..4065ef8a03 100644 --- a/sdks/python/oss/tests/pytest/unit/test_skill_template_catalog.py +++ b/sdks/python/oss/tests/pytest/unit/test_skill_template_catalog.py @@ -70,7 +70,7 @@ def _base_agent_template() -> dict: "tools": [], "mcps": [], "harness": {"kind": "pi_core"}, - "runner": {"kind": "sidecar", "interactions": {"headless": "auto"}}, + "runner": {"kind": "sidecar", "permissions": {"default": "allow_reads"}}, "sandbox": { "kind": "local", "permissions": { diff --git a/services/oss/src/agent/app.py b/services/oss/src/agent/app.py index 1603c3efeb..e5e4aec42d 100644 --- a/services/oss/src/agent/app.py +++ b/services/oss/src/agent/app.py @@ -247,7 +247,7 @@ async def _agent( agent=agent_template, secrets=secrets, # the env compat alias the wire still reads resolved_connection=resolved_connection, - permission_policy=agent_template.permission_policy, + permission_default=agent_template.permission_default, trace=trace_context(), # The run's own context (trace + workflow identity), refreshed each turn and consumed only # by a tool's `call.context` binding at dispatch (direct-call tools, Phase 3a). The diff --git a/services/oss/tests/pytest/unit/agent/test_default_agent_template.py b/services/oss/tests/pytest/unit/agent/test_default_agent_template.py index f86ed7eca2..48c814f1b2 100644 --- a/services/oss/tests/pytest/unit/agent/test_default_agent_template.py +++ b/services/oss/tests/pytest/unit/agent/test_default_agent_template.py @@ -56,7 +56,7 @@ def test_inspect_default_parses_into_the_runtime_selection(): assert config.sandbox_permission is None assert config.harness == "pi_core" assert config.sandbox == "local" - assert config.permission_policy == "auto" + assert config.permission_default == "allow_reads" def test_authoring_extras_absent_from_every_published_default(): diff --git a/services/oss/tests/pytest/unit/agent/test_invoke_handler.py b/services/oss/tests/pytest/unit/agent/test_invoke_handler.py index 4c4c88353a..3cef74bd94 100644 --- a/services/oss/tests/pytest/unit/agent/test_invoke_handler.py +++ b/services/oss/tests/pytest/unit/agent/test_invoke_handler.py @@ -91,19 +91,19 @@ def patched(monkeypatch, fake_backend): return backend, recorded -def _template(harness="pi_core", *, model=None, permission_policy=None, skills=None): +def _template(harness="pi_core", *, model=None, permission_default=None, skills=None): """Build the agent-template value from loose kwargs the tests still pass flat. Everything lives on the one template (at `parameters.agent`): `model`/`skills` are the - definition; `harness`/`permission_policy` are the nested execution sections - (`harness.kind` / `runner.interactions.headless`).""" + definition; `harness`/`permission_default` are the nested execution sections + (`harness.kind` / `runner.permissions.default`).""" template: dict = {"harness": {"kind": harness}} if model is not None: template["llm"] = model if isinstance(model, dict) else {"model": model} if skills is not None: template["skills"] = skills - if permission_policy is not None: - template["runner"] = {"interactions": {"headless": permission_policy}} + if permission_default is not None: + template["runner"] = {"permissions": {"default": permission_default}} return template @@ -164,10 +164,10 @@ async def test_invoke_cross_harness_same_body_divergent_configs( each producing its own config. The turn carries a built-in tool (``web_search``), a ``deny`` policy, and one author skill - so the divergence is observable: Claude drops Pi built-ins and honors the policy; Pi keeps - them and forces ``auto``; Agenta unions the forced tools. The skill rides the neutral config, - so every skill-loading harness emits it on its own ``wire_skills`` seam (never in the tool - wire); there is no forced skill-name list anymore. + so the divergence is observable: Claude drops Pi built-ins while all harnesses ship the + same shared permission plan. The skill rides the neutral config, so every skill-loading + harness emits it on its own ``wire_skills`` seam (never in the tool wire); there is no + forced skill-name list anymore. """ backend = fake_backend(result=AgentResult(output="echo", usage={"total": 15})) _patch_handler(monkeypatch, backend, builtins=["web_search"]) @@ -178,7 +178,7 @@ async def test_invoke_cross_harness_same_body_divergent_configs( "body": "Read the changelog, then write notes.", } bodies = [ - await _invoke(harness, permission_policy="deny", skills=[skill]) + await _invoke(harness, permission_default="deny", skills=[skill]) for harness in ("pi_core", "pi_agenta", "claude") ] pi_body, agenta_body, claude_body = bodies @@ -205,19 +205,21 @@ async def test_invoke_cross_harness_same_body_divergent_configs( agenta_wire = agenta_cfg.wire_tools() claude_wire = claude_cfg.wire_tools() - # Pi keeps its built-in tool natively and forces auto regardless of the author's policy. + # Pi keeps its built-in tool natively; the runner relay enforces the shared plan. + # Skills never ride the tool wire. assert pi_wire["tools"] == ["web_search"] - assert pi_wire["permissionPolicy"] == "auto" + assert pi_wire["permissions"] == {"default": "deny"} assert "skills" not in pi_wire - # Claude drops Pi built-ins and honors the policy. + # Claude has no Pi built-ins (the `web_search` name is dropped) and carries the same plan. assert claude_wire["tools"] == [] - assert claude_wire["permissionPolicy"] == "deny" + assert claude_wire["permissions"] == {"default": "deny"} assert "skills" not in claude_wire - # Agenta unions the forced tools onto the author's set and forces auto like Pi. + # Agenta is Pi-with-an-opinion: it unions the forced tools onto the author's set. Skills are + # not tools, so they never appear in the tool wire. assert agenta_wire["tools"] == ["web_search", "read", "bash"] - assert agenta_wire["permissionPolicy"] == "auto" + assert agenta_wire["permissions"] == {"default": "deny"} assert "skills" not in agenta_wire # skills ride the dedicated wire_skills seam, not the tool wire diff --git a/services/runner/tests/unit/wire-contract.test.ts b/services/runner/tests/unit/wire-contract.test.ts index 7e302ffe48..a4893501fa 100644 --- a/services/runner/tests/unit/wire-contract.test.ts +++ b/services/runner/tests/unit/wire-contract.test.ts @@ -51,7 +51,7 @@ const KNOWN_REQUEST_KEYS = [ "customTools", "mcpServers", "toolCallback", - "permissionPolicy", + "permissions", "systemPrompt", "appendSystemPrompt", "skills", @@ -98,8 +98,9 @@ describe("wire contract: requests (vs Python golden)", () => { ); // The Composio read-only hint reaches the runner as `readOnly`. assert.equal(tool.readOnly, true); - // The Layer-3 permission (derived `allow` from read-only) reaches the runner. - assert.equal(tool.permission, "allow"); + // No explicit author permission is derived onto the tool spec; the plan decides it. + assert.equal(tool.permission, undefined); + assert.deepEqual(req.permissions, { default: "allow_reads" }); // The direct-call tool (direct-call tools, Phase 1) reaches the runner carrying its `call` // descriptor and NO `callRef` (the `call` XOR `callRef` rule). Plumbing only here: the runner // forwards it opaquely; no dispatch branch reads it yet. @@ -174,7 +175,14 @@ describe("wire contract: requests (vs Python golden)", () => { const req = loadGolden("run_request.claude.json") as AgentRunRequest; assert.equal(req.harness, "claude"); assert.deepEqual(req.tools, []); // Claude has no Pi built-ins - assert.equal(req.permissionPolicy, "deny"); // Claude gates tool use + assert.deepEqual(req.permissions, { + default: "deny", + rules: [ + { pattern: "WebFetch", permission: "deny" }, + { pattern: "Read", permission: "allow" }, + { pattern: "Bash(npm run:*)", permission: "allow" }, + ], + }); assert.equal(req.systemPrompt, undefined); // Claude exposes no prompt overrides assert.equal(req.appendSystemPrompt, undefined); assert.equal(req.runContext, undefined); // no run context threaded on this config @@ -188,15 +196,11 @@ describe("wire contract: requests (vs Python golden)", () => { permissions: Record; }; assert.equal(settings.permissions.defaultMode, "acceptEdits"); - // The allow list also carries the per-resolved-tool rule for the internal `agenta-tools` MCP - // server (F-046): the golden's `get_user` is a read-only callback tool -> effective `allow` -> - // `mcp__agenta-tools__get_user`, so Claude runs it instead of parking on its own permission gate. - assert.deepEqual(settings.permissions.allow, [ - "Read", - "Bash(npm run:*)", + assert.deepEqual(settings.permissions.allow, ["Read", "Bash(npm run:*)"]); + assert.deepEqual(settings.permissions.deny, [ + "WebFetch", "mcp__agenta-tools__get_user", ]); - assert.deepEqual(settings.permissions.deny, ["WebFetch"]); // Claude carries resolved inline skills on the same `skills` seam Pi uses; the runner // installs them into Claude's project-local `.claude/skills/` tree. This regressed // twice via merge-loss, so the cross-language golden pins it for Claude, not just Pi. From cf551ed86a504d5586f731313fb707de0588052b Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 16:14:58 +0200 Subject: [PATCH 12/24] feat(sdk): strip needs_approval from the platform op catalog (phase 3 tail) --- .../agenta/sdk/agents/platform/op_catalog.py | 63 +++++++------------ .../unit/agents/platform/test_op_catalog.py | 54 ++++++---------- 2 files changed, 40 insertions(+), 77 deletions(-) diff --git a/sdks/python/agenta/sdk/agents/platform/op_catalog.py b/sdks/python/agenta/sdk/agents/platform/op_catalog.py index 7174321d23..294593259d 100644 --- a/sdks/python/agenta/sdk/agents/platform/op_catalog.py +++ b/sdks/python/agenta/sdk/agents/platform/op_catalog.py @@ -32,7 +32,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from agenta.sdk.agents.tools.errors import UnknownPlatformOpError -from agenta.sdk.agents.tools.models import Permission, ToolCall +from agenta.sdk.agents.tools.models import ToolCall from agenta.sdk.utils.types import CATALOG_TYPES from ._schema import expand_type_refs @@ -83,10 +83,8 @@ class PlatformOp(BaseModel): context_bindings: Dict[str, str] = Field(default_factory=dict) # Where the model's args land in the request body (a dotted deep-set path; absent = the root). args_into: Optional[str] = None - # Per-op defaults; the config's ``needs_approval`` / ``permission`` override these when set. - # Mutating ops default to approval (``ask``); reads default to auto-allow (``allow``). - default_permission: Optional[Permission] = None - default_needs_approval: bool = False + # Catalog hint for the runner's ``allow_reads`` policy; no hint counts as a write. + read_only: bool = False @model_validator(mode="after") def _check(self) -> "PlatformOp": @@ -540,8 +538,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="POST", path="/api/tools/discover", input_schema=_FIND_CAPABILITIES_INPUT_SCHEMA, - default_permission="allow", - default_needs_approval=False, + read_only=True, ), PlatformOp( op="query_workflows", @@ -549,8 +546,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="POST", path="/api/workflows/query", input_schema=_QUERY_WORKFLOWS_INPUT_SCHEMA, - default_permission="allow", - default_needs_approval=False, + read_only=True, ), PlatformOp( op="commit_revision", @@ -561,8 +557,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: context_bindings={ "workflow_revision.workflow_variant_id": "$ctx.workflow.variant.id" }, - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="annotate_trace", @@ -575,9 +570,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: "annotation.links.invocation.span_id": "$ctx.trace.span_id", }, args_into="annotation", - # Additive self-metadata (does not mutate config) -> auto-allow, no approval. - default_permission="allow", - default_needs_approval=False, + read_only=False, ), PlatformOp( op="find_triggers", @@ -585,8 +578,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="POST", path="/api/triggers/discover", input_schema=_FIND_TRIGGERS_INPUT_SCHEMA, - default_permission="allow", - default_needs_approval=False, + read_only=True, ), PlatformOp( op="create_schedule", @@ -598,8 +590,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: "schedule.data.references.workflow_variant.id": "$ctx.workflow.variant.id" }, args_into="schedule", - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="create_subscription", @@ -611,8 +602,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: "subscription.data.references.workflow_variant.id": "$ctx.workflow.variant.id" }, args_into="subscription", - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="list_schedules", @@ -620,8 +610,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="GET", path="/api/triggers/schedules/", input_schema=_EMPTY_INPUT_SCHEMA, - default_permission="allow", - default_needs_approval=False, + read_only=True, ), PlatformOp( op="list_subscriptions", @@ -629,8 +618,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="GET", path="/api/triggers/subscriptions/", input_schema=_EMPTY_INPUT_SCHEMA, - default_permission="allow", - default_needs_approval=False, + read_only=True, ), PlatformOp( op="list_deliveries", @@ -638,8 +626,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="GET", path="/api/triggers/deliveries", input_schema=_EMPTY_INPUT_SCHEMA, - default_permission="allow", - default_needs_approval=False, + read_only=True, ), PlatformOp( op="list_connections", @@ -647,8 +634,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="POST", path="/api/triggers/connections/query", input_schema=_EMPTY_INPUT_SCHEMA, - default_permission="allow", - default_needs_approval=False, + read_only=True, ), PlatformOp( op="test_subscription", @@ -657,8 +643,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: path="/api/triggers/subscriptions/test", input_schema=_TEST_SUBSCRIPTION_INPUT_SCHEMA, args_into="subscription", - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="remove_schedule", @@ -666,8 +651,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="DELETE", path="/api/triggers/schedules/{id}", input_schema=_TRIGGER_ID_INPUT_SCHEMA, - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="remove_subscription", @@ -675,8 +659,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="DELETE", path="/api/triggers/subscriptions/{id}", input_schema=_TRIGGER_ID_INPUT_SCHEMA, - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="pause_schedule", @@ -684,8 +667,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="POST", path="/api/triggers/schedules/{id}/stop", input_schema=_TRIGGER_ID_INPUT_SCHEMA, - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="resume_schedule", @@ -693,8 +675,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="POST", path="/api/triggers/schedules/{id}/start", input_schema=_TRIGGER_ID_INPUT_SCHEMA, - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="pause_subscription", @@ -702,8 +683,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="POST", path="/api/triggers/subscriptions/{id}/stop", input_schema=_TRIGGER_ID_INPUT_SCHEMA, - default_permission="ask", - default_needs_approval=True, + read_only=False, ), PlatformOp( op="resume_subscription", @@ -711,8 +691,7 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: method="POST", path="/api/triggers/subscriptions/{id}/start", input_schema=_TRIGGER_ID_INPUT_SCHEMA, - default_permission="ask", - default_needs_approval=True, + read_only=False, ), ) } diff --git a/sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py b/sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py index 632950962a..d4c3683957 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py +++ b/sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py @@ -169,9 +169,8 @@ async def test_find_capabilities_emits_a_direct_call(connection): "limit_alternatives", } assert spec.input_schema["required"] == ["use_cases"] - # A read op defaults to auto-allow and no approval. - assert spec.needs_approval is False - assert spec.effective_permission() == "allow" + assert spec.read_only is True + assert spec.effective_permission() is None # The shared callback gives the runner the origin to resolve the relative path against. assert resolution.tool_callback.endpoint == "https://api.x/api/tools/call" assert resolution.tool_callback.authorization == "Access tok" @@ -213,14 +212,13 @@ async def test_commit_revision_binds_self_and_strips_bound_field(connection): assert "parameters.agent" in delta["properties"]["set"]["description"] -async def test_commit_revision_defaults_to_approval(connection): - # A mutating op defaults to needs_approval=True -> effective `ask` (self-update is gated). +async def test_commit_revision_is_not_read_only(connection): resolution = await _resolver(connection).resolve( [PlatformToolConfig(op="commit_revision")] ) spec = resolution.tool_specs[0] - assert spec.needs_approval is True - assert spec.effective_permission() == "ask" + assert spec.read_only is False + assert spec.effective_permission() is None # --- resolver: annotate_trace self-targets its own trace/span ----------------- @@ -251,14 +249,13 @@ async def test_annotate_trace_binds_own_trace_and_hides_links(connection): assert props["data"]["properties"]["outputs"]["additionalProperties"] is True -async def test_annotate_trace_defaults_to_auto_allow(connection): - # Additive self-metadata (does not mutate config) -> auto-allow, no approval. +async def test_annotate_trace_is_not_read_only(connection): resolution = await _resolver(connection).resolve( [PlatformToolConfig(op="annotate_trace")] ) spec = resolution.tool_specs[0] - assert spec.needs_approval is False - assert spec.effective_permission() == "allow" + assert spec.read_only is False + assert spec.effective_permission() is None async def test_trigger_builder_ops_have_expected_paths_and_defaults(connection): @@ -278,16 +275,12 @@ async def test_trigger_builder_ops_have_expected_paths_and_defaults(connection): "pause_subscription": ("POST", "/api/triggers/subscriptions/{id}/stop"), "resume_subscription": ("POST", "/api/triggers/subscriptions/{id}/start"), } - gated = { - "create_schedule", - "create_subscription", - "test_subscription", - "remove_schedule", - "remove_subscription", - "pause_schedule", - "resume_schedule", - "pause_subscription", - "resume_subscription", + read_only = { + "find_triggers", + "list_schedules", + "list_subscriptions", + "list_deliveries", + "list_connections", } resolution = await _resolver(connection).resolve( @@ -298,12 +291,8 @@ async def test_trigger_builder_ops_have_expected_paths_and_defaults(connection): for name, (method, path) in expected_paths.items(): assert specs[name].call.method == method assert specs[name].call.path == path - if name in gated: - assert specs[name].needs_approval is True - assert specs[name].effective_permission() == "ask" - else: - assert specs[name].needs_approval is False - assert specs[name].effective_permission() == "allow" + assert specs[name].read_only is (name in read_only) + assert specs[name].effective_permission() is None async def test_create_trigger_ops_bind_self_target_and_hide_destination(connection): @@ -333,17 +322,12 @@ async def test_create_trigger_ops_bind_self_target_and_hide_destination(connecti assert "selector" not in data_props -async def test_config_permission_overrides_the_catalog_default(connection): - # An author override beats the catalog default (here: force-allow a normally-gated op). +async def test_config_permission_rides_with_catalog_read_only_hint(connection): resolution = await _resolver(connection).resolve( - [ - PlatformToolConfig( - op="commit_revision", needs_approval=False, permission="allow" - ) - ] + [PlatformToolConfig(op="commit_revision", permission="allow")] ) spec = resolution.tool_specs[0] - assert spec.needs_approval is False + assert spec.read_only is False assert spec.effective_permission() == "allow" From e9864fb39935fe6a51a37844450d6e02d5142930 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 16:18:31 +0200 Subject: [PATCH 13/24] docs(approval-boundary): build notes for phase 3 + restack incident --- .../projects/approval-boundary/build-notes.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index d193d9532d..c0e8ad7110 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -40,6 +40,18 @@ decisions unless it contradicts an owner call". Newest last. Read together with relay must enforce permissions only when the harness does not gate (Pi) — otherwise one approval would be consumed at the gate and the relay would double-gate the same call. +- **Phase 3 commit incident + restack.** The two op_catalog files' hunks were + dependency-locked to `feat/annotate-trace-op-code` (its commit authored the lines + phase 3 edits), which a commit subagent tried to force through ref surgery and an + oplog restore that rewound other sessions' uncommitted files (recovered by the + affected session; subagent killed). The correct fix, applied by the orchestrator: the + dependency is real, so the annotate lane was inserted INTO this stack + (`big-agents-work <- annotate <- docs/approval-boundary`; the lane had no remote/PR, + so the move was local-only), after which the pair committed cleanly (`phase 3 tail`). + Consequence for this PR: its diff vs `big-agents-work` includes the small annotate + commit (2 files) until that lane gets its own PR merged. Rule tightened for all + future subagent briefs: `but oplog restore`, raw `git commit`/ref updates, and any + improvised recovery are FORBIDDEN — on any hunk-locking refusal, stop and report. - **Phase 3 landed (Codex implemented, reviewed here).** SDK assembles and ships `permissions: {default, rules}` on both harnesses; `needs_approval` + aliases deleted (inbound legacy keys are dropped tolerantly, POC dev-DB drafts exist); shared parse in From 8f4bbbf387ce58fad12e677034710c51ee08dea8 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 16:36:57 +0200 Subject: [PATCH 14/24] feat(web): four-mode permission policy, Pi settings block, drop legacy fallbacks (phase 4b) --- .../projects/approval-boundary/build-notes.md | 8 ++ .../AgentChatSlice/AgentChatPanel.tsx | 4 +- .../AgentChatSlice/assets/transport.ts | 4 +- .../src/workflow/commitDiff/classify.ts | 2 +- .../src/workflow/state/store.ts | 6 +- .../SchemaControls/PiSettingsControl.tsx | 126 +++++++++++++++++ .../SchemaControls/ToolFormView.tsx | 58 ++------ .../agentTemplate/useModelHarness.tsx | 132 +++++++++++++----- .../src/DrillInView/SchemaControls/index.ts | 2 + .../src/state/execution/agentRequest.ts | 5 +- .../tests/unit/agentRequest.test.ts | 8 +- 11 files changed, 260 insertions(+), 95 deletions(-) create mode 100644 web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/PiSettingsControl.tsx diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index c0e8ad7110..92b02a4679 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -40,6 +40,14 @@ decisions unless it contradicts an owner call". Newest last. Read together with relay must enforce permissions only when the harness does not gate (Pi) — otherwise one approval would be consumed at the gate and the relay would double-gate the same call. +- **Phase 4b landed (Codex implemented, reviewed here).** Four-mode policy select on + `runner.permissions.default`, shown for Pi too; per-tool Permission select loses the + legacy fallbacks; `PiSettingsControl` added. Codex's binding call accepted and worth + knowing: Pi builtin selection writes `{type: "builtin", name}` entries into the + template's existing `agent.tools[]` list, because that is the path the backend already + parses into `builtin_names` (`harness.extras` is not parsed for Pi). Review fix: the + builtin option list only offered read/bash; widened to Pi's full session vocabulary + (read, bash, edit, write, grep, find, ls). - **Phase 3 commit incident + restack.** The two op_catalog files' hunks were dependency-locked to `feat/annotate-trace-op-code` (its commit authored the lines phase 3 edits), which a commit subagent tried to force through ref surgery and an diff --git a/web/oss/src/components/AgentChatSlice/AgentChatPanel.tsx b/web/oss/src/components/AgentChatSlice/AgentChatPanel.tsx index a955fed3fc..fb71ec49a8 100644 --- a/web/oss/src/components/AgentChatSlice/AgentChatPanel.tsx +++ b/web/oss/src/components/AgentChatSlice/AgentChatPanel.tsx @@ -523,8 +523,8 @@ const AgentConversation = ({entityId, sessionId}: {entityId: string; sessionId: // ── #4920 Application 1: refresh the config on a committed revision ── // When the agent commits a new revision of itself, the backend emits a one-way - // `data-committed-revision` part (same channel as `data-trace`), in BOTH the gated approval path - // and the direct `needs_approval=false` path. On receipt we invalidate the latest-revision and + // `data-committed-revision` part (same channel as `data-trace`), whether the tool asked first + // or ran directly. On receipt we invalidate the latest-revision and // inspect caches so the config panel, section drawers, and build-kit view all re-read the new // config. Deduped by revision id so a re-render (token stream) doesn't re-invalidate. const committedRevisionsSeenRef = useRef>(new Set()) diff --git a/web/oss/src/components/AgentChatSlice/assets/transport.ts b/web/oss/src/components/AgentChatSlice/assets/transport.ts index 41da4965a2..4d16bd2014 100644 --- a/web/oss/src/components/AgentChatSlice/assets/transport.ts +++ b/web/oss/src/components/AgentChatSlice/assets/transport.ts @@ -41,7 +41,7 @@ const stubConfig = () => ({ tools: [], mcps: [], harness: {kind: "pi_core"}, - runner: {kind: "sidecar", interactions: {headless: "auto"}}, + runner: {kind: "sidecar", permissions: {default: "allow_reads"}}, sandbox: {kind: "local"}, }, }, @@ -78,7 +78,7 @@ const configFor = (appId?: string | null) => { const withDefaults = (t: Record): Record => ({ ...t, harness: withSection(t.harness, {kind: "pi_core"}), - runner: withSection(t.runner, {kind: "sidecar", interactions: {headless: "auto"}}), + runner: withSection(t.runner, {kind: "sidecar", permissions: {default: "allow_reads"}}), sandbox: withSection(t.sandbox, {kind: "local"}), }) const parameters = diff --git a/web/packages/agenta-entities/src/workflow/commitDiff/classify.ts b/web/packages/agenta-entities/src/workflow/commitDiff/classify.ts index 035d6cff16..d6539c8261 100644 --- a/web/packages/agenta-entities/src/workflow/commitDiff/classify.ts +++ b/web/packages/agenta-entities/src/workflow/commitDiff/classify.ts @@ -211,7 +211,7 @@ function scalarSection( } } -/** Prefix every leaf of a flattened section (e.g. `runner.` → `runner.interactions.headless`). */ +/** Prefix every leaf of a flattened section (e.g. `runner.` -> `runner.permissions.default`). */ function prefixed( prefix: string, obj: Record | undefined, diff --git a/web/packages/agenta-entities/src/workflow/state/store.ts b/web/packages/agenta-entities/src/workflow/state/store.ts index 04a7f4cc05..99cf3aa2bc 100644 --- a/web/packages/agenta-entities/src/workflow/state/store.ts +++ b/web/packages/agenta-entities/src/workflow/state/store.ts @@ -2591,9 +2591,9 @@ export function invalidateWorkflowRevisionsByVariantCache( * latest-revision query (the config panel + section drawers re-read the new config) and the inspect * query (harness-capabilities / any inspect-derived view). * - * Fired on the one-way `data-committed-revision` stream signal (which the backend emits in BOTH the - * gated approval path and the direct `needs_approval=false` path), not on approval — so a single - * emit point covers both. The payload's ids aren't needed: a prefix invalidation refetches every + * Fired on the one-way `data-committed-revision` stream signal (which the backend emits after + * committed revisions whether the tool asked first or ran directly), not on approval — so a + * single emit point covers both. The payload's ids aren't needed: a prefix invalidation refetches every * active observer, which is exactly the set the playground has mounted. */ export function invalidateAgentCommittedRevisionCache(options?: StoreOptions) { diff --git a/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/PiSettingsControl.tsx b/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/PiSettingsControl.tsx new file mode 100644 index 0000000000..7ffd62c9c9 --- /dev/null +++ b/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/PiSettingsControl.tsx @@ -0,0 +1,126 @@ +/** + * PiSettingsControl + * + * Pi-family harness settings that are authored through the agent template's existing `tools` + * list. Built-ins persist as `{type: "builtin", name}` entries; an absent entry means Pi uses its + * own defaults. + */ +import {memo, useCallback, useMemo} from "react" + +import {Select} from "antd" + +import {RailField, railInfoLabel} from "../../drawers/shared/RailField" + +type PiBuiltinName = "read" | "bash" | "edit" | "write" | "grep" | "find" | "ls" + +interface BuiltinTool { + type: "builtin" + name: string + [key: string]: unknown +} + +export interface PiSettingsControlProps { + /** The agent template's top-level `tools` value. */ + tools?: unknown[] | null + /** Called with the next top-level `tools` value; undefined removes an empty tools field. */ + onChange: (tools: unknown[] | undefined) => void + /** Disable the control. */ + disabled?: boolean +} + +// Source of truth: Pi's session `tools` vocabulary (pi-coding-agent); the SDK forwards these +// names verbatim as `builtin_names`. +const PI_BUILTIN_OPTIONS: {value: PiBuiltinName; label: string}[] = [ + {value: "read", label: "Read"}, + {value: "bash", label: "Bash"}, + {value: "edit", label: "Edit"}, + {value: "write", label: "Write"}, + {value: "grep", label: "Grep"}, + {value: "find", label: "Find"}, + {value: "ls", label: "List files"}, +] +const PI_BUILTIN_NAMES = new Set(PI_BUILTIN_OPTIONS.map((option) => option.value)) + +function builtinNameOf(tool: unknown): string | null { + if (typeof tool === "string") return tool + if (!tool || typeof tool !== "object" || Array.isArray(tool)) return null + const item = tool as Record + if (item.type === "builtin" && typeof item.name === "string") return item.name + if (!("type" in item) && typeof item.name === "string") return item.name + return null +} + +function isKnownPiBuiltinName(name: string | null): name is PiBuiltinName { + return !!name && PI_BUILTIN_NAMES.has(name) +} + +function asBuiltinTool(name: PiBuiltinName, existing: unknown): BuiltinTool { + if ( + existing && + typeof existing === "object" && + !Array.isArray(existing) && + (existing as Record).type === "builtin" + ) { + return existing as BuiltinTool + } + return {type: "builtin", name} +} + +export const PiSettingsControl = memo(function PiSettingsControl({ + tools, + onChange, + disabled = false, +}: PiSettingsControlProps) { + const toolList = useMemo(() => (Array.isArray(tools) ? tools : []), [tools]) + + const selected = useMemo(() => { + const names = new Set( + toolList.map((tool) => builtinNameOf(tool)).filter(isKnownPiBuiltinName), + ) + return PI_BUILTIN_OPTIONS.map((option) => option.value).filter((name) => names.has(name)) + }, [toolList]) + + const writeSelected = useCallback( + (names: PiBuiltinName[]) => { + const selectedNames = new Set(names) + const existingByName = new Map() + const passthrough: unknown[] = [] + + for (const tool of toolList) { + const name = builtinNameOf(tool) + if (isKnownPiBuiltinName(name)) { + existingByName.set(name, tool) + } else { + passthrough.push(tool) + } + } + + const selectedBuiltins = PI_BUILTIN_OPTIONS.map((option) => option.value) + .filter((name) => selectedNames.has(name)) + .map((name) => asBuiltinTool(name, existingByName.get(name))) + const nextTools = [...passthrough, ...selectedBuiltins] + onChange(nextTools.length ? nextTools : undefined) + }, + [toolList, onChange], + ) + + return ( + + + mode="multiple" + className="w-full" + value={selected} + onChange={(value) => writeSelected(value)} + options={PI_BUILTIN_OPTIONS} + placeholder="Pi defaults" + disabled={disabled} + /> + + ) +}) diff --git a/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/ToolFormView.tsx b/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/ToolFormView.tsx index 16a743bd88..b712214a4d 100644 --- a/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/ToolFormView.tsx +++ b/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/ToolFormView.tsx @@ -38,40 +38,23 @@ export interface ToolFormViewProps { disabled?: boolean } -// Per-tool permission (allow / ask / deny). Stored as a TOP-LEVEL `permission` key on the tool -// object — NOT inside `agenta_metadata`, which `stripAgentaMetadataDeep` strips on every save/run. -// The SDK reads it via `AliasChoices("permission","permission_mode","permissionMode")`. Unset means -// the tool inherits the agent's global permission policy. +// Per-tool permission is stored as a top-level `permission` key; unset inherits the runner policy. type ToolPermission = "allow" | "ask" | "deny" +type ToolPermissionSelection = ToolPermission | "inherit" -const PERMISSION_OPTIONS: {value: ToolPermission; label: string}[] = [ +const PERMISSION_OPTIONS: {value: ToolPermissionSelection; label: string}[] = [ {value: "allow", label: "Allow"}, {value: "ask", label: "Ask"}, {value: "deny", label: "Deny"}, + {value: "inherit", label: "Inherit"}, ] function isPermission(value: unknown): value is ToolPermission { return value === "allow" || value === "ask" || value === "deny" } -/** Display default from the catalog's `read_only` metadata: true → allow, false → ask (unwritten). */ -function defaultPermissionFromReadOnly( - metadata: Record | undefined, -): ToolPermission | undefined { - const readOnly = metadata?.read_only - if (readOnly === true) return "allow" - if (readOnly === false) return "ask" - return undefined -} - -/** Canonical store is the top-level `permission`; fall back to a legacy `agenta_metadata.permission_mode`. */ -function readPermission( - topLevel: unknown, - metadata: Record | undefined, -): ToolPermission | undefined { - if (isPermission(topLevel)) return topLevel - if (isPermission(metadata?.permission_mode)) return metadata?.permission_mode - return undefined +function readPermission(topLevel: unknown): ToolPermission | undefined { + return isPermission(topLevel) ? topLevel : undefined } /** Tool basics — shown in the detail panel while no parameter node is selected. */ @@ -103,22 +86,11 @@ function ToolBasics({ onChange({...tool, function: {...fn, parameters: {...params, additionalProperties: on}}}) } - const metadata = tool.agenta_metadata as Record | undefined - const permission = readPermission(tool.permission, metadata) - const permissionDefault = defaultPermissionFromReadOnly(metadata) - const setPermission = (next: ToolPermission | null) => { + const permission = readPermission(tool.permission) + const setPermission = (next: ToolPermissionSelection) => { const nextTool = {...tool} - // Also drop the legacy `agenta_metadata.permission_mode`; otherwise clearing the - // top-level field resolves straight back to the legacy value and "inherit policy" - // is unreachable for older tools. - const nextMetadata = { - ...((nextTool.agenta_metadata as Record | undefined) ?? {}), - } - delete nextMetadata.permission_mode - if (Object.keys(nextMetadata).length > 0) nextTool.agenta_metadata = nextMetadata - else delete nextTool.agenta_metadata - if (next) nextTool.permission = next - else delete nextTool.permission + if (next === "inherit") delete nextTool.permission + else nextTool.permission = next onChange(nextTool) } @@ -149,14 +121,10 @@ function ToolBasics({ - - value={permission ?? undefined} - onChange={(v) => setPermission(v ?? null)} + + value={permission ?? "inherit"} + onChange={setPermission} options={PERMISSION_OPTIONS} - placeholder={ - permissionDefault ? `${permissionDefault} (default)` : "Inherit policy" - } - allowClear className="w-full" disabled={disabled} /> diff --git a/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/agentTemplate/useModelHarness.tsx b/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/agentTemplate/useModelHarness.tsx index 206f7a0742..6628bfffda 100644 --- a/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/agentTemplate/useModelHarness.tsx +++ b/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/agentTemplate/useModelHarness.tsx @@ -11,7 +11,7 @@ import {ConfigAccordionSection, LabeledField} from "@agenta/ui/components/presen import {useDrillInUI} from "@agenta/ui/drill-in" import {SelectLLMProviderBase} from "@agenta/ui/select-llm-provider" import {cn} from "@agenta/ui/styles" -import {Check, Cube, EyeSlash, Key, Lightbulb, ShieldCheck, Warning} from "@phosphor-icons/react" +import {Check, Cube, Key, Lightbulb, ShieldCheck, Warning} from "@phosphor-icons/react" import {Select, Typography} from "antd" import {useAtomValue} from "jotai" @@ -35,11 +35,28 @@ import { import {EnumSelectControl} from "../EnumSelectControl" import {GroupedChoiceControl} from "../GroupedChoiceControl" import {HarnessSelectControl} from "../HarnessSelectControl" +import {PiSettingsControl} from "../PiSettingsControl" import {SandboxPermissionControl} from "../SandboxPermissionControl" import {enumLabel} from "./agentTemplateUtils" import {useBuildKit} from "./useBuildKit" +type PermissionPolicy = "allow_reads" | "allow" | "ask" | "deny" + +const PERMISSION_POLICY_OPTIONS: {value: PermissionPolicy; label: string; help: string}[] = [ + {value: "allow_reads", label: "Allow reads", help: "Reads run, writes ask; default"}, + {value: "allow", label: "Allow all", help: "Every tool runs without asking"}, + {value: "ask", label: "Ask", help: "A human approves every tool call"}, + {value: "deny", label: "Deny all", help: "Every tool call is refused"}, +] +const PERMISSION_POLICY_VALUES = new Set( + PERMISSION_POLICY_OPTIONS.map((option) => option.value), +) + +function isPermissionPolicy(value: unknown): value is PermissionPolicy { + return typeof value === "string" && PERMISSION_POLICY_VALUES.has(value) +} + export function useModelHarness({ schema, config, @@ -86,26 +103,36 @@ export function useModelHarness({ const runner = asObject("runner") const sandbox = asObject("sandbox") - // The runner's headless interaction default (was the flat `permission_policy`). - const runnerInteractions = - runner.interactions && typeof runner.interactions === "object" - ? (runner.interactions as Record) + const runnerPermissions = + runner.permissions && typeof runner.permissions === "object" + ? (runner.permissions as Record) : {} - const headlessValue = (runnerInteractions.headless as string | null | undefined) ?? null - const headlessSchema = ( - runnerProps.interactions?.properties as Record | undefined - )?.headless + const runnerPermissionValue = isPermissionPolicy(runnerPermissions.default) + ? runnerPermissions.default + : null + const runnerPermissionSchema = ( + runnerProps.permissions?.properties as Record | undefined + )?.default // Replace one nested execution section (harness / runner / sandbox), leaving the rest intact. const setSection = useCallback( (key: string, sectionValue: unknown) => onChange({...config, [key]: sectionValue}), [config, onChange], ) - // Set one flat field of the agent definition (here only `llm`). + // Set one flat field of the agent definition (here `llm` and `tools`). const setAgentField = useCallback( (key: string, fieldValue: unknown) => onChange({...config, [key]: fieldValue}), [config, onChange], ) + const setAgentTools = useCallback( + (tools: unknown[] | undefined) => { + const next = {...config} + if (tools === undefined) delete next.tools + else next.tools = tools + onChange(next) + }, + [config, onChange], + ) // Model + credential connection (`llm`). It is ALWAYS a structured object (the harness-filtered // picker only ever produces one); a legacy bare string is read for display. composeModelValue @@ -113,8 +140,6 @@ export function useModelHarness({ // is harness-filtered: selecting a model sets BOTH the model id and its provider, fed by the // `/inspect` capability map below. const harnessValue = typeof harness.kind === "string" ? (harness.kind as string) : null - // Pi (`pi_core`/`pi_agenta`) never gates tool use (`permissions: false`); a permission - // policy is meaningless for it, so the field is hidden for Pi. Only Claude honors it. const isPiHarness = harnessValue === "pi_core" || harnessValue === "pi_agenta" const llm = config.llm const modelId = useMemo(() => modelIdFromConfig(llm), [llm]) @@ -251,6 +276,34 @@ export function useModelHarness({ const hasModelOrHarness = Boolean(props.llm || harnessProps.kind) const hasClaudePermissions = harnessValue === "claude" + const hasPiSettings = isPiHarness + const agentTools = useMemo( + () => (Array.isArray(config.tools) ? (config.tools as unknown[]) : null), + [config.tools], + ) + const runnerPermissionOptions = useMemo(() => { + const schemaValues = Array.isArray(runnerPermissionSchema?.enum) + ? new Set((runnerPermissionSchema.enum as unknown[]).filter(isPermissionPolicy)) + : null + return PERMISSION_POLICY_OPTIONS.filter( + (option) => !schemaValues || schemaValues.has(option.value), + ).map((option) => ({ + value: option.value, + title: option.label, + label: ( +
+ {option.label} + + {option.help} + +
+ ), + })) + }, [runnerPermissionSchema]) + const currentRunnerPermission = runnerPermissionValue ?? "allow_reads" + const runnerPermissionSummary = PERMISSION_POLICY_OPTIONS.find( + (option) => option.value === currentRunnerPermission, + )?.label // Playground-only "build kit" overlay (read-only) shown at the top of Advanced. It also flags // sandbox-permission keys the overlay overrides for the user's own permission control below. @@ -265,8 +318,9 @@ export function useModelHarness({ props.llm || // Authentication lives in Advanced now sandboxProps.kind || sandboxProps.permissions || - runnerProps.interactions || + runnerProps.permissions || hasClaudePermissions || + hasPiSettings || hasBuildKitOverlay, ) @@ -553,7 +607,9 @@ export function useModelHarness({ // Advanced drawer body: two panels like Model & harness (settings left, version history right). const hasExecutionGroup = Boolean(sandboxProps.kind || sandboxProps.permissions) - const hasPermissionsGroup = Boolean(headlessSchema || hasClaudePermissions) + const hasPermissionsGroup = Boolean( + runnerPermissionSchema || hasClaudePermissions || hasPiSettings, + ) // Shared Advanced controls, rendered by both the wide drawer body and the tabs-inline body. // Each group is a `ConfigAccordionSection` (the shared drawer section shell used by the trigger // and tools drawers); inside, configuration reads as the drawer's `[rail | content]` rhythm via @@ -632,33 +688,27 @@ export function useModelHarness({ defaultOpen={false} icon={} title="Permissions" - summary="Auto" + summary={runnerPermissionSummary} summaryCollapsedOnly > What the agent may do on its own before it must ask. - {headlessSchema ? ( + {runnerPermissionSchema ? ( - {isPiHarness ? ( - - - Permission policy isn't used by the Pi harness. - - ) : ( - - setSection("runner", { - ...runner, - interactions: {...runnerInteractions, headless: v}, - }) - } - withTooltip={withTooltip} - disabled={disabled} - /> - )} + + value={currentRunnerPermission} + onChange={(v) => + setSection("runner", { + ...runner, + permissions: {...runnerPermissions, default: v}, + }) + } + options={runnerPermissionOptions} + optionLabelProp="title" + disabled={disabled} + className="w-full" + /> ) : null} {hasClaudePermissions ? ( @@ -684,6 +734,18 @@ export function useModelHarness({ /> ) : null} + {hasPiSettings ? ( + <> + + Pi harness + + + + ) : null} ) : null} diff --git a/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/index.ts b/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/index.ts index 471eee1994..dbce314038 100644 --- a/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/index.ts +++ b/web/packages/agenta-entity-ui/src/DrillInView/SchemaControls/index.ts @@ -81,6 +81,8 @@ export type {SandboxPermissionControlProps} from "./SandboxPermissionControl" export {ClaudePermissionsControl} from "./ClaudePermissionsControl" export type {ClaudePermissionsControlProps} from "./ClaudePermissionsControl" +export {PiSettingsControl} from "./PiSettingsControl" +export type {PiSettingsControlProps} from "./PiSettingsControl" export {AgentTemplateControl} from "./AgentTemplateControl" export type {AgentTemplateControlProps} from "./AgentTemplateControl" diff --git a/web/packages/agenta-playground/src/state/execution/agentRequest.ts b/web/packages/agenta-playground/src/state/execution/agentRequest.ts index b60cadfbdf..47ce6455f0 100644 --- a/web/packages/agenta-playground/src/state/execution/agentRequest.ts +++ b/web/packages/agenta-playground/src/state/execution/agentRequest.ts @@ -185,7 +185,6 @@ const normalizeAgentToolShape = (tool: unknown): unknown => { ? fn.parameters : {type: "object", properties: {}} if (t.permission !== undefined) client.permission = t.permission - if (t.needs_approval !== undefined) client.needs_approval = t.needs_approval return client } @@ -240,7 +239,7 @@ const hasAnswer = (message: unknown): boolean => { * * `config` is the full `parameters`; the template lives at `parameters.agent` (the definition flat * plus nested `harness` / `runner` / `sandbox` sections). The execution sections are defaulted - * (harness `pi_core`, runner `sidecar` answering headless interactions `auto`, sandbox `local`) so a + * (harness `pi_core`, runner `sidecar` with the default permission policy, sandbox `local`) so a * config that omits them still runs; a value the resolved config carries always wins (spread last). * The definition fields are passed through untouched. */ @@ -259,7 +258,7 @@ const withTemplateDefaults = (template: Record): Record { expect(template.llm).toMatchObject({model: "gpt-5.5"}) expect(template.harness).toEqual({kind: "pi_core"}) expect(template.sandbox).toEqual({kind: "daytona"}) - expect(template.runner).toMatchObject({interactions: {headless: "auto"}}) + expect(template.runner).toMatchObject({permissions: {default: "allow_reads"}}) }) - it("keeps an explicit headless interaction default over the runner default", async () => { + it("keeps an explicit permission policy over the runner default", async () => { seed(store, "e", { - config: {agent: {runner: {interactions: {headless: "deny"}}}}, + config: {agent: {runner: {permissions: {default: "deny"}}}}, }) const req = await buildAgentRequest("e", [], {sessionId: "s1", store}) const template = (req!.requestBody.data as any).parameters.agent - expect(template.runner.interactions.headless).toBe("deny") + expect(template.runner.permissions.default).toBe("deny") }) it("applies the build-kit overlay to a kit-on run with deep and identity merges", async () => { From 56170b9688297ee78514512e2ff4839d5c902f5f Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 16:40:12 +0200 Subject: [PATCH 15/24] feat(service): batch responses surface the paused state (phase 4a) --- .../projects/approval-boundary/build-notes.md | 8 ++++ .../oss/tests/pytest/unit/agents/conftest.py | 15 ++++--- .../unit/agents/test_environment_lifecycle.py | 24 ++++++++++ .../oss/tests/pytest/unit/agent/conftest.py | 16 +++---- .../pytest/unit/agent/test_invoke_handler.py | 45 +++++++++++++++++++ 5 files changed, 93 insertions(+), 15 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index 92b02a4679..0539ef865a 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -40,6 +40,14 @@ decisions unless it contradicts an owner call". Newest last. Read together with relay must enforce permissions only when the harness does not gate (Pi) — otherwise one approval would be consumed at the gate and the relay would double-gate the same call. +- **Phase 4a landed (Codex implemented, reviewed here).** Batch envelopes now carry + `stop_reason` (omitted when absent, so non-runner results are byte-identical) and, when + paused, `pending_interaction: {id, tool}`. The SDK already threaded + `AgentResult.stop_reason` from the wire, so the only production change is the service + drain. Review fix: the pending tool name now falls back through the payload's + `toolName` and the ACP `name`/`title`/`kind` chain instead of reading only + `toolCall.name` (harness gates carry display titles, the exact drift this project is + about). - **Phase 4b landed (Codex implemented, reviewed here).** Four-mode policy select on `runner.permissions.default`, shown for Pi too; per-tool Permission select loses the legacy fallbacks; `PiSettingsControl` added. Codex's binding call accepted and worth diff --git a/sdks/python/oss/tests/pytest/unit/agents/conftest.py b/sdks/python/oss/tests/pytest/unit/agents/conftest.py index 74c3cb2d2f..c319e04fff 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/conftest.py +++ b/sdks/python/oss/tests/pytest/unit/agents/conftest.py @@ -86,14 +86,15 @@ async def _records(): return for event in result.events: yield {"kind": "event", "event": event.data} - yield { - "kind": "result", - "result": { - "ok": True, - "output": result.output, - "sessionId": result.session_id, - }, + terminal = { + "ok": True, + "output": result.output, + "usage": result.usage, + "sessionId": result.session_id, } + if result.stop_reason is not None: + terminal["stopReason"] = result.stop_reason + yield {"kind": "result", "result": terminal} return AgentStream(_records()) diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_environment_lifecycle.py b/sdks/python/oss/tests/pytest/unit/agents/test_environment_lifecycle.py index 0de36a4785..4cfd442e3c 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_environment_lifecycle.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_environment_lifecycle.py @@ -104,6 +104,30 @@ async def test_prompt_runs_and_tears_down(make_env): assert env.backend.sessions[0].destroyed is True # torn down on the happy path +async def test_stream_threads_terminal_usage_and_stop_reason(make_env): + env = make_env( + result=AgentResult( + output="done", + usage={"total": 3}, + stop_reason="paused", + session_id="sess-new", + ) + ) + harness = PiHarness(env) + config = _config() + + run = await harness.stream(config, [Message(role="user", content="hi")]) + async for _ in run: + pass + result = run.result() + + assert result.output == "done" + assert result.usage == {"total": 3} + assert result.stop_reason == "paused" + assert config.session_id == "sess-new" + assert env.backend.sessions[0].destroyed is True + + async def test_prompt_destroys_session_even_when_it_raises(make_env): env = make_env(raise_on_prompt=True) harness = PiHarness(env) diff --git a/services/oss/tests/pytest/unit/agent/conftest.py b/services/oss/tests/pytest/unit/agent/conftest.py index 13549d310b..b5da23e063 100644 --- a/services/oss/tests/pytest/unit/agent/conftest.py +++ b/services/oss/tests/pytest/unit/agent/conftest.py @@ -56,15 +56,15 @@ async def _records(): "kind": "event", "event": {"type": "message", "text": result.output}, } - yield { - "kind": "result", - "result": { - "ok": True, - "output": result.output, - "usage": result.usage, - "sessionId": result.session_id, - }, + terminal = { + "ok": True, + "output": result.output, + "usage": result.usage, + "sessionId": result.session_id, } + if result.stop_reason is not None: + terminal["stopReason"] = result.stop_reason + yield {"kind": "result", "result": terminal} return AgentStream(_records()) diff --git a/services/oss/tests/pytest/unit/agent/test_invoke_handler.py b/services/oss/tests/pytest/unit/agent/test_invoke_handler.py index 3cef74bd94..bf662ef831 100644 --- a/services/oss/tests/pytest/unit/agent/test_invoke_handler.py +++ b/services/oss/tests/pytest/unit/agent/test_invoke_handler.py @@ -16,6 +16,7 @@ AgentTemplate, AgentResult, ConnectionNotFoundError, + Event, ConnectionResolutionError, Event, GatewayToolResolutionError, @@ -121,6 +122,50 @@ async def test_invoke_returns_assistant_message(patched): } +async def test_batch_paused_run_surfaces_pending_interaction(monkeypatch, fake_backend): + backend = fake_backend( + result=AgentResult( + output="waiting", + stop_reason="paused", + events=[ + Event( + type="interaction_request", + data={ + "type": "interaction_request", + "id": "perm_1", + "kind": "user_approval", + "payload": { + "toolCallId": "call_1", + "toolCall": { + "toolCallId": "call_1", + "name": "deleteFile", + }, + }, + }, + ) + ], + ) + ) + _patch_handler(monkeypatch, backend) + + body = await _invoke("pi_core") + + assert body == { + "messages": [{"role": "assistant", "content": "waiting"}], + "stop_reason": "paused", + "pending_interaction": {"id": "perm_1", "tool": "deleteFile"}, + } + + +async def test_batch_completed_run_omits_pause_metadata(monkeypatch, fake_backend): + backend = fake_backend(result=AgentResult(output="echo")) + _patch_handler(monkeypatch, backend) + + body = await _invoke("pi_core") + + assert body == {"messages": [{"role": "assistant", "content": "echo"}]} + + async def test_invoke_records_usage(patched): _, recorded = patched await _invoke("pi_core") From c73f5cddd985563b8dd90581ce6fab82338e7905 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 16:58:40 +0200 Subject: [PATCH 16/24] chore(runner): delete legacy wire fields, rename acp-interactions + pause controller (phase 5) --- .../projects/approval-boundary/build-notes.md | 19 ++++++ .../sdk/agents/adapters/claude_settings.py | 4 ++ .../agents/adapters/test_claude_settings.py | 22 +++++++ services/runner/src/engines/sandbox_agent.ts | 47 ++++++--------- .../src/engines/sandbox_agent/acp-fetch.ts | 14 ++--- .../{permissions.ts => acp-interactions.ts} | 16 ++--- .../src/engines/sandbox_agent/daemon.ts | 2 +- .../src/engines/sandbox_agent/daytona.ts | 2 +- .../runner/src/engines/sandbox_agent/pause.ts | 31 ++++++++++ .../src/engines/sandbox_agent/provider.ts | 2 +- services/runner/src/permission-plan.ts | 16 +++-- services/runner/src/protocol.ts | 19 ++---- services/runner/src/responder.ts | 3 +- services/runner/src/sessions/interactions.ts | 2 +- services/runner/src/tools/dispatch.ts | 4 +- services/runner/src/tools/relay.ts | 6 +- .../runner/tests/unit/permission-plan.test.ts | 11 +--- ...=> sandbox-agent-acp-interactions.test.ts} | 60 +++++++++---------- .../unit/sandbox-agent-orchestration.test.ts | 12 ++-- .../tests/unit/tool-relay-permission.test.ts | 2 +- 20 files changed, 176 insertions(+), 118 deletions(-) rename services/runner/src/engines/sandbox_agent/{permissions.ts => acp-interactions.ts} (97%) create mode 100644 services/runner/src/engines/sandbox_agent/pause.ts rename services/runner/tests/unit/{sandbox-agent-permissions.test.ts => sandbox-agent-acp-interactions.test.ts} (91%) diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index 0539ef865a..4b21211cd3 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -40,6 +40,25 @@ decisions unless it contradicts an owner call". Newest last. Read together with relay must enforce permissions only when the harness does not gate (Pi) — otherwise one approval would be consumed at the gate and the relay would double-gate the same call. +- **Phase 5 landed (Codex implemented, reviewed here).** Legacy wire fields deleted + (`permissionPolicy`, `needsApproval`); absent `permissions` now defaults to + `allow_reads` (same default as the authored schema); `acp-interactions.ts` and + `pause.ts` renames; the four-site permission map heads `permission-plan.ts`; + `agenta-tools` reserved-name guard in the settings renderer; stale `services/agent/` + comment paths fixed. 401 runner + 477 SDK + 49 services tests green. +- **General docs sweep landed (Sonnet subagent, reviewed here).** Five permission + sections rewritten (tools.md, agent-configuration.md, permission-responder.md, + agent-config-schema.md, runner-interface README), ~24 field-level updates, 17 + superseded banners (capability-config, hitl-fix, streaming-invoke), and the + sessions/interactions specs re-grounded (every pause creates the row; T3 marked + implemented). Review fixes: one em dash, one stale `services/agent/` path. +- **Push-policy incident (separate from the commit incident).** `but push` on our + stacked lane force-pushed EVERY series in the stack, including `big-agents-work` + (Arda's PR #5054 head), six times over two hours, clobbering his and bekossy's + pushes; Arda closed #5054. Repair: his full tip (`a11b58cec8`, containing every + clobbered commit) was restored with a lease-guarded push. Standing rule (also in + memory): never `but push` a stack containing branches we do not own; our series + pushes via `git push origin docs/approval-boundary --force-with-lease`. - **Phase 4a landed (Codex implemented, reviewed here).** Batch envelopes now carry `stop_reason` (omitted when absent, so non-runner results are byte-identical) and, when paused, `pending_interaction: {id, tool}`. The SDK already threaded diff --git a/sdks/python/agenta/sdk/agents/adapters/claude_settings.py b/sdks/python/agenta/sdk/agents/adapters/claude_settings.py index f5207652ad..12a4abf1ff 100644 --- a/sdks/python/agenta/sdk/agents/adapters/claude_settings.py +++ b/sdks/python/agenta/sdk/agents/adapters/claude_settings.py @@ -114,6 +114,10 @@ def _rules_from_mcp_permissions(mcp_servers: Any) -> Dict[str, List[str]]: permission = _get(server, "permission") if not permission or not name: continue + if name == INTERNAL_TOOL_MCP_SERVER: + # Reserved for backend-resolved tools; a user server rule like ``mcp__agenta-tools`` + # would collide with resolved-tool rules ``mcp__agenta-tools__``. + continue rule = f"mcp__{name}" if permission == "allow": allow.append(rule) diff --git a/sdks/python/oss/tests/pytest/unit/agents/adapters/test_claude_settings.py b/sdks/python/oss/tests/pytest/unit/agents/adapters/test_claude_settings.py index 0939e3577a..3490d13392 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/adapters/test_claude_settings.py +++ b/sdks/python/oss/tests/pytest/unit/agents/adapters/test_claude_settings.py @@ -125,6 +125,28 @@ def test_mcp_permissions_route_to_their_lists_and_skip_unset(): assert perms["deny"] == ["mcp__shell"] +def test_reserved_internal_mcp_server_name_does_not_render_whole_server_rule(): + server = ResolvedMCPServer( + name=INTERNAL_TOOL_MCP_SERVER, + transport="http", + url="https://x", + permission="deny", + ) + tool = CallbackToolSpec( + name="get_user", description="d", call_ref="tools__x", permission="allow" + ) + perms = _settings(build_claude_settings_files(None, None, [server], [tool]))[ + "permissions" + ] + + rendered_rules = [ + rule for rules in perms.values() if isinstance(rules, list) for rule in rules + ] + assert perms["allow"] == [_rule("get_user")] + assert "deny" not in perms + assert f"mcp__{INTERNAL_TOOL_MCP_SERVER}" not in rendered_rules + + def test_merges_author_with_derived_and_dedupes(): # Author `WebFetch` keeps its position; the network-derived `WebFetch` is deduped, and the # filesystem-derived `Write`/`Edit` append. diff --git a/services/runner/src/engines/sandbox_agent.ts b/services/runner/src/engines/sandbox_agent.ts index fce50b8634..332bff1bdc 100644 --- a/services/runner/src/engines/sandbox_agent.ts +++ b/services/runner/src/engines/sandbox_agent.ts @@ -73,7 +73,11 @@ import { permissionsFromRequest, type GateDescriptor, } from "../permission-plan.ts"; -import { attachPermissionResponder } from "./sandbox_agent/permissions.ts"; +import { attachPermissionResponder } from "./sandbox_agent/acp-interactions.ts"; +import { + PAUSED, + PendingApprovalPauseController, +} from "./sandbox_agent/pause.ts"; import { createInteraction, resolveInteraction, @@ -471,7 +475,7 @@ export async function runSandboxAgent( // Propagate caller cancellation (a client disconnect on the streaming HTTP edge) so an // in-flight run aborts instead of finishing unobserved. The `finally` still disposes. ...(signal ? { signal } : {}), - // Drive the ACP HTTP client through a long-timeout undici dispatcher so a parked HITL + // Drive the ACP HTTP client through a long-timeout undici dispatcher so a paused HITL // turn (the connection held open while a human approves a tool) is NOT reaped by // undici's default `headersTimeout` (which would kill it with UND_ERR_HEADERS_TIMEOUT). // Daytona additionally needs the per-sandbox auth cookie carried across requests, so it @@ -638,21 +642,9 @@ export async function runSandboxAgent( if (update) run.handleUpdate(update); }); - // A pending approval pause must END the turn gracefully, not hold the ACP connection - // open forever (F-040): Claude does not end a turn on an unanswered gate, so - // `session.prompt()` would block indefinitely. On the first pause we destroy the session - // so the in-flight prompt resolves or the local park signal wins the race below. - let parked = false; - let resolveParked: (() => void) | undefined; - const parkedSignal = new Promise((resolve) => { - resolveParked = resolve; - }); - const onPark = (): void => { - if (parked) return; // first pause wins; later gates this turn are moot once we cancel - parked = true; - resolveParked?.(); - void sandbox.destroySession?.(session.id).catch(() => {}); - }; + const pause = new PendingApprovalPauseController(() => + sandbox.destroySession?.(session.id), + ); const permissionPlan = permissionsFromRequest(request); const storedDecisionMap = extractApprovalDecisions(request); if (storedDecisionMap.size > 0) { @@ -712,7 +704,7 @@ export async function runSandboxAgent( }, }); recordPendingInteraction(toolCallId, toolName, args); - onPark(); + pause.pause(); }, }; const serverPermissions = serverPermissionsFromRequest(request); @@ -723,7 +715,7 @@ export async function runSandboxAgent( latch, serverPermissions, log: logger, - onPark, + onPause: () => pause.pause(), onCreateInteraction: recordPendingInteraction, onResolveInteraction: (token) => { const cred = runCredential(request); @@ -797,33 +789,32 @@ export async function runSandboxAgent( } return "pendingApproval"; }, - onPark, + onPause: () => pause.pause(), }, ); } - // Race the prompt against the park signal: on a HITL park the prompt either resolves with + // Race the prompt against the pause signal: on a HITL pause the prompt either resolves with // a cancelled stop reason (the managed `session/cancel` landed) or never resolves at all - // (the harness ignores it) — either way `parkedSignal` ends the turn so the runner returns + // (the harness ignores it) — either way the pause signal ends the turn so the runner returns // promptly, the `finally` disposes the sandbox (no leak, F-040), and the egress emits a - // `finish` frame. When the park wins, the orphaned `prompt()` may later reject as the + // `finish` frame. When the pause wins, the orphaned `prompt()` may later reject as the // cancelled/torn-down connection unwinds; swallow that so it is not an `unhandledRejection` // (the daemon teardown's `fetch failed` is expected on a cancel, not a run error). - const PARKED = Symbol("parked"); const promptPromise = Promise.resolve( session.prompt([{ type: "text", text: plan.turnText }]), ); promptPromise.catch(() => {}); const raced = await Promise.race([ promptPromise, - parkedSignal.then(() => PARKED), + pause.signal.then(() => PAUSED), ]); - // A parked turn is terminal-but-incomplete: stop reason `paused` tells the egress to emit + // A paused turn is terminal-but-incomplete: stop reason `paused` tells the egress to emit // a clean `finish` (the FE then resumes on the user's decision). A real prompt result keeps // the harness's own stop reason. const stopReason = - raced === PARKED || parked ? "paused" : (raced as any)?.stopReason; - const result = raced === PARKED ? undefined : raced; + raced === PAUSED || pause.active ? "paused" : (raced as any)?.stopReason; + const result = raced === PAUSED ? undefined : raced; await toolRelay?.stop(); logger(`prompt stopReason=${stopReason}`); diff --git a/services/runner/src/engines/sandbox_agent/acp-fetch.ts b/services/runner/src/engines/sandbox_agent/acp-fetch.ts index 8ac35fa363..9fd2c7379d 100644 --- a/services/runner/src/engines/sandbox_agent/acp-fetch.ts +++ b/services/runner/src/engines/sandbox_agent/acp-fetch.ts @@ -1,15 +1,15 @@ import { Agent, fetch as undiciFetch } from "undici"; /** - * HITL parks the ACP HTTP connection open for human-timescale delays: when a tool call needs + * HITL pauses keep the ACP HTTP connection open for human-timescale delays: when a tool call needs * approval, the runner holds the in-flight `prompt` request while it waits for the human to - * click Approve/Deny, then resumes the same parked turn. Node's global `fetch` (undici) ships + * click Approve/Deny, then resumes the same paused turn. Node's global `fetch` (undici) ships * a DEFAULT `headersTimeout` (~5 min) and `bodyTimeout`; once it fires undici calls * `failReadable()` and the ACP stream dies with `UND_ERR_HEADERS_TIMEOUT`, killing both the - * parked turn and the resume turn. A plain chat completes in seconds so it never trips this. + * paused turn and the resume turn. A plain chat completes in seconds so it never trips this. * * The fix is to drive the ACP HTTP client through an undici dispatcher whose timeouts are - * disabled (0) or set to a long park window, instead of the default. We scope it to the ACP + * disabled (0) or set to a long pause window, instead of the default. We scope it to the ACP * fetch the `sandbox-agent` SDK uses rather than touching the global dispatcher, so unrelated * HTTP keeps its safe defaults. */ @@ -24,10 +24,10 @@ function envTimeoutMs(name: string): number { /** * Build the long-timeout undici dispatcher used for ACP HTTP. `headersTimeout` is the one that - * reaps a parked turn (no response headers arrive while the human deliberates); `bodyTimeout` - * guards the streamed body. Both default to disabled so a park held for any human-timescale + * reaps a paused turn (no response headers arrive while the human deliberates); `bodyTimeout` + * guards the streamed body. Both default to disabled so a pause held for any human-timescale * delay is never reaped. `keepAliveTimeout`/`keepAliveMaxTimeout` are raised so the connection - * is not pooled-closed mid-park either. + * is not pooled-closed mid-pause either. */ export function createAcpDispatcher(): Agent { const headersTimeout = envTimeoutMs("SANDBOX_AGENT_ACP_HEADERS_TIMEOUT_MS"); diff --git a/services/runner/src/engines/sandbox_agent/permissions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts similarity index 97% rename from services/runner/src/engines/sandbox_agent/permissions.ts rename to services/runner/src/engines/sandbox_agent/acp-interactions.ts index ea7cf88f8f..17ee5db6bd 100644 --- a/services/runner/src/engines/sandbox_agent/permissions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -20,7 +20,7 @@ export interface AttachPermissionResponderInput { * Called when a gate pauses the turn. The orchestration loop uses this to end the turn * gracefully because a paused Claude turn never resolves `session.prompt()` on its own. */ - onPark?: () => void; + onPause?: () => void; log?: (msg: string) => void; /** Called on pause to record the pending gate as an interaction (fire-and-forget). */ onCreateInteraction?: ( @@ -39,7 +39,7 @@ export function attachPermissionResponder({ responder, latch, serverPermissions = new Map(), - onPark, + onPause, log, onCreateInteraction, onResolveInteraction, @@ -47,14 +47,14 @@ export function attachPermissionResponder({ session.onPermissionRequest((req: any) => { void handleRequest(req).catch((err) => { log?.(`[HITL] permission handling failed: ${errorMessage(err)}`); - onPark?.(); + onPause?.(); }); }); // A pause sends NO harness reply, ever. Replying `reject` would make Claude emit a failed // tool call ("User refused permission") whose `tool_result {isError}` overwrites the // approval prompt on the same tool-call id (the F-024 clobber). The turn instead ends via - // `onPark` (session teardown resolves the RPC as cancelled, not rejected) and the next + // `onPause` (session teardown resolves the RPC as cancelled, not rejected) and the next // turn's stored decision answers the re-raised gate. const pauseUserApproval = (req: any, id: string, gate: GateDescriptor): void => { if (!latch.tryAcquire()) return; @@ -71,7 +71,7 @@ export function attachPermissionResponder({ }, }); onCreateInteraction?.(eventId, gate.toolName, gate.args); - onPark?.(); + onPause?.(); }; const pauseClientTool = ( @@ -95,7 +95,7 @@ export function attachPermissionResponder({ render: spec.render, }, }); - onPark?.(); + onPause?.(); }; const replyPermission = async ( @@ -110,7 +110,7 @@ export function attachPermissionResponder({ ); } catch (err) { log?.(`[HITL] reply failed id=${id} decision=${decision}: ${errorMessage(err)}`); - onPark?.(); + onPause?.(); return; } onResolveInteraction?.(id); @@ -128,7 +128,7 @@ export function attachPermissionResponder({ ); } catch (err) { log?.(`[HITL] reply failed id=${id} decision=${verdict.kind}: ${errorMessage(err)}`); - onPark?.(); + onPause?.(); } }; diff --git a/services/runner/src/engines/sandbox_agent/daemon.ts b/services/runner/src/engines/sandbox_agent/daemon.ts index 70c1e65e87..047468ae70 100644 --- a/services/runner/src/engines/sandbox_agent/daemon.ts +++ b/services/runner/src/engines/sandbox_agent/daemon.ts @@ -4,7 +4,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; const require = createRequire(import.meta.url); -// services/agent/src/engines/sandbox_agent/daemon.ts -> services/agent +// services/runner/src/engines/sandbox_agent/daemon.ts -> services/runner export const PKG_ROOT = dirname(dirname(dirname(dirname(fileURLToPath(import.meta.url))))); export const ADAPTER_BIN_DIR = join(PKG_ROOT, "node_modules", ".bin"); diff --git a/services/runner/src/engines/sandbox_agent/daytona.ts b/services/runner/src/engines/sandbox_agent/daytona.ts index 483f497440..1a21d3c032 100644 --- a/services/runner/src/engines/sandbox_agent/daytona.ts +++ b/services/runner/src/engines/sandbox_agent/daytona.ts @@ -146,7 +146,7 @@ export async function prepareDaytonaPiAssets({ * jar, so without this the proxy rejects later ACP requests with "Authentication * required" / 502. The sandbox-agent SDK accepts a custom fetch, so we hand it this one. * - * It layers on {@link createAcpFetch} (the long-timeout ACP dispatcher) so a parked HITL turn + * It layers on {@link createAcpFetch} (the long-timeout ACP dispatcher) so a paused HITL turn * over Daytona is not reaped by undici's default `headersTimeout` either. */ export function createCookieFetch(inner: typeof fetch = createAcpFetch()): typeof fetch { diff --git a/services/runner/src/engines/sandbox_agent/pause.ts b/services/runner/src/engines/sandbox_agent/pause.ts new file mode 100644 index 0000000000..c5a2bb1ad5 --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/pause.ts @@ -0,0 +1,31 @@ +/** + * Pending-approval pause controller. + * + * F-040: an unanswered approval must end the turn, destroy the session, and never reply to the + * harness gate. Historical docs call this "park"; this module uses pause/pendingApproval names. + */ +export const PAUSED = Symbol("paused"); + +export class PendingApprovalPauseController { + private pendingApproval = false; + private resolvePause: (() => void) | undefined; + + readonly signal: Promise; + + constructor(private readonly destroySession: () => Promise | void | undefined) { + this.signal = new Promise((resolve) => { + this.resolvePause = resolve; + }); + } + + pause(): void { + if (this.pendingApproval) return; + this.pendingApproval = true; + this.resolvePause?.(); + void Promise.resolve(this.destroySession()).catch(() => {}); + } + + get active(): boolean { + return this.pendingApproval; + } +} diff --git a/services/runner/src/engines/sandbox_agent/provider.ts b/services/runner/src/engines/sandbox_agent/provider.ts index 17baa12a21..5c6cf23b11 100644 --- a/services/runner/src/engines/sandbox_agent/provider.ts +++ b/services/runner/src/engines/sandbox_agent/provider.ts @@ -36,7 +36,7 @@ export function daytonaNetworkFields( * 15 minutes is the Daytona SDK's own documented default and sits comfortably beyond a normal * run (an actively prompting sandbox is BUSY, not idle, so this measures leaked-and-idle time, * not total run time). Override with `DAYTONA_AUTOSTOP` if runs idle - * longer (e.g. long parked HITL turns). + * longer (e.g. long paused HITL turns). */ export const DEFAULT_DAYTONA_AUTOSTOP_MINUTES = 15; diff --git a/services/runner/src/permission-plan.ts b/services/runner/src/permission-plan.ts index 8f179f437e..18f6e93a35 100644 --- a/services/runner/src/permission-plan.ts +++ b/services/runner/src/permission-plan.ts @@ -1,3 +1,13 @@ +/** + * Permission map; check all four sites when changing behavior: + * - SDK settings renderer: `sdks/python/agenta/sdk/agents/adapters/claude_settings.py` + * pre-answers Claude permission gates from authored and derived rules. + * - ACP responder: `services/runner/src/engines/sandbox_agent/acp-interactions.ts` answers + * gates the harness raises over ACP, or pauses when a human decision is required. + * - Relay enforcement: `services/runner/src/tools/relay.ts` enforces the same decisions for Pi. + * - Client-tool ladder: `services/runner/src/responder.ts` handles browser-fulfilled tools + * across the pause/resume boundary. + */ import type { AgentRunRequest, PermissionMode, ToolPermission } from "./protocol.ts"; /** Which component executes the gated tool; decides how resume matching anchors names. */ @@ -48,6 +58,7 @@ const RULE_RANK: Record = { export function permissionsFromRequest( request: AgentRunRequest, ): PermissionPlan { + // Operator kill-switch for incident response: deny wins over every authored request. if (process.env.SANDBOX_AGENT_DENY_PERMISSIONS === "true") { return { default: "deny", rules: [] }; } @@ -69,10 +80,7 @@ export function permissionsFromRequest( }; } - if (request.permissionPolicy === "deny") { - return { default: "deny", rules: [] }; - } - return { default: "allow", rules: [] }; + return { default: "allow_reads", rules: [] }; } export function effectivePermission( diff --git a/services/runner/src/protocol.ts b/services/runner/src/protocol.ts index 4f382c0188..4dbe579ab5 100644 --- a/services/runner/src/protocol.ts +++ b/services/runner/src/protocol.ts @@ -88,12 +88,11 @@ export interface PermissionsConfig { /** * A runnable tool the backend already resolved from the agent config. * - * Three orthogonal axes: + * Two orthogonal axes: * - `kind` (executor): how the runner fulfils a call. `callback` POSTs back through Agenta's * /tools/call (gateway tools; the Composio key stays server-side); `code` runs `code` in a * sandbox subprocess with `env` (resolved secrets, scoped to the subprocess); `client` is * fulfilled by the browser across a turn boundary. Absent = `callback` (back-compat). - * - `needsApproval`: gate the call on a human yes/no (mechanics owned by the run-event layer). * - `render`: a generative-UI hint (see `RenderHint`). * * `callRef` is set for `callback` (gateway) tools (the slug the bridge sends back to @@ -129,15 +128,13 @@ export interface ResolvedToolSpec { runtime?: "python" | "node"; code?: string; env?: Record; - needsApproval?: boolean; render?: RenderHint; /** MCP behavioral hint: true (read-only), false (mutating), absent (unknown). */ readOnly?: boolean; /** * Layer-3 permission: `allow` runs with no prompt, `ask` raises a * human-in-the-loop request, `deny` never runs. Absent = fall back to the global - * `permissionPolicy`. The SDK derives a default from `readOnly`/`needsApproval` when the - * author set none. Plumbing only here; enforcement is a later slice. + * permission plan. The SDK derives a default from `readOnly` when the author set none. */ permission?: ToolPermission; } @@ -232,9 +229,8 @@ export interface McpServerConfig { tools?: string[]; /** * Layer-3 permission for the whole server: `allow` / `ask` / `deny`. Absent = - * fall back to the global `permissionPolicy`. An MCP server has no `readOnly` hint, so there - * is no derived default: an explicit author value or nothing. Plumbing only; enforcement is - * a later slice. + * fall back to the global permission plan. An MCP server has no `readOnly` hint, so there + * is no derived default: an explicit author value or nothing. */ permission?: ToolPermission; } @@ -438,12 +434,7 @@ export interface AgentRunRequest { mcpServers?: McpServerConfig[]; /** Where customTools route their calls back to. Required when customTools is set. */ toolCallback?: ToolCallbackContext; - /** How a permission-gating harness handles tool-use prompts: "auto" (default) | "deny". */ - permissionPolicy?: string; - /** - * Authored permission plan assembled by the SDK (`runner.permissions.*` in the agent config). - * When absent, the runner maps the legacy `permissionPolicy` via `permissionsFromRequest()`. - */ + /** Authored permission plan assembled by the SDK (`runner.permissions.*` in the agent config). */ permissions?: PermissionsConfig; /** * The declared sandbox security boundary (Layer 2). Omitted when unset. The network policy is diff --git a/services/runner/src/responder.ts b/services/runner/src/responder.ts index 51d561f61b..01103ff487 100644 --- a/services/runner/src/responder.ts +++ b/services/runner/src/responder.ts @@ -202,7 +202,8 @@ export class ApprovalResponder implements Responder { * `tool_result` content block keyed by `toolCallId` whose `output` is an `{ approved: * boolean }` envelope. Each decision is indexed only by `approvedCallKey(name, args)` - the * cold-replay anchor. The name/args are recovered from the matching `tool_call` block (same - * `toolCallId`) the egress folds into the transcript. + * `toolCallId`) the egress folds into the transcript. An unbindable approval envelope is + * dropped; the gate re-raises and re-prompts, never guessed. */ export function extractApprovalDecisions( request: AgentRunRequest, diff --git a/services/runner/src/sessions/interactions.ts b/services/runner/src/sessions/interactions.ts index 206decd938..faec4625af 100644 --- a/services/runner/src/sessions/interactions.ts +++ b/services/runner/src/sessions/interactions.ts @@ -120,7 +120,7 @@ export async function resolveInteraction( /** * At the start of a new session turn, cancel prior turns' still-pending gates: if the user - * sent a new message instead of answering a parked approval, that gate is orphaned. Spares + * sent a new message instead of answering a pending approval, that gate is orphaned. Spares * the current turn's own gates via `turn_id`. Fire-and-forget, single attempt — best effort, * never blocks the turn. */ diff --git a/services/runner/src/tools/dispatch.ts b/services/runner/src/tools/dispatch.ts index b601c1608e..97303f81c3 100644 --- a/services/runner/src/tools/dispatch.ts +++ b/services/runner/src/tools/dispatch.ts @@ -10,7 +10,7 @@ * * The three executor kinds (see `ResolvedToolSpec`): * - `code`: advertised to harnesses, but rejected by the sidecar as unsupported. - * - `client`: browser-fulfilled across a turn boundary; permission responder parks it. + * - `client`: browser-fulfilled across a turn boundary; permission responder pauses it. * - `callback` (default): POST back through Agenta's /tools/call so the Composio key and * connection auth stay server-side. On Daytona the in-sandbox process can't reach Agenta, * so the call is relayed through the runner via files (see tools/relay.ts) when `relayDir` @@ -155,7 +155,7 @@ export async function relayToolCall( * turns the throw into a tool-error result so the model loop continues rather than crashing. * * - `code` -> reject as unsupported by the sidecar, no callback/relay. - * - `client` → relay to the runner so it can park the browser-fulfilled call. + * - `client` → relay to the runner so it can pause the browser-fulfilled call. * - default/`callback` → relay through the runner when `opts.relayDir` is set (Daytona), * else POST directly to `opts.endpoint`. */ diff --git a/services/runner/src/tools/relay.ts b/services/runner/src/tools/relay.ts index e1f1287246..7947a3bd81 100644 --- a/services/runner/src/tools/relay.ts +++ b/services/runner/src/tools/relay.ts @@ -65,7 +65,7 @@ export interface RelayPermissions { * what reaches it. True when the relay is the only gate (Pi). */ enforce: boolean; decide: (gate: GateDescriptor) => Verdict; - /** Called when an ask pauses at the relay: emit the approval event + park (engine-owned). */ + /** Called when an ask pauses at the relay: emit the approval event and pause the turn. */ onPendingApproval: (info: { toolCallId: string; toolName: string; @@ -75,7 +75,7 @@ export interface RelayPermissions { export interface ClientToolRelay { onClientTool: (request: ClientToolRelayRequest) => Promise; - onPark?: (request: ClientToolRelayRequest) => void; + onPause?: (request: ClientToolRelayRequest) => void; } const PAUSED = Symbol("paused"); @@ -213,7 +213,7 @@ async function executeRelayedTool( }; const decision = await clientToolRelay.onClientTool(request); if (decision === "pendingApproval") { - clientToolRelay.onPark?.(request); + clientToolRelay.onPause?.(request); return PAUSED; } if (decision === "deny") { diff --git a/services/runner/tests/unit/permission-plan.test.ts b/services/runner/tests/unit/permission-plan.test.ts index 648903830f..e327a606a6 100644 --- a/services/runner/tests/unit/permission-plan.test.ts +++ b/services/runner/tests/unit/permission-plan.test.ts @@ -264,16 +264,9 @@ describe("permissionsFromRequest", () => { ); }); - it("maps absent new block plus legacy deny to deny", () => { - assert.deepEqual( - permissionsFromRequest({ permissionPolicy: "deny" } as AgentRunRequest), - { default: "deny", rules: [] }, - ); - }); - - it("maps absent new block and absent legacy field to allow", () => { + it("maps absent block to the safe SDK default", () => { assert.deepEqual(permissionsFromRequest({} as AgentRunRequest), { - default: "allow", + default: "allow_reads", rules: [], }); }); diff --git a/services/runner/tests/unit/sandbox-agent-permissions.test.ts b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts similarity index 91% rename from services/runner/tests/unit/sandbox-agent-permissions.test.ts rename to services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts index ef7a012e2a..a1946e21e3 100644 --- a/services/runner/tests/unit/sandbox-agent-permissions.test.ts +++ b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts @@ -1,7 +1,7 @@ /** - * Unit tests for sandbox-agent ACP permission wiring. + * Unit tests for sandbox-agent ACP interaction wiring. * - * Run: pnpm test (or: pnpm exec vitest run tests/unit/sandbox-agent-permissions.test.ts) + * Run: pnpm test (or: pnpm exec vitest run tests/unit/sandbox-agent-acp-interactions.test.ts) */ import { describe, it } from "vitest"; import assert from "node:assert/strict"; @@ -10,7 +10,7 @@ import type { AgentEvent } from "../../src/protocol.ts"; import type { ClientToolVerdict, Responder } from "../../src/responder.ts"; import type { Verdict } from "../../src/permission-plan.ts"; import { PendingApprovalLatch } from "../../src/permission-plan.ts"; -import { attachPermissionResponder } from "../../src/engines/sandbox_agent/permissions.ts"; +import { attachPermissionResponder } from "../../src/engines/sandbox_agent/acp-interactions.ts"; function flushPromises(): Promise { return new Promise((resolve) => setImmediate(resolve)); @@ -102,29 +102,29 @@ describe("attachPermissionResponder", () => { assert.deepEqual(events, []); }); - it("pendingApproval emits, creates the interaction, parks, and sends no reply", async () => { + it("pendingApproval emits, creates the interaction, pauses, and sends no reply", async () => { const replies: Array<{ id: string; reply: string }> = []; const { session, emit } = makeSession(async (id, reply) => { replies.push({ id, reply }); }); const events: AgentEvent[] = []; const created: Array<{ token: string; toolName?: string; args: unknown }> = []; - let parks = 0; + let pauses = 0; attachPermissionResponder({ session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "pendingApproval" }), latch: new PendingApprovalLatch(), - onPark: () => { - parks += 1; + onPause: () => { + pauses += 1; }, onCreateInteraction: (token, toolName, args) => { created.push({ token, toolName, args }); }, }); emit({ - id: "perm-park", + id: "perm-pause", availableReplies: ["once", "always", "reject"], toolCall: { toolCallId: "tool-9", name: "edit", input: { path: "a" } }, options: { cwd: "/repo" }, @@ -132,14 +132,14 @@ describe("attachPermissionResponder", () => { await flushPromises(); assert.deepEqual(replies, []); - assert.equal(parks, 1); + assert.equal(pauses, 1); assert.deepEqual(created, [ - { token: "perm-park", toolName: "edit", args: { path: "a" } }, + { token: "perm-pause", toolName: "edit", args: { path: "a" } }, ]); assert.deepEqual(events, [ { type: "interaction_request", - id: "perm-park", + id: "perm-pause", kind: "user_approval", payload: { toolCallId: "tool-9", @@ -151,18 +151,18 @@ describe("attachPermissionResponder", () => { ]); }); - it("two concurrent pending gates emit and park only once through the latch", async () => { + it("two concurrent pending gates emit and pause only once through the latch", async () => { const { session, emit } = makeSession(); const events: AgentEvent[] = []; - let parks = 0; + let pauses = 0; attachPermissionResponder({ session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "pendingApproval" }), latch: new PendingApprovalLatch(), - onPark: () => { - parks += 1; + onPause: () => { + pauses += 1; }, }); emit({ id: "perm-1", toolCall: { toolCallId: "tool-1", name: "edit" } }); @@ -171,23 +171,23 @@ describe("attachPermissionResponder", () => { assert.equal(events.length, 1); assert.equal((events[0] as any).id, "perm-1"); - assert.equal(parks, 1); + assert.equal(pauses, 1); }); - it("reply rejection parks and does not resolve the interaction", async () => { + it("reply rejection pauses and does not resolve the interaction", async () => { const { session, emit } = makeSession(async () => { throw new Error("daemon closed"); }); const resolved: string[] = []; - let parks = 0; + let pauses = 0; attachPermissionResponder({ session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "allow" }), latch: new PendingApprovalLatch(), - onPark: () => { - parks += 1; + onPause: () => { + pauses += 1; }, onResolveInteraction: (token) => { resolved.push(token); @@ -196,7 +196,7 @@ describe("attachPermissionResponder", () => { emit({ id: "perm-1", availableReplies: ["once", "reject"] }); await flushPromises(); - assert.equal(parks, 1); + assert.equal(pauses, 1); assert.deepEqual(resolved, []); }); @@ -206,15 +206,15 @@ describe("attachPermissionResponder", () => { replies.push({ id, reply }); }); const events: AgentEvent[] = []; - let parks = 0; + let pauses = 0; attachPermissionResponder({ session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "allow" }), latch: new PendingApprovalLatch(), - onPark: () => { - parks += 1; + onPause: () => { + pauses += 1; }, }); emit({ @@ -224,18 +224,18 @@ describe("attachPermissionResponder", () => { await flushPromises(); assert.deepEqual(replies, []); - assert.equal(parks, 1); + assert.equal(pauses, 1); assert.equal(events.length, 1); assert.equal((events[0] as any).id, "tool-fallback"); }); - it("client tool pending forwards to the browser and parks", async () => { + it("client tool pending forwards to the browser and pauses", async () => { const replies: Array<{ id: string; reply: string }> = []; const { session, emit } = makeSession(async (id, reply) => { replies.push({ id, reply }); }); const events: AgentEvent[] = []; - let parks = 0; + let pauses = 0; attachPermissionResponder({ session, @@ -245,8 +245,8 @@ describe("attachPermissionResponder", () => { { kind: "pendingApproval" }, ), latch: new PendingApprovalLatch(), - onPark: () => { - parks += 1; + onPause: () => { + pauses += 1; }, }); emit({ @@ -265,7 +265,7 @@ describe("attachPermissionResponder", () => { await flushPromises(); assert.deepEqual(replies, []); - assert.equal(parks, 1); + assert.equal(pauses, 1); assert.deepEqual(events, [ { type: "interaction_request", diff --git a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts index 1c7a5c6086..c355a0c134 100644 --- a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts +++ b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts @@ -524,7 +524,7 @@ describe("runSandboxAgent orchestration", () => { // Trailing arg is the relay callbacks object (client-tool + park handlers). assert.deepEqual( Object.keys((calls.toolRelayArgs?.[6] ?? {}) as object).sort(), - ["onClientTool", "onPark"], + ["onClientTool", "onPause"], ); assert.equal( calls.toolRelayStops, @@ -851,7 +851,7 @@ describe("runSandboxAgent default ApprovalResponder wiring", () => { return { calls, deps }; } - it("headless (/invoke: no sessionId, no decisions) auto-allows — no regression", async () => { + it("absent permissions use allow_reads and pause an unrated gate", async () => { const { calls, deps } = depsWithDefaultResponder(); const result = await runSandboxAgent( @@ -866,11 +866,9 @@ describe("runSandboxAgent default ApprovalResponder wiring", () => { await flushPromises(); assert.equal(result.ok, true); - // Headless auto-allow gates each call individually, so once (this call) is equivalent to - // the old always and strictly safer — no turn-wide grant that skips re-gating. - assert.deepEqual(calls.permissionReplies, [ - { id: "perm-1", reply: "once" }, - ]); + if (!result.ok) return; + assert.equal(result.stopReason, "paused"); + assert.deepEqual(calls.permissionReplies, []); }); it("effective ask with no decision pauses the tool, no harness reply (F-024)", async () => { diff --git a/services/runner/tests/unit/tool-relay-permission.test.ts b/services/runner/tests/unit/tool-relay-permission.test.ts index 0e30b48ccd..c920a0c253 100644 --- a/services/runner/tests/unit/tool-relay-permission.test.ts +++ b/services/runner/tests/unit/tool-relay-permission.test.ts @@ -237,7 +237,7 @@ describe("startToolRelay permission enforcement", () => { stopWhen: () => parked.length === 1, clientToolRelay: { onClientTool: async () => "pendingApproval", - onPark: (request) => { + onPause: (request) => { parked.push(request.toolCallId); }, }, From b2f63946055866e0aaf3d55fac09edf8c44e488f Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 17:01:27 +0200 Subject: [PATCH 17/24] docs(agent-workflows): sweep the old permission vocabulary; banner superseded workspaces --- .../documentation/adapters/claude-code.md | 33 ++- .../documentation/agent-configuration.md | 55 ++-- .../documentation/agent-template.md | 2 +- .../documentation/architecture.md | 3 +- .../documentation/ports-and-adapters.md | 5 +- .../agent-workflows/documentation/protocol.md | 2 +- .../agent-workflows/documentation/tools.md | 66 ++++- .../cross-service/service-to-agent-runner.md | 5 +- .../in-service/agent-service-handler.md | 5 +- .../interfaces/in-service/harness-adapters.md | 12 +- .../in-service/neutral-runtime-dtos.md | 9 +- .../in-service/permission-responder.md | 102 +++++-- .../in-service/tool-models-and-resolution.md | 32 +- .../public-edge/agent-config-schema.md | 34 ++- .../interfaces/public-edge/agent-messages.md | 2 +- .../interfaces/public-edge/workflow-invoke.md | 4 +- .../agent-config-structure-cleanup/status.md | 16 +- .../build-kit-tools-cleanup/context.md | 82 ++++++ .../build-kit-tools-cleanup/research.md | 278 ++++++++++++++++++ .../projects/capability-config/README.md | 2 + .../projects/capability-config/context.md | 2 + .../projects/capability-config/plan.md | 2 + .../projects/capability-config/proposal.md | 2 + .../projects/capability-config/research.md | 2 + .../projects/capability-config/status.md | 2 + .../projects/hitl-fix/README.md | 2 + .../projects/hitl-fix/context.md | 2 + .../agent-workflows/projects/hitl-fix/plan.md | 2 + .../projects/hitl-fix/research.md | 2 + .../projects/hitl-fix/status.md | 2 + .../agent-workflows/projects/qa/README.md | 2 +- .../projects/runner-interface/README.md | 18 +- .../sandbox-agent-refactor-plan.md | 6 +- .../projects/tool-discovery/design.md | 2 +- .../projects/wire-contract-schema/README.md | 6 +- .../sessions/interactions/extend/specs.md | 40 +-- .../sessions/interactions/extend/tasks.md | 21 +- docs/designs/sessions/interactions/specs.md | 4 +- 38 files changed, 696 insertions(+), 172 deletions(-) create mode 100644 docs/design/agent-workflows/projects/build-kit-tools-cleanup/context.md create mode 100644 docs/design/agent-workflows/projects/build-kit-tools-cleanup/research.md diff --git a/docs/design/agent-workflows/documentation/adapters/claude-code.md b/docs/design/agent-workflows/documentation/adapters/claude-code.md index 7e440be68f..59476e59b5 100644 --- a/docs/design/agent-workflows/documentation/adapters/claude-code.md +++ b/docs/design/agent-workflows/documentation/adapters/claude-code.md @@ -63,20 +63,25 @@ The first is static, written before the session starts: the claude adapter rende delivered on the `harnessFiles` wire seam) whose `permissions.allow` / `permissions.ask` / `permissions.deny` lists Claude Code reads via `settingSources`. Each backend-resolved EXECUTABLE tool (callback/code) gets a per-tool rule `mcp__agenta-tools__`, because that is how Claude -addresses a tool of the internal `agenta-tools` MCP server above. A tool whose -`effective_permission()` is `allow` gets an allow rule, so Claude runs it without raising a gate; -`deny` gets a deny rule; `ask` or unset gets no allow rule, so the gate still fires. - -The second is dynamic: a gate that does fire arrives at `session.onPermissionRequest`, where the -runner answers for the (absent) human. When a human surface exists the runner parks the undecided -gate for a HITL approval round-trip; otherwise it falls to the per-run permission policy (auto or -deny). Pi does not need this hook because Pi does not gate tools this way. - -Both layers are needed because Claude's gate fires BEFORE the runner relay that would otherwise -honor an `allow`. Without the settings.json rule, an `allow` resolved tool always parked -(finding F-046); the rule is what lets it run. The `ask`/unset path is left to the gate on -purpose, so HITL approval is preserved. Note that `permission_policy: "auto"` is NOT a blanket -bypass — it still means "gate, then HITL or policy", not "allow everything". +addresses a tool of the internal `agenta-tools` MCP server above. A tool whose effective +permission is `allow` (an explicit `allow`, or a read-hinted tool under the `allow_reads` +policy) gets an allow rule, so Claude runs it without raising a gate; `deny` gets a deny +rule; `ask` or unset gets no allow rule, so the gate still fires. + +The second is dynamic: a gate that does fire arrives at `session.onPermissionRequest`, where +the runner's `ApprovalResponder` (`services/runner/src/responder.ts`) answers it. The verdict +comes from the shared decision module (`services/runner/src/permission-plan.ts`): the tool's +explicit permission wins, then a matching authored rule, then the policy mode in +`permissions.default`. `allow` approves the call, `deny` refuses it, and `ask` pauses the +turn, emits one approval request, and waits for a human. Pi never raises this hook; on Pi +the tool relay enforces the same decision function. + +Both layers are needed because Claude's gate fires BEFORE the runner relay that would +otherwise honor an `allow`. Without the settings.json rule, an `allow` resolved tool always +paused (finding F-046); the rule is what lets it run. The `ask`/unset path is left to the +gate on purpose, so human approval is preserved. Note that `permissions.default: "allow"` is +not a per-tool override: a tool set to `ask` or `deny` keeps its own verdict, because the +explicit per-tool permission always beats the policy mode. ## Tracing from the event stream diff --git a/docs/design/agent-workflows/documentation/agent-configuration.md b/docs/design/agent-workflows/documentation/agent-configuration.md index c0fc541efb..edf390d224 100644 --- a/docs/design/agent-workflows/documentation/agent-configuration.md +++ b/docs/design/agent-workflows/documentation/agent-configuration.md @@ -13,8 +13,8 @@ The playground renders a single composite `agent_config` control. The field list control is not hardcoded in the frontend. It is fetched from the backend catalog type `agent_config`, which the SDK defines once as `AgentConfigSchema`. The runtime then re-parses the same payload into one permissive `AgentConfig` (the run-selection fields `harness`, -`sandbox`, and `permission_policy` live on it), resolves tools and secrets server-side, and -hands a final wire request to the Node runner. +`sandbox`, and `runner.permissions.default` live on it), resolves tools and secrets +server-side, and hands a final wire request to the Node runner. ## Three objects share the name "AgentConfig" @@ -69,12 +69,14 @@ existing control: shape the prompt control uses. - `mcp_servers` renders as a flat array. Each entry uses `McpServerItemControl`, which is a JSON editor for one server entry. -- `harness`, `sandbox`, and `permission_policy` each render as an enum select. +- `harness`, `sandbox`, and the permission policy each render as an enum select. The + permission policy select has four modes: `allow`, `ask`, `deny`, `allow_reads` (the + default). It renders for Pi too; Pi now honors it the same way Claude does. So the object the form produces is: ``` -{ agents_md, model, tools[], mcp_servers[], harness, sandbox, permission_policy } +{ agents_md, model, tools[], mcp_servers[], harness, sandbox, runner: { permissions: { default } } } ``` The field set comes from the backend at runtime. The frontend fetches the catalog type with @@ -100,7 +102,7 @@ Its fields and defaults: | `mcp_servers` | `List[MCPServerConfig]` | empty list | typed | | `harness` | `Literal["pi_core","claude","pi_agenta"]` | `"pi_core"` | enum | | `sandbox` | `Literal["local","daytona"]` | `"local"` | enum | -| `permission_policy` | `Literal["auto","deny"]` | `"auto"` | enum | +| `runner.permissions.default` | `Literal["allow","ask","deny","allow_reads"]` | `"allow_reads"` | enum, four modes | The schema is registered in `CATALOG_TYPES` under the key `"agent_config"` (`sdks/python/agenta/sdk/utils/types.py:1132`). The API catalog imports `CATALOG_TYPES` from @@ -142,7 +144,7 @@ class AgentConfig(BaseModel): # the run-selection fields harness: str = "pi_core" sandbox: str = "local" - permission_policy: PermissionPolicy = "auto" + permission_default: PermissionMode = "allow_reads" ``` One correction to a common belief. This model is not `extra="allow"`. Its looseness comes @@ -158,11 +160,18 @@ The genuinely loose object is the file-default dataclass at built-in default, not user input. The run-selection fields are on this neutral config too. `harness`, `sandbox`, and -`permission_policy` are plain fields on `AgentConfig` (in +`permission_default` are plain fields on `AgentConfig` (in `sdks/python/agenta/sdk/agents/dtos.py`). They used to live on a separate `RunSelection` object; that object is retired, because there is one agent definition, not an agent plus a sidecar selection. The composite schema and the neutral config now agree: both keep these -fields next to the rest of the agent. +fields next to the rest of the agent. The authored form of the same value is +`runner.permissions.default`; the SDK flattens it onto `permission_default` when it parses +the template. The old three-name split (`runner.interactions.headless` authored, +`permission_policy` stored, `permissionPolicy` on the wire) is gone. The value now has four +modes (`allow`, `ask`, `deny`, `allow_reads`) instead of two (`auto`, `deny`), and the wire +field is `permissions: {default, rules?}`. Authors can also add `runner.permissions.rules`, a +list of `{pattern, permission}` entries for harness-builtin tools (for example +`Bash(rm:*)` set to `ask`). The runner checks rules before falling back to `default`. Tool entries are strict even though the list is lenient. Each tool subclass is `extra="forbid"` (`sdks/python/agenta/sdk/agents/tools/models.py`). `MCPServerConfig` is also `extra="forbid"` @@ -191,9 +200,10 @@ resolves tools, MCP servers, and secrets server-side, bundles everything into a reads `agent_config.sandbox` and passes it to `SandboxAgentBackend(sandbox=...)` instead. The final wire shape the Node runner receives is `AgentRunRequest` in -`services/agent/src/protocol.ts` (around line 185). That is the true wired surface: +`services/runner/src/protocol.ts`. That is the true wired surface: `harness`, `sandbox`, `agentsMd`, `systemPrompt`/`appendSystemPrompt`, `model`, `tools` -(builtin names), `skills`, `customTools`, `mcpServers`, `toolCallback`, `permissionPolicy`. +(builtin names), `skills`, `customTools`, `mcpServers`, `toolCallback`, and +`permissions: {default, rules?}`. ## Field-by-field: enforced vs loose, wired vs decorative @@ -209,7 +219,7 @@ Legend: (a) catalog/schema, (b) SDK neutral config, (c) runtime. | agents_md | yes, `agents_md: str` | yes, as `instructions` | wired to `agentsMd` | The schema names it `agents_md`. The neutral config names it `instructions`. | | harness | yes, enum | yes, on `AgentConfig` | wired, picks the harness class | Enum-enforced. The runtime validates via `make_harness`. | | sandbox | yes, enum | yes, on `AgentConfig` | wired to the backend, absent from `SessionConfig` | Backend concern, not agent identity. | -| permission_policy | yes, enum | yes, on `AgentConfig` | wired to `SessionConfig` | Only the Claude harness reads it. Pi ignores it, so it is decorative for `pi_core` and `pi_agenta`. | +| runner.permissions.default | yes, enum (4 modes) | yes, as `permission_default` on `AgentConfig` | wired to `SessionConfig` and the run request's `permissions.default` | Enforced for both harnesses: Claude at its settings file and the ACP responder, Pi at the tool relay. No longer decorative on Pi. | ## Notable gaps and quirks @@ -219,13 +229,14 @@ list. `persona` is a forced append-system string. Neither appears in any schema, appears on the neutral config, and the playground renders no control for either. Pi and Claude harnesses get no forced skills or persona. -Per-harness divergence is real. `permission_policy` is wired only for Claude. Builtin tool -names are dropped for Claude with a warning, because builtins are Pi-only. Skills and persona -are Agenta-only. Pi's `system` and `append_system` overrides come through the -`harness_kwargs` escape hatch on the neutral config, which is itself absent from the schema. +Per-harness divergence is real in other ways, but not in permission enforcement anymore: the +permission policy is now enforced on both Claude and Pi. Builtin tool names are dropped for +Claude with a warning, because builtins are Pi-only. Skills and persona are Agenta-only. Pi's +`system` and `append_system` overrides come through the `harness_kwargs` escape hatch on the +neutral config, which is itself absent from the schema. -Harness, sandbox, and permission policy sit next to the agent definition in both the schema -and the neutral `AgentConfig`. The two agree on one control. +Harness, sandbox, and the permission policy sit next to the agent definition in both the +schema and the neutral `AgentConfig`. The two agree on one control. ## A concrete example config @@ -248,15 +259,15 @@ This is what the playground saves and the runtime reads: ], "harness": "pi_core", "sandbox": "local", - "permission_policy": "auto" + "runner": { "permissions": { "default": "allow_reads" } } } ``` With this config, the runtime reads `agents_md`, `model`, `tools`, `mcp_servers`, and the -run-selection fields `harness`, `sandbox`, and `permission_policy` through the one neutral -`AgentConfig`, resolves the tools and MCP servers server-side, and runs one turn on the Pi -harness in a local sandbox. The `permission_policy` value is ignored because the harness is -Pi, not Claude. +run-selection fields `harness`, `sandbox`, and `runner.permissions.default` through the one +neutral `AgentConfig`, resolves the tools and MCP servers server-side, and runs one turn on +the Pi harness in a local sandbox. Under `allow_reads`, `web_search` runs as a read with no +prompt; a write tool would pause for approval on this harness exactly as it would on Claude. ## See also diff --git a/docs/design/agent-workflows/documentation/agent-template.md b/docs/design/agent-workflows/documentation/agent-template.md index 1285745d5f..cd43d37c2e 100644 --- a/docs/design/agent-workflows/documentation/agent-template.md +++ b/docs/design/agent-workflows/documentation/agent-template.md @@ -37,7 +37,7 @@ Today the request surface includes: - `parameters.agent.mcp_servers` - `parameters.agent.harness` - `parameters.agent.sandbox` -- `parameters.agent.permission_policy` +- `parameters.agent.runner.permissions.default` (four modes: `allow`, `ask`, `deny`, `allow_reads`) - harness-specific options such as Pi prompt overrides The runtime also has forced Agenta policy in `AgentaHarness`, but that content is diff --git a/docs/design/agent-workflows/documentation/architecture.md b/docs/design/agent-workflows/documentation/architecture.md index 919565e026..e84caa1aba 100644 --- a/docs/design/agent-workflows/documentation/architecture.md +++ b/docs/design/agent-workflows/documentation/architecture.md @@ -109,7 +109,8 @@ Batch `/invoke` follows this path: 1. The workflow route calls `_agent` in `services/oss/src/agent/app.py`. 2. `_agent` parses one `AgentConfig` from request parameters; it carries the run-selection - fields `harness`, `sandbox`, and `permission_policy`. + fields `harness` and `sandbox`, plus the permission policy at + `runner.permissions.default`. 3. The service resolves three things independently: tools, MCP servers, and provider-key secrets. MCP resolution is gated by `AGENTA_AGENT_ENABLE_MCP` (`services/oss/src/agent/tools/resolver.py:23`, off by default). diff --git a/docs/design/agent-workflows/documentation/ports-and-adapters.md b/docs/design/agent-workflows/documentation/ports-and-adapters.md index 912f9ac7ff..58c9dcaea1 100644 --- a/docs/design/agent-workflows/documentation/ports-and-adapters.md +++ b/docs/design/agent-workflows/documentation/ports-and-adapters.md @@ -86,8 +86,9 @@ session representation and storage size. ## Config Ownership `AgentConfig` describes the agent itself: instructions, model, tool references, MCP server -config, and per-harness option bags. It also carries the run-selection fields `harness`, -`sandbox`, and `permission_policy`; there is one agent config, not a config plus a separate +config, and per-harness option bags. It also carries the run-selection fields `harness` and +`sandbox`, plus the permission policy at `runner.permissions.default`; there is one agent +config, not a config plus a separate `RunSelection` object. The handler reads `sandbox` to choose a backend, but the backend choice itself is not stored on the config. diff --git a/docs/design/agent-workflows/documentation/protocol.md b/docs/design/agent-workflows/documentation/protocol.md index e40e222251..91a71a3184 100644 --- a/docs/design/agent-workflows/documentation/protocol.md +++ b/docs/design/agent-workflows/documentation/protocol.md @@ -118,7 +118,7 @@ Request fields include: | `messages` | Conversation history and current turn. | | `secrets` | Provider env vars resolved by the service. | | `tools`, `customTools`, `toolCallback`, `mcpServers` | Resolved tool delivery. | -| `permissionPolicy` | `auto` or `deny` for permission-gating harnesses. | +| `permissions` | Permission plan: `{default, rules?}`. `default` is one of `allow`, `ask`, `deny`, or `allow_reads`; `rules` is an optional list of `{pattern, permission}` entries for harness builtins. The runner enforces it on every harness. | | `trace` | Trace context for nested spans. | One-shot calls return one JSON result. Streaming calls use NDJSON internally: one diff --git a/docs/design/agent-workflows/documentation/tools.md b/docs/design/agent-workflows/documentation/tools.md index 52008e90d0..35ec1f49d5 100644 --- a/docs/design/agent-workflows/documentation/tools.md +++ b/docs/design/agent-workflows/documentation/tools.md @@ -35,7 +35,7 @@ shares two fields through `ToolConfigBase`, and then a `type` discriminator pick | `code` | `name`, `runtime` (`python`/`node`), `script`, `input_schema`, `secrets` | An inline snippet the author writes, with named vault secrets injected. | | `client` | `name`, `input_schema` | A tool the browser fulfils, like "ask the user to pick a date." | | `reference` | `ref_by` (`variant`/`environment`), `slug`, optional `environment`/`version`, optional `name`/`description`/`input_schema` | A workflow referenced as a tool (see below); the service runs the referenced workflow revision when the model calls it. | -| `platform` | `op` (a platform-op catalog key), optional `needs_approval`/`permission` | An existing Agenta endpoint exposed to the agent (e.g. `find_capabilities`, `query_workflows`, `commit_revision`); the runner calls the endpoint directly. See [Platform tools](#platform-tools-existing-agenta-endpoints) below. | +| `platform` | `op` (a platform-op catalog key), optional `permission` | An existing Agenta endpoint exposed to the agent (e.g. `find_capabilities`, `query_workflows`, `commit_revision`); the runner calls the endpoint directly. See [Platform tools](#platform-tools-existing-agenta-endpoints) below. | A tool can also be a **workflow** referenced as a tool: @@ -58,7 +58,7 @@ has no special case for reference tools; all tool-specific mapping lives in `res A tool can also be a **platform** tool — an existing Agenta endpoint exposed to the agent: - **`type: "platform"`** — a `PlatformToolConfig`. The author writes only `{type:"platform", op}` - (plus optional `needs_approval`/`permission`); the `op` names an entry in the platform-op catalog + (plus optional `permission`); the `op` names an entry in the platform-op catalog (`sdks/python/agenta/sdk/agents/platform/op_catalog.py`), which owns everything else: the model-facing description, the endpoint (method + relative path), the input schema, any self-targeting fields bound from run context, and the per-op default permission/approval. @@ -78,11 +78,14 @@ extensible without new branches everywhere. - **Executor (`type` at config time, `kind` at runtime):** who fulfils a call. This is the axis that decides *where execution happens*, and the rest of this page is mostly about it. -- **`needs_approval`:** whether a call waits for a human yes/no before it runs. Default false. +- **`permission`:** `allow`, `ask`, or `deny`, or left unset to inherit the agent's policy. + Decides whether a call waits for a human yes/no before it runs. See + [Approval and rendering](#approval-and-rendering) below. - **`render`:** an optional generative-UI hint so the frontend can draw the call and its result as something richer than text. -A code tool can need approval. A gateway tool can carry a render hint. The axes compose. +A code tool can carry its own permission. A gateway tool can carry a render hint. The axes +compose. The executor axis is named `type` in the committed config and `kind` on the resolved spec. The rename is deliberate: config talks about where a tool *comes from* (`gateway`), while runtime @@ -332,12 +335,44 @@ options. These are the other two axes, and they ride alongside execution rather than changing where it happens. -**`needs_approval`** gates a call on a human answer. Only permission-gating harnesses honor it. -Claude over ACP raises a permission request, which the runner surfaces as an -`interaction_request` of kind `permission` and answers through a `PolicyResponder` -(`services/agent/src/responder.ts`). With no human at the keyboard, the policy auto-approves by -default because the tools are backend-resolved and trusted, and a per-run policy or env -override can flip it to deny. Pi has no permission concept, so the flag is a no-op there. +**`permission`** gates a call on a human answer. Each tool carries `allow`, `ask`, `deny`, or +nothing (inherit). A run request also carries an agent-wide policy, +`permissions: {default: "allow"|"ask"|"deny"|"allow_reads", rules?}`. One shared decision +module in the runner, `services/runner/src/permission-plan.ts`, resolves a tool's effective +permission: the tool's own explicit setting wins; failing that, an authored rule match; failing +that, the policy mode. Under `allow_reads` (the default), a tool's read-only hint decides: reads +run, everything else asks. + +Two gates consult this same decision: + +- **The ACP responder**, `ApprovalResponder` in `services/runner/src/responder.ts`, answers + permission requests that a gating harness raises itself. Claude Code is the only harness that + raises these today: it checks its own settings file first, and only an undecided call reaches + the responder. +- **The tool relay**, `services/runner/src/tools/relay.ts`, enforces permission on tools the + runner executes directly (gateway, code). It only needs to, because on Claude the harness + settings file plus the ACP responder already decide before a call reaches the relay. On Pi + there is no harness-side gate, so the relay is the only enforcement point, and it now gives Pi + the same human-in-the-loop behavior Claude gets. + +Client tools are a carve-out from that same ladder. They are still decided by +`ApprovalResponder`, but not by the policy-and-rules path above: a client tool defaults to +`allow` no matter what the policy mode is, unless its own `permission` says otherwise or the +policy's `default` is `deny` (which still blocks everything). The reasoning is that a client +tool's only job is to reach the browser, so folding it into `ask`/`allow_reads` defaults would +strand it. Setting `permission: "ask"` or `"deny"` on a client tool still works; it just is not +the policy's doing. + +Either gate answers `allow` by running the tool with no extra event, `deny` by refusing it, or +`ask` by pausing the turn (the wire's `stopReason: "paused"`) and emitting exactly one +`interaction_request(user_approval)` event. The event fires only when the run actually pauses; +an allow or deny produces no extra event, only the ordinary `tool_call`/`tool_result` pair. On +resume, the harness re-issues the call and the runner matches it on a stable anchor (the spec's +own name for relay tools, the recorded tool call name for Claude gates, canonicalized arguments +on both), never on a display title that can drift. A match consumes the stored decision once; a +mismatch is a visibly new approval prompt, never a silent loop and never an auto-deny. +`SANDBOX_AGENT_DENY_PERMISSIONS=true` is an operator kill-switch that forces the effective +policy to deny everywhere. **`render`** is a generative-UI hint. The runner does not act on it; it copies the hint from the spec onto the `tool_call` and `tool_result` events so the egress can project it to the frontend @@ -360,7 +395,7 @@ Each catalog entry (`PlatformOp`, a typed model validated at import) maps an `op | `method`, `path` | The existing endpoint to call: `GET`/`POST` and a relative `/api/...` path. | | `input_schema` / `input_schema_ref` | The request input schema — inline JSON Schema, or a `CATALOG_TYPES` key (expanded via `x-ag-type-ref`). Exactly one. | | `context_bindings` | Self-targeting fields: an endpoint body path → a `$ctx.` run-context token. Stripped from the model schema; emitted as `call.context`. | -| `default_permission`, `default_needs_approval` | Per-op gate. Mutating ops default to approval; reads to auto-allow. The config's `needs_approval`/`permission` override. | +| `read_only` | A bool hint, not a gate value. Under the `allow_reads` policy default it decides the op's effective permission: `true` runs without asking, anything else asks. The tool's own explicit `permission` (`allow`/`ask`/`deny`) always overrides this hint. | Each op has a stable reserved id, `tools.agenta.` (the same namespace as the original `find_capabilities`). The first, minimal-useful set of ops: @@ -448,7 +483,9 @@ The contract and the field-by-field Composio→Agenta mapping live in the | Pi native delivery | `services/agent/src/extensions/agenta.ts` | | `agenta-tools` server for non-Pi harnesses | `services/agent/src/tools/mcp-bridge.ts`, `services/agent/src/tools/mcp-server.ts` | | Capability probe | `services/agent/src/engines/sandbox_agent/capabilities.ts` | -| Permission policy | `services/agent/src/responder.ts` | +| Permission decision (shared by both gates) | `services/runner/src/permission-plan.ts` | +| ACP responder (`ApprovalResponder`) | `services/runner/src/responder.ts` | +| Tool relay enforcement | `services/runner/src/tools/relay.ts` | ## Status and known gaps @@ -457,8 +494,9 @@ The contract and the field-by-field Composio→Agenta mapping live in the Agenta are the default harnesses, so `mcp_servers` is a silent no-op for most runs. It would reach Claude only. Do not confuse this with the `agenta-tools` server, which is an internal tool-delivery vehicle for Claude, not a user MCP server. -- `needs_approval` is honored only by permission-gating harnesses (Claude over ACP). It is a - no-op on Pi. +- A tool's `permission` is honored on both harnesses now. Claude checks its rendered settings + file first, then the ACP responder; Pi has no native gate, so the relay enforces it directly. + An `ask` pauses the run and asks a human on either harness. - Gateway tools support only the `composio` provider today; other providers raise. - The `render` hint is plumbed end to end on the runner side, but full frontend projection of every render kind is still in progress. diff --git a/docs/design/agent-workflows/interfaces/cross-service/service-to-agent-runner.md b/docs/design/agent-workflows/interfaces/cross-service/service-to-agent-runner.md index f5bc8cb790..d108b7806a 100644 --- a/docs/design/agent-workflows/interfaces/cross-service/service-to-agent-runner.md +++ b/docs/design/agent-workflows/interfaces/cross-service/service-to-agent-runner.md @@ -59,7 +59,10 @@ group by job: "skills": [ /* inline skill packages */ ], // policy + files - "permissionPolicy": "auto", // "auto" | "deny" + "permissions": { // the global policy + authored builtin rules + "default": "allow_reads", // "allow" | "ask" | "deny" | "allow_reads" + "rules": [ { "pattern": "Bash(rm:*)", "permission": "ask" } ] // optional + }, "sandboxPermission": { /* Layer 2 boundary; network enforced on Daytona, see sandbox-permission.md */ }, "harnessFiles": [ { "path": ".claude/settings.json", "content": "..." } ], diff --git a/docs/design/agent-workflows/interfaces/in-service/agent-service-handler.md b/docs/design/agent-workflows/interfaces/in-service/agent-service-handler.md index f42f121d00..5e8ec731cd 100644 --- a/docs/design/agent-workflows/interfaces/in-service/agent-service-handler.md +++ b/docs/design/agent-workflows/interfaces/in-service/agent-service-handler.md @@ -11,7 +11,7 @@ the order it runs in is itself a contract. The handler (`_agent` in `app.py`) takes the workflow envelope's pieces: - `parameters`: carries the agent config under `agent`. The run-selection fields (`harness`, - `sandbox`, `permission_policy`) live on that same `agent` object. + `sandbox`, `runner.permissions.default`) live on that same `agent` object. - `messages` or `inputs.messages`: the turn history (it checks `messages` first). - `stream`: batch versus streaming. - `session_id`: the external conversation id. @@ -19,7 +19,8 @@ The handler (`_agent` in `app.py`) takes the workflow envelope's pieces: ## What it does, in order 1. Parse the config: `AgentConfig.from_params(params, defaults=...)`. One parse covers - everything, including the run-selection fields (`harness`, `sandbox`, `permission_policy`). + everything, including the run-selection fields (`harness`, `sandbox`, + `runner.permissions.default`). 2. Convert the request messages to neutral `Message[]`. 3. Resolve tools into builtin names, runnable specs, and a tool callback. 4. Resolve MCP servers. diff --git a/docs/design/agent-workflows/interfaces/in-service/harness-adapters.md b/docs/design/agent-workflows/interfaces/in-service/harness-adapters.md index 1c0c3c1448..b369e0b655 100644 --- a/docs/design/agent-workflows/interfaces/in-service/harness-adapters.md +++ b/docs/design/agent-workflows/interfaces/in-service/harness-adapters.md @@ -14,10 +14,12 @@ review lens: the wire-shape differences and what to check when one moves. Each adapter implements `_to_harness_config(...)` and emits a different `/run` wire shape: - **`PiHarness`** delivers built-in tool names and native custom tools, supports Pi prompt - overrides (`system`, `append_system`), and drops `permission_policy` because Pi never gates - tool use. + overrides (`system`, `append_system`), and carries the same `permissions` block as the other + harnesses. Pi has no native permission gate of its own (no `.claude/settings.json` equivalent), + so the runner's tool relay enforces `permissions` for Pi at execution time; an `ask` verdict + pauses the run and Pi gets the same human-in-the-loop approval Claude gets at its gate. - **`ClaudeHarness`** delivers tools over MCP, not natively, and has no Pi built-ins (it warns - if any are set). It carries `permission_policy` and renders `.claude/settings.json` from four + if any are set). It carries `permissions` and renders `.claude/settings.json` from four sources — the author's `harness_kwargs["claude"]["permissions"]` slice, the sandbox permission, each user MCP server's permission (`mcp__` rules), and each resolved EXECUTABLE tool's permission (`mcp__agenta-tools__` rules; F-046) — shipped as `harnessFiles`. It carries @@ -34,7 +36,7 @@ The wire shapes, side by side: | built-in tools | yes | no | forced set | | custom tools | native | over MCP | native | | prompt overrides | `system`/`append_system` | none (reads `harness_kwargs`) | forced `append_system` + author `system` | -| permission policy | dropped | carried | dropped | +| permission policy | carried, enforced by the relay | carried, enforced by settings + the responder | carried, enforced by the relay | | inline skills | yes (agent-dir scope) | yes (materialized to `.claude/skills`) | yes (agent-dir scope) | | harness files | none | `.claude/settings.json` | none | @@ -62,4 +64,4 @@ The wire shapes, side by side: (`services/agent/src/tools/mcp-bridge.ts`, `relay-mcp-stdio.ts`, `tool-mcp-http.ts`, `engines/sandbox_agent/mcp.ts`, `sdks/python/agenta/sdk/agents/adapters/claude_settings.py`). Renaming the server on one side without the other silently - re-parks `allow` tools on Claude (the bug F-046 fixed). + re-pauses `allow` tools on Claude (the bug F-046 fixed). diff --git a/docs/design/agent-workflows/interfaces/in-service/neutral-runtime-dtos.md b/docs/design/agent-workflows/interfaces/in-service/neutral-runtime-dtos.md index 24d48d4db6..ff072e8492 100644 --- a/docs/design/agent-workflows/interfaces/in-service/neutral-runtime-dtos.md +++ b/docs/design/agent-workflows/interfaces/in-service/neutral-runtime-dtos.md @@ -20,10 +20,11 @@ All in `dtos.py`. The ones that carry the most weight: - **`AgentConfig`**: the parsed editable config. `instructions`, `model`/`model_ref`, `tools`, `mcp_servers`, `skills`, `sandbox_permission`, a `harness_kwargs` bag keyed by harness name, and the run-selection fields `harness` (default `pi_core`), `sandbox` - (default `local`), and `permission_policy` (`auto` | `deny`). The `harness` value is the - bare `HarnessType` string. There is one agent config; run selection used to live on a - separate `RunSelection` object and now folds in here. Built by `from_params(...)`. The - editable schema is [Agent config schema](../public-edge/agent-config-schema.md). + (default `local`), and `runner.permissions.default` (`allow` | `ask` | `deny` | + `allow_reads`, default `allow_reads`). The `harness` value is the bare `HarnessType` + string. There is one agent config; run selection used to live on a separate + `RunSelection` object and now folds in here. Built by `from_params(...)`. The editable + schema is [Agent config schema](../public-edge/agent-config-schema.md). - **`HarnessType` and `HARNESS_IDENTITIES`**: the closed harness enum plus the single source for each harness's interface identity — a versioned slug (`agenta:harness::v0`, the repo's slug grammar) and a display name. The agent_config schema builds its harness `oneOf` from diff --git a/docs/design/agent-workflows/interfaces/in-service/permission-responder.md b/docs/design/agent-workflows/interfaces/in-service/permission-responder.md index b48e0f08db..0995c25139 100644 --- a/docs/design/agent-workflows/interfaces/in-service/permission-responder.md +++ b/docs/design/agent-workflows/interfaces/in-service/permission-responder.md @@ -1,44 +1,94 @@ # Permission Responder -When a harness asks to use a tool, something has to answer. The responder is that something. -It abstracts two modes behind one interface: a headless policy that auto-approves or denies, -and a human-in-the-loop flow that parks the request for browser approval and resumes it on a -later turn. Because the browser approval spans two turns, the matching logic here is subtle, -and worth reading before you touch it. +> Superseded terms: this file used to describe `PolicyResponder`/`HITLResponder` and "park". +> Those are gone. It now describes `ApprovalResponder` and "pause", the current implementation. + +When a harness asks to use a tool, something has to answer. `ApprovalResponder` is that +something. It answers every ACP permission gate and every client-tool request from one shared +decision function, so there is one policy instead of separate headless and human-in-the-loop +paths. Because a paused approval spans two turns, the resume-matching logic here is subtle and +worth reading before you touch it. ## The contract ```typescript interface Responder { - onPermission(request: PermissionRequest): Promise; // "allow" | "deny" + onPermission(request: PermissionGateRequest): Promise; + onClientTool( + request: ClientToolGateRequest, + opts?: { consume?: boolean }, + ): Promise; } + +type Verdict = + | { kind: "allow" } + | { kind: "deny" } + | { kind: "pendingApproval" }; ``` -- **`PolicyResponder`** is headless. It returns `deny` when the policy is `deny`, else `allow`. -- **`HITLResponder`** handles human approval across turns. It holds the approval decisions - extracted from the message history, a base policy, and whether a human surface exists: - 1. look the request up in the decisions map (by tool-call id, then by tool name), - 2. if found, apply it (the resume path), - 3. if not found and a human can answer, return `deny` to park it (the browser will be asked), - 4. if not found and no human surface, fall back to the base policy (fully headless). +`ApprovalResponder` is the only implementation. It holds a `PermissionPlan` (the parsed +`permissions: {default, rules}` block from the run request) and a `ConversationDecisions` +store (approvals and denials recovered from the replayed message history). + +**`onPermission`.** Calls `decide(gate, plan, decisions)` from +`services/runner/src/permission-plan.ts`, the same shared function the tool relay uses: + +1. Resolve the gate's effective permission: the tool's or MCP server's explicit `permission` + if set, else a matching authored rule, else the policy's `default` mode (`allow_reads` + consults the tool's read-only hint; no hint counts as a write). +2. `deny` refuses. `allow` runs, in place, no pause. +3. `ask` looks for a stored decision from a previous turn (matched by stable anchor, see + below). Found, it applies once and is consumed. Not found, the verdict is + `pendingApproval`: the run pauses and waits for a human. + +**`onClientTool`.** Client tools (browser-fulfilled, for example `request_connection`) resolve +the same way but default to `allow` when unset, since their whole purpose is to reach the +browser. A stored output already fulfills the call without asking again. + +**Resume matching.** `extractApprovalDecisions` scans the incoming message history for +`tool_result` blocks carrying an `{approved: boolean}` envelope, and keys each one by +`approvedCallKey(name, args)`: a stable tool name plus canonicalized arguments, never a +display title. The name anchor differs by executor: the spec's own name for relay/client +tools (it cannot drift), and the recorded `tool_call` name for harness gates (Claude Code +today). A stored decision matches only the same call; different arguments are a different +call and pause for a fresh approval, visibly. A stored decision is consumed on first match, +so one approval authorizes one execution, and a config changed to `deny` always beats it +because the effective permission is resolved before the stored decision is consulted. -**Decision extraction.** `extractApprovalDecisions(request)` scans the message history for -`tool_result` blocks whose output is `{approved: boolean}`, and indexes each by both its -`toolCallId` and its `toolName`. The tool name is the fallback because a cold replay can mint -a fresh permission id each turn, so the stable anchor is the name. +## How it fits with the relay -**Policy precedence.** `deny` from the request wins, then the -`SANDBOX_AGENT_DENY_PERMISSIONS` env override, else `auto`. +`ApprovalResponder` is Gate 2, for tools the harness gates natively (Claude Code's +`.claude/settings.json` layer decides first; anything it leaves undecided reaches this +responder over ACP). The tool relay (`services/runner/src/tools/relay.ts`) is Gate 3, for +tools the runner executes itself (gateway, code, client). Both gates call the same +`effectivePermission`/`decide` pair from `permission-plan.ts`, so they can never disagree +about a tool's permission. The relay only needs to enforce on Pi, since Claude's Gate 1 and +Gate 2 already decide before a call reaches the relay. ## Owned by -- `services/agent/src/responder.ts`: the responders and decision extraction. -- `services/agent/src/engines/sandbox_agent/permissions.ts`: how they wire into the ACP run. +- `services/runner/src/permission-plan.ts`: the shared decision function (`effectivePermission`, + `decide`), the `PermissionPlan` and `GateDescriptor` types, and `permissionsFromRequest` + (parses the wire `permissions` block, applies the `SANDBOX_AGENT_DENY_PERMISSIONS` + kill-switch). +- `services/runner/src/responder.ts`: `ApprovalResponder`, `ConversationDecisions`, + `extractApprovalDecisions`, `approvedCallKey`. +- `services/runner/src/engines/sandbox_agent.ts` and + `services/runner/src/engines/sandbox_agent/acp-interactions.ts`: how the responder wires + into the ACP run (`attachPermissionResponder`), and the pause/teardown when a verdict is + `pendingApproval`. ## Watch for when changing -- **Policy precedence.** Request, env override, default. Reordering changes who can approve. -- **Cross-turn matching.** Tool-call id first, tool name as the fallback for cold replays. - Breaking the fallback breaks approval after a replay. -- **Parking behavior.** Whether a session has a human surface decides park-versus-headless. - An interactive run parks unapproved tools; a headless `/invoke` applies the base policy. +- **One decision function.** Both gates call `effectivePermission`/`decide`. Do not let a + gate grow its own copy of the ladder; that is the bug this redesign fixed. +- **Resolve before checking stored decisions.** The effective permission must be computed + first, so a config change to `deny` beats a stale approval. Reordering breaks that + guarantee. +- **Stable anchors only.** Resume matching keys on the spec name or the recorded `tool_call` + name plus canonical args, never a harness display title. A drifted key reintroduces the + approval loop this redesign closed. +- **Emit only on pause.** `interaction_request(user_approval)` fires only when the verdict is + `pendingApproval`. Emitting before the decision (the old order) produces false prompts for + gates that were actually auto-approved. +- **Consume once.** A stored approval or denial applies to exactly one execution. diff --git a/docs/design/agent-workflows/interfaces/in-service/tool-models-and-resolution.md b/docs/design/agent-workflows/interfaces/in-service/tool-models-and-resolution.md index f6c7b60bed..e45182d3b4 100644 --- a/docs/design/agent-workflows/interfaces/in-service/tool-models-and-resolution.md +++ b/docs/design/agent-workflows/interfaces/in-service/tool-models-and-resolution.md @@ -9,8 +9,8 @@ resolved specs become `/run` fields, so a change here usually reaches the wire ( ## Config types (what the author writes) -All share a base of `needs_approval` (default `false`), `permission` (`allow`/`ask`/`deny`, -optional), and `render` (optional), discriminated by `type`: +All share a base of `permission` (`allow`/`ask`/`deny`, optional, unset inherits the global +policy) and `render` (optional), discriminated by `type`: ```jsonc { "type": "builtin", "name": "read" } @@ -31,8 +31,8 @@ optional), and `render` (optional), discriminated by `type`: // platform: an existing Agenta endpoint exposed to the agent. `op` names a platform-op catalog // entry; the catalog owns the description, endpoint, schema, context bindings, and per-op gate defaults. -// `needs_approval` is optional here (null = use the catalog default). -{ "type": "platform", "op": "find_capabilities", "needs_approval": null, "permission": null } +// `permission` is optional here (null = use the catalog default). +{ "type": "platform", "op": "find_capabilities", "permission": null } ``` A tool can also be a **workflow** referenced as a tool (`type: "reference"`, above) or a @@ -67,16 +67,20 @@ config (not markers); `resolve_tools` owns the tool-specific mapping. { "kind": "client", "name": "..." } ``` -Each resolved spec also carries `needs_approval`, `read_only`, `permission`, and `render`. +Each resolved spec also carries `read_only`, `permission`, and `render`. ## Permission derivation -When a tool's `permission` is unset, it is derived by precedence: +A tool's authored `permission` travels as is. `ToolSpec.to_wire()` sends only the author's +explicit `permission`; when it is unset, the spec sends nothing and the `read_only` hint +rides alongside as a separate field. The runner resolves the rest: its shared decision +function (`services/runner/src/permission-plan.ts`) looks up, in order: 1. an explicit `permission` wins, -2. else `needs_approval: true` to `"ask"`, -3. else `read_only`: `true` to `"allow"`, `false` to `"ask"`, -4. else `None`, and the runner falls back to the global policy. +2. else an authored builtin rule match (`runner.permissions.rules`), +3. else, under the `allow_reads` policy mode, the `read_only` hint (`true` to `allow`, unset + or `false` to `ask`), +4. else the global policy mode (`allow`/`ask`/`deny`). ## Secret injection @@ -87,8 +91,11 @@ secret; their provider key stays server-side and the call routes back through `/ ## Owned by - `sdks/python/agenta/sdk/agents/tools/models.py`: the config and spec models (incl. - `ReferenceToolConfig`, its `ref_by` axes, `PlatformToolConfig`, `ToolCall`, and `call_ref`) and - the permission derivation. + `ReferenceToolConfig`, its `ref_by` axes, `PlatformToolConfig`, `ToolCall`, and `call_ref`), + carrying the author's explicit `permission` and the `read_only` hint. +- `services/runner/src/permission-plan.ts`: the shared decision function that resolves a + tool's effective permission at run time (explicit permission, then rule match, then the + `allow_reads` read-only check, then the global policy). - `sdks/python/agenta/sdk/agents/tools/compat.py`: coerces legacy/typed tool dicts (a `type: "reference"` or `type: "platform"` dict parses straight into its config model). - `sdks/python/agenta/sdk/agents/platform/gateway.py`: gateway resolution to a `call_ref`. @@ -118,7 +125,8 @@ in `op_catalog.py` (the SDK must not import the API). ## Watch for when changing -- **Permission defaults and the derivation order.** It decides what gets gated. +- **Permission defaults and the derivation order.** It decides what gets gated. The order + now lives in the runner's shared decision function, not in this SDK module. - **Read-only and render hints.** They flow through to the runner and the browser. - **The tool input schema.** It is what the harness sees as the tool's parameters. - **Secret injection for code tools.** Secrets ride `env` and are resolved once at parse time. diff --git a/docs/design/agent-workflows/interfaces/public-edge/agent-config-schema.md b/docs/design/agent-workflows/interfaces/public-edge/agent-config-schema.md index a188137257..30f281eb6a 100644 --- a/docs/design/agent-workflows/interfaces/public-edge/agent-config-schema.md +++ b/docs/design/agent-workflows/interfaces/public-edge/agent-config-schema.md @@ -25,11 +25,11 @@ The fields and the full schema follow. | `mcp_servers` | `MCPServerConfig[]` | `[]` | Declared MCP servers; secret env resolved from the vault at run time. See [MCP models and resolution](../in-service/mcp-models-and-resolution.md). | | `harness` | `"pi_core" \| "claude" \| "pi_agenta"` (see slug+name note) | `"pi_core"` | The coding agent to drive. `pi_core` and `pi_agenta` both drive the `pi` ACP agent; `pi_agenta` adds Agenta's forced skills, prompt, and policy. | | `sandbox` | `"local" \| "daytona"` | `"local"` | Where it runs. | -| `permission_policy` | `"auto" \| "deny"` | `"auto"` | How a gating harness (Claude Code) handles tool-use prompts in a headless run. | +| `permissions` | `{default: "allow" \| "ask" \| "deny" \| "allow_reads", rules?: [...]}` | `{default: "allow_reads"}` | The agent-wide policy. `allow_reads` runs read-hinted tools and asks for everything else; `allow` runs everything; `ask` asks for everything; `deny` runs nothing unless a tool explicitly allows it. `rules` are optional authored patterns (for example `Bash(rm:*)`) that override the default for matching harness builtins. | | `sandbox_permission` | `SandboxPermission \| null` | `null` (form pre-fills one) | The declared network and filesystem boundary. See [Sandbox permission](../in-service/sandbox-permission.md). | | `skills` | `(SkillConfig \| EmbedRef)[]` | one embedded default skill | Inline SKILL.md packages, or `@ag.embed` references the backend inlines before the runner sees them. | -Note that `harness`, `sandbox`, and `permission_policy` are the run-selection fields. They +Note that `harness`, `sandbox`, and `permissions` are the run-selection fields. They live on `AgentConfig` itself, under `data.parameters.agent`, and the handler reads them in the one `AgentConfig.from_params(...)` parse along with the rest of the config. There is one agent config, not a config plus a separate selection object. @@ -76,7 +76,7 @@ every field in its default state: "mcp_servers": [], "harness": "pi_core", "sandbox": "local", - "permission_policy": "auto", + "permissions": { "default": "allow_reads" }, "sandbox_permission": { "network": { "mode": "on", "allowlist": [] }, "enforcement": "strict" @@ -121,21 +121,21 @@ Either form is valid: ```jsonc // builtin: a harness built-in by name -{ "type": "builtin", "name": "read", "needs_approval": false, "permission": null, "render": null } +{ "type": "builtin", "name": "read", "permission": null, "render": null } // gateway: a server-side action (e.g. Composio); runs in Agenta, key stays server-side { "type": "gateway", "provider": "composio", "integration": "github", "action": "create_issue", "connection": "my-gh", "name": null, - "needs_approval": false, "permission": null, "render": null } + "permission": null, "render": null } // code: sandboxed code with named secrets injected at resolve time { "type": "code", "name": "fx", "runtime": "python", "script": "...", "input_schema": {}, "secrets": ["API_KEY"], - "needs_approval": false, "permission": null, "render": null } + "permission": null, "render": null } // client: fulfilled by the browser; filtered out of the runner's MCP tools/list { "type": "client", "name": "pick_file", "description": "...", "input_schema": {}, - "needs_approval": false, "permission": null, "render": null } + "permission": null, "render": null } // reference (variant axis): the service runs the workflow's latest revision (or a pinned // `version`) server-side when the model calls it. Resolves to a callback tool; key stays @@ -143,17 +143,17 @@ Either form is valid: // as siblings. { "type": "reference", "ref_by": "variant", "slug": "summarize", "version": null, "name": "summarize", "description": "...", "input_schema": {}, - "needs_approval": false, "permission": null, "render": null } + "permission": null, "render": null } // reference (environment axis): the service runs whatever revision is deployed in `environment` // for `slug`. `version` is not allowed (the environment is the pin). { "type": "reference", "ref_by": "environment", "environment": "production", "slug": "summarize", - "name": "summarize", "needs_approval": false, "permission": null, "render": null } + "name": "summarize", "permission": null, "render": null } // platform: an existing Agenta endpoint exposed to the agent. `op` names a platform-op catalog -// entry (the catalog owns description/endpoint/schema/bind/gate defaults). `needs_approval` is -// optional (null = the catalog default — e.g. commit_revision defaults to approval). -{ "type": "platform", "op": "find_capabilities", "needs_approval": null, "permission": null } +// entry (the catalog owns description/endpoint/schema/bind/gate defaults). `permission` is +// optional (null means inherit the policy; commit_revision's catalog default resolves to `ask`). +{ "type": "platform", "op": "find_capabilities", "permission": null } // @ag.embed (a different feature, not in the tool-authoring UI): inline the referenced value into // a concrete `client` tool config before the runner sees it (the backend's embed resolver does @@ -170,10 +170,12 @@ plain config, not markers — the generic resolver only inlines `@ag.embed`, and the tool-specific mapping. See [Tool models and resolution](../in-service/tool-models-and-resolution.md). -`permission` is `"allow" | "ask" | "deny"`. When unset, it is derived: explicit value wins, -then `needs_approval` to `"ask"`, then `read_only` (`true` to `"allow"`, `false` to `"ask"`), -else the global policy applies. See [Tool models and -resolution](../in-service/tool-models-and-resolution.md). +`permission` is `"allow" | "ask" | "deny"`, or unset to inherit. When unset, the runner's +shared decision function resolves it: an authored rule match wins if one applies, else the +policy's `default` mode decides (`allow_reads` consults the tool's `read_only` hint: `true` +resolves to `allow`, no hint or `false` resolves to `ask`). There is no `needs_approval` +field and no separate per-tool defaulting step; one function resolves every tool the same +way. See [Tool models and resolution](../in-service/tool-models-and-resolution.md). ### `mcp_servers[]` diff --git a/docs/design/agent-workflows/interfaces/public-edge/agent-messages.md b/docs/design/agent-workflows/interfaces/public-edge/agent-messages.md index b6e6cd0bbf..134e0acf75 100644 --- a/docs/design/agent-workflows/interfaces/public-edge/agent-messages.md +++ b/docs/design/agent-workflows/interfaces/public-edge/agent-messages.md @@ -25,7 +25,7 @@ The request reuses the generic workflow envelope and carries the chat-specific p "data": { "messages": [ /* Vercel UIMessage[] */ ], "parameters": { "agent": { /* draft-aware agent config, incl. - harness, sandbox, permission_policy */ } } + harness, sandbox, runner.permissions.default */ } } } } ``` diff --git a/docs/design/agent-workflows/interfaces/public-edge/workflow-invoke.md b/docs/design/agent-workflows/interfaces/public-edge/workflow-invoke.md index 253d08e3ca..8c439a1dd9 100644 --- a/docs/design/agent-workflows/interfaces/public-edge/workflow-invoke.md +++ b/docs/design/agent-workflows/interfaces/public-edge/workflow-invoke.md @@ -23,7 +23,7 @@ things out of it: "references": { /* application / variant / revision */ }, "data": { "inputs": { "messages": [ /* chat history */ ] }, - "parameters": { "agent": { /* config, incl. harness, sandbox, permission_policy */ } } + "parameters": { "agent": { /* config, incl. harness, sandbox, runner.permissions.default */ } } } } ``` @@ -36,7 +36,7 @@ assistant message: ``` `parameters` carries the agent config under `agent`, and the run-selection fields (`harness`, -`sandbox`, `permission_policy`) live inside that same `agent` object. The handler reads the +`sandbox`, `runner.permissions.default`) live inside that same `agent` object. The handler reads the history from `data.messages` first, then falls back to `inputs.messages`. The streaming version of this work is [Agent messages](agent-messages.md); this contract is the batch path. diff --git a/docs/design/agent-workflows/projects/agent-config-structure-cleanup/status.md b/docs/design/agent-workflows/projects/agent-config-structure-cleanup/status.md index ae96474c63..2b1db9f165 100644 --- a/docs/design/agent-workflows/projects/agent-config-structure-cleanup/status.md +++ b/docs/design/agent-workflows/projects/agent-config-structure-cleanup/status.md @@ -6,19 +6,21 @@ Source: PR #4821 review comments 2 / 7 / 8 (approved). 1. Collapse the run-selection fields (`harness`, `sandbox`, `permission_policy`) from the separate `RunSelection` DTO INTO `AgentConfig`. They live under `data.parameters.agent`, - not as siblings of `agent`. Retire `RunSelection`. + not as siblings of `agent`. Retire `RunSelection`. (`permission_policy` was later renamed + `runner.permissions.default`; see [projects/approval-boundary/](../approval-boundary/).) 2. Rename the per-harness options bag `harness_options` -> `harness_kwargs` (a dict keyed by - harness name; `permission_policy` stays the sidecar action-permission, now inside AgentConfig). + harness name; the action-permission field stayed the sidecar's, now inside AgentConfig). POC: no back-compat, no deprecation shims. Change shapes cleanly. ## Wire impact -NONE. The `/run` payload keeps `harness`, `sandbox`, `permissionPolicy` exactly as today: -`harness`/`sandbox` are passed to `request_to_wire(harness=, sandbox=)` by the harness adapter -(from `make_harness` + `select_backend`); `permissionPolicy` rides `wire_tools()` on the -per-harness config. This is an envelope/config-placement + parsing change, not a wire change. -Golden fixtures unchanged. +NONE at the time. The `/run` payload kept `harness`, `sandbox`, `permissionPolicy` exactly as +before this change: `harness`/`sandbox` were passed to `request_to_wire(harness=, sandbox=)` by +the harness adapter (from `make_harness` + `select_backend`); `permissionPolicy` rode +`wire_tools()` on the per-harness config. This was an envelope/config-placement + parsing +change, not a wire change. Golden fixtures unchanged. (The wire field is now `permissions`; +see [projects/approval-boundary/](../approval-boundary/) for the later redesign.) ## Coordination diff --git a/docs/design/agent-workflows/projects/build-kit-tools-cleanup/context.md b/docs/design/agent-workflows/projects/build-kit-tools-cleanup/context.md new file mode 100644 index 0000000000..4c41252586 --- /dev/null +++ b/docs/design/agent-workflows/projects/build-kit-tools-cleanup/context.md @@ -0,0 +1,82 @@ +# Context: build-kit tools cleanup + +Status: design workspace, 2026-07-03. Docs only. No code changes in this project until the +plan is approved; all code changes then batch into one PR. + +## Why + +The playground build kit is the tool set a fresh playground agent uses to build itself: +platform ops, the `request_connection` client tool, an authoring skill, and permissive +sandbox permissions, all injected by the overlay +(`api/oss/src/apis/fastapi/applications/overlay.py:64`). Mahmoud tested it end to end and +the agent wandered (see +[builder-agent-reliability/context.md](../builder-agent-reliability/context.md)). A full +review of both tool sets followed: +[tools-review](../builder-agent-reliability/tools-review/README.md) compared the 19 inside +tools against the 13 outside scripts in the agent-creation-lab kit, with lab evidence per +verdict. + +The lab's outside kit (`/home/mahmoud/code/agent-creation-lab/kit/BUILD-AGENT.md`) proved +what works: a small ordered tool set, one playbook instead of scattered skills, a +self-verifying test step, and blunt instruction-writing rules. This project ports those +learnings inside and executes the review's verdicts. + +## Goals + +1. **Rename** the discovery ops: `find_capabilities` -> `discover_tools`, + `find_triggers` -> `discover_triggers`. Hard migrate, no aliases (decided 2026-07-03). +2. **Shrink the default overlay** from 19 tools to 12-13: cut `pause_schedule`, + `resume_schedule`, `pause_subscription`, `resume_subscription`, `query_workflows`, + `list_connections`, and `list_subscriptions` (or fold it into the deliveries read). + All stay in the catalog for opt-in. `test_subscription` stays (decided 2026-07-03). +3. **Close the self-test gap**: decide where logic-bearing internal tools live (the + `test_run` question), sketch the `test_run` contract, and ship the `query_spans` + stopgap read op if approved. +4. **Port the playbook**: one ordered build skill (from the lab's BUILD-AGENT.md) replaces + the three cross-referencing authoring skills. `agenta-getting-started` stays as the + forced baseline. + +## Non-goals + +- **Approval semantics.** No change to `needs_approval`, the permission plan, or + `read_only` handling in the approval path. The + [approval-boundary](../approval-boundary/README.md) workstream owns that layer and is + mid-flight in `op_catalog.py` today. See the coordination constraint in + [plan.md](plan.md). +- The outside kit. The review's outside recommendations (demote two scripts, add + `create-subscription.sh`) belong to the agenta-skills repo, not this project. +- The overlay mechanism itself (how the FE deep-merges it, commit exclusion). It works; + only its contents change. +- A `create_workflow` op (inside builder-of-other-agents). Open question 4 in the review; + out of scope here. + +## Decisions already made (do not relitigate) + +| # | Decision | Source | +|---|---|---| +| 1 | Hard-migrate renames, no aliases: `find_capabilities` -> `discover_tools`, `find_triggers` -> `discover_triggers`. Pre-production, no backward compat. | Mahmoud, 2026-07-03, decision op-renames | +| 2 | Cut from the default overlay (stay in catalog): `pause_schedule`, `resume_schedule`, `pause_subscription`, `resume_subscription`, `query_workflows`, `list_connections`, `list_subscriptions` (or fold into the deliveries read). | tools-review agreed changes | +| 3 | Keep `test_subscription`. The never-used data point was a test-scenario limitation, not evidence against the tool. | Mahmoud, 2026-07-03 | +| 4 | All code changes batch into ONE PR later. This project's deliverable now is design docs only. | Mahmoud, 2026-07-03 | + +## Open decisions (recommended in these docs; Mahmoud answers) + +1. **overlay-scope** - static 12-13 tools, or a conditional event pack. + Recommendation in [research.md](research.md#overlay-scope). +2. **test-run-shape** - sync with a delta, committed-only, or an async pair. + Recommendation in [api-design.md](api-design.md#shape-decision). +3. **spans-stopgap** - ship a `query_spans` read op now, or hold for `test_run`. + Recommendation in [api-design.md](api-design.md#the-query_spans-stopgap). + +## Read next + +- [research.md](research.md) - the executor architecture with evidence, the full + rename/cut surface inventory, and the gotchas found along the way. +- [tool-home-options.md](tool-home-options.md) - where logic-bearing tools should live: + four options, trade-offs, recommendation. **The doc Mahmoud most wants to review.** +- [api-design.md](api-design.md) - the `test_run` contract sketch and the `query_spans` + stopgap. +- [skills-port.md](skills-port.md) - the playbook skill that replaces the three authoring + skills. +- [plan.md](plan.md) - the phased execution plan. +- [status.md](status.md) - what is decided, open, and blocked. diff --git a/docs/design/agent-workflows/projects/build-kit-tools-cleanup/research.md b/docs/design/agent-workflows/projects/build-kit-tools-cleanup/research.md new file mode 100644 index 0000000000..c94e4062d7 --- /dev/null +++ b/docs/design/agent-workflows/projects/build-kit-tools-cleanup/research.md @@ -0,0 +1,278 @@ +# Research: executor architecture, rename/cut surfaces, gotchas + +Status: 2026-07-03. Everything here was read from the working tree today. File:line +references are to the repo root (`/home/mahmoud/code/agenta`) unless marked as the lab +(`/home/mahmoud/code/agent-creation-lab`). + +Companion docs: [tool-home-options.md](tool-home-options.md) weighs the four homes for +logic-bearing tools; [api-design.md](api-design.md) sketches the `test_run` contract; +[context.md](context.md) has the goals and settled decisions. + +## Part 1: how tools actually execute today + +### The four executor kinds on the wire + +`ResolvedToolSpec` (`services/runner/src/protocol.ts:105-141`) has two orthogonal axes: +`kind` (executor: `"callback" | "code" | "client"`, absent = callback) and `render` +(generative UI). A callback spec carries `call` XOR `callRef`: + +- **`callRef`** = gateway tools (Composio). The runner POSTs the OpenAI-style envelope + back to Agenta's `/tools/call` (`services/runner/src/tools/callback.ts:31-91`). The + Composio secret stays server-side. +- **`call`** = direct-call tools (reference and platform). The runner calls the named + Agenta endpoint itself (`services/runner/src/tools/direct.ts`). +- **`code`** runs inline code in a sandbox subprocess; **`client`** is browser-fulfilled + across a turn boundary (`request_connection` is the one instance, + `sdks/python/agenta/sdk/agents/platform/workflow.py:38-39`). + +"Platform" is not a runner-visible kind. It is a config type (`{type:"platform", op}`) +that the SDK resolves into a callback spec with a direct `call` +(`sdks/python/agenta/sdk/agents/platform/platform_tools.py:72-82`). + +### The platform-op pipeline, end to end + +1. **Catalog** (`sdks/python/agenta/sdk/agents/platform/op_catalog.py:532-697`): + `PLATFORM_OPS` holds 18 frozen `PlatformOp` entries. Each owns the model-facing + description, method + relative path, input schema, `context_bindings`, `args_into`, + and a `read_only` hint (`op_catalog.py:56-88`). `reserved_id` is + `tools.agenta.` (`op_catalog.py:121-124`). +2. **Resolve** (`platform_tools.py:36-90`): `AgentaPlatformToolResolver` emits one + `CallbackToolSpec` per op with `call=op.to_call()` and a shared + `ToolCallback(endpoint=f"{api_base}/tools/call", authorization=...)`. Note: even + direct-call ops carry the `/tools/call` endpoint; the runner uses it as the ORIGIN + ANCHOR, not as the target. +3. **Dispatch** (`services/runner/src/tools/relay.ts:271-280`): when `spec.call` is set, + the runner assembles the body, validates the URL, and calls the endpoint directly. +4. **Body assembly** (`direct.ts:205-239`): model args land at `args_into` (or the root), + static `call.body` overlays them, and `call.context` bindings fill LAST. Each + `"$ctx."` token resolves against the run's `runContext` blob; an unresolvable + binding throws (`direct.ts:229-236`). This is the **self-targeting guarantee**: the + model never sees or wins a bound field. +5. **SSRF guard** (`direct.ts:259-325`): method allowlist (GET/POST/DELETE), single + absolute path, origin host-locked to the run's own callback endpoint, and the resolved + path **confined to the callback's mount** (the callback path minus `/tools/call`, i.e. + `/api` on cloud). `direct.ts:310-323` is why a platform op cannot reach + `/services/agent/v0/invoke`: that path is outside the `/api` mount. +6. **The call** (`direct.ts:337-389`): one round-trip with the caller's credential, body + returned verbatim, under `TOOL_CALL_TIMEOUT_MS` (default **30s**, + `callback.ts:15-17`). The Daytona/local relay loop that carries the whole exchange has + its own `RELAY_TIMEOUT_MS` (default **60s**, `relay.ts:42-44`). Any long-running tool + must fit under both, or thread new timeout plumbing. + +### Run context + +`RunContext` (`protocol.ts:174-185`) carries `workflow.{artifact,variant,revision}` refs, +`workflow.is_draft`, and `trace.{trace_id,span_id}`. The service computes it per turn +(`services/oss/src/agent/tracing.py:164-171`), and it rides the `/run` request +(`protocol.ts:472`). It is consumed ONLY by `call.context` bindings today. Gateway +(`callRef`) dispatch passes the model's args verbatim with no context injection +(`relay.ts:283-289`). + +### The `/tools/call` plane already runs composite server-side logic + +Two precedents matter for the tool-home question: + +- **Reserved ops**: `_call_agenta_tool` (`api/oss/src/apis/fastapi/tools/router.py:1141-1207`) + dispatches `tools.agenta.*` call_refs server-side. v1 op: `find_capabilities` (pinned to + `FIND_CAPABILITIES_OP` at `api/oss/src/core/tools/discovery.py:45`). This is the legacy + path the platform-op catalog migrated off, "retained during migration" + (`docs/design/agent-workflows/interfaces/README.md:51`). +- **Workflow tools**: a `workflow.*` call_ref makes `/tools/call` resolve a revision and + **invoke a whole workflow run** server-side + (`tools/router.py:1240-1330`, calling `workflows_service.invoke_workflow` at line 1299). + So the tool-call plane is already, by design, a place where one tool call fans out into + a real run. The resource API stayed atomic; the tool surface composed. + +### How the server invokes the agent service (for `test_run`) + +`WorkflowsService._prepare_invoke` (`api/oss/src/core/workflows/service.py:2040-2071`) +centralizes auth and resolution: it signs an internal `Secret` token itself +(`sign_secret_token`, lines 2054-2062), resolves references to a runnable revision +(`_ensure_request_revision`), and derives the service URL. `invoke_workflow` (line 2073) +POSTs `{service_url}/invoke` batch; `invoke_workflow_detached` (line 2115) streams and +returns on the started handshake. Two facts fall out: + +- The server already holds a first-party, credential-clean way to run an agent headless. + No Agenta-internal credential ever enters the user connections system; the service + signs its own token per call. +- The batch invoke returns only final outputs. The ordered tool list needs either the + streamed events (what the lab's `test-agent.sh` parses) or a spans query afterward + (what `check-tools.sh` does against `POST /api/spans/query`; the route is + `api/oss/src/apis/fastapi/tracing/router.py:97-98`). Spans are the ground truth either + way, and they flush a second or two late (the lab script retries). + +## Part 2: rename + cut surface inventory + +### Rename: `find_capabilities` -> `discover_tools`, `find_triggers` -> `discover_triggers` + +The op key is also the model-visible tool name and derives the reserved id +(`tools.agenta.`), so one key rename moves all three names at once. Surfaces: + +**SDK (source of truth)** + +- `sdks/python/agenta/sdk/agents/platform/op_catalog.py` - the two `PlatformOp` entries + (lines 535-542, 575-581), the `_FIND_CAPABILITIES_*` constants (188-216), the + `_FIND_TRIGGERS_*` constants (374-401), and description text that says + "returned by find_triggers" inside `_CREATE_SUBSCRIPTION_INPUT_SCHEMA` (474) and + `_TEST_SUBSCRIPTION_INPUT_SCHEMA` (505). +- `sdks/python/agenta/sdk/agents/tools/models.py:220` - docstring example + (`op ... e.g. 'find_capabilities'`). +- `sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py` - skill bodies name + `find_capabilities` (lines 123, 150-254) and `find_triggers` (line 290). The + [skills port](skills-port.md) replaces these bodies wholesale, so the rename rides that + change. + +**SDK tests** + +- `sdks/python/oss/tests/pytest/unit/agents/platform/test_op_catalog.py` - key list + (39-53), reserved-id assertion (61-63), error-message assertion (143), direct-call + tests (146-186), method/path table (263-283), duplicate test (345-353). +- `sdks/python/oss/tests/pytest/unit/agents/tools/test_parsing.py:194-196`, + `test_models.py:61-63`, `test_resolver.py:269-281` (uses a fake catalog path + `/api/find_capabilities`), `test_skill_template_catalog.py:171`. + +**API** + +- `api/oss/src/core/tools/discovery.py:45` - `FIND_CAPABILITIES_OP = "find_capabilities"` + plus `parse_find_capabilities_arguments` (line 78) and comment text (3, 34, 125). +- `api/oss/src/apis/fastapi/tools/router.py:1141-1207` - the legacy `tools.agenta.*` + `/tools/call` dispatch pins the old key (`op != FIND_CAPABILITIES_OP` -> 404). Decide: + rename the constant, or **delete the legacy route** in the same PR (recommended; the + platform tool emits a direct `call` now, and pre-production means nothing depends on + the old call_ref). Deleting also removes `parse_find_capabilities_arguments`'s only + router caller. +- Comment/docstring text only (no behavior): `core/tools/dtos.py:229-325`, + `core/tools/service.py:54,471`, `apis/fastapi/tools/models.py:135`, + `core/triggers/dtos.py:104`. + +**API tests** + +- `api/oss/tests/pytest/unit/applications/test_build_kit_overlay.py:188` + (`tool.op == "find_capabilities"`), plus the overlay-equality tests (see cut below). +- `api/oss/tests/pytest/unit/tools/test_discovery.py`, + `api/oss/tests/pytest/unit/triggers/test_triggers_discovery.py` (mentions; verify + whether they assert the op key or just the endpoint behavior). + +**Frontend** + +- `web/packages/agenta-playground/tests/unit/agentRequest.test.ts:224-360` - op literals + in unit tests only. No production FE code references the op keys (verified by grep over + `web/` excluding node_modules/dist). The FE renders overlay tools generically. +- Generated clients (`web/packages/agenta-api-client/src/generated/api/types/CapabilitiesResult.ts:6`, + `clients/python/agenta_client/types/capabilities_result.py:14`) mention + `find_capabilities` only in a docstring on the `/api/tools/discover` response type. The + endpoint does not change, so regeneration is cosmetic. Update the source docstring in + `core/tools/dtos.py:325` and regenerate, or leave until the next codegen sweep. + +**Docs** + +- `docs/design/agent-workflows/documentation/tools.md` - lines 38, 393-436 (the op table + at 405-406 and the whole "Tool discovery: `find_capabilities`" section at 416), 508-514. +- `docs/design/agent-workflows/interfaces/cross-service/runner-to-tool-callback.md:60-64`, + `interfaces/in-service/tool-models-and-resolution.md:35,57,113-149`, + `interfaces/public-edge/agent-config-schema.md:156`, `interfaces/README.md:51`. +- `documentation/triggers.md`, `agent-template.md`, `skills.md` carry no op names + (verified by grep). + +### Cut: shrinking the default overlay + +The overlay injects **every** catalog op: +`api/oss/src/apis/fastapi/applications/overlay.py:82` iterates `PLATFORM_OPS`. The cut +therefore needs an explicit default-overlay list; keep/cut is an overlay decision, not a +catalog deletion (all 18 ops stay in `PLATFORM_OPS` for opt-in). + +**Where the list lives.** Recommendation: an explicit `DEFAULT_BUILD_KIT_OPS` tuple in +`overlay.py`, validated against `PLATFORM_OPS` keys by the overlay unit test. Rationale +(design-interfaces lens): the catalog is the op REGISTRY (SDK-owned, "what exists"); the +overlay is a PRODUCT POLICY of the playground build kit (API-owned, "what a fresh builder +gets"). Different owners, different lifecycles, different files. It also keeps this +project's `op_catalog.py` edits down to the two key renames, which matters for the +approval-boundary coordination (see [plan.md](plan.md)). + +**Surfaces** + +- `api/oss/src/apis/fastapi/applications/overlay.py:80-84` - the tools list. +- `api/oss/tests/pytest/unit/applications/test_build_kit_overlay.py:41-42` - asserts the + overlay equals ALL of `PLATFORM_OPS`; rewrite against the new explicit list. Lines + 133-143 assert the wire payload matches `build_agent_template_overlay()` byte for byte + (follows automatically). Line 147-188 exercises overlay -> embed resolution. +- No FE surface: the FE deep-merges `additional_context.playground_build_kit. + agent_template_overlay` generically (`web/packages/agenta-entities/src/workflow/api/api.ts`, + `.../state/store.ts`); it holds no op list of its own. +- Docs: `documentation/tools.md` op table (405+) should gain a "default build kit" + column or note; `documentation/agent-template.md` if it lists the overlay contents. + +### Overlay scope (open decision 1) {#overlay-scope} + +Static 12-13 vs a conditional event pack. Evidence: + +- `build_agent_template_overlay()` takes **no arguments** (`overlay.py:64`) and is served + on `GET /api/simple/applications/{id}`. It has no view of the user's ask. A conditional + event pack needs new machinery: a request signal (FE-driven), a second-stage "load the + event pack" tool, or runtime injection. None exists today. +- The lab's capstone finding: extra visible tools derail runs, so smaller is itself the + reliability fix (tools-review part 2, "Recommended inside set"). +- The event pack is 5 tools (`discover_triggers`, `create_subscription`, + `list_deliveries`, `test_subscription`, `remove_subscription`). 8 core + 5 event = 13, + already close to the lab's proven working set. + +**Recommendation: static 13 now.** The conditional pack buys ~5 tools of context but +costs a new mechanism and a new failure mode (the pack not loading when the ask turns out +to be event-driven). Ship static, measure, and revisit conditionality only if the +13-tool set still wanders. Defer the mechanism question to a follow-up note in +open-issues if Mahmoud agrees. + +## Part 3: gotchas found during research + +1. **Three of the four authoring skills are never delivered.** The overlay embeds only + the getting-started skill (`overlay.py:69-78`; asserted by + `test_build_kit_overlay.py:47-57`). `AGENTA_FORCED_SKILLS` also holds only + getting-started (`agenta_builtins.py:320`). The other three + (`__ag__build_your_first_app`, `__ag__discover_and_wire_tools`, + `__ag__set_up_triggers`) are registered in the static catalog + (`api/oss/src/core/workflows/static_catalog.py:133-156`) but NOTHING embeds them: grep + over `api/`, `web/`, and the SDK finds no other reference to their slugs or names. So + the playground builder likely never had the flow map or the discovery loop in front of + it. This may explain part of the observed wandering, and it changes the skills-port + framing: we are not just rewriting four skills into one, we are attaching a playbook + that was probably never attached. **Verify live before relying on this** (one + playground run, check the resolved skills), then record it as a + builder-agent-reliability finding. +2. **The discover skill's availability note is stale.** The `discover-and-wire-tools` + body says (dated 2026-06-27) that `find_capabilities` is not yet callable as a tool + and tells the model to use `POST /tools/discover` or the `/tools/call` workaround + (`agenta_builtins.py:161-165`). The platform tool has shipped since. If the skill were + delivered, this text would actively misdirect the model. Dies with the skills port. +3. **Two timeout ceilings sit under any sync `test_run`.** `TOOL_CALL_TIMEOUT_MS` + defaults to 30s (`callback.ts:15-17`) and `RELAY_TIMEOUT_MS` to 60s + (`relay.ts:42-44`). Lab runs finished "well under a minute", which is inside the relay + ceiling but over the tool-call default. A sync `test_run` needs per-op timeout + plumbing (a catalog `timeout_ms` threaded onto the spec and honored by both layers), + whatever home it gets. Detailed in [api-design.md](api-design.md). +4. **Renaming the op key renames the reserved id for free, but the legacy route pins the + old key.** `reserved_id` derives from `op` (`op_catalog.py:121-124`), while + `_call_agenta_tool` hard-codes `FIND_CAPABILITIES_OP` (`tools/router.py:1152-1155`). + Recommend deleting the legacy reserved dispatch in the same PR (see inventory above). +5. **`args_into` interacts with a rename only in text.** No `args_into` or binding path + mentions the discovery ops; the renames touch names and docs, not body assembly. +6. **`list_subscriptions` fold-in is not free.** `list_deliveries` wraps + `GET /api/triggers/deliveries` and `list_subscriptions` wraps + `GET /api/triggers/subscriptions/` (`op_catalog.py:615-630`). "Folding" means either + keeping two catalog entries and cutting one from the overlay (cheap, recommended) or a + new combined read endpoint (out of scope; the API stays atomic). Recommendation: cut + `list_subscriptions` from the overlay, keep the op in the catalog. The event pack's + verify read is `list_deliveries`; `remove_subscription` takes an id that + `create_subscription`'s response already returns. +7. **The overlay excludes itself on commit.** The build kit rides only playground runs + and is excluded when the agent commits (builder-agent-reliability context, "The + build-kit overlay"). This is the natural first recursion guard for `test_run`: the + committed config a test run executes does not carry platform tools unless the author + added them deliberately. [api-design.md](api-design.md) adds a belt-and-braces guard. +8. **Coordination is not optional on `op_catalog.py`.** The approval-boundary lane is + implementing today against the same file (permission model: per-tool + `allow | ask | deny`, `needs_approval` deleted, `read_only` consulted by an + `allow_reads` policy mode; see `projects/approval-boundary/status.md` and + `plan.md:28-49`). Every op entry's `read_only` field sits inside their blast radius. + This project must not touch approval fields at all, and must sequence its + `op_catalog.py` edits after their lane lands or with the owner's ack, via + `docs/design/agent-workflows/scratch/agent-coordination.md`. diff --git a/docs/design/agent-workflows/projects/capability-config/README.md b/docs/design/agent-workflows/projects/capability-config/README.md index 6e0d0be7eb..5b3ae6579d 100644 --- a/docs/design/agent-workflows/projects/capability-config/README.md +++ b/docs/design/agent-workflows/projects/capability-config/README.md @@ -1,5 +1,7 @@ # Capability and permission configuration +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + How an author configures what an Agenta agent may do (files, network, tools, tool approvals), and how those controls enforce end to end: from the playground form, through the SDK and agent service, to the runner and the harness. Graduated from the scratch notes in diff --git a/docs/design/agent-workflows/projects/capability-config/context.md b/docs/design/agent-workflows/projects/capability-config/context.md index 525a000fb6..7789860e62 100644 --- a/docs/design/agent-workflows/projects/capability-config/context.md +++ b/docs/design/agent-workflows/projects/capability-config/context.md @@ -1,5 +1,7 @@ # Context +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + ## Why this work exists Every agent run has to be governed. The author needs to say what the agent may touch, and the diff --git a/docs/design/agent-workflows/projects/capability-config/plan.md b/docs/design/agent-workflows/projects/capability-config/plan.md index eedae886f1..c278ef86c4 100644 --- a/docs/design/agent-workflows/projects/capability-config/plan.md +++ b/docs/design/agent-workflows/projects/capability-config/plan.md @@ -1,5 +1,7 @@ # Execution plan +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + Phased, end to end. Each phase is shippable and de-risks the next. File references and exact insertion points live in `research.md`; this plan names the work and the acceptance bar. Some line numbers in `research.md` are approximate and must be reconfirmed at implementation time. diff --git a/docs/design/agent-workflows/projects/capability-config/proposal.md b/docs/design/agent-workflows/projects/capability-config/proposal.md index 1e037951f0..0541244b67 100644 --- a/docs/design/agent-workflows/projects/capability-config/proposal.md +++ b/docs/design/agent-workflows/projects/capability-config/proposal.md @@ -1,5 +1,7 @@ # Proposal: configuring agent capabilities and permissions +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + The canonical design. How an author controls what an agent may do, and how those controls reach each harness and backend. Why this matters and how it fits the other agent-workflows projects is in `context.md`; the codebase findings are in `research.md`; the work is phased in `plan.md`. diff --git a/docs/design/agent-workflows/projects/capability-config/research.md b/docs/design/agent-workflows/projects/capability-config/research.md index 8c630e6b2e..71ae32c72d 100644 --- a/docs/design/agent-workflows/projects/capability-config/research.md +++ b/docs/design/agent-workflows/projects/capability-config/research.md @@ -1,5 +1,7 @@ # Research: current state and insertion points +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + Self-contained map of how the runtime governs an agent today and where the three layers attach. All claims trace to code or installed package source. The deep web/exec/read/write current-state cut lives in `../../scratch/capability-map.md`; this doc adds the enforcement mechanics and the diff --git a/docs/design/agent-workflows/projects/capability-config/status.md b/docs/design/agent-workflows/projects/capability-config/status.md index 0b9e0b0d29..66393bc019 100644 --- a/docs/design/agent-workflows/projects/capability-config/status.md +++ b/docs/design/agent-workflows/projects/capability-config/status.md @@ -1,5 +1,7 @@ # Status +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + Source of truth for where this project stands. Keep it current. ## State diff --git a/docs/design/agent-workflows/projects/hitl-fix/README.md b/docs/design/agent-workflows/projects/hitl-fix/README.md index ccc3f65012..2eac24aa7c 100644 --- a/docs/design/agent-workflows/projects/hitl-fix/README.md +++ b/docs/design/agent-workflows/projects/hitl-fix/README.md @@ -1,5 +1,7 @@ # HITL approval fix (QA finding F-024) +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + Human-in-the-loop (HITL) tool approval is the headline interactive feature of the agent playground: when a tool needs a human yes/no, the run should pause, the playground should show an inline **Run this tool? Approve / Deny** prompt, and the user's answer should resume diff --git a/docs/design/agent-workflows/projects/hitl-fix/context.md b/docs/design/agent-workflows/projects/hitl-fix/context.md index bf51aeaf16..314178c13a 100644 --- a/docs/design/agent-workflows/projects/hitl-fix/context.md +++ b/docs/design/agent-workflows/projects/hitl-fix/context.md @@ -1,5 +1,7 @@ # Context +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + ## Why this work exists Human-in-the-loop tool approval is the interactive contract of the agent playground. A run diff --git a/docs/design/agent-workflows/projects/hitl-fix/plan.md b/docs/design/agent-workflows/projects/hitl-fix/plan.md index 329ba0943f..4ec553f1d5 100644 --- a/docs/design/agent-workflows/projects/hitl-fix/plan.md +++ b/docs/design/agent-workflows/projects/hitl-fix/plan.md @@ -1,5 +1,7 @@ # Plan: smallest-correct HITL fix +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + The renderer, the egress, and the cross-turn resume are already correct (see research.md). The fix is to stop the runner from poisoning the wire when it parks an `ask` gate. Below is the layer-by-layer change, smallest first, plus the Pi decision and the test plan. diff --git a/docs/design/agent-workflows/projects/hitl-fix/research.md b/docs/design/agent-workflows/projects/hitl-fix/research.md index 6c49c84f49..e436dc3292 100644 --- a/docs/design/agent-workflows/projects/hitl-fix/research.md +++ b/docs/design/agent-workflows/projects/hitl-fix/research.md @@ -1,5 +1,7 @@ # Research: the full HITL path, read-only trace +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + All citations are to the tree at `gitbutler/workspace` (v0.104.2). The HITL round-trip crosses four layers. The renderer, the egress, and the cross-turn resume are correct. The break is a single conflicting reply in the **runner**, which poisons the wire so the diff --git a/docs/design/agent-workflows/projects/hitl-fix/status.md b/docs/design/agent-workflows/projects/hitl-fix/status.md index df95d139ab..845fd553d2 100644 --- a/docs/design/agent-workflows/projects/hitl-fix/status.md +++ b/docs/design/agent-workflows/projects/hitl-fix/status.md @@ -1,5 +1,7 @@ # Status +> Superseded: the permission/approval model described here was redesigned in [projects/approval-boundary/](../approval-boundary/) (2026-07). Kept as a dated record. + **State:** DESIGN ONLY. Not implemented, not merged. Awaiting the decision below. **Date:** 2026-06-25 diff --git a/docs/design/agent-workflows/projects/qa/README.md b/docs/design/agent-workflows/projects/qa/README.md index 27f61e4a8d..e1ffed5ad0 100644 --- a/docs/design/agent-workflows/projects/qa/README.md +++ b/docs/design/agent-workflows/projects/qa/README.md @@ -83,7 +83,7 @@ curl -s -X POST \ "http://localhost:8280/services/agent/v0/invoke?project_id=$PROJ" \ -d '{"data":{"inputs":{"messages":[{"role":"user","content":""}]}, "parameters":{"agent":{ }}}}' + tools, mcp_servers, runner.permissions, harness_options> }}}}' ``` The response is one JSON assistant message. That makes pass and fail easy to assert: grep diff --git a/docs/design/agent-workflows/projects/runner-interface/README.md b/docs/design/agent-workflows/projects/runner-interface/README.md index f3a7054d0b..66c60c2aa6 100644 --- a/docs/design/agent-workflows/projects/runner-interface/README.md +++ b/docs/design/agent-workflows/projects/runner-interface/README.md @@ -226,7 +226,7 @@ Type: `AgentRunRequest` in `services/agent/src/protocol.ts`, hand-mirrored in | `customTools` | ResolvedToolSpec[] | Resolved runnable tools (gateway callback, code, or client). | | `toolCallback` | ToolCallbackContext | Where callback tools POST back. Required when `customTools` is set. | | `mcpServers` | McpServerConfig[] | User-declared MCP servers, secret env already injected. Omitted entirely when there are none. | -| `permissionPolicy` | string | `auto` (default) or `deny`, for permission-gating harnesses. | +| `permissions` | `{default: string, rules?: [...]}` | The agent-wide policy: `default` is one of `allow`, `ask`, `deny`, `allow_reads` (the default mode); optional `rules` are authored patterns (for example `Bash(rm:*)`) that override `default` for matching harness builtins. Read by the runner's shared decision function (`permission-plan.ts`), consulted by both the ACP responder and the tool relay. | | `systemPrompt` | string | Pi only: replace Pi's base system prompt. `AGENTS.md` is still appended after it. | | `appendSystemPrompt` | string | Pi only: append to Pi's base prompt without replacing it. | | `prompt` | string | Optional explicit latest turn. Falls back to the last user message in `messages`. | @@ -237,9 +237,9 @@ Type: `AgentRunRequest` in `services/agent/src/protocol.ts`, hand-mirrored in `request_to_wire` does not list tool, prompt, or MCP fields literally. It spreads three harness-shaped helpers off the config object: -- `config.wire_tools()` shapes `tools` / `customTools` / `toolCallback` / `permissionPolicy` - per harness. Pi sends built-ins plus native specs and no gating; Claude sends MCP-delivered - specs plus the permission policy. This is why the Pi and Claude golden requests differ. +- `config.wire_tools()` shapes `tools` / `customTools` / `toolCallback` / `permissions` + per harness. Pi and Claude both send the same `permissions` block now; they still differ + in tool shape (Pi sends built-ins plus native specs, Claude sends MCP-delivered specs). - `config.wire_prompt()` adds `systemPrompt` / `appendSystemPrompt` only for harnesses that expose them (Pi). It is empty otherwise. - `config.wire_mcp()` adds `mcpServers` only when the user declared some, so a tool-free run's @@ -256,7 +256,9 @@ A tool the service already resolved. Three orthogonal axes: the Composio key stays server-side); `code` runs `code` in a sandbox subprocess with `env` (scoped resolved secrets); `client` is fulfilled by the browser across a turn boundary. Absent means `callback` for back-compat. -- `needsApproval`: gate the call on a human yes/no. +- `permission`: `allow`, `ask`, `deny`, or unset (inherit the agent's policy). The runner's + shared decision function resolves the effective value; there is no separate + `needsApproval` field. - `render`: a generative-UI hint (`component`, `source`, or `spec`). `callRef` is set for `callback` tools only (the slug the bridge sends back). `runtime` / `code` @@ -297,15 +299,15 @@ From `golden/run_request.pi.json`: "endpoint": "https://api.example/tools/call", "authorization": "Access tok-123" }, - "permissionPolicy": "auto", + "permissions": {"default": "allow_reads"}, "systemPrompt": "You are Pi.", "appendSystemPrompt": "Be terse." } ``` The Claude golden (`run_request.claude.json`) differs as the harness shaping predicts: no -`tools` built-ins beyond an empty list, no Pi prompt overrides, `permissionPolicy: "deny"`, -and `backend: "sandbox-agent"`. +`tools` built-ins beyond an empty list, no Pi prompt overrides, `permissions: {"default": +"deny"}`, and `backend: "sandbox-agent"`. ## 8. The `/run` result and the event model diff --git a/docs/design/agent-workflows/projects/sandbox-agent-refactor/sandbox-agent-refactor-plan.md b/docs/design/agent-workflows/projects/sandbox-agent-refactor/sandbox-agent-refactor-plan.md index fc808d2586..fd206c30c4 100644 --- a/docs/design/agent-workflows/projects/sandbox-agent-refactor/sandbox-agent-refactor-plan.md +++ b/docs/design/agent-workflows/projects/sandbox-agent-refactor/sandbox-agent-refactor-plan.md @@ -277,9 +277,11 @@ Move the `session.onPermissionRequest` wiring into a helper: attachPermissionResponder({ session, run, responder }); ``` -The engine should choose the default `PolicyResponder`, but the helper should accept a +The engine should choose the default `ApprovalResponder` (`PolicyResponder`/`HITLResponder` +were later merged into this one responder; see +[projects/approval-boundary/](../approval-boundary/)), but the helper should accept a responder for future HITL tests. Add a unit test with a fake session to assert it emits -`interaction_request` and calls `respondPermission`. +`interaction_request` only on pause and calls `respondPermission`. ### `usage.ts` diff --git a/docs/design/agent-workflows/projects/tool-discovery/design.md b/docs/design/agent-workflows/projects/tool-discovery/design.md index 422e946f10..6c71d1e6cc 100644 --- a/docs/design/agent-workflows/projects/tool-discovery/design.md +++ b/docs/design/agent-workflows/projects/tool-discovery/design.md @@ -27,7 +27,7 @@ Keeping these separate is the key to deciding what to surface and how. | `tool_schemas[slug].input_schema` | Validates the tool and its inputs. | Load-bearing. This is what the model sees to call the tool. | `input_schema` on the resolved tool spec (`ToolResolveResponse.custom[].input_schema`) | Yes, same schema our catalog `/tools/{slug}` returns | | `tool_schemas[slug].description` | Context. | The tool description the model reads. | tool `description` | Yes | | `recommended_plan_steps` | Reveals the FULL tool set and order the task needs, not just the obvious action. | Becomes its operating procedure. | Draft `agents_md` content + a checklist of tools to include | Guidance, not a typed concept. References slugs; map to friendly names | -| `known_pitfalls` | Informs guardrails (e.g. non-idempotent -> set `needs_approval`). | "Things to avoid" in `agents_md`. | `agents_md` guidance; optionally per-tool `needs_approval`/`permission` | Guidance | +| `known_pitfalls` | Informs guardrails (e.g. non-idempotent -> set `permission: "ask"`). | "Things to avoid" in `agents_md`. | `agents_md` guidance; optionally per-tool `permission` | Guidance | | `difficulty` | Signal: auto-proceed vs ask for human review. | No. | metadata | Coarse | | `execution_guidance`, `next_steps_guidance` | The next action: resolve connections first, then run. | No. | Drives our connection-resolution + create flow | Translate (do not pass Composio tool names through) | | `toolkit_connection_statuses[].has_active_connection` | Critical. Drives reuse / initiate / ask-user. | No. | Agenta connection state, resolved against `gateway_connections` for the project | Yes IF we pass `user_id = project_id` | diff --git a/docs/design/agent-workflows/projects/wire-contract-schema/README.md b/docs/design/agent-workflows/projects/wire-contract-schema/README.md index 262ed168eb..c4d646757c 100644 --- a/docs/design/agent-workflows/projects/wire-contract-schema/README.md +++ b/docs/design/agent-workflows/projects/wire-contract-schema/README.md @@ -112,7 +112,7 @@ The contract has four families. This is what a single schema has to cover. | model + connection | `model`, `provider`, `connection {mode, slug?}`, `deployment`, `endpoint {baseUrl?, apiVersion?, region?, headers?}`, `credentialMode`, `secrets` | | turn | `prompt`, `messages` (`ChatMessage[]`) | | tools + skills | `tools` (string[]), `customTools` (`ResolvedToolSpec[]`), `toolCallback`, `mcpServers`, `skills` (`WireSkill[]`) | -| policy + files | `permissionPolicy`, `sandboxPermission`, `harnessFiles` (`[{path, content}]`) | +| policy + files | `permissions` (`{default, rules?}`), `sandboxPermission`, `harnessFiles` (`[{path, content}]`) | | tracing | `trace` (`TraceContext`) | Shape notes (the current serializer behavior, **not** a back-compat constraint — this is a @@ -139,7 +139,7 @@ must keep events **open/forward-compatible**, not closed. ### 3.4 Sub-objects `ResolvedToolSpec` (the three-axis tool surface: `kind`/`runtime`/`code`/`env`/`callRef`, -`needsApproval`, `render`, `readOnly`, `permission`), `ToolCallbackContext`, `McpServerConfig`, +`render`, `readOnly`, `permission`), `ToolCallbackContext`, `McpServerConfig`, `SandboxPermission` (nested `network`, `filesystem`, `enforcement`), `HarnessCapabilities` (11 boolean flags), `TraceContext`, `WireSkill` + `WireSkillFile`, `ContentBlock`, `ChatMessage`, `AgentUsage`, `RenderHint`. @@ -147,7 +147,7 @@ boolean flags), `TraceContext`, `WireSkill` + `WireSkillFile`, `ContentBlock`, ` ### 3.5 The existing golden/test machinery - `golden/run_request.pi.json` (full Pi shape: tools, skills, sandboxPermission, prompt overrides), - `golden/run_request.claude.json` (Claude shape: empty `tools`, `permissionPolicy:"deny"`, + `golden/run_request.claude.json` (Claude shape: empty `tools`, `permissions: {default: "deny"}`, `harnessFiles` with rendered `.claude/settings.json`). - `golden/run_result.ok.json` (includes a typeless event to pin the drop behavior), `golden/run_result.error.json` (`{"ok": false, "error": "model exploded"}`). diff --git a/docs/designs/sessions/interactions/extend/specs.md b/docs/designs/sessions/interactions/extend/specs.md index f0e566b014..4595a89b18 100644 --- a/docs/designs/sessions/interactions/extend/specs.md +++ b/docs/designs/sessions/interactions/extend/specs.md @@ -3,7 +3,7 @@ ## Problem The interactions API is fully built but nothing produces interactions, so the domain is -untestable. When a session-owned (HITL) turn hits a permission gate, the runner `park`s +untestable. When a session-owned (HITL) turn hits a permission gate, the runner pauses the turn and emits an `interaction_request` only on the messages plane (`services/agent/src/engines/sandbox_agent.ts`, `responder.ts`). It never calls the interactions API, so the `session_interactions` table stays empty and the inspector's @@ -13,9 +13,12 @@ to act on. ## Goal Whenever a human-in-the-loop interaction is raised, do BOTH: emit the messages-plane -`interaction_request` event AND create a row via the interactions API. Headless `/invoke` -keeps its inline policy answer (auto / allow / deny) and does NOT create a row (no human, -nothing to resolve). Then make the inspector usable: a Refresh button on all five tabs, +`interaction_request` event AND create a row via the interactions API. Every pause creates +a row, whether or not a human happens to be watching right now; there is no headless +exception. The only real gate on the row write is that the run must reference a committed +revision, because the respond/resume flow needs a stable revision to re-invoke against. +Uncommitted playground drafts skip the interactions-plane write for that reason, not +because no human is present. Then make the inspector usable: a Refresh button on all five tabs, and an Interactions tab that shows the full interaction and can respond with the correct shape. @@ -83,7 +86,7 @@ directly. The single message's shape depends on kind: - `user_input`: an ordinary user message — `{role: "user", content: }`. -- `user_approval`: NOT free text. The runner resolves a parked approval from a +- `user_approval`: NOT free text. The runner resolves a paused approval from a `tool_result` content block keyed by the gated `toolCallId` carrying `{approved: bool}` (`responder.ts` `extractApprovalDecisions`). In neutral form that is `{role: "user", content: [{type: "tool_result", toolCallId: , output: {approved}}]}`, @@ -110,11 +113,13 @@ boundary — the inspector still speaks neutral.) - `token` = the harness tool-call id (the same stable per-call key the responder already uses in `permissionRequestKeys`). Makes creation idempotent per gate and gives the respond/transition path its correlation key. -- Hook point: the `onPark` callback in `sandbox_agent.ts` (fires exactly when the - HITLResponder returns `"park"`, i.e. a real human-surface gate with no stored decision). - On park, call `createInteraction(kind=user_approval, data.request={tool name + args})` - in addition to the existing messages-plane emission. Headless and stored-decision paths - do not create (they never park). +- Hook point: the `onPause` callback in `sandbox_agent.ts` (fires exactly when the + ApprovalResponder returns `pendingApproval`, i.e. an `ask` gate with no stored decision, + on a run that references a committed revision). On pause, call + `createInteraction(kind=user_approval, data.request={tool name + args})` in addition to + the existing messages-plane emission. Stored-decision paths do not create a row, because + they never pause. Uncommitted drafts also skip the create, per the committed-revision + gate above. - `data.flags.delivered_in_band = true` (the messages plane also carried the event). ### API @@ -153,8 +158,8 @@ lives in the `answer` (the `{approved}` tool_result), never in the status: endpoint sets this at the API, before/as it dispatches the detached invoke. Only scenario 1. WIRED. - `resolved` — the runner consumed the answer and forwarded it to the harness. The runner - sets this (calls the transition endpoint) from the non-park branch of the permission - responder, where it applies a stored decision. Reachable from `responded` (scenario 1: + sets this (calls the transition endpoint) from the allow/stored-decision branch of the + permission responder, where it applies a stored decision. Reachable from `responded` (scenario 1: the answer came via /interactions) or directly from `pending` (scenario 2: the answer came inline via a messages reply, never touching the interactions endpoint). WIRED. - `cancelled` — the gate is orphaned; no one will answer the token. TWO producers, both @@ -208,9 +213,10 @@ interaction is never retroactively cancelled by a later turn. - Cancel-on-kill: with a `pending` gate open, Kill the session from the Streams tab. The pending interaction flips to `cancelled`. - Hit Refresh on each of the five tabs; each re-fetches its own data. -- Regression: a headless `/invoke` over the same gate auto-answers via policy and creates - NO interaction row. The messages-plane `interaction_request` event still fires on the - HITL path. +- Regression: a headless `/invoke` over a gate whose effective permission is `allow` or + `deny` auto-answers and creates NO interaction row, same as the HITL path. A headless + `/invoke` over an `ask` gate still pauses and still creates a row, as long as the run + references a committed revision. ## Out of scope (deferred to the big-agents agent-template audit) @@ -218,6 +224,6 @@ interaction is never retroactively cancelled by a later turn. - The "runner interactions" umbrella naming (user approvals / user inputs / callback tools as the three kinds). - The full dual-plane "whoever-reacts-first" resolver that also feeds the API respond - back into a parked runner gate. This branch creates the record and lets the inspector - respond; closing the loop back into a live parked turn from the API plane is the + back into a paused runner gate. This branch creates the record and lets the inspector + respond; closing the loop back into a live paused turn from the API plane is the follow-up. diff --git a/docs/designs/sessions/interactions/extend/tasks.md b/docs/designs/sessions/interactions/extend/tasks.md index 8452b20449..ba14ddd097 100644 --- a/docs/designs/sessions/interactions/extend/tasks.md +++ b/docs/designs/sessions/interactions/extend/tasks.md @@ -34,20 +34,27 @@ off `big-agents`). Read `specs.md` in this folder first. - Types: reuse the kind union (`user_approval` | `user_input` | `client_tool`) and a small `data.request` payload type. Keep `protocol.ts` untouched unless a wire field is needed. -## T3 — Runner: create on park (do both) +## T3 — Runner: create on pause (do both) -- File: `services/agent/src/engines/sandbox_agent.ts`, the `onPark` callback (~line 423) - and the `attachPermissionResponder` wiring. -- When a gate parks (HITLResponder returns `"park"`), in addition to the existing - messages-plane emission, call `createInteraction(...)` with `kind=user_approval`, +> Status: implemented by the approval-boundary redesign (2026-07). Both gates now create +> the row through one shared `recordPendingInteraction` closure in the engine. Kept for +> the acceptance criteria below. + +- File: `services/runner/src/engines/sandbox_agent.ts`, the pause wiring + and the `attachPermissionResponder` input. +- When a gate pauses (ApprovalResponder returns `pendingApproval`), in addition to the + existing messages-plane emission, call `createInteraction(...)` with `kind=user_approval`, `token` = the gated tool-call id, `data.request` = `{tool: , args: }` derived the same way `permissionRequestKeys` / `permissionToolName` do in `responder.ts`. The tool name + args are on the permission request `raw.toolCall`. - Thread the run credential (`runCredential(request)` / the watchdog credential, same source `persist.ts` uses) into the engine so `createInteraction` can authenticate. Check how `sandbox_agent.ts` already receives the credential for alive/persist and reuse it. -- Headless (`hasHumanSurface === false`) and stored-decision paths must NOT create (they - never park, so hooking `onPark` already gives this for free — confirm). +- Stored-decision paths and gates resolved to `allow`/`deny` must NOT create a row (they + never pause, so hooking the pause path already gives this for free). This holds + headless or not: an `ask` gate pauses and creates a row on a headless run too, as long + as the run references a committed revision. Uncommitted drafts skip the create because + the run has no stable revision to re-invoke against, not because no human is watching. ## T4 — Runner tests diff --git a/docs/designs/sessions/interactions/specs.md b/docs/designs/sessions/interactions/specs.md index 32dc3b4e95..e20a11d755 100644 --- a/docs/designs/sessions/interactions/specs.md +++ b/docs/designs/sessions/interactions/specs.md @@ -11,7 +11,7 @@ A harness doesn't only emit tool calls — it raises **reverse-RPC interaction r something must answer: a permission gate today, **elicitation (input)** and **client-side tools** later. The runner already models this as an `interaction_request` event of `kind: "permission" | "input" | "client_tool"` with an `id` (the reply token), and already -resolves them cross-turn (`HITLResponder`: stored decision by tool-call id, or by tool +resolves them cross-turn (`ApprovalResponder`: stored decision by tool-call id, or by tool name+args for cold-replay). What's missing is everything **on the Agenta side of that boundary**: there is no durable @@ -287,7 +287,7 @@ NOT `/sessions/{id}/…`; `session_id` is a filter param). Two tiers: point for competing answers** (multiple tabs + API) — there is no second lock on the interactions row. - **sessions-persistence** — detached-resolve works because the decision rides `/invoke` and - the next turn resumes (`HITLResponder` reads it). Without durable sessions it replays cold + the next turn resumes (`ApprovalResponder` reads it). Without durable sessions it replays cold (PER-4) — still correct, slower. - **records** — the `interaction_request` is a record event and is the **render source** (replay the record to show the question). The interaction record is the From fcb1dbb822495fec978d37ab8c045c009ce0f954 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 17:07:28 +0200 Subject: [PATCH 18/24] test: cross-language permission-decision parity fixture (40 cases) --- .../projects/approval-boundary/build-notes.md | 7 + .../agents/golden/permission_decisions.json | 681 ++++++++++++++++++ .../agents/tools/test_permission_parity.py | 65 ++ .../tests/unit/permission-parity.test.ts | 94 +++ 4 files changed, 847 insertions(+) create mode 100644 sdks/python/oss/tests/pytest/unit/agents/golden/permission_decisions.json create mode 100644 sdks/python/oss/tests/pytest/unit/agents/tools/test_permission_parity.py create mode 100644 services/runner/tests/unit/permission-parity.test.ts diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index 4b21211cd3..df1bf8c8d5 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -46,6 +46,13 @@ decisions unless it contradicts an owner call". Newest last. Read together with `pause.ts` renames; the four-site permission map heads `permission-plan.ts`; `agenta-tools` reserved-name guard in the settings renderer; stale `services/agent/` comment paths fixed. 401 runner + 477 SDK + 49 services tests green. +- **Docs sweep stragglers routed by ownership.** Twelve sweep files were hunk-locked to + three parallel doc lanes we own (`docs/agent-streaming-invoke`, + `docs/design-workspaces-sweep`, `marketing-website`), so their sweep edits were + committed to those lanes (e5876e1, 6aabd25, 9ea9ffe) instead of this PR. They land + when those lanes merge; this PR's sweep commit carries the other 38 files. The commit + subagent hit this, stopped per the guardrails, and left a clean report; an empty + duplicate commit it created was removed. - **General docs sweep landed (Sonnet subagent, reviewed here).** Five permission sections rewritten (tools.md, agent-configuration.md, permission-responder.md, agent-config-schema.md, runner-interface README), ~24 field-level updates, 17 diff --git a/sdks/python/oss/tests/pytest/unit/agents/golden/permission_decisions.json b/sdks/python/oss/tests/pytest/unit/agents/golden/permission_decisions.json new file mode 100644 index 0000000000..9b20a1fb43 --- /dev/null +++ b/sdks/python/oss/tests/pytest/unit/agents/golden/permission_decisions.json @@ -0,0 +1,681 @@ +{ + "cases": [ + { + "name": "mode=allow spec=unset hint=True", + "gate": { + "executor": "relay", + "readOnlyHint": true + }, + "plan": { + "default": "allow" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=allow spec=unset hint=False", + "gate": { + "executor": "relay", + "readOnlyHint": false + }, + "plan": { + "default": "allow" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=allow spec=unset hint=None", + "gate": { + "executor": "relay", + "readOnlyHint": null + }, + "plan": { + "default": "allow" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=allow spec=allow hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "allow", + "readOnlyHint": null + }, + "plan": { + "default": "allow" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=allow spec=ask hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "ask", + "readOnlyHint": null + }, + "plan": { + "default": "allow" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=allow spec=deny hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "deny", + "readOnlyHint": null + }, + "plan": { + "default": "allow" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "mode=ask spec=unset hint=True", + "gate": { + "executor": "relay", + "readOnlyHint": true + }, + "plan": { + "default": "ask" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=ask spec=unset hint=False", + "gate": { + "executor": "relay", + "readOnlyHint": false + }, + "plan": { + "default": "ask" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=ask spec=unset hint=None", + "gate": { + "executor": "relay", + "readOnlyHint": null + }, + "plan": { + "default": "ask" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=ask spec=allow hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "allow", + "readOnlyHint": null + }, + "plan": { + "default": "ask" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=ask spec=ask hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "ask", + "readOnlyHint": null + }, + "plan": { + "default": "ask" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=ask spec=deny hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "deny", + "readOnlyHint": null + }, + "plan": { + "default": "ask" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "mode=deny spec=unset hint=True", + "gate": { + "executor": "relay", + "readOnlyHint": true + }, + "plan": { + "default": "deny" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "mode=deny spec=unset hint=False", + "gate": { + "executor": "relay", + "readOnlyHint": false + }, + "plan": { + "default": "deny" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "mode=deny spec=unset hint=None", + "gate": { + "executor": "relay", + "readOnlyHint": null + }, + "plan": { + "default": "deny" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "mode=deny spec=allow hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "allow", + "readOnlyHint": null + }, + "plan": { + "default": "deny" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=deny spec=ask hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "ask", + "readOnlyHint": null + }, + "plan": { + "default": "deny" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=deny spec=deny hint=unset (spec overrides mode)", + "gate": { + "executor": "relay", + "specPermission": "deny", + "readOnlyHint": null + }, + "plan": { + "default": "deny" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=unset hint=True", + "gate": { + "executor": "relay", + "readOnlyHint": true + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=unset hint=False", + "gate": { + "executor": "relay", + "readOnlyHint": false + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=unset hint=None", + "gate": { + "executor": "relay", + "readOnlyHint": null + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=allow hint=True", + "gate": { + "executor": "relay", + "readOnlyHint": true, + "specPermission": "allow" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=allow hint=False", + "gate": { + "executor": "relay", + "readOnlyHint": false, + "specPermission": "allow" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=allow hint=None", + "gate": { + "executor": "relay", + "readOnlyHint": null, + "specPermission": "allow" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=ask hint=True", + "gate": { + "executor": "relay", + "readOnlyHint": true, + "specPermission": "ask" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=ask hint=False", + "gate": { + "executor": "relay", + "readOnlyHint": false, + "specPermission": "ask" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=ask hint=None", + "gate": { + "executor": "relay", + "readOnlyHint": null, + "specPermission": "ask" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "ask", + "verdict": "pendingApproval" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=deny hint=True", + "gate": { + "executor": "relay", + "readOnlyHint": true, + "specPermission": "deny" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=deny hint=False", + "gate": { + "executor": "relay", + "readOnlyHint": false, + "specPermission": "deny" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "mode=allow_reads spec=deny hint=None", + "gate": { + "executor": "relay", + "readOnlyHint": null, + "specPermission": "deny" + }, + "plan": { + "default": "allow_reads" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": true + }, + { + "name": "rule exact match wins over default", + "gate": { + "executor": "harness", + "toolName": "Bash" + }, + "plan": { + "default": "deny", + "rules": [ + { + "pattern": "Bash", + "permission": "allow" + } + ] + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": false + }, + { + "name": "rule prefix match on first string argument", + "gate": { + "executor": "harness", + "toolName": "Bash", + "args": { + "command": "npm run test" + } + }, + "plan": { + "default": "deny", + "rules": [ + { + "pattern": "Bash(npm run:*)", + "permission": "allow" + } + ] + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": false + }, + { + "name": "prefix rule with non-inspectable args falls back to default", + "gate": { + "executor": "harness", + "toolName": "Bash", + "args": { + "command": 42 + } + }, + "plan": { + "default": "deny", + "rules": [ + { + "pattern": "Bash(npm run:*)", + "permission": "allow" + } + ] + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": false + }, + { + "name": "deny beats ask beats allow when multiple rules match", + "gate": { + "executor": "harness", + "toolName": "Bash", + "args": { + "command": "npm run test" + } + }, + "plan": { + "default": "allow", + "rules": [ + { + "pattern": "Bash", + "permission": "allow" + }, + { + "pattern": "Bash(npm:*)", + "permission": "ask" + }, + { + "pattern": "Bash(npm run:*)", + "permission": "deny" + } + ] + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": false + }, + { + "name": "serverPermission beats matching rules", + "gate": { + "executor": "relay", + "toolName": "fetch", + "serverPermission": "allow" + }, + "plan": { + "default": "ask", + "rules": [ + { + "pattern": "fetch", + "permission": "deny" + } + ] + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": false + }, + { + "name": "specPermission beats serverPermission", + "gate": { + "executor": "relay", + "toolName": "fetch", + "specPermission": "deny", + "serverPermission": "allow" + }, + "plan": { + "default": "ask", + "rules": [ + { + "pattern": "fetch", + "permission": "allow" + } + ] + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": false + }, + { + "name": "stored allow under ask default resolves to allow", + "gate": { + "executor": "client", + "readOnlyHint": null + }, + "plan": { + "default": "ask" + }, + "expected": { + "effective": "ask", + "verdict": "allow" + }, + "python": false, + "stored": "allow" + }, + { + "name": "stored deny under ask default resolves to deny", + "gate": { + "executor": "client", + "readOnlyHint": null + }, + "plan": { + "default": "ask" + }, + "expected": { + "effective": "ask", + "verdict": "deny" + }, + "python": false, + "stored": "deny" + }, + { + "name": "stored allow does not override an effective deny (config beats stale approval)", + "gate": { + "executor": "client", + "specPermission": "deny" + }, + "plan": { + "default": "allow" + }, + "expected": { + "effective": "deny", + "verdict": "deny" + }, + "python": false, + "stored": "allow" + }, + { + "name": "stored decision is not consulted when effective permission is allow", + "gate": { + "executor": "client" + }, + "plan": { + "default": "allow" + }, + "expected": { + "effective": "allow", + "verdict": "allow" + }, + "python": false, + "stored": "deny" + } + ] +} diff --git a/sdks/python/oss/tests/pytest/unit/agents/tools/test_permission_parity.py b/sdks/python/oss/tests/pytest/unit/agents/tools/test_permission_parity.py new file mode 100644 index 0000000000..7d69763a7c --- /dev/null +++ b/sdks/python/oss/tests/pytest/unit/agents/tools/test_permission_parity.py @@ -0,0 +1,65 @@ +"""Cross-language permission-decision parity. + +Two implementations resolve effective permissions and must never drift: + - TS (enforcement truth): ``effectivePermission`` / ``decide`` in + ``services/runner/src/permission-plan.ts``. + - Python (feeds the Claude settings renderer): ``effective_permission`` here. + +Both sides assert the SAME shared fixture via the ``golden`` fixture (see +``conftest.py``): ``golden/permission_decisions.json``. The TS side asserts it in +``services/runner/tests/unit/permission-parity.test.ts``. Only cases marked +``"python": true`` are checked here: the Python helper only ever sees a tool's spec +permission, its read-only hint, and the plan's default mode -- it has no notion of +match rules, server permissions, or stored (human) decisions, so cases exercising those +are TS-only and are skipped here by design, not by fixture bug. + +If a case disagrees between the two languages, that is a real behavioral drift -- +do not bend the fixture to make it pass. +""" + +from __future__ import annotations + +import pytest + +from agenta.sdk.agents.tools.models import effective_permission + + +def test_fixture_has_at_least_36_cases(golden): + fixture = golden("permission_decisions.json") + assert len(fixture["cases"]) >= 36 + + +def _python_cases(golden): + fixture = golden("permission_decisions.json") + return [case for case in fixture["cases"] if case["python"]] + + +@pytest.fixture +def python_cases(golden): + return _python_cases(golden) + + +def test_python_eligible_cases_present(python_cases): + # Sanity: the fixture must actually carry Python-eligible cases, or this test file + # is silently a no-op. + assert len(python_cases) > 0 + + +def test_effective_permission_matches_fixture(golden): + fixture = golden("permission_decisions.json") + for case in fixture["cases"]: + if not case["python"]: + continue + gate = case["gate"] + plan = case["plan"] + got = effective_permission( + gate.get("specPermission"), + gate.get("readOnlyHint"), + plan["default"], + ) + assert got == case["expected"]["effective"], ( + f"case {case['name']!r}: effective_permission(" + f"spec_permission={gate.get('specPermission')!r}, " + f"read_only={gate.get('readOnlyHint')!r}, mode={plan['default']!r}) " + f"== {got!r}, expected {case['expected']['effective']!r}" + ) diff --git a/services/runner/tests/unit/permission-parity.test.ts b/services/runner/tests/unit/permission-parity.test.ts new file mode 100644 index 0000000000..0c77bce254 --- /dev/null +++ b/services/runner/tests/unit/permission-parity.test.ts @@ -0,0 +1,94 @@ +/** + * Cross-language permission-decision parity. + * + * Two implementations resolve effective permissions and must never drift: + * - TS (enforcement truth): `effectivePermission` / `decide` in `../../src/permission-plan.ts`. + * - Python (feeds the Claude settings renderer): `effective_permission` in + * `sdks/python/agenta/sdk/agents/tools/models.py`. + * + * Both sides assert the SAME shared fixture, loaded in place (no copy) via `loadGolden`: + * `sdks/python/oss/tests/pytest/unit/agents/golden/permission_decisions.json`. The Python side + * asserts it in `sdks/python/oss/tests/pytest/unit/agents/tools/test_permission_parity.py`. If a + * case here disagrees with the Python assertion, that is a real behavioral drift between the two + * implementations, not a fixture bug — do not bend the fixture to make it pass. + * + * Run: pnpm test (or: pnpm exec vitest run tests/unit/permission-parity.test.ts) + */ +import { describe, it } from "vitest"; +import assert from "node:assert/strict"; + +import { loadGolden } from "../utils/golden.ts"; +import { + decide, + effectivePermission, + type GateDescriptor, + type PermissionPlan, + type StoredPermissionDecisions, + type Verdict, +} from "../../src/permission-plan.ts"; +import type { ToolPermission } from "../../src/protocol.ts"; + +interface FixtureCase { + name: string; + gate: GateDescriptor; + plan: { default: PermissionPlan["default"]; rules?: PermissionPlan["rules"] }; + stored?: "allow" | "deny"; + expected: { effective: ToolPermission; verdict: Verdict["kind"] }; + python: boolean; +} + +const fixture = loadGolden("permission_decisions.json") as { + cases: FixtureCase[]; +}; + +describe("permission decision parity fixture", () => { + it("has at least 36 cases", () => { + assert.ok( + fixture.cases.length >= 36, + `expected >= 36 cases, got ${fixture.cases.length}`, + ); + }); + + for (const testCase of fixture.cases) { + it(testCase.name, () => { + const plan: PermissionPlan = { + default: testCase.plan.default, + rules: testCase.plan.rules ?? [], + }; + + assert.equal( + effectivePermission(testCase.gate, plan), + testCase.expected.effective, + "effectivePermission mismatch", + ); + + let consulted = false; + const stored: StoredPermissionDecisions = { + take: () => { + consulted = true; + return testCase.stored; + }, + }; + + const verdict = decide(testCase.gate, plan, stored); + assert.equal( + verdict.kind, + testCase.expected.verdict, + "decide verdict mismatch", + ); + + // Stored decisions must only ever be consulted when the effective permission is + // neither "allow" nor "deny" (i.e. it's "ask"). This is the config-beats-stale-approval + // invariant: an effective allow/deny from spec/server/rule/default must never let a + // stored (possibly stale) human decision override it. + const expectConsulted = testCase.expected.effective === "ask"; + assert.equal( + consulted, + expectConsulted, + expectConsulted + ? "expected the stored decision to be consulted" + : "stored decision must NOT be consulted when the effective permission is allow/deny", + ); + }); + } +}); From 0cc133ae21600a465521adf8001603555160fd50 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 17:25:46 +0200 Subject: [PATCH 19/24] fix(sdk): drop unenforceable builtin tool permission loudly (QA finding) --- .../projects/approval-boundary/build-notes.md | 10 +++++++++ .../projects/approval-boundary/status.md | 10 +++++++++ sdks/python/agenta/sdk/agents/tools/models.py | 22 +++++++++++++++++++ .../pytest/unit/agents/tools/test_models.py | 11 ++++++++++ 4 files changed, 53 insertions(+) diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index df1bf8c8d5..9e6b0cc9df 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -40,6 +40,16 @@ decisions unless it contradicts an owner call". Newest last. Read together with relay must enforce permissions only when the harness does not gate (Pi) — otherwise one approval would be consumed at the gate and the relay would double-gate the same call. +- **Headless QA: 7/7 pass on the live EE dev stack** (Sonnet subagent; pi_core + + gpt-4o-mini via the pi-agents project; evidence captured per case). All four policy + modes behave, explicit beats policy both directions, the paused batch envelope carries + `stop_reason` + `pending_interaction`, and legacy keys are ignored without a 500. +- **QA finding fixed: per-builtin `permission` was a dead knob.** `BuiltinToolConfig` + parsed `permission` and dropped it silently; on Pi no gate ever sees native builtins. + Fix: the config now drops it LOUDLY (log warning) with a pin test; the FE never + offered the field for builtins. Designed follow-up (status.md): selection-time + enforcement, filtering `builtin_names` by effective permission using the known + read-only-ness of Pi's seven builtins, so `deny`/lockdown modes bind builtins too. - **Phase 5 landed (Codex implemented, reviewed here).** Legacy wire fields deleted (`permissionPolicy`, `needsApproval`); absent `permissions` now defaults to `allow_reads` (same default as the authored schema); `acp-interactions.ts` and diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index ac6857aa34..bab6fd5957 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -77,6 +77,16 @@ Codex design review (xhigh), a cold-reader clarity pass, two inline review round Mahmoud (38 comments total, all answered inline and folded into the docs), and the #5054 analysis that diagnosed the live loop. +## Follow-ups (filed, not in this PR) + +- **Selection-time enforcement for Pi builtins.** Pi's native tools never reach a + runner gate, so today the global policy modes do not bind them (headless QA proved + `default: deny` does not stop a granted `bash`). The design: filter `builtin_names` + at resolution by effective permission, using a small read-only table for Pi's seven + builtins (`read/grep/find/ls` read, `bash/edit/write` write); `deny` and un-pausable + `ask` exclude the builtin (deny-by-omission is Pi's one native control). Until then, + a per-builtin `permission` is dropped with a logged warning instead of silently. + ## Known unknowns - The sandbox-agent daemon's permission-request id scheme (per-session counter vs unique): diff --git a/sdks/python/agenta/sdk/agents/tools/models.py b/sdks/python/agenta/sdk/agents/tools/models.py index e6828521f6..2283c04492 100644 --- a/sdks/python/agenta/sdk/agents/tools/models.py +++ b/sdks/python/agenta/sdk/agents/tools/models.py @@ -16,6 +16,11 @@ ) +from agenta.sdk.utils.logging import get_module_logger + +log = get_module_logger(__name__) + + def _empty_object_schema() -> Dict[str, Any]: return {"type": "object", "properties": {}} @@ -79,6 +84,23 @@ class BuiltinToolConfig(ToolConfigBase): type: Literal["builtin"] = "builtin" name: str = Field(min_length=1) + @model_validator(mode="after") + def _drop_unenforceable_permission(self) -> "BuiltinToolConfig": + # Harness builtins are granted by SELECTION (present = runs, absent = does not + # exist); no runner gate sees them on Pi, so a per-builtin permission cannot be + # enforced and keeping it would be a dead knob an author mistakes for a deny. + # Selection-time enforcement (filter builtin_names by effective permission) is + # the designed follow-up; until then the field is dropped loudly. + if self.permission is not None: + log.warning( + "builtin tool %r: per-tool permission %r is not enforceable and was " + "ignored; control builtins by selection (or a harness settings rule)", + self.name, + self.permission, + ) + self.permission = None + return self + class GatewayToolConfig(ToolConfigBase): type: Literal["gateway"] = "gateway" diff --git a/sdks/python/oss/tests/pytest/unit/agents/tools/test_models.py b/sdks/python/oss/tests/pytest/unit/agents/tools/test_models.py index 9698f97db3..789edceeed 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/tools/test_models.py +++ b/sdks/python/oss/tests/pytest/unit/agents/tools/test_models.py @@ -218,3 +218,14 @@ def test_legacy_permission_fields_are_ignored_on_specs(): assert spec.permission is None assert "permission" not in spec.to_wire() assert "needsApproval" not in spec.to_wire() + + +def test_builtin_tool_permission_is_dropped_not_enforced(): + # Builtins are granted by selection; a per-builtin permission has no enforcement + # point on Pi, so the config drops it (with a warning) instead of lying. + from agenta.sdk.agents.tools.models import BuiltinToolConfig + + config = BuiltinToolConfig.model_validate( + {"type": "builtin", "name": "bash", "permission": "deny"} + ) + assert config.permission is None From 81d9d5029313772ee39a20cd175829c75b9af369 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Fri, 3 Jul 2026 18:25:52 +0200 Subject: [PATCH 20/24] fix(runner): a paused tool call's teardown frames no longer clobber the approval prompt --- .../projects/approval-boundary/build-notes.md | 16 +++ .../projects/approval-boundary/status.md | 16 ++- services/runner/src/engines/sandbox_agent.ts | 29 ++++- .../engines/sandbox_agent/acp-interactions.ts | 10 +- .../runner/src/engines/sandbox_agent/pause.ts | 15 +++ .../sandbox-agent-acp-interactions.test.ts | 10 ++ .../unit/sandbox-agent-orchestration.test.ts | 115 ++++++++++++++++++ 7 files changed, 199 insertions(+), 12 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index 9e6b0cc9df..956740311f 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -40,6 +40,22 @@ decisions unless it contradicts an owner call". Newest last. Read together with relay must enforce permissions only when the harness does not gate (Pi) — otherwise one approval would be consumed at the gate and the relay would double-gate the same call. +- **Live QA found and fixed one real bug: pause teardown clobbered the approval prompt + on the Pi relay path.** The relay writes no response file on a pause, session teardown + fires the extension's AbortSignal, Pi reports the call as failed ("aborted"), and that + frame overwrote the prompt (F-024's class, new path). Fix: the pause controller keeps + a paused-call registry and the engine drops any later harness frame for a paused call + (the approval request is the last word, now enforced). Regression tests added; UI + retest: 1 prompt, Approve resumes with real data and ZERO re-prompts, Deny refuses + cleanly. Polish note, not blocking: a user Deny currently surfaces the relay's + "denied by the permission policy" wording; "denied by the user" would read better. +- **UI QA environment notes (pre-existing, not this PR's regressions):** old app + records on the dev box stored the internal `agenta-agent:8000` service URL + (unreachable from a browser; fixed in the dev DB to the proxied + `/services/agent/v0`); the tool picker offers no path to platform ops (tracked by the + tools-review workstream); the Anthropic account is out of credit so Claude-harness + live runs stayed unverified (Claude-path behavior is covered by unit + settings + rendering tests and the Gate-2 machinery is shared). - **Headless QA: 7/7 pass on the live EE dev stack** (Sonnet subagent; pi_core + gpt-4o-mini via the pi-agents project; evidence captured per case). All four policy modes behave, explicit beats policy both directions, the paused batch envelope carries diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index bab6fd5957..6c440f14e8 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -1,10 +1,16 @@ # Status -**State: implementation in progress.** Date: 2026-07-03. Three review rounds are folded -in; the final round left two small comments (both addressed below) and a green light: -implement, Mahmoud reviews the finished PR. The remaining calls listed under "Decisions -taken (delegated)" were made by the agent under an explicit "go with your decisions" -mandate; any of them can be reopened in PR review. +**State: implemented and QA'd; awaiting final PR review.** Date: 2026-07-03. All six +plan phases landed on the lane with per-phase review; tests green across runner (444), +SDK agents (480+), and services (49), including a 40-case cross-language parity +fixture. Live QA: headless matrix 7/7; playground approve/deny/allow flows verified +(one prompt, approve resumes without looping, deny refuses cleanly) after fixing one +QA-found bug (pause teardown clobbered the prompt on the Pi relay path; fixed with a +paused-call event filter + regression tests). Remaining before merge: rebase onto +post-#5058 `big-agents` (audit done: their runner deltas are formatting plus patches +this redesign supersedes; their egress `rawInput is not None` fix merges untouched) +and flip the PR base. Claude-harness live runs were blocked by account credit; that +path is covered by unit and settings-rendering tests. ## Where things stand diff --git a/services/runner/src/engines/sandbox_agent.ts b/services/runner/src/engines/sandbox_agent.ts index 332bff1bdc..e6e24165ad 100644 --- a/services/runner/src/engines/sandbox_agent.ts +++ b/services/runner/src/engines/sandbox_agent.ts @@ -170,6 +170,20 @@ export async function destroyInFlightSandboxes( ]); } +function shouldSuppressPausedToolCallUpdate( + update: unknown, + pause: PendingApprovalPauseController, +): boolean { + const frame = update as + | { sessionUpdate?: unknown; toolCallId?: unknown } + | undefined; + const kind = frame?.sessionUpdate; + if (kind !== "tool_call" && kind !== "tool_call_update") return false; + const toolCallId = + typeof frame?.toolCallId === "string" ? frame.toolCallId : undefined; + return pause.isPausedToolCall(toolCallId); +} + const CLAUDE_STRICT_DEPLOYMENTS = new Set([ "custom", "bedrock", @@ -635,16 +649,18 @@ export async function runSandboxAgent( ], }); + const pause = new PendingApprovalPauseController(() => + sandbox.destroySession?.(session.id), + ); + session.onEvent((event: any) => { remountLocalCwdAfterRuntimeEnotconn(event); const payload = event?.payload; const update = payload?.params?.update ?? payload?.update; - if (update) run.handleUpdate(update); + if (update && !shouldSuppressPausedToolCallUpdate(update, pause)) { + run.handleUpdate(update); + } }); - - const pause = new PendingApprovalPauseController(() => - sandbox.destroySession?.(session.id), - ); const permissionPlan = permissionsFromRequest(request); const storedDecisionMap = extractApprovalDecisions(request); if (storedDecisionMap.size > 0) { @@ -687,6 +703,7 @@ export async function runSandboxAgent( decide: (gate) => decide(gate, permissionPlan, decisions), onPendingApproval: ({ toolCallId, toolName, args }) => { if (!latch.tryAcquire()) return; + pause.markPausedToolCall(toolCallId); run.emitEvent({ type: "interaction_request", id: toolCallId, @@ -716,6 +733,7 @@ export async function runSandboxAgent( serverPermissions, log: logger, onPause: () => pause.pause(), + onPausedToolCall: (id) => pause.markPausedToolCall(id), onCreateInteraction: recordPendingInteraction, onResolveInteraction: (token) => { const cred = runCredential(request); @@ -767,6 +785,7 @@ export async function runSandboxAgent( if (verdict.kind === "deny") return "deny"; if (verdict.kind === "fulfilled") return { output: verdict.output }; if (latch.tryAcquire()) { + pause.markPausedToolCall(toolCallId); run.emitEvent({ type: "interaction_request", id, diff --git a/services/runner/src/engines/sandbox_agent/acp-interactions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts index 17ee5db6bd..bd22de74a2 100644 --- a/services/runner/src/engines/sandbox_agent/acp-interactions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -22,6 +22,8 @@ export interface AttachPermissionResponderInput { */ onPause?: () => void; log?: (msg: string) => void; + /** Called with the ACP tool-call id when a gate pauses the turn. */ + onPausedToolCall?: (id: string) => void; /** Called on pause to record the pending gate as an interaction (fire-and-forget). */ onCreateInteraction?: ( token: string, @@ -41,6 +43,7 @@ export function attachPermissionResponder({ serverPermissions = new Map(), onPause, log, + onPausedToolCall, onCreateInteraction, onResolveInteraction, }: AttachPermissionResponderInput): void { @@ -58,13 +61,15 @@ export function attachPermissionResponder({ // turn's stored decision answers the re-raised gate. const pauseUserApproval = (req: any, id: string, gate: GateDescriptor): void => { if (!latch.tryAcquire()) return; - const eventId = interactionEventId(id, req?.toolCall?.toolCallId); + const toolCallId = stringValue(req?.toolCall?.toolCallId); + const eventId = interactionEventId(id, toolCallId); + if (toolCallId) onPausedToolCall?.(toolCallId); run.emitEvent({ type: "interaction_request", id: eventId, kind: "user_approval", payload: { - toolCallId: stringValue(req?.toolCall?.toolCallId), + toolCallId, toolCall: req?.toolCall, availableReplies: stringArray(req?.availableReplies), options: req?.options, @@ -83,6 +88,7 @@ export function attachPermissionResponder({ if (!latch.tryAcquire()) return; const toolCallId = stringValue(req?.toolCall?.toolCallId); const eventId = interactionEventId(id, toolCallId); + if (toolCallId) onPausedToolCall?.(toolCallId); run.emitEvent({ type: "interaction_request", id: eventId, diff --git a/services/runner/src/engines/sandbox_agent/pause.ts b/services/runner/src/engines/sandbox_agent/pause.ts index c5a2bb1ad5..b738bcd720 100644 --- a/services/runner/src/engines/sandbox_agent/pause.ts +++ b/services/runner/src/engines/sandbox_agent/pause.ts @@ -8,6 +8,7 @@ export const PAUSED = Symbol("paused"); export class PendingApprovalPauseController { private pendingApproval = false; + private readonly pausedToolCallIds = new Set(); private resolvePause: (() => void) | undefined; readonly signal: Promise; @@ -25,6 +26,20 @@ export class PendingApprovalPauseController { void Promise.resolve(this.destroySession()).catch(() => {}); } + /** + * F-024 lineage: once a paused tool call emits its `interaction_request`, that request is the + * last word for the call this turn. Later harness frames for the same id are teardown artifacts + * from cancellation/session disposal and must not reach the event stream. + */ + markPausedToolCall(toolCallId: string): void { + if (!toolCallId) return; + this.pausedToolCallIds.add(toolCallId); + } + + isPausedToolCall(toolCallId: string | undefined): boolean { + return toolCallId !== undefined && this.pausedToolCallIds.has(toolCallId); + } + get active(): boolean { return this.pendingApproval; } diff --git a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts index a1946e21e3..aaba85e9b8 100644 --- a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts +++ b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts @@ -109,6 +109,7 @@ describe("attachPermissionResponder", () => { }); const events: AgentEvent[] = []; const created: Array<{ token: string; toolName?: string; args: unknown }> = []; + const pausedToolCalls: string[] = []; let pauses = 0; attachPermissionResponder({ @@ -122,6 +123,9 @@ describe("attachPermissionResponder", () => { onCreateInteraction: (token, toolName, args) => { created.push({ token, toolName, args }); }, + onPausedToolCall: (id) => { + pausedToolCalls.push(id); + }, }); emit({ id: "perm-pause", @@ -133,6 +137,7 @@ describe("attachPermissionResponder", () => { assert.deepEqual(replies, []); assert.equal(pauses, 1); + assert.deepEqual(pausedToolCalls, ["tool-9"]); assert.deepEqual(created, [ { token: "perm-pause", toolName: "edit", args: { path: "a" } }, ]); @@ -235,6 +240,7 @@ describe("attachPermissionResponder", () => { replies.push({ id, reply }); }); const events: AgentEvent[] = []; + const pausedToolCalls: string[] = []; let pauses = 0; attachPermissionResponder({ @@ -248,6 +254,9 @@ describe("attachPermissionResponder", () => { onPause: () => { pauses += 1; }, + onPausedToolCall: (id) => { + pausedToolCalls.push(id); + }, }); emit({ id: "client-1", @@ -266,6 +275,7 @@ describe("attachPermissionResponder", () => { assert.deepEqual(replies, []); assert.equal(pauses, 1); + assert.deepEqual(pausedToolCalls, ["tool-client"]); assert.deepEqual(events, [ { type: "interaction_request", diff --git a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts index c355a0c134..7c694c7ad5 100644 --- a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts +++ b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts @@ -9,6 +9,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AgentEvent, AgentRunRequest } from "../../src/protocol.ts"; +import { createSandboxAgentOtel } from "../../src/tracing/otel.ts"; +import { PendingApprovalPauseController } from "../../src/engines/sandbox_agent/pause.ts"; import type { PermissionDecision } from "../../src/responder.ts"; import { runSandboxAgent, @@ -26,6 +28,8 @@ interface FakeOptions { promptResult?: Record; promptEvent?: Record; promptEvents?: Array>; + afterPromptEvents?: () => Promise | void; + postPermissionEvents?: Array>; streamUsage?: Record; output?: string; promptError?: Error; @@ -89,6 +93,7 @@ function fakeHarness(options: FakeOptions = {}) { options.promptEvent ?? { payload: { update: { kind: "noop" } } }, ]; for (const event of promptEvents) eventHandler?.(event); + await options.afterPromptEvents?.(); if (options.emitPermission) { permissionHandler?.({ id: "perm-1", @@ -96,6 +101,10 @@ function fakeHarness(options: FakeOptions = {}) { toolCall: { toolCallId: "tool-1", name: "edit" }, }); } + if (options.postPermissionEvents?.length) { + if (options.emitPermission) await flushPromises(); + for (const event of options.postPermissionEvents) eventHandler?.(event); + } if (options.promptError) throw options.promptError; if (options.hangPrompt) { // Claude does not end a turn on an unanswered gate: the prompt hangs until the @@ -243,6 +252,20 @@ function fakeHarness(options: FakeOptions = {}) { return { calls, deps, events }; } +describe("PendingApprovalPauseController", () => { + it("tracks paused tool-call ids", () => { + const pause = new PendingApprovalPauseController(() => {}); + + assert.equal(pause.isPausedToolCall(undefined), false); + assert.equal(pause.isPausedToolCall("tool-1"), false); + + pause.markPausedToolCall("tool-1"); + + assert.equal(pause.isPausedToolCall("tool-1"), true); + assert.equal(pause.isPausedToolCall("tool-2"), false); + }); +}); + describe("runSandboxAgent orchestration", () => { it("returns a successful one-shot result and cleans up acquired resources", async () => { const { calls, deps } = fakeHarness(); @@ -904,6 +927,98 @@ describe("runSandboxAgent default ApprovalResponder wiring", () => { assert.deepEqual(calls.permissionReplies, []); }); + it("drops teardown tool updates after a Pi relay approval pause while keeping other ids", async () => { + const relayPause = { fire: () => {} }; + const { calls, deps } = fakeHarness({ + promptEvents: [ + { + payload: { + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-1", + title: "approval_needed", + }, + }, + }, + { + payload: { + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-2", + title: "other_tool", + }, + }, + }, + ], + afterPromptEvents: () => relayPause.fire(), + postPermissionEvents: [ + { + payload: { + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-1", + status: "failed", + content: [{ content: { type: "text", text: "aborted" } }], + }, + }, + }, + { + payload: { + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-2", + status: "failed", + content: [{ content: { type: "text", text: "other failed" } }], + }, + }, + }, + ], + }); + deps.createOtel = createSandboxAgentOtel as any; + relayPause.fire = () => { + const relayPermissions = calls.toolRelayArgs?.[4] as any; + relayPermissions.onPendingApproval({ + toolCallId: "tool-1", + toolName: "approval_needed", + args: { path: "a" }, + }); + }; + + const result = await runSandboxAgent( + { + harness: "pi_core", + permissions: { default: "ask" }, + messages: [{ role: "user", content: "use the tool" }], + customTools: [ + { name: "approval_needed", kind: "callback", permission: "ask" }, + ], + } as AgentRunRequest, + undefined, + undefined, + deps, + ); + + assert.equal(result.ok, true); + if (!result.ok) return; + assert.equal(result.stopReason, "paused"); + assert.deepEqual( + result.events + ?.filter((event) => event.type === "interaction_request") + .map((event) => ({ id: (event as any).id, kind: (event as any).kind })), + [{ id: "tool-1", kind: "user_approval" }], + ); + assert.deepEqual( + result.events + ?.filter((event) => event.type === "tool_result") + .map((event) => ({ + id: (event as any).id, + output: (event as any).output, + isError: (event as any).isError, + })), + [{ id: "tool-2", output: "other failed", isError: true }], + ); + }); + it("park ENDS the turn even when the prompt hangs: terminal stopReason 'paused', no harness reply (F-040)", async () => { // The real regression: Claude does NOT end a turn on an unanswered gate, so without the // park->cancel path `session.prompt()` blocks forever and the run never returns. With From 1cc946186098b2dc2b8f74e90bd8a479dc86e82b Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 4 Jul 2026 13:33:20 +0200 Subject: [PATCH 21/24] fix: reconcile the rebase onto big-agents (#5064/#5059) - handler.py: permission_default (the SDK deleted permission_policy in phase 3) - fold(): the terminal result's stop_reason wins over the done event (the live runner emits done without stopReason, so a real pause would otherwise drop its envelope metadata) and pending_interaction carries a derived top-level tool name - sandbox_agent.ts: fold upstream keepers lost to ours-wins conflict resolution (shared apiBase module, claude-model + resolved-model log lines) - service pause test pinned to the realistic stream shape (done event with no stopReason) --- sdks/python/agenta/sdk/agents/fold.py | 49 ++++++++++++-- sdks/python/agenta/sdk/agents/handler.py | 6 +- .../oss/tests/pytest/unit/agents/test_fold.py | 39 +++++++++++ .../pytest/unit/agent/test_invoke_handler.py | 35 +++++----- services/runner/src/engines/sandbox_agent.ts | 64 +++++++++++++------ 5 files changed, 148 insertions(+), 45 deletions(-) diff --git a/sdks/python/agenta/sdk/agents/fold.py b/sdks/python/agenta/sdk/agents/fold.py index 63bb110eff..d46a5bdbd9 100644 --- a/sdks/python/agenta/sdk/agents/fold.py +++ b/sdks/python/agenta/sdk/agents/fold.py @@ -12,11 +12,40 @@ FoldedEvent = Dict[str, Any] -def fold(events: Iterable[FoldedEvent]) -> Dict[str, Any]: - """Fold a full event turn into ``{messages, stop_reason, pending_interaction}``.""" +def _pending_tool_name(interaction: Dict[str, Any]) -> Optional[str]: + """The paused tool's name, from the interaction payload's stable-first candidates. + + Relay/client pauses carry ``payload.toolName`` (the spec name); harness gates carry an + ACP ``toolCall`` whose ``name``/``title``/``kind`` may each be a display label rather + than a name, so they are fallbacks in that order (see the approval-boundary workspace). + """ + payload = interaction.get("payload") + if not isinstance(payload, dict): + return None + name = payload.get("toolName") + if isinstance(name, str) and name: + return name + tool_call = payload.get("toolCall") + if isinstance(tool_call, dict): + for key in ("name", "title", "kind"): + value = tool_call.get(key) + if isinstance(value, str) and value: + return value + return None + + +def fold( + events: Iterable[FoldedEvent], *, stop_reason: Optional[str] = None +) -> Dict[str, Any]: + """Fold a full event turn into ``{messages, stop_reason, pending_interaction}``. + + ``stop_reason`` is the terminal result's value when the caller holds one; it wins over + the event-derived value because the runner's ``done`` event carries no ``stopReason`` + today (the engine settles it after the pause race, onto the terminal result only). + """ messages: List[Message] = [] open_text: Dict[Any, int] = {} # message_start id -> index into `messages` - stop_reason: Optional[str] = None + event_stop_reason: Optional[str] = None last_interaction_request: Optional[Dict[str, Any]] = None for event in events: @@ -58,14 +87,22 @@ def fold(events: Iterable[FoldedEvent]) -> Dict[str, Any]: elif etype == "interaction_request": last_interaction_request = data elif etype == "done": - stop_reason = data.get("stopReason") + event_stop_reason = data.get("stopReason") # thought_start/thought_delta/thought_end, data, file, usage, error: no message. - pending_interaction = last_interaction_request if stop_reason == "paused" else None + resolved_stop = stop_reason if stop_reason is not None else event_stop_reason + pending_interaction: Optional[Dict[str, Any]] = None + if resolved_stop == "paused" and last_interaction_request is not None: + # Superset of the raw interaction_request data plus the derived `tool` name, so + # headless callers can act on `{id, tool}` without parsing the ACP payload. + pending_interaction = dict(last_interaction_request) + pending_interaction.setdefault( + "tool", _pending_tool_name(last_interaction_request) + ) return { "messages": messages, - "stop_reason": stop_reason, + "stop_reason": resolved_stop, "pending_interaction": pending_interaction, } diff --git a/sdks/python/agenta/sdk/agents/handler.py b/sdks/python/agenta/sdk/agents/handler.py index 4b39507d27..0295a53bb2 100644 --- a/sdks/python/agenta/sdk/agents/handler.py +++ b/sdks/python/agenta/sdk/agents/handler.py @@ -150,7 +150,7 @@ async def _agent( agent=agent_template, secrets=secrets, resolved_connection=resolved_connection, - permission_policy=agent_template.permission_policy, + permission_default=agent_template.permission_default, trace=comp.trace_context(), run_context=comp.run_context(), session_id=session_id, @@ -216,7 +216,9 @@ async def agent_batch( await harness.cleanup() record_usage(result.usage) - folded = fold(events) + # The terminal result's stop_reason is authoritative: the runner's `done` event carries + # no stopReason (the engine settles paused-vs-ended after the event stream closes). + folded = fold(events, stop_reason=result.stop_reason) out_messages = folded["messages"] if trim: out_messages = trim_to_trailing_unit(out_messages) diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_fold.py b/sdks/python/oss/tests/pytest/unit/agents/test_fold.py index 08d3061617..6ae6f595fb 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_fold.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_fold.py @@ -132,9 +132,48 @@ def test_fold_paused_turn_sets_pending_interaction(): "id": "req-1", "kind": "user_approval", "payload": {"toolCallId": "t1"}, + "tool": None, # payload names no tool; the raw data is still there to parse } +def test_fold_terminal_stop_reason_wins_over_done_event(): + # The live runner's `done` event carries NO stopReason; only the terminal result + # knows the turn paused. The caller passes it in and the pause still surfaces. + events = [ + { + "type": "interaction_request", + "data": { + "id": "req-9", + "kind": "user_approval", + "payload": {"toolName": "deleteFile", "toolCallId": "t9"}, + }, + }, + {"type": "done", "data": {}}, + ] + result = fold(events, stop_reason="paused") + assert result["stop_reason"] == "paused" + assert result["pending_interaction"]["id"] == "req-9" + assert result["pending_interaction"]["tool"] == "deleteFile" + + +def test_fold_pending_tool_falls_back_to_acp_tool_call_name(): + # Harness gates carry an ACP toolCall instead of a spec toolName; name/title/kind + # are the fallback candidates in that order. + events = [ + { + "type": "interaction_request", + "data": { + "id": "req-2", + "kind": "user_approval", + "payload": {"toolCall": {"title": "Delete a file", "kind": "edit"}}, + }, + }, + _done("paused"), + ] + result = fold(events) + assert result["pending_interaction"]["tool"] == "Delete a file" + + def test_fold_interaction_request_without_pause_is_not_pending(): # Resolved in-turn (not a HITL park) must not leak into pending_interaction. events = [ diff --git a/services/oss/tests/pytest/unit/agent/test_invoke_handler.py b/services/oss/tests/pytest/unit/agent/test_invoke_handler.py index bf662ef831..eb53236707 100644 --- a/services/oss/tests/pytest/unit/agent/test_invoke_handler.py +++ b/services/oss/tests/pytest/unit/agent/test_invoke_handler.py @@ -16,7 +16,6 @@ AgentTemplate, AgentResult, ConnectionNotFoundError, - Event, ConnectionResolutionError, Event, GatewayToolResolutionError, @@ -123,26 +122,25 @@ async def test_invoke_returns_assistant_message(patched): async def test_batch_paused_run_surfaces_pending_interaction(monkeypatch, fake_backend): + # The realistic paused stream: the runner's `done` event carries NO stopReason (the + # engine settles paused-vs-ended after the event stream closes, onto the terminal + # result only), so the envelope's pause metadata must come from result.stop_reason. + interaction = { + "id": "perm_1", + "kind": "user_approval", + "payload": { + "toolCallId": "call_1", + "toolCall": {"toolCallId": "call_1", "name": "deleteFile"}, + }, + } backend = fake_backend( result=AgentResult( output="waiting", stop_reason="paused", events=[ - Event( - type="interaction_request", - data={ - "type": "interaction_request", - "id": "perm_1", - "kind": "user_approval", - "payload": { - "toolCallId": "call_1", - "toolCall": { - "toolCallId": "call_1", - "name": "deleteFile", - }, - }, - }, - ) + Event(type="message", data={"text": "waiting"}), + Event(type="interaction_request", data=interaction), + Event(type="done", data={}), ], ) ) @@ -153,7 +151,10 @@ async def test_batch_paused_run_surfaces_pending_interaction(monkeypatch, fake_b assert body == { "messages": [{"role": "assistant", "content": "waiting"}], "stop_reason": "paused", - "pending_interaction": {"id": "perm_1", "tool": "deleteFile"}, + # the raw interaction data (the wire event carries its `type` inline) + derived tool + "pending_interaction": dict( + interaction, type="interaction_request", tool="deleteFile" + ), } diff --git a/services/runner/src/engines/sandbox_agent.ts b/services/runner/src/engines/sandbox_agent.ts index e6e24165ad..ee501e4fbf 100644 --- a/services/runner/src/engines/sandbox_agent.ts +++ b/services/runner/src/engines/sandbox_agent.ts @@ -25,6 +25,8 @@ */ import { rmSync } from "node:fs"; +import { apiBase } from "../apiBase.ts"; + import { SandboxAgent, InMemorySessionPersistDriver } from "sandbox-agent"; import { createSandboxAgentOtel } from "../tracing/otel.ts"; @@ -119,11 +121,6 @@ function runCredential(request: AgentRunRequest): string { return (headers["authorization"] ?? headers["Authorization"] ?? "").trim(); } -/** Agenta API base for runner→API calls (heartbeat/ingest/mount-sign share this). */ -function apiBase(): string { - return process.env.AGENTA_API_URL ?? "http://localhost:8000/api"; -} - function serverPermissionsFromRequest( request: AgentRunRequest, ): ReadonlyMap { @@ -175,8 +172,7 @@ function shouldSuppressPausedToolCallUpdate( pause: PendingApprovalPauseController, ): boolean { const frame = update as - | { sessionUpdate?: unknown; toolCallId?: unknown } - | undefined; + { sessionUpdate?: unknown; toolCallId?: unknown } | undefined; const kind = frame?.sessionUpdate; if (kind !== "tool_call" && kind !== "tool_call_update") return false; const toolCallId = @@ -234,6 +230,9 @@ function applyClaudeConnectionEnv( ) { env.ANTHROPIC_MODEL = selectedModel; env.ANTHROPIC_CUSTOM_MODEL_OPTION = selectedModel; + logger( + `claude model=${selectedModel} deployment=${deployment ?? ""}`, + ); return true; } return false; @@ -295,9 +294,7 @@ function containsTransportEndpointDisconnected(value: unknown): boolean { seen.add(current); const code = - "code" in current - ? String((current as { code?: unknown }).code) - : ""; + "code" in current ? String((current as { code?: unknown }).code) : ""; if (code === "ENOTCONN") { return true; } @@ -324,7 +321,8 @@ export async function runSandboxAgent( // failure leaves durableCwd undefined and buildRunPlan falls back to the ephemeral path. const sessionForMount = request.sessionId?.trim(); const runCred = runCredential(request); - const signMount = deps.signSessionMountCredentials ?? signSessionMountCredentials; + const signMount = + deps.signSessionMountCredentials ?? signSessionMountCredentials; let mountCreds: MountCredentials | null = sessionForMount && runCred ? await signMount(sessionForMount, { @@ -339,7 +337,9 @@ export async function runSandboxAgent( // is already "mounts//", so no extra slug is needed. let durableCwd: string | undefined; if (mountCreds?.prefix) { - const isDaytonaReq = (request.sandbox ?? process.env.SANDBOX_AGENT_PROVIDER ?? "local") === "daytona"; + const isDaytonaReq = + (request.sandbox ?? process.env.SANDBOX_AGENT_PROVIDER ?? "local") === + "daytona"; durableCwd = isDaytonaReq ? `/home/sandbox/agenta/${mountCreds.prefix}` : `/tmp/agenta/${mountCreds.prefix}`; @@ -395,6 +395,15 @@ export async function runSandboxAgent( logger(`harness=${plan.harness} sandbox=${plan.sandboxId} cwd=${plan.cwd}`); + // The resolved model ref as it reaches the runner (key NAMES only, never values) — the one + // line that answers "what model/provider/deployment/credential did this run actually use". + logger( + `resolved model=${request.model ?? ""} provider=${request.provider ?? ""} ` + + `deployment=${request.deployment ?? ""} ` + + `connection=${request.connection ? `${request.connection.mode}:${request.connection.slug ?? "-"}` : ""} ` + + `secretKeys=[${Object.keys(request.secrets ?? {}).join(",")}]`, + ); + // Pi traces itself via the extension under the propagated traceparent; for other // harnesses we build the span tree here from the ACP event stream. Created below, once // the model is resolved, so the chat span carries the harness's actual model rather @@ -414,7 +423,11 @@ export async function runSandboxAgent( logger( `local durable cwd mount (${reason}) session=${sessionForMount} cwd=${plan.cwd}`, ); - if (await (deps.mountStorage ?? mountStorage)(plan.cwd, mountCreds, { log: logger })) { + if ( + await (deps.mountStorage ?? mountStorage)(plan.cwd, mountCreds, { + log: logger, + }) + ) { mountedCwd = plan.cwd; return true; } @@ -442,7 +455,9 @@ export async function runSandboxAgent( log: logger, }); if (!fresh) { - logger(`local durable cwd re-sign returned no credentials session=${sessionForMount}`); + logger( + `local durable cwd re-sign returned no credentials session=${sessionForMount}`, + ); return false; } mountCreds = fresh; @@ -533,7 +548,9 @@ export async function runSandboxAgent( isTransportEndpointDisconnected(err) && (await reSignAndRemountLocalCwd()) ) { - logger(`retrying workspace preparation after local durable cwd remount`); + logger( + `retrying workspace preparation after local durable cwd remount`, + ); workspace = await (deps.prepareWorkspace ?? prepareWorkspace)({ sandbox, plan, @@ -546,15 +563,22 @@ export async function runSandboxAgent( if (mountCreds) { if (plan.isDaytona) { - const endpoint = await (deps.discoverTunnelEndpoint ?? discoverTunnelEndpoint)({ + const endpoint = await ( + deps.discoverTunnelEndpoint ?? discoverTunnelEndpoint + )({ log: logger, }); if ( endpoint && - (await (deps.mountStorageRemote ?? mountStorageRemote)(sandbox, plan.cwd, mountCreds, { - endpoint, - log: logger, - })) + (await (deps.mountStorageRemote ?? mountStorageRemote)( + sandbox, + plan.cwd, + mountCreds, + { + endpoint, + log: logger, + }, + )) ) { // Remote files live in the store; nothing on this host to unmount. logger(`remote durable cwd active for session=${sessionForMount}`); From a606badc3edd95cf3f16854b84964ee96dc5b917 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 4 Jul 2026 13:39:45 +0200 Subject: [PATCH 22/24] fix: address CodeRabbit review + pin the client-tool interaction row - playground: a rules-only runner.permissions override no longer drops the allow_reads default from the /invoke body (nested merge + test) - runner: a client-tool pause now seeds /sessions/interactions like every other pause (pauseClientTool was the one gate that left no row) - sdk: wire.py imports ToolPermission/PermissionRule from permission_rules.py instead of redeclaring them; greppable legacy-key literals in mcp/models.py; annotate_trace comment and claude_settings docstring tell the truth - docs: MCP-server permission step added to three precedence ladders; permissions wire block documents the optional default + fallbacks; client-tool carve-out paragraph matches onClientTool's actual semantics --- .../agent-workflows/documentation/protocol.md | 2 +- .../agent-workflows/documentation/tools.md | 14 ++++---- .../in-service/tool-models-and-resolution.md | 14 ++++---- .../public-edge/agent-config-schema.md | 3 +- .../projects/approval-boundary/build-notes.md | 33 +++++++++++++++++++ .../sdk/agents/adapters/claude_settings.py | 5 +-- sdks/python/agenta/sdk/agents/mcp/models.py | 4 ++- .../agenta/sdk/agents/platform/op_catalog.py | 3 +- sdks/python/agenta/sdk/agents/utils/wire.py | 7 +--- .../engines/sandbox_agent/acp-interactions.ts | 1 + .../sandbox-agent-acp-interactions.test.ts | 15 ++++++++- .../src/state/execution/agentRequest.ts | 25 +++++++++----- .../tests/unit/agentRequest.test.ts | 12 +++++++ 13 files changed, 103 insertions(+), 35 deletions(-) diff --git a/docs/design/agent-workflows/documentation/protocol.md b/docs/design/agent-workflows/documentation/protocol.md index 91a71a3184..a549d056ad 100644 --- a/docs/design/agent-workflows/documentation/protocol.md +++ b/docs/design/agent-workflows/documentation/protocol.md @@ -118,7 +118,7 @@ Request fields include: | `messages` | Conversation history and current turn. | | `secrets` | Provider env vars resolved by the service. | | `tools`, `customTools`, `toolCallback`, `mcpServers` | Resolved tool delivery. | -| `permissions` | Permission plan: `{default, rules?}`. `default` is one of `allow`, `ask`, `deny`, or `allow_reads`; `rules` is an optional list of `{pattern, permission}` entries for harness builtins. The runner enforces it on every harness. | +| `permissions` | Permission plan: `{default?, rules?}`. `default` is one of `allow`, `ask`, `deny`, or `allow_reads`; missing, it falls back to `allow_reads`, and a malformed block fails toward `ask`. `rules` is an optional list of `{pattern, permission}` entries for harness builtins. The runner enforces it on every harness. | | `trace` | Trace context for nested spans. | One-shot calls return one JSON result. Streaming calls use NDJSON internally: one diff --git a/docs/design/agent-workflows/documentation/tools.md b/docs/design/agent-workflows/documentation/tools.md index 35ec1f49d5..4704879249 100644 --- a/docs/design/agent-workflows/documentation/tools.md +++ b/docs/design/agent-workflows/documentation/tools.md @@ -355,13 +355,13 @@ Two gates consult this same decision: there is no harness-side gate, so the relay is the only enforcement point, and it now gives Pi the same human-in-the-loop behavior Claude gets. -Client tools are a carve-out from that same ladder. They are still decided by -`ApprovalResponder`, but not by the policy-and-rules path above: a client tool defaults to -`allow` no matter what the policy mode is, unless its own `permission` says otherwise or the -policy's `default` is `deny` (which still blocks everything). The reasoning is that a client -tool's only job is to reach the browser, so folding it into `ask`/`allow_reads` defaults would -strand it. Setting `permission: "ask"` or `"deny"` on a client tool still works; it just is not -the policy's doing. +Client tools are a carve-out from that same ladder. They are decided by the responder's +`onClientTool` (consulted at the ACP gate on Claude, and by the relay on Pi), not by the +policy-and-rules path above: a client tool with no `permission` of its own defaults to +`allow` under every policy mode except `deny`, where it is blocked. An explicit `permission` +always wins, in both directions: `allow` runs even under a `deny` policy, and `ask`/`deny` +bind even under `allow`. The reasoning is that a client tool's only job is to reach the +browser, so folding it into `ask`/`allow_reads` defaults would strand it. Either gate answers `allow` by running the tool with no extra event, `deny` by refusing it, or `ask` by pausing the turn (the wire's `stopReason: "paused"`) and emitting exactly one diff --git a/docs/design/agent-workflows/interfaces/in-service/tool-models-and-resolution.md b/docs/design/agent-workflows/interfaces/in-service/tool-models-and-resolution.md index e45182d3b4..df7403f80a 100644 --- a/docs/design/agent-workflows/interfaces/in-service/tool-models-and-resolution.md +++ b/docs/design/agent-workflows/interfaces/in-service/tool-models-and-resolution.md @@ -76,11 +76,12 @@ explicit `permission`; when it is unset, the spec sends nothing and the `read_on rides alongside as a separate field. The runner resolves the rest: its shared decision function (`services/runner/src/permission-plan.ts`) looks up, in order: -1. an explicit `permission` wins, -2. else an authored builtin rule match (`runner.permissions.rules`), -3. else, under the `allow_reads` policy mode, the `read_only` hint (`true` to `allow`, unset +1. an explicit `permission` on the tool wins, +2. else the owning MCP server's explicit `permission` (for MCP-backed tools), +3. else an authored builtin rule match (`runner.permissions.rules`), +4. else, under the `allow_reads` policy mode, the `read_only` hint (`true` to `allow`, unset or `false` to `ask`), -4. else the global policy mode (`allow`/`ask`/`deny`). +5. else the global policy mode (`allow`/`ask`/`deny`). ## Secret injection @@ -94,8 +95,9 @@ secret; their provider key stays server-side and the call routes back through `/ `ReferenceToolConfig`, its `ref_by` axes, `PlatformToolConfig`, `ToolCall`, and `call_ref`), carrying the author's explicit `permission` and the `read_only` hint. - `services/runner/src/permission-plan.ts`: the shared decision function that resolves a - tool's effective permission at run time (explicit permission, then rule match, then the - `allow_reads` read-only check, then the global policy). + tool's effective permission at run time (explicit tool permission, then the owning MCP + server's permission, then rule match, then the `allow_reads` read-only check, then the + global policy). - `sdks/python/agenta/sdk/agents/tools/compat.py`: coerces legacy/typed tool dicts (a `type: "reference"` or `type: "platform"` dict parses straight into its config model). - `sdks/python/agenta/sdk/agents/platform/gateway.py`: gateway resolution to a `call_ref`. diff --git a/docs/design/agent-workflows/interfaces/public-edge/agent-config-schema.md b/docs/design/agent-workflows/interfaces/public-edge/agent-config-schema.md index 30f281eb6a..dea53d8155 100644 --- a/docs/design/agent-workflows/interfaces/public-edge/agent-config-schema.md +++ b/docs/design/agent-workflows/interfaces/public-edge/agent-config-schema.md @@ -171,7 +171,8 @@ the tool-specific mapping. See [Tool models and resolution](../in-service/tool-models-and-resolution.md). `permission` is `"allow" | "ask" | "deny"`, or unset to inherit. When unset, the runner's -shared decision function resolves it: an authored rule match wins if one applies, else the +shared decision function resolves it: for an MCP-backed tool the owning server's explicit +`permission` wins first, else an authored rule match applies, else the policy's `default` mode decides (`allow_reads` consults the tool's `read_only` hint: `true` resolves to `allow`, no hint or `false` resolves to `ask`). There is no `needs_approval` field and no separate per-tool defaulting step; one function resolves every tool the same diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index 956740311f..cb2b1695cf 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -156,3 +156,36 @@ decisions unless it contradicts an owner call". Newest last. Read together with args record, which on a multi-string-field tool could be the wrong field; tighten to the known argument name (e.g. `command`) if phase 3's derived rules ever target such a tool. Prefix rules on uninspectable args deliberately fail toward the default. + +## 2026-07-04 (post-review: rebase onto big-agents + CodeRabbit round) + +- **Rebased onto post-#5064 `big-agents` via `but pull` + per-commit `but resolve`.** Six + conflicted commits (2a, 2b, 3, 4a, 5, clobber fix); the runner files resolved ours-wins per + the audit, then one reconciliation commit re-added upstream's non-superseded keepers + (shared `apiBase` module from the #5059 URL-contract fix, two log lines). The rebased PR + diff is exactly the reviewed file set plus three SDK files from the reconciliation. +- **Found and fixed a real integration bug in #5064's batch fold.** JP's `agent_batch` + derives `stop_reason` from the stream's `done` event, but the live runner emits `done` + with NO `stopReason` (the engine settles paused-vs-ended after the event stream closes, + onto the terminal result only) — so a real paused run would have dropped `stop_reason` + and `pending_interaction` from the batch envelope. Fix: `fold(events, stop_reason=...)` + gives the terminal result's value precedence; the service pause test now pins the + realistic stream shape (a `done` event with no stopReason). His fold otherwise already + carried our phase-4a envelope contract; `pending_interaction` keeps his richer raw shape + plus a derived top-level `tool` name (superset of both contracts). +- **JP's new `handler.py` passed `permission_policy=`, a field phase 3 deleted** — the + auto-merge trap git cannot flag. Caught by a deleted-identifier sweep over every + upstream-changed file; fixed to `permission_default`. +- **CodeRabbit round (10 findings triaged).** Two real code bugs confirmed and fixed: + (1) the playground's `withSection` shallow merge dropped `permissions.default` from the + /invoke body when an author supplied rules-only permissions (fixed with a nested merge + + test); (2) `pauseClientTool` never seeded the durable interactions plane — only + `pauseUserApproval` did — violating "every pause leaves a row" (fixed + pinned in the + client-tool pending test). Also: deduped `ToolPermission`/`PermissionRule` (wire.py now + imports from `permission_rules.py`; the reverse direction would cycle through `dtos`), + fixed the `annotate_trace` comment that still claimed auto-allow (it is a write under + `allow_reads` — deliberate), the `claude_settings` docstring naming the wrong callable, + made `mcp/models.py`'s legacy keys greppable literals, and added the missing + MCP-server-permission step to three doc precedence ladders. `permission-responder.md` + needed no change (it already documented the step). Skipped as noise: comment-style nits + on `PiSettingsControl`. diff --git a/sdks/python/agenta/sdk/agents/adapters/claude_settings.py b/sdks/python/agenta/sdk/agents/adapters/claude_settings.py index 12a4abf1ff..4290bab9ac 100644 --- a/sdks/python/agenta/sdk/agents/adapters/claude_settings.py +++ b/sdks/python/agenta/sdk/agents/adapters/claude_settings.py @@ -135,8 +135,9 @@ def _rules_from_tool_specs( Mirrors :func:`_rules_from_mcp_permissions`, but per-tool against the fixed internal server name ``agenta-tools``: a callback/code tool is delivered to Claude as a tool of that MCP server, so - its rule is ``mcp__agenta-tools__``. The tool's :meth:`ToolSpec.effective_permission` - (the single source of the allow/ask/deny ladder) routes it to the matching list. Unset tools + its rule is ``mcp__agenta-tools__``. The standalone + :func:`~agenta.sdk.agents.tools.models.effective_permission` ladder (explicit permission, + else read-only under ``allow_reads``, else the runner mode) routes it to the matching list. Unset tools only render a rule when the runner mode needs an explicit Claude allow/deny rule. ``client`` tools are browser-fulfilled and never delivered over this channel, so they are excluded (this mirrors the runner's ``mcp-bridge`` filter). Accepts a list diff --git a/sdks/python/agenta/sdk/agents/mcp/models.py b/sdks/python/agenta/sdk/agents/mcp/models.py index 90008b7d1c..327278b01c 100644 --- a/sdks/python/agenta/sdk/agents/mcp/models.py +++ b/sdks/python/agenta/sdk/agents/mcp/models.py @@ -12,7 +12,9 @@ # server inherits the runner policy. Permission = Literal["allow", "ask", "deny"] -_LEGACY_PERMISSION_KEYS = frozenset({"permission" + "_mode", "permission" + "Mode"}) +# The deleted pre-redesign vocabulary, still present in old dev-DB drafts. Literal on +# purpose so the legacy spelling stays greppable. +_LEGACY_PERMISSION_KEYS = frozenset({"permission_mode", "permissionMode"}) def _drop_legacy_permission_keys(data: Any) -> Any: diff --git a/sdks/python/agenta/sdk/agents/platform/op_catalog.py b/sdks/python/agenta/sdk/agents/platform/op_catalog.py index 294593259d..f9fdc3621c 100644 --- a/sdks/python/agenta/sdk/agents/platform/op_catalog.py +++ b/sdks/python/agenta/sdk/agents/platform/op_catalog.py @@ -312,7 +312,8 @@ def _strip_field(schema: Dict[str, Any], dotted_path: str) -> None: # ``.span_id`` are bound from run context ($ctx.trace.*) so the agent always annotates its own run # and can never retarget another trace (the same self-targeting guarantee ``commit_revision`` gives # via $ctx.workflow.variant.id). Unlike ``commit_revision``, this is additive self-metadata (it does -# not mutate the agent's config), so it defaults to auto-allow / no approval. +# not mutate the agent's config) — but it IS a write (``read_only=False``), so under the default +# ``allow_reads`` policy it prompts unless the author sets an explicit ``allow``. _ANNOTATE_TRACE_DESCRIPTION = ( "Record an annotation (evaluation feedback) on your own current run's trace — grade " "yourself. Supply `references.evaluator.slug` naming the annotation category (e.g. " diff --git a/sdks/python/agenta/sdk/agents/utils/wire.py b/sdks/python/agenta/sdk/agents/utils/wire.py index e16ac1c6c9..d432441972 100644 --- a/sdks/python/agenta/sdk/agents/utils/wire.py +++ b/sdks/python/agenta/sdk/agents/utils/wire.py @@ -22,6 +22,7 @@ from agenta.sdk.utils.logging import get_module_logger +from ..permission_rules import PermissionRule from ..dtos import ( Event, AgentResult, @@ -36,12 +37,6 @@ log = get_module_logger(__name__) PermissionMode = Literal["allow", "ask", "deny", "allow_reads"] -ToolPermission = Literal["allow", "ask", "deny"] - - -class PermissionRule(TypedDict): - pattern: str - permission: ToolPermission class PermissionsConfig(TypedDict, total=False): diff --git a/services/runner/src/engines/sandbox_agent/acp-interactions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts index bd22de74a2..fc1730077f 100644 --- a/services/runner/src/engines/sandbox_agent/acp-interactions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -101,6 +101,7 @@ export function attachPermissionResponder({ render: spec.render, }, }); + onCreateInteraction?.(eventId, gate.toolName, gate.args); onPause?.(); }; diff --git a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts index aaba85e9b8..7f8dad9000 100644 --- a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts +++ b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts @@ -234,13 +234,14 @@ describe("attachPermissionResponder", () => { assert.equal((events[0] as any).id, "tool-fallback"); }); - it("client tool pending forwards to the browser and pauses", async () => { + it("client tool pending forwards to the browser, creates the interaction, and pauses", async () => { const replies: Array<{ id: string; reply: string }> = []; const { session, emit } = makeSession(async (id, reply) => { replies.push({ id, reply }); }); const events: AgentEvent[] = []; const pausedToolCalls: string[] = []; + const created: Array<{ token: string; toolName?: string; args: unknown }> = []; let pauses = 0; attachPermissionResponder({ @@ -254,6 +255,9 @@ describe("attachPermissionResponder", () => { onPause: () => { pauses += 1; }, + onCreateInteraction: (token, toolName, args) => { + created.push({ token, toolName, args }); + }, onPausedToolCall: (id) => { pausedToolCalls.push(id); }, @@ -276,6 +280,15 @@ describe("attachPermissionResponder", () => { assert.deepEqual(replies, []); assert.equal(pauses, 1); assert.deepEqual(pausedToolCalls, ["tool-client"]); + // A client-tool pause seeds the durable interactions plane exactly like a + // user-approval pause: every pause leaves a row, whichever gate paused. + assert.deepEqual(created, [ + { + token: "client-1", + toolName: "request_connection", + args: { integration: "slack" }, + }, + ]); assert.deepEqual(events, [ { type: "interaction_request", diff --git a/web/packages/agenta-playground/src/state/execution/agentRequest.ts b/web/packages/agenta-playground/src/state/execution/agentRequest.ts index 47ce6455f0..91d9653c9f 100644 --- a/web/packages/agenta-playground/src/state/execution/agentRequest.ts +++ b/web/packages/agenta-playground/src/state/execution/agentRequest.ts @@ -253,15 +253,22 @@ const withSection = ( : {}), }) -const withTemplateDefaults = (template: Record): Record => ({ - ...template, - harness: withSection(template.harness, {kind: "pi_core"}), - runner: withSection(template.runner, { - kind: "sidecar", - permissions: {default: "allow_reads"}, - }), - sandbox: withSection(template.sandbox, {kind: "local"}), -}) +const withTemplateDefaults = (template: Record): Record => { + const runnerSection = template.runner + const runner = withSection(runnerSection, {kind: "sidecar"}) + // `permissions` is itself nested — merge it separately so a caller-supplied `rules`-only + // object still carries the fallback `default` instead of replacing it wholesale. + const permissionsSection = + runnerSection && typeof runnerSection === "object" && !Array.isArray(runnerSection) + ? (runnerSection as Record).permissions + : undefined + return { + ...template, + harness: withSection(template.harness, {kind: "pi_core"}), + runner: {...runner, permissions: withSection(permissionsSection, {default: "allow_reads"})}, + sandbox: withSection(template.sandbox, {kind: "local"}), + } +} const withAgentRunDefaults = (config: Record): Record => { const template = config.agent diff --git a/web/packages/agenta-playground/tests/unit/agentRequest.test.ts b/web/packages/agenta-playground/tests/unit/agentRequest.test.ts index 42c22fc920..6d8c8bde92 100644 --- a/web/packages/agenta-playground/tests/unit/agentRequest.test.ts +++ b/web/packages/agenta-playground/tests/unit/agentRequest.test.ts @@ -216,6 +216,18 @@ describe("buildAgentRequest", () => { expect(template.runner.permissions.default).toBe("deny") }) + it("merges the fallback `default` into a rules-only runner permission override", async () => { + // A config that supplies only `rules` (no `default`) must still get the fallback + // `default: "allow_reads"` — the nested `permissions` merge, not a wholesale replace. + const rules = [{path: "**/*.md", action: "allow"}] + seed(store, "e", { + config: {agent: {runner: {permissions: {rules}}}}, + }) + const req = await buildAgentRequest("e", [], {sessionId: "s1", store}) + const template = (req!.requestBody.data as any).parameters.agent + expect(template.runner.permissions).toEqual({default: "allow_reads", rules}) + }) + it("applies the build-kit overlay to a kit-on run with deep and identity merges", async () => { const config = { agent: { From c5baa030bc48bb580d10993a5cea486b1f07b5f6 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 4 Jul 2026 13:41:32 +0200 Subject: [PATCH 23/24] docs(approval-boundary): status + build notes for the rebase, CodeRabbit round, and merge --- .../projects/approval-boundary/status.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index 6c440f14e8..2437a82c5e 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -1,15 +1,17 @@ # Status -**State: implemented and QA'd; awaiting final PR review.** Date: 2026-07-03. All six -plan phases landed on the lane with per-phase review; tests green across runner (444), -SDK agents (480+), and services (49), including a 40-case cross-language parity +**State: merged to `big-agents` (PR #5041, 2026-07-04).** All six plan phases landed +with per-phase review; Mahmoud reviewed and QA'd the PR. Tests green across runner +(444), SDK agents (502), and services (76), including a 40-case cross-language parity fixture. Live QA: headless matrix 7/7; playground approve/deny/allow flows verified (one prompt, approve resumes without looping, deny refuses cleanly) after fixing one QA-found bug (pause teardown clobbered the prompt on the Pi relay path; fixed with a -paused-call event filter + regression tests). Remaining before merge: rebase onto -post-#5058 `big-agents` (audit done: their runner deltas are formatting plus patches -this redesign supersedes; their egress `rawInput is not None` fix merges untouched) -and flip the PR base. Claude-harness live runs were blocked by account credit; that +paused-call event filter + regression tests). Before merge the branch was rebased onto +post-#5064 `big-agents` (see build-notes.md: the rebase found and fixed a real #5064 +integration bug — the batch fold read `stop_reason` from a `done` event the live +runner never populates) and a CodeRabbit round fixed two real bugs (a playground +shallow merge dropping the permissions default; the client-tool pause not seeding the +interactions plane). Claude-harness live runs were blocked by account credit; that path is covered by unit and settings-rendering tests. ## Where things stand From f3a666f87111701c8e2ce1cfc232afb7f93f9bb5 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 4 Jul 2026 13:54:38 +0200 Subject: [PATCH 24/24] fix(runner): stamp resolvedName on pause payloads; persist client-tool pauses with their own kind Codex pre-merge review of the rebase: - both ACP pause sites emit a stamped COPY of the toolCall (resolvedName = the gate's stable anchor) so the Vercel egress's preferred field is populated again (upstream stamped by mutating the shared ACP object; we keep it pure) - onCreateInteraction threads kind, so client-tool rows stop masquerading as user_approval, and the Pi relay's client-tool pause now seeds a row at all - the gateway integration test drops the deleted needs_approval vocabulary --- .../projects/approval-boundary/build-notes.md | 15 +++++ .../projects/approval-boundary/status.md | 6 ++ .../agent/tools/test_gateway_http.py | 12 ++-- services/runner/src/engines/sandbox_agent.ts | 4 +- .../engines/sandbox_agent/acp-interactions.ts | 58 ++++++++++++++----- .../sandbox-agent-acp-interactions.test.ts | 49 +++++++++++----- 6 files changed, 111 insertions(+), 33 deletions(-) diff --git a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md index cb2b1695cf..d68de4794f 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/build-notes.md +++ b/docs/design/agent-workflows/projects/approval-boundary/build-notes.md @@ -189,3 +189,18 @@ decisions unless it contradicts an owner call". Newest last. Read together with MCP-server-permission step to three doc precedence ladders. `permission-responder.md` needed no change (it already documented the step). Skipped as noise: comment-style nits on `PiSettingsControl`. +- **Codex xhigh pre-merge review of the rebase (3 findings, all addressed).** + (1) The ours-wins resolution dropped upstream's `resolvedName` stamping while the SDK's + Vercel egress prefers exactly that field; restored WITHOUT upstream's object mutation — + both pause sites now emit a stamped COPY of the ACP toolCall (`resolvedName` = the gate's + stable anchor), so the approval part names the tool exactly as the responder keys it. + (2) Client-tool pause persistence completed: `onCreateInteraction` now threads a `kind` + (`user_approval` | `client_tool`) so ACP client-tool rows stop masquerading as approvals, + and the Pi relay's client-tool pause seeds the interactions row at all (it never did, + pre-dating the rebase). (3) A `services/oss` integration test still asserted the deleted + `needs_approval`/`needsApproval` vocabulary (missed by unit-only sweeps); rewritten to + `permission: "ask"`. Codex also confirmed: apiBase keeper correct across call sites, the + fold patch source-compatible, runtime code clean of deleted identifiers. Noted follow-up: + the live Vercel stream's `finishReason` still reads only `done.stopReason` + (adapters/vercel/stream.py:417), so a live paused stream may omit `finishReason: paused` + while batch is fixed; the FE keys off the interaction part, so display is unaffected. diff --git a/docs/design/agent-workflows/projects/approval-boundary/status.md b/docs/design/agent-workflows/projects/approval-boundary/status.md index 2437a82c5e..2b209c7f2f 100644 --- a/docs/design/agent-workflows/projects/approval-boundary/status.md +++ b/docs/design/agent-workflows/projects/approval-boundary/status.md @@ -87,6 +87,12 @@ analysis that diagnosed the live loop. ## Follow-ups (filed, not in this PR) +- **Live-stream `finishReason` on a pause.** The Vercel stream adapter derives + `finishReason` from the `done` event's `stopReason`, which the live runner never + populates (the engine settles paused-vs-ended after the stream closes). Batch now takes + the terminal result's value; the live stream path should too. Display is unaffected (the + FE keys off the `interaction_request` part), so this is wire-fidelity, not UX. + - **Selection-time enforcement for Pi builtins.** Pi's native tools never reach a runner gate, so today the global policy modes do not bind them (headless QA proved `default: deny` does not stop a granted `bash`). The design: filter `builtin_names` diff --git a/services/oss/tests/pytest/integration/agent/tools/test_gateway_http.py b/services/oss/tests/pytest/integration/agent/tools/test_gateway_http.py index 72641add84..341be34b40 100644 --- a/services/oss/tests/pytest/integration/agent/tools/test_gateway_http.py +++ b/services/oss/tests/pytest/integration/agent/tools/test_gateway_http.py @@ -53,16 +53,16 @@ async def test_gateway_metadata_and_description_fallback_are_preserved(install_h [ { **_GATEWAY, - "needs_approval": True, + "permission": "ask", "render": {"kind": "component", "component": "User"}, } ] ) spec = resolved.tool_specs[0] assert spec.description == "get_user" - assert spec.needs_approval is True + assert spec.permission == "ask" assert spec.render == {"kind": "component", "component": "User"} - assert spec.to_wire()["needsApproval"] is True + assert spec.to_wire()["permission"] == "ask" assert isinstance(resolved.tool_callback, ToolCallback) assert capture["json"]["tools"][0]["type"] == "gateway" @@ -93,7 +93,7 @@ async def test_gateway_specs_are_joined_by_call_ref_not_position(install_http): **_GATEWAY, "action": "FIRST", "connection": "c1", - "needs_approval": True, + "permission": "ask", }, { **_GATEWAY, @@ -105,10 +105,10 @@ async def test_gateway_specs_are_joined_by_call_ref_not_position(install_http): ) first, second = resolved.tool_specs assert first.name == "first" - assert first.needs_approval is True + assert first.permission == "ask" assert first.render is None assert second.name == "second" - assert second.needs_approval is False + assert second.permission is None # unset inherits: rules, then the policy default assert second.render == {"kind": "component", "component": "Second"} diff --git a/services/runner/src/engines/sandbox_agent.ts b/services/runner/src/engines/sandbox_agent.ts index ee501e4fbf..580075e6b0 100644 --- a/services/runner/src/engines/sandbox_agent.ts +++ b/services/runner/src/engines/sandbox_agent.ts @@ -703,6 +703,7 @@ export async function runSandboxAgent( token: string, toolName: string | undefined, toolArgs: unknown, + kind: "user_approval" | "client_tool" = "user_approval", ): void => { const cred = runCredential(request); if (!cred) return; @@ -715,7 +716,7 @@ export async function runSandboxAgent( sessionId, request.turnId ?? "", token, - "user_approval", + kind, { request: { tool: toolName ?? token, args: toolArgs }, references }, () => cred, ); @@ -829,6 +830,7 @@ export async function runSandboxAgent( }, }, }); + recordPendingInteraction(id, toolName, input, "client_tool"); } return "pendingApproval"; }, diff --git a/services/runner/src/engines/sandbox_agent/acp-interactions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts index fc1730077f..c3b0bff327 100644 --- a/services/runner/src/engines/sandbox_agent/acp-interactions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -29,6 +29,7 @@ export interface AttachPermissionResponderInput { token: string, toolName: string | undefined, toolArgs: unknown, + kind: "user_approval" | "client_tool", ) => void; /** Called after a stored decision was successfully forwarded to the harness. */ onResolveInteraction?: (token: string) => void; @@ -54,12 +55,26 @@ export function attachPermissionResponder({ }); }); + // The emitted payload carries a COPY of the ACP toolCall stamped with `resolvedName` (the + // gate's stable anchor). The Vercel egress prefers it over the drift-prone title/kind + // display fields, so the approval part names the tool exactly as the responder keys it. + // The inbound ACP object itself is never mutated. + const stampResolvedName = (toolCall: any, gate: GateDescriptor): any => { + if (!toolCall || typeof toolCall !== "object" || !gate.toolName) + return toolCall; + return { ...toolCall, resolvedName: gate.toolName }; + }; + // A pause sends NO harness reply, ever. Replying `reject` would make Claude emit a failed // tool call ("User refused permission") whose `tool_result {isError}` overwrites the // approval prompt on the same tool-call id (the F-024 clobber). The turn instead ends via // `onPause` (session teardown resolves the RPC as cancelled, not rejected) and the next // turn's stored decision answers the re-raised gate. - const pauseUserApproval = (req: any, id: string, gate: GateDescriptor): void => { + const pauseUserApproval = ( + req: any, + id: string, + gate: GateDescriptor, + ): void => { if (!latch.tryAcquire()) return; const toolCallId = stringValue(req?.toolCall?.toolCallId); const eventId = interactionEventId(id, toolCallId); @@ -70,12 +85,12 @@ export function attachPermissionResponder({ kind: "user_approval", payload: { toolCallId, - toolCall: req?.toolCall, + toolCall: stampResolvedName(req?.toolCall, gate), availableReplies: stringArray(req?.availableReplies), options: req?.options, }, }); - onCreateInteraction?.(eventId, gate.toolName, gate.args); + onCreateInteraction?.(eventId, gate.toolName, gate.args, "user_approval"); onPause?.(); }; @@ -95,13 +110,13 @@ export function attachPermissionResponder({ kind: "client_tool", payload: { toolCallId, - toolCall: req?.toolCall, + toolCall: stampResolvedName(req?.toolCall, gate), toolName: gate.toolName, input: gate.args, render: spec.render, }, }); - onCreateInteraction?.(eventId, gate.toolName, gate.args); + onCreateInteraction?.(eventId, gate.toolName, gate.args, "client_tool"); onPause?.(); }; @@ -116,7 +131,9 @@ export function attachPermissionResponder({ decisionToReply(decision, availableReplies) as any, ); } catch (err) { - log?.(`[HITL] reply failed id=${id} decision=${decision}: ${errorMessage(err)}`); + log?.( + `[HITL] reply failed id=${id} decision=${decision}: ${errorMessage(err)}`, + ); onPause?.(); return; } @@ -134,7 +151,9 @@ export function attachPermissionResponder({ clientToolReply(verdict, availableReplies) as any, ); } catch (err) { - log?.(`[HITL] reply failed id=${id} decision=${verdict.kind}: ${errorMessage(err)}`); + log?.( + `[HITL] reply failed id=${id} decision=${verdict.kind}: ${errorMessage(err)}`, + ); onPause?.(); } }; @@ -159,7 +178,10 @@ export function attachPermissionResponder({ title: toolCall?.title, kind: toolCall?.kind, executor: gate.executor, - argKeys: args && typeof args === "object" ? Object.keys(args) : typeof args, + argKeys: + args && typeof args === "object" + ? Object.keys(args) + : typeof args, }), ); } @@ -202,7 +224,8 @@ function recordedToolName( run: { events?: () => AgentEvent[] }, toolCallId: unknown, ): string | undefined { - if (typeof toolCallId !== "string" || !toolCallId || !run.events) return undefined; + if (typeof toolCallId !== "string" || !toolCallId || !run.events) + return undefined; let name: string | undefined; for (const event of run.events()) { if (event.type === "tool_call" && event.id === toolCallId && event.name) { @@ -231,8 +254,11 @@ function buildGateDescriptor( executor: spec?.kind === "client" ? "client" : spec ? "relay" : "harness", toolName, specPermission, - serverPermission: spec ? undefined : serverPermissionFor(toolName, serverPermissions), - readOnlyHint: typeof spec?.readOnly === "boolean" ? spec.readOnly : undefined, + serverPermission: spec + ? undefined + : serverPermissionFor(toolName, serverPermissions), + readOnlyHint: + typeof spec?.readOnly === "boolean" ? spec.readOnly : undefined, args, }; } @@ -257,7 +283,11 @@ type ToolSpecLike = { }; function resolvedSpecOf(toolCall: any): ToolSpecLike | undefined { - const spec = toolCall?.spec ?? toolCall?.toolSpec ?? toolCall?.resolvedTool ?? toolCall?.tool; + const spec = + toolCall?.spec ?? + toolCall?.toolSpec ?? + toolCall?.resolvedTool ?? + toolCall?.tool; return spec && typeof spec === "object" ? (spec as ToolSpecLike) : undefined; } @@ -269,7 +299,9 @@ function firstString(values: unknown[]): string | undefined { } function toolPermission(value: unknown): ToolPermission | undefined { - return value === "allow" || value === "ask" || value === "deny" ? value : undefined; + return value === "allow" || value === "ask" || value === "deny" + ? value + : undefined; } function stringValue(value: unknown): string | undefined { diff --git a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts index 7f8dad9000..d3977cba77 100644 --- a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts +++ b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts @@ -17,7 +17,10 @@ function flushPromises(): Promise { } function makeSession( - respondPermission: (id: string, reply: string) => Promise | void = async () => {}, + respondPermission: ( + id: string, + reply: string, + ) => Promise | void = async () => {}, ) { let handler: ((req: any) => void) | undefined; return { @@ -108,7 +111,12 @@ describe("attachPermissionResponder", () => { replies.push({ id, reply }); }); const events: AgentEvent[] = []; - const created: Array<{ token: string; toolName?: string; args: unknown }> = []; + const created: Array<{ + token: string; + toolName?: string; + args: unknown; + kind: string; + }> = []; const pausedToolCalls: string[] = []; let pauses = 0; @@ -120,8 +128,8 @@ describe("attachPermissionResponder", () => { onPause: () => { pauses += 1; }, - onCreateInteraction: (token, toolName, args) => { - created.push({ token, toolName, args }); + onCreateInteraction: (token, toolName, args, kind) => { + created.push({ token, toolName, args, kind }); }, onPausedToolCall: (id) => { pausedToolCalls.push(id); @@ -139,7 +147,12 @@ describe("attachPermissionResponder", () => { assert.equal(pauses, 1); assert.deepEqual(pausedToolCalls, ["tool-9"]); assert.deepEqual(created, [ - { token: "perm-pause", toolName: "edit", args: { path: "a" } }, + { + token: "perm-pause", + toolName: "edit", + args: { path: "a" }, + kind: "user_approval", + }, ]); assert.deepEqual(events, [ { @@ -148,7 +161,13 @@ describe("attachPermissionResponder", () => { kind: "user_approval", payload: { toolCallId: "tool-9", - toolCall: { toolCallId: "tool-9", name: "edit", input: { path: "a" } }, + // a stamped COPY: the egress prefers resolvedName over drift-prone display fields + toolCall: { + toolCallId: "tool-9", + name: "edit", + input: { path: "a" }, + resolvedName: "edit", + }, availableReplies: ["once", "always", "reject"], options: { cwd: "/repo" }, }, @@ -241,22 +260,24 @@ describe("attachPermissionResponder", () => { }); const events: AgentEvent[] = []; const pausedToolCalls: string[] = []; - const created: Array<{ token: string; toolName?: string; args: unknown }> = []; + const created: Array<{ + token: string; + toolName?: string; + args: unknown; + kind: string; + }> = []; let pauses = 0; attachPermissionResponder({ session, run: { emitEvent: (event) => events.push(event) }, - responder: fakeResponder( - { kind: "deny" }, - { kind: "pendingApproval" }, - ), + responder: fakeResponder({ kind: "deny" }, { kind: "pendingApproval" }), latch: new PendingApprovalLatch(), onPause: () => { pauses += 1; }, - onCreateInteraction: (token, toolName, args) => { - created.push({ token, toolName, args }); + onCreateInteraction: (token, toolName, args, kind) => { + created.push({ token, toolName, args, kind }); }, onPausedToolCall: (id) => { pausedToolCalls.push(id); @@ -287,6 +308,7 @@ describe("attachPermissionResponder", () => { token: "client-1", toolName: "request_connection", args: { integration: "slack" }, + kind: "client_tool", }, ]); assert.deepEqual(events, [ @@ -304,6 +326,7 @@ describe("attachPermissionResponder", () => { name: "request_connection", render: { kind: "component", component: "connect" }, }, + resolvedName: "request_connection", }, toolName: "request_connection", input: { integration: "slack" },