From b62fc13548ef03008bb391c03c35937b9304fe6b Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Tue, 7 Jul 2026 02:30:55 +0200 Subject: [PATCH 1/2] feat(sdk): build-an-agent reference files + hardened playbook rules Claude-Session: https://claude.ai/code/session_01N2djTMgXnpk84EqtugHDJB --- .../sdk/agents/adapters/agenta_builtins.py | 336 +++++++++++++++++- .../agenta/sdk/agents/skills/parsing.py | 9 +- .../test_agenta_builtins_reference_files.py | 89 +++++ 3 files changed, 415 insertions(+), 19 deletions(-) create mode 100644 sdks/python/oss/tests/pytest/unit/agents/test_agenta_builtins_reference_files.py diff --git a/sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py b/sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py index 3ab6af412b..800a8f3f81 100644 --- a/sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py +++ b/sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py @@ -1,4 +1,6 @@ """The Agenta harness's forced defaults: the things ``AgentaHarness`` always applies. +(``ClaudeHarness`` shares the AGENTS.md preamble and forced platform skills; the persona and +forced tools remain Pi-only — see :mod:`.harnesses`.) ``AgentaHarness`` is Pi with an opinion. It is the same engine as :class:`PiHarness`, but every run carries a fixed set of Agenta-shipped extras the author cannot turn off: @@ -27,7 +29,7 @@ from typing import List, Optional -from ..skills import SkillTemplate +from ..skills import SkillFile, SkillTemplate # The base AGENTS.md preamble. The author's own ``instructions`` are appended after this, so # the final AGENTS.md is ``AGENTA_PREAMBLE`` + the author's project conventions. @@ -99,6 +101,262 @@ body=_GETTING_STARTED_BODY, ) +# Bundled reference file: the exact shape of `parameters.agent`. Sourced field-by-field from +# `AgentTemplateSchema` + the `ToolConfig` union + `SkillTemplate` (sdks/python/agenta/sdk/utils/ +# types.py and agents/tools/models.py, skills/models.py) so the model reads the real shape instead +# of guessing against an `additionalProperties: true` commit schema. A drift test (test_agenta_ +# builtins_reference_files.py) asserts this text names every top-level template field and every +# tool `type`, so a schema that grows without updating this file fails CI. +_CONFIG_SCHEMA_REFERENCE = """\ +# The agent config, field by field + +Read this before your first `commit_revision`, and whenever a commit fails validation and you +need to check the shape. + +`parameters.agent` is one object. You edit it by sending only the changed fields under +`commit_revision`'s `workflow_revision.delta.set.parameters.agent`. The portable definition — +`instructions`, `llm`, `tools`, `mcps`, `skills` — is flat on it; the execution parts — +`harness`, `runner`, `sandbox` — are nested sub-objects. Almost every sub-object rejects unknown +keys, so a misplaced or misspelled field fails the commit rather than being silently ignored. + +## The whole object + +```json +{ + "instructions": { "agents_md": "" }, + "llm": { "model": "gpt-5.5", "provider": "openai", "connection": { "mode": "agenta" } }, + "tools": [], + "mcps": [], + "skills": [], + "harness": { "kind": "pi_agenta" }, + "runner": { "kind": "sidecar", "permissions": { "default": "allow_reads" } }, + "sandbox": { "kind": "local" } +} +``` + +You are a `pi_agenta` agent. Keep `harness`, `runner`, `sandbox`, and `llm` as they are unless +the user asks to change one. + +## The fields you decide + +### instructions + +`instructions.agents_md` — a Markdown string, your AGENTS.md: who you are and what you do. On the +`pi_agenta` harness a fixed Agenta preamble and persona are prepended automatically, so write only +your own project conventions here. One or two sentences for a simple agent; an explicit numbered +procedure for a multi-tool or scheduled one (see the instruction-writing section of SKILL.md). + +### llm + +- `model` — the model. How you NAME it depends on the harness (this is the trap): + - `pi_core` / `pi_agenta`: a real model id, e.g. `gpt-5.5` or `anthropic/claude-...` + (provider/id selection). + - `claude`: an alias — `default`, `sonnet`, `opus`, or `haiku` — never a raw model id. +- `provider` — the provider family (`openai`, `anthropic`, ...); inferred from the model string + when unset. The `claude` harness reaches `anthropic` only. +- `connection` — `{ "mode": "agenta" | "self_managed", "slug": "" }`. `agenta` + uses an Agenta vault connection (omit `slug` for the project default); `self_managed` means the + harness owns its own auth. Omit the whole object for the project default. +- `extras` — neutral model knobs passed through unchanged (e.g. `reasoning_effort`). + +### tools + +A list of tool entries, each discriminated on `type`. Every entry may also carry two shared +optional fields: `render` (a UI hint) and `permission` (`allow` / `ask` / `deny`, overriding the +runner default for that one tool). The six `type` values: + +- `builtin` — a harness built-in: `{ "type": "builtin", "name": "read" }`. (A per-builtin + `permission` is dropped — builtins are granted by selection, not gated.) +- `gateway` — a server-side gateway action (Composio). Do not hand-write it: run `discover_tools` + and copy what it returns, adding the `connection` slug once the connection is ready. + `{ "type": "gateway", "provider": "composio", "integration": "github", + "action": "GET_AN_ISSUE", "connection": "" }`. `name` is optional. +- `code` — sandboxed code you supply: `{ "type": "code", "name": "...", "runtime": + "python"|"node", "script": "...", "input_schema": {...}, "secrets": [...] }`. +- `client` — a tool the caller fulfills: `{ "type": "client", "name": "...", "description": + "...", "input_schema": {...} }`. +- `reference` — another Agenta workflow run as a tool: `{ "type": "reference", "ref_by": + "variant"|"environment", "slug": "...", "version": "...", "environment": "...", "name": "...", + "input_schema": {...} }`. `ref_by="variant"` takes the slug's latest revision (or a pinned + `version`); `ref_by="environment"` takes whatever is deployed in `environment` (and must not + set `version`). +- `platform` — an existing Agenta endpoint exposed as a tool: `{ "type": "platform", "op": + "discover_tools" }`. The catalog owns everything else about it. + +### mcps + +Declared MCP servers. Each: `{ "name": ..., "transport": "stdio"|"http", "command"/"args" +(stdio) or "url" (http), "env", "secrets", "tools", "permission" }`. Secret env resolves from the +vault at run time; tokens never live in the config. + +### skills + +A list; each entry is either an inline skill template or an `@ag.embed` reference. + +An inline skill template: + +```json +{ "name": "clear-writing", + "description": "When to use this skill (one line — the trigger).", + "body": "# Title\\n\\nThe know-how, in Markdown.", + "files": [ { "path": "references/checklist.md", "content": "...", "executable": false } ], + "disable_model_invocation": false, + "allow_executable_files": false } +``` + +- `name` — required, kebab-case, <=64 chars (`^[a-z0-9]+(-[a-z0-9]+)*$`). +- `description` — required, <=1024 chars: the trigger the model matches. +- `body` — required, the SKILL.md Markdown after the composed frontmatter, <=50000 chars. +- `files` — optional bundled files, each `{ path, content, executable? }`. `path` is a relative + POSIX path (no leading `/`, no backslash, no `..` segment, and not `SKILL.md`), <=255 chars. + `content` is inline UTF-8, <=200000 chars. `executable` marks +x, honored only when + `allow_executable_files` is set and the sandbox policy allows it. A folder is just `/`-joined + segments in `path`; there is no separate folder object. +- `disable_model_invocation` — hide from the prompt (invoke only via `/skill:name`). +- `allow_executable_files` — default deny; the sandbox policy must also allow execution. + +An `@ag.embed` reference points at a stored skill the backend inlines into that same shape before +the run: + +```json +{ "@ag.embed": { "@ag.references": { "workflow": { "slug": "" } }, + "@ag.selector": { "path": "parameters.skill" } } } +``` + +## The execution parts (keep as-is unless asked) + +- `harness` — `{ "kind": "pi_core" | "pi_agenta" | "claude", "permissions": {...}, "extras": + {...} }`. `permissions` gates tool use on gating harnesses (Claude): `{ "default_mode": + "default"|"acceptEdits"|"plan"|"bypassPermissions", "allow": [...], "ask": [...], "deny": + [...] }`. Pi harnesses leave `permissions` empty and read prompt overrides (`system` / + `append_system`) from `extras`. +- `runner` — `{ "kind": "sidecar", "permissions": { "default": "allow"|"ask"|"deny"| + "allow_reads" }, "extras": {...} }`. `allow_reads` (the default) runs read-hinted tools and + asks for everything else. +- `sandbox` — `{ "kind": "local" | "daytona", "permissions": {...}, "extras": {...} }`. + `permissions` (optional) is the security boundary: `{ "network": { "mode": "on"|"off"| + "allowlist", "allowlist": [""] }, "filesystem": "on"|"readonly"|"off", "enforcement": + "strict"|"best_effort" }`. + +## How a delta commits (merge semantics) + +`commit_revision` sends `workflow_revision.delta.set` and an optional `delta.remove`: + +- `set` **deep-merges** onto your current config: a nested object key you leave out keeps its old + value. +- **Lists replace wholesale.** `tools`, `skills`, and `mcps` are NOT merged item by item — the + list you send REPLACES the old one. To add one tool, send the full list (your current entries + plus the new one). Sending only the new tool wipes the rest, including the platform ops you + configure yourself with — which severs them on your next run. +- `remove` takes dotted paths, e.g. `parameters.agent.tools`. + +## Mistakes that fail the commit + +- `slug` or `content` as top-level fields on a skill entry. The skill's Markdown goes in `body`; + a bundled file's text goes in that file's `content` inside `files`. Top-level `slug`/`content` + are unknown keys and the commit fails. +- `harness.kind: "claude"` paired with a non-Anthropic `provider`. Claude reaches `anthropic` + only; the run's Model & Harness never resolves and it never runs. +- Any unknown key in `llm`, `instructions`, `harness`, `runner`, `sandbox`, a skill entry, or a + tool entry. These objects reject extras, so a typo'd field name fails the commit. +- Dropping `harness`, `runner`, or `sandbox` from a fresh full-object commit. Prefer a narrow + `delta.set` that touches only what you change, so the boilerplate survives the deep merge. + +## Mistakes that commit fine but break the run + +- A raw model id on the `claude` harness (Claude selects by alias) or an alias like `sonnet` on a + `pi_core`/`pi_agenta` harness (Pi selects by provider/id). The model field is a free string, so + the commit succeeds — the run then silently falls back to a default model. Match the naming to + the harness and check `test_run`'s `resolved` block to catch a fallback. +""" + +# Bundled reference file: the `inputs_fields` template language. Verified against the runtime +# resolver (agenta.sdk.utils.resolvers.resolve_target_fields / resolve_json_selector) and the +# triggers dispatcher context builder (api/oss/src/tasks/asyncio/triggers/dispatcher.py + +# core/triggers/dtos.py TRIGGER_CONTEXT_FIELDS / SUBSCRIPTION_CONTEXT_FIELDS + the synthetic +# schedule event in core/triggers/service.py). Reality matched the external kit 1:1. +_TRIGGER_INPUTS_REFERENCE = """\ +# What a schedule or subscription passes to the run + +Read this when you create a schedule or subscription (`create_schedule` / `create_subscription`) +and need to control the `inputs_fields` template — the inputs your agent receives on each fire. + +## The template (`inputs_fields`) + +Both trigger kinds carry an optional `inputs_fields` template. On each fire the platform walks it +and resolves every leaf against the fire context (below): + +- A leaf string starting with `$` is a **JSON Path** over the context (it must begin `$`, `$.`, + or `$[`). +- A leaf string starting with `/` is a **JSON Pointer** over the context. +- **Every other leaf passes through literally** — plain strings, numbers, nested objects. There + is **no string interpolation**: `"Summarize $.event.attributes"` stays that literal text. A + selector must be the WHOLE leaf, not embedded inside a larger string. +- A selector that resolves to nothing becomes `null` (no error). +- If you **omit** `inputs_fields` entirely, the run receives the whole context object as its + inputs. + +## The fire context + +```json +{ + "event": { "event_id", "event_type", "timestamp", "created_at", "attributes" }, + "subscription": { "id", "name", "tags", "meta", "created_at", "updated_at" }, + "scope": { "project_id" } +} +``` + +`subscription` holds the firing schedule's or subscription's own header fields — for a schedule +too, the key is still `subscription`. Only these keys are exposed; connection internals and +secrets never reach the template. + +## A schedule fire (synthetic event) + +On a cron tick the `event` is synthetic: + +- `event.event_id` — `":"` (the dedup key). +- `event.event_type` — the `event_key` you gave `create_schedule`. +- `event.attributes` — `{ "timestamp": "" }`, nothing more. + +A schedule has no payload worth mapping; the useful part of the template is the literal message +you want the agent to receive. + +## A subscription fire (provider event) + +On a provider event: + +- `event.event_type` — the provider trigger slug. +- `event.attributes` — the provider's event payload (the GitHub issue, the Slack message). This + is the part you map into the run. + +## The canonical pattern + +Your agent reads its task from `inputs.messages` (the same shape `test_run` uses). Give every +trigger an explicit imperative `messages` entry so the run starts from a command, not an empty +context. + +A schedule that runs a fixed job every fire: + +```json +{ "messages": [ { "role": "user", "content": "Run the daily digest now." } ] } +``` + +A subscription that hands the agent the provider payload alongside a fixed instruction (no +interpolation, so the payload rides as a SIBLING key, not inlined into the message): + +```json +{ + "messages": [ { "role": "user", "content": "Triage the GitHub issue in inputs.event." } ], + "event": "$.event.attributes" +} +``` + +Passing no `inputs_fields` at all gives the agent the raw context object as inputs — fine for a +smoke test, but a real agent should get an explicit `messages` entry so the run starts from an +imperative instruction. +""" + + _BUILD_AN_AGENT_BODY = """\ # Build an Agenta agent @@ -123,6 +381,11 @@ Everything else is fixed unless the user explicitly asks to change it. Configure yourself with `commit_revision` by setting `parameters.agent` fields; do not create a separate app. +Read `references/config-schema.md` before your first `commit_revision`: it gives the exact shape +of every field, the tool-entry types, the skill-entry shape, the delta merge semantics, and the +mistakes that fail a commit. Read `references/trigger-inputs.md` before you write a schedule or +subscription's `inputs_fields`. + ## Decision table | The ask... | Needs | What to add | @@ -142,25 +405,41 @@ 2. Decide from the table. Most agents need only instructions. If the ask needs outside actions, call `discover_tools` with one short fragment per capability, such as "list github issues" or "post a slack message". -3. Read discovery as a search result, not an oracle. Confirm the matched integration and the - action are both right. If a "new telegram message" search returns a Slack event, reword the - fragment or choose the right alternative. +3. Read discovery as a search result, not an oracle. It is a high-recall keyword match over the + live catalog, so check three things before wiring anything: + - Per-integration connection state is authoritative, not the headline match. The primary match + can be the wrong integration while reporting ready, and the tool you wanted can sit in + `alternatives` with `needs_auth`. Trust the per-integration connection block, not the top + `ready` line. + - Right integration is not enough — read the matched event's description. A fragment like "new + github issue" can match a `..._ARTIFACT_CREATED` event on the shared word "created" with a + ready connection. Confirm the matched action or event actually does what the user asked. + - If nothing in the match or its alternatives plausibly corresponds, stop and tell the user the + integration does not support that action yet. Never wire the closest keyword hit. 4. If a needed connection is not ready, call `request_connection` for that integration and stop. Give the user the connection request and wait for them. Re-run `discover_tools` after they connect; do not silently create, fake, or skip connections. 5. Configure yourself. Put the chosen `capability.tool` entries and needed alternatives in `tools`, write `instructions.agents_md`, and call `commit_revision`. This is an approval stop. If the commit is denied or fails, earlier connections or triggers are not undone. -6. Verify with `test_run`. First warn the user that this is a real run: external write - tools may perform their action if approved. Then call `test_run` with - `inputs.messages` as a blunt instruction-framed test message and - `expectations.terminal_tool` set to the final tool that proves success. Read `verdict`, - `verdict_reason`, `tools`, and `approvals`; a 200 response is not proof. Pass only - when `verdict` is `pass`. If `approvals` is non-empty, this is an approval stop: - report the waiting gate and wait for the user. If `verdict` is `incomplete`, rewrite - `instructions.agents_md` as a blunter numbered procedure, call `commit_revision`, and - run `test_run` again. Use `query_spans` after schedules or subscriptions fire to read - back the SCHEDULED run spans. +6. Verify with `test_run`. First warn the user that this is a real run: external write tools may + perform their action if approved. Then call `test_run` with `inputs.messages` as a blunt + instruction-framed test message and `expectations.terminal_tool` set to the final tool that + proves success. Read `verdict`, `verdict_reason`, `tools`, `approvals`, and `resolved`; a 200 + response is not proof. The four verdicts: + - `pass` — the terminal tool ran and returned; done. + - `incomplete` — the run stopped short (did the early reads, then wandered or stopped before the + terminal action). Rewrite `instructions.agents_md` as a blunter numbered procedure, call + `commit_revision`, and run `test_run` again. + - `unconfirmed` — the terminal tool's completion could not be proven: it was dispatched but + never returned a result (the stalled-approval signature), or no `expectations.terminal_tool` + was set. A tool NAME appearing in the executed list is not proof it completed. If `approvals` + is non-empty this is an approval stop: report the waiting gate and wait for the user. + - `failed` — a tool errored or the run failed outright; read `verdict_reason` and fix. + + For an EXTERNAL WRITE, even a returned result is only truly confirmed by reading the side effect + back (fetch the channel history, re-read the issue). Use `query_spans` to read back SCHEDULED + run spans after a schedule or subscription fires. 7. Add a trigger only if asked. For schedules, cron is UTC, five fields, with a one-minute floor; convert the user's timezone yourself, then stop for approval before `create_schedule`: say what you are about to create and wait for the gate. After approval, call `create_schedule`, then @@ -169,7 +448,9 @@ and wait for the gate. After approval, call `create_subscription`, and confirm with `list_deliveries`. `test_subscription` waits for a real event, so warn the user before using it in a chat turn. Use `remove_schedule` or `remove_subscription` only when cleaning up a wrong - trigger. + trigger. Shape the run's inputs with `inputs_fields` (see `references/trigger-inputs.md`). + Triggers do NOT follow a new revision: after any later `commit_revision`, existing schedules and + subscriptions still point at the old revision, so re-point them to the new one. 8. Report short: what you became, what is connected, what is scheduled, what you verified, and what still needs the human. @@ -188,6 +469,13 @@ - Pin concrete ids, such as channel id and repo, instead of telling the agent to re-resolve them. - Make the final numbered step the terminal side effect, such as the post or write. - Say "finish by doing step N" so the run does not stop after the early read steps. +- Write the persona as an explicit imperative — who the agent is and what it does, stated as a + command, not a vague topic. On ambiguous input the harness falls back to a generic coding + assistant instead of doing the job. (Same reason a test message is phrased as a command.) +- Prefer narrow, filtered tools over list dumps. A huge list payload (e.g. a `LIST_ALL_*` action) + pushes the run to reach for a shell or code tool to sift it, which trips a separate + code-execution approval gate and derails the run. Pick the narrowest action (a `FIND_*` or + `GET_A_*` over a `LIST_ALL_*`), resolve an id once, and pin it into the instructions. ## Prefer wired tools @@ -197,6 +485,18 @@ `remove_subscription`) over harness builtins. Touch Terminal, RemoteTrigger, File tools, or raw HTTP only when your wired tools cannot do the job, and say so when you do. +## When something fails + +- A denied or failed `commit_revision` does not undo earlier connections or triggers; they still + exist. Do not redo them. +- A validation error on commit names the fields that are wrong. Fix those named fields and + re-commit; do not start over. `references/config-schema.md` has the "mistakes that fail" list. +- After any commit, existing schedules and subscriptions still point at the previous revision. + Re-point them so they run the new config. +- If `test_run`'s `resolved` harness or model differs from what you committed, the config silently + fell back (usually a harness/model/provider mismatch). Fix it against `references/config-schema.md` + and re-test. + ## Footguns - Empty output is not enough to fail a run; read the `test_run` verdict, tools, approvals, @@ -215,6 +515,12 @@ "automate, connect tools for, schedule, or subscribe an agent." ), body=_BUILD_AN_AGENT_BODY, + files=[ + SkillFile(path="references/config-schema.md", content=_CONFIG_SCHEMA_REFERENCE), + SkillFile( + path="references/trigger-inputs.md", content=_TRIGGER_INPUTS_REFERENCE + ), + ], ) # Platform skills every pi_agenta run carries, regardless of the author's config. These are the diff --git a/sdks/python/agenta/sdk/agents/skills/parsing.py b/sdks/python/agenta/sdk/agents/skills/parsing.py index ca02759e82..045ea4a103 100644 --- a/sdks/python/agenta/sdk/agents/skills/parsing.py +++ b/sdks/python/agenta/sdk/agents/skills/parsing.py @@ -31,7 +31,10 @@ def _unresolved_embed_message(value: Any) -> str | None: Walks nested mappings and sequences so an embed buried in a field (e.g. ``{"body": "@{{...}}"}`` or a bundled file's ``content``) is caught here and surfaces the clear, typed error rather than - slipping past into a confusing strict-model ``ValidationError``. + slipping past into a confusing strict-model ``ValidationError``. Strings are checked only for + the ``@{{`` snippet token: a structural embed is a mapping keyed ``@ag.embed``, and the literal + text "@ag.embed" legitimately appears in skill documentation (e.g. the build-an-agent + config-schema reference). """ if isinstance(value, Mapping): if _AG_EMBED_MARKER in value: @@ -49,9 +52,7 @@ def _unresolved_embed_message(value: Any) -> str | None: message = _unresolved_embed_message(nested) if message is not None: return message - elif isinstance(value, str) and ( - _AG_EMBED_MARKER in value or _AG_SNIPPET_MARKER in value - ): + elif isinstance(value, str) and _AG_SNIPPET_MARKER in value: return ( "Skill entry contains an unresolved embed token. Embeds resolve server-side before " "parsing; this usually means resolution was opted out (flags.resolve=False) or no " diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_agenta_builtins_reference_files.py b/sdks/python/oss/tests/pytest/unit/agents/test_agenta_builtins_reference_files.py new file mode 100644 index 0000000000..86a20bce1e --- /dev/null +++ b/sdks/python/oss/tests/pytest/unit/agents/test_agenta_builtins_reference_files.py @@ -0,0 +1,89 @@ +"""Drift protection for the ``build-an-agent`` skill's bundled reference files. + +``references/config-schema.md`` and ``references/trigger-inputs.md`` are hand-written docs that +describe the real ``parameters.agent`` shape (``AgentTemplateSchema`` + the ``ToolConfig`` union + +``SkillTemplate``). Hand-written docs drift silently when the schema grows. These tests assert the +config-schema reference still names every top-level template field and every tool ``type`` +discriminator, and that the bundled ``SkillTemplate`` file paths validate — so a schema change that +is not mirrored in the doc fails CI instead of shipping a stale reference to the builder agent. +""" + +from __future__ import annotations + +from typing import get_args + +from agenta.sdk.agents.adapters.agenta_builtins import BUILD_AN_AGENT_SKILL +from agenta.sdk.agents.skills import SkillFile +from agenta.sdk.agents.tools.models import ToolConfig +from agenta.sdk.utils.types import AgentTemplateSchema + + +def _file(path: str) -> SkillFile: + for bundled in BUILD_AN_AGENT_SKILL.files: + if bundled.path == path: + return bundled + raise AssertionError(f"{path!r} is not bundled with build-an-agent") + + +def _agent_template_top_fields() -> set[str]: + return set(AgentTemplateSchema.model_fields.keys()) + + +def _tool_type_discriminators() -> set[str]: + # ToolConfig is Annotated[Union[...], Field(discriminator="type")]; the first get_args peels the + # Annotated wrapper, the second peels the Union into its concrete member models. + union = get_args(ToolConfig)[0] + types: set[str] = set() + for member in get_args(union): + type_field = member.model_fields.get("type") + if type_field is None: + continue + types |= set(get_args(type_field.annotation)) + return types + + +def test_build_an_agent_bundles_the_two_reference_files(): + paths = {bundled.path for bundled in BUILD_AN_AGENT_SKILL.files} + assert paths == {"references/config-schema.md", "references/trigger-inputs.md"} + + +def test_bundled_file_paths_revalidate(): + # Reconstructing each SkillFile re-runs the safe-path validator; a path that ever regressed to + # absolute / escaping / SKILL.md would raise here. + for bundled in BUILD_AN_AGENT_SKILL.files: + SkillFile(path=bundled.path, content=bundled.content) + + +def test_config_schema_names_every_top_level_template_field(): + content = _file("references/config-schema.md").content + missing = sorted( + field for field in _agent_template_top_fields() if field not in content + ) + assert not missing, ( + f"config-schema.md does not mention template field(s): {missing}" + ) + + +def test_config_schema_names_every_tool_type_discriminator(): + content = _file("references/config-schema.md").content + types = _tool_type_discriminators() + # Sanity: the union really carries the six documented arms. + assert types == {"builtin", "gateway", "code", "client", "reference", "platform"} + missing = sorted( + tool_type + for tool_type in types + if f"`{tool_type}`" not in content and f'"{tool_type}"' not in content + ) + assert not missing, f"config-schema.md does not document tool type(s): {missing}" + + +def test_reference_files_ride_the_wire(): + wire = BUILD_AN_AGENT_SKILL.to_wire() + wire_paths = {entry["path"] for entry in wire["files"]} + assert wire_paths == {"references/config-schema.md", "references/trigger-inputs.md"} + + +def test_trigger_inputs_reference_documents_the_context_shape(): + content = _file("references/trigger-inputs.md").content + for key in ("event", "subscription", "scope", "attributes", "inputs_fields"): + assert key in content, f"trigger-inputs.md omits {key!r}" From 23794a1d42fc27704faeea8036cbf18f5fa03d70 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Tue, 7 Jul 2026 13:18:02 +0200 Subject: [PATCH 2/2] docs(sdk): example requests + review-round fixes in the build-an-agent references Mahmoud's PR #5106 review applied: harness-neutral wording, llm keep-as-is framing, honest no-server-validation failure modes (one 'mistakes that break your agent' section grounded in the real run-time failure points), instructions+skills examples first with a resend-your-entire-list caveat, reference-tool variant/latest/pinned examples, and the remediation rewrite. Claude-Session: https://claude.ai/code/session_01N2djTMgXnpk84EqtugHDJB --- .../sdk/agents/adapters/agenta_builtins.py | 267 +++++++++++++++--- .../test_agenta_builtins_reference_files.py | 18 ++ 2 files changed, 251 insertions(+), 34 deletions(-) diff --git a/sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py b/sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py index 800a8f3f81..1297cebfce 100644 --- a/sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py +++ b/sdks/python/agenta/sdk/agents/adapters/agenta_builtins.py @@ -110,14 +110,15 @@ _CONFIG_SCHEMA_REFERENCE = """\ # The agent config, field by field -Read this before your first `commit_revision`, and whenever a commit fails validation and you -need to check the shape. +Read this before your first `commit_revision`, and whenever a run misbehaves after a commit and +you need to check the shape. `parameters.agent` is one object. You edit it by sending only the changed fields under `commit_revision`'s `workflow_revision.delta.set.parameters.agent`. The portable definition — `instructions`, `llm`, `tools`, `mcps`, `skills` — is flat on it; the execution parts — -`harness`, `runner`, `sandbox` — are nested sub-objects. Almost every sub-object rejects unknown -keys, so a misplaced or misspelled field fails the commit rather than being silently ignored. +`harness`, `runner`, `sandbox` — are nested sub-objects. The commit does NOT validate this shape: +a misplaced or misspelled field commits fine and only bites when the agent next runs. Get the +shape right from this reference, and verify with `test_run` after every commit. ## The whole object @@ -134,20 +135,25 @@ } ``` -You are a `pi_agenta` agent. Keep `harness`, `runner`, `sandbox`, and `llm` as they are unless -the user asks to change one. +The example above shows one common setup; your own `harness` may be `pi_agenta`, `claude`, or +`pi_core`. Whatever it is, keep `harness`, `runner`, `sandbox`, and `llm` as they are unless the +user explicitly asks to change one. ## The fields you decide ### instructions -`instructions.agents_md` — a Markdown string, your AGENTS.md: who you are and what you do. On the -`pi_agenta` harness a fixed Agenta preamble and persona are prepended automatically, so write only -your own project conventions here. One or two sentences for a simple agent; an explicit numbered -procedure for a multi-tool or scheduled one (see the instruction-writing section of SKILL.md). +`instructions.agents_md` — a Markdown string, your AGENTS.md: who you are and what you do. Write +only your own project conventions here — the platform supplies its own baseline framing (on +`pi_agenta` and `claude`, a fixed Agenta preamble is prepended automatically). One or two +sentences for a simple agent; an explicit numbered procedure for a multi-tool or scheduled one +(see the instruction-writing section of SKILL.md). ### llm +You almost never touch `llm` in a delta. Keep it exactly as it is unless the user explicitly asks +to change the model, provider, or connection. The rules below matter only when they do ask: + - `model` — the model. How you NAME it depends on the harness (this is the trap): - `pi_core` / `pi_agenta`: a real model id, e.g. `gpt-5.5` or `anthropic/claude-...` (provider/id selection). @@ -175,11 +181,15 @@ "python"|"node", "script": "...", "input_schema": {...}, "secrets": [...] }`. - `client` — a tool the caller fulfills: `{ "type": "client", "name": "...", "description": "...", "input_schema": {...} }`. -- `reference` — another Agenta workflow run as a tool: `{ "type": "reference", "ref_by": - "variant"|"environment", "slug": "...", "version": "...", "environment": "...", "name": "...", - "input_schema": {...} }`. `ref_by="variant"` takes the slug's latest revision (or a pinned - `version`); `ref_by="environment"` takes whatever is deployed in `environment` (and must not - set `version`). +- `reference` — another Agenta workflow run as a tool. `ref_by` picks which revision runs: + - `{ "type": "reference", "ref_by": "variant", "slug": "my-summarizer" }` — runs that + variant's LATEST revision, following every new commit. + - `{ "type": "reference", "ref_by": "variant", "slug": "my-summarizer", "version": "3" }` — + pins revision 3; later commits do not change what runs. + - `{ "type": "reference", "ref_by": "environment", "slug": "my-summarizer", + "environment": "production" }` — runs whatever is deployed in that environment (and must + NOT set `version`; the environment is the pin). + Optionally add the model-facing surface: `name`, `description`, and `input_schema`. - `platform` — an existing Agenta endpoint exposed as a tool: `{ "type": "platform", "op": "discover_tools" }`. The catalog owns everything else about it. @@ -250,24 +260,150 @@ configure yourself with — which severs them on your next run. - `remove` takes dotted paths, e.g. `parameters.agent.tools`. -## Mistakes that fail the commit +## Mistakes that break your agent + +The commit accepts whatever you send — none of these return a validation error. Each one commits +fine and then bites later, at a different spot. Your two detectors are `test_run` (read the +`resolved` block and the executed tool list, not just the status) and a skill that fails to load +on the next run. - `slug` or `content` as top-level fields on a skill entry. The skill's Markdown goes in `body`; - a bundled file's text goes in that file's `content` inside `files`. Top-level `slug`/`content` - are unknown keys and the commit fails. + a bundled file's text goes in that file's `content` inside `files`. Bites at RUN time: skill + parsing rejects the unknown keys and the run fails to load the skill. +- Any unknown or misspelled key in a skill entry or a tool entry. Same failure point: the run + rejects the entry when it parses the config, not the commit. - `harness.kind: "claude"` paired with a non-Anthropic `provider`. Claude reaches `anthropic` - only; the run's Model & Harness never resolves and it never runs. -- Any unknown key in `llm`, `instructions`, `harness`, `runner`, `sandbox`, a skill entry, or a - tool entry. These objects reject extras, so a typo'd field name fails the commit. -- Dropping `harness`, `runner`, or `sandbox` from a fresh full-object commit. Prefer a narrow - `delta.set` that touches only what you change, so the boilerplate survives the deep merge. + only. Bites at RUN time: the run's Model & Harness never resolves and the agent never runs. +- A raw model id on the `claude` harness (Claude selects by alias) or an alias like `sonnet` on a + `pi_core`/`pi_agenta` harness (Pi selects by provider/id). Bites silently: the run falls back + to a default model with no error. Only `test_run`'s `resolved` block shows the fallback. +- Sending a short `tools`/`skills`/`mcps` list. Bites on the NEXT run: lists replace wholesale, + so everything you left out is gone — including the platform ops you configure yourself with. +- Rebuilding the whole `parameters.agent` object instead of a narrow delta. Prefer a `delta.set` + that touches only what you change, so `harness`, `runner`, `sandbox`, and `llm` survive the + deep merge untouched. -## Mistakes that commit fine but break the run +## Example requests -- A raw model id on the `claude` harness (Claude selects by alias) or an alias like `sonnet` on a - `pi_core`/`pi_agenta` harness (Pi selects by provider/id). The model field is a free string, so - the commit succeeds — the run then silently falls back to a default model. Match the naming to - the harness and check `test_run`'s `resolved` block to catch a fallback. +These are complete `commit_revision` payloads, ready to adapt. Field names match exactly; only the +values are placeholders. Most of the time you change only `instructions` and `skills` — the first +two examples cover the common case. + +Instructions only, the minimal two-step case — just a persona and a task: + +```json +{ + "workflow_revision": { + "message": "Set the agent's persona: triage inbound support emails.", + "delta": { + "set": { + "parameters": { + "agent": { + "instructions": { + "agents_md": "You triage inbound support emails. For each email: (1) classify it as bug, billing, or question; (2) draft a one-paragraph reply; (3) hand off billing issues instead of answering them." + } + } + } + } + } + } +} +``` + +Adding a skill entry — an inline skill template with one bundled reference file. `skills` replaces +wholesale too, so include your existing entries (this example assumes the list was empty): + +```json +{ + "workflow_revision": { + "message": "Add a code-review-checklist skill.", + "delta": { + "set": { + "parameters": { + "agent": { + "skills": [ + { + "name": "code-review-checklist", + "description": "Use when reviewing a pull request for style and correctness issues.", + "body": "# Code review checklist\\n\\nWalk every changed file against `references/checklist.md` before approving.\\n", + "files": [ + { + "path": "references/checklist.md", + "content": "- No commented-out code\\n- Tests cover the new branch\\n- Error messages are actionable\\n" + } + ] + } + ] + } + } + } + } + } +} +``` + +Adding ONE gateway tool — `tools` replaces wholesale, so resend every entry you already have (the +forced builtins, your existing platform ops, any `@ag.embed` tool) plus the new one. The gateway +entry is copied from what `discover_tools` returned, with the `connection` slug filled in. +CAVEAT: the list below is SHORTENED to keep the example readable — in a real commit, resend your +ENTIRE current tools list, every entry you have, not this subset: + +```json +{ + "workflow_revision": { + "message": "Add the GitHub create-issue tool alongside the existing build-kit tools.", + "delta": { + "set": { + "parameters": { + "agent": { + "tools": [ + { "type": "builtin", "name": "read" }, + { "type": "builtin", "name": "bash" }, + { "type": "platform", "op": "discover_tools" }, + { "type": "platform", "op": "commit_revision" }, + { "type": "platform", "op": "test_run" }, + { "@ag.embed": { "@ag.references": { "workflow": { "slug": "__ag__some_tool" } }, + "@ag.selector": { "path": "parameters.tool" } } }, + { + "type": "gateway", + "provider": "composio", + "integration": "github", + "action": "GITHUB_CREATE_AN_ISSUE", + "connection": "conn_9f3a1c", + "name": "create_github_issue" + } + ] + } + } + } + } + } +} +``` + +Dropping one field with `delta.remove` — a dotted path, no `set` required: + +```json +{ + "workflow_revision": { + "message": "Drop the reasoning_effort override; use the model's default again.", + "delta": { + "remove": ["parameters.agent.llm.extras.reasoning_effort"] + } + } +} +``` + +Don't forget: + +- Re-send the complete list for `tools`, `skills`, and `mcps`. A one-entry list wipes the rest, + including the platform ops you rely on every run. +- Copy `@ag.embed` entries through unchanged; do not try to inline or edit what they point at. +- `message` is a real commit message. Say what changed and why, not a placeholder. +- After the user connects an integration, re-run `discover_tools` before committing the tool, so + the `connection` slug is current. +- Keep `harness`, `runner`, `sandbox`, and `llm` out of `delta.set` unless you are intentionally + changing them; a narrow delta preserves them through the deep merge. """ # Bundled reference file: the `inputs_fields` template language. Verified against the runtime @@ -354,6 +490,60 @@ Passing no `inputs_fields` at all gives the agent the raw context object as inputs — fine for a smoke test, but a real agent should get an explicit `messages` entry so the run starts from an imperative instruction. + +## Example requests + +These are complete `create_schedule` / `create_subscription` payloads, ready to adapt. Write the +fields exactly as shown here — top-level `name` / `data` / ... — not wrapped in an outer key. + +`create_schedule` — a weekday-morning cron tick with a fixed instruction: + +```json +{ + "name": "daily-digest", + "description": "Runs the daily support digest every weekday morning.", + "data": { + "event_key": "daily_digest_tick", + "schedule": "0 8 * * 1-5", + "inputs_fields": { + "messages": [ + { "role": "user", "content": "Run the daily digest now." } + ] + } + } +} +``` + +`create_subscription` — a provider event, with the payload mapped alongside a fixed instruction: + +```json +{ + "name": "github-issue-triage", + "description": "Fires when a new issue is opened in the connected repo.", + "connection_id": "conn_9f3a1c", + "data": { + "event_key": "GITHUB_ISSUE_OPENED", + "trigger_config": { "owner": "agenta-ai", "repo": "agenta" }, + "inputs_fields": { + "messages": [ + { "role": "user", "content": "Triage the GitHub issue in inputs.event." } + ], + "event": "$.event.attributes" + } + } +} +``` + +Don't forget: + +- A selector must be the WHOLE leaf. `"Summarize $.event.attributes"` stays that literal text; put + the selector on its own key instead. +- Give every trigger an explicit imperative `messages` entry so the run starts from a command, not + raw context. +- `schedule` is five fields, UTC, with a one-minute floor. Convert the user's timezone yourself + before writing it. +- Map the provider payload as a sibling key, such as `"event": "$.event.attributes"`, never inlined + into the message text. """ @@ -381,10 +571,16 @@ Everything else is fixed unless the user explicitly asks to change it. Configure yourself with `commit_revision` by setting `parameters.agent` fields; do not create a separate app. -Read `references/config-schema.md` before your first `commit_revision`: it gives the exact shape -of every field, the tool-entry types, the skill-entry shape, the delta merge semantics, and the -mistakes that fail a commit. Read `references/trigger-inputs.md` before you write a schedule or -subscription's `inputs_fields`. +Read `references/config-schema.md` before your first `commit_revision`. It gives: + +- the exact shape of every field, +- the tool-entry types, +- the skill-entry shape, +- the delta merge semantics, +- the mistakes that break your agent. + +Read `references/trigger-inputs.md` before you write a schedule or subscription's +`inputs_fields`. ## Decision table @@ -489,8 +685,11 @@ - A denied or failed `commit_revision` does not undo earlier connections or triggers; they still exist. Do not redo them. -- A validation error on commit names the fields that are wrong. Fix those named fields and - re-commit; do not start over. `references/config-schema.md` has the "mistakes that fail" list. +- The commit does not validate your config: a wrong shape commits fine and surfaces at run time — + a skill fails to load, the model silently falls back, a tool goes missing. So verify with + `test_run` after every commit: read `resolved` and the executed tool list, and when something + is off, check the shape against `references/config-schema.md` and re-commit the fix; do not + start over. - After any commit, existing schedules and subscriptions still point at the previous revision. Re-point them so they run the new config. - If `test_run`'s `resolved` harness or model differs from what you committed, the config silently diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_agenta_builtins_reference_files.py b/sdks/python/oss/tests/pytest/unit/agents/test_agenta_builtins_reference_files.py index 86a20bce1e..4af9d4f981 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_agenta_builtins_reference_files.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_agenta_builtins_reference_files.py @@ -87,3 +87,21 @@ def test_trigger_inputs_reference_documents_the_context_shape(): content = _file("references/trigger-inputs.md").content for key in ("event", "subscription", "scope", "attributes", "inputs_fields"): assert key in content, f"trigger-inputs.md omits {key!r}" + + +def test_config_schema_has_example_commit_revision_requests(): + content = _file("references/config-schema.md").content + assert "## Example requests" in content + # A complete, copy-adaptable `commit_revision` payload, not just a field-shape snippet. + assert '"workflow_revision"' in content + assert '"delta"' in content + + +def test_trigger_inputs_has_example_trigger_requests(): + content = _file("references/trigger-inputs.md").content + assert "## Example requests" in content + # Complete `create_schedule` / `create_subscription` args payloads, written the way the + # model must emit them: unwrapped, matching the resolved input schema (not args_into). + assert '"event_key"' in content + assert '"schedule"' in content + assert '"connection_id"' in content