diff --git a/migrations/versions/017_add_run_metadata.py b/migrations/versions/017_add_run_metadata.py new file mode 100644 index 00000000..5fe21404 --- /dev/null +++ b/migrations/versions/017_add_run_metadata.py @@ -0,0 +1,28 @@ +"""Add run_metadata column to automation_runs. + +Stores additional execution metadata captured after a run completes, such as +structured semantic task outcomes parsed from preset conversation finish actions. + +Revision ID: 017 +Revises: 016 +Create Date: 2026-08-12 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + + +revision: str = "017" +down_revision: str = "016" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column("automation_runs", sa.Column("run_metadata", sa.JSON, nullable=True)) + + +def downgrade() -> None: + op.drop_column("automation_runs", "run_metadata") diff --git a/openhands/automation/models.py b/openhands/automation/models.py index de33ea83..86bc25d4 100644 --- a/openhands/automation/models.py +++ b/openhands/automation/models.py @@ -193,6 +193,11 @@ class AutomationRun(Base): # Uses generic JSON type for cross-database compatibility (PostgreSQL + SQLite) event_payload: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # Additional metadata captured during run execution. + # For preset automations this may include the semantic task outcome parsed + # from the final conversation action. + run_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # Timestamps created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), diff --git a/openhands/automation/presets/plugin/sdk_main.py b/openhands/automation/presets/plugin/sdk_main.py index 7c259b15..daba6e92 100644 --- a/openhands/automation/presets/plugin/sdk_main.py +++ b/openhands/automation/presets/plugin/sdk_main.py @@ -86,9 +86,8 @@ # may be missing here even when the agent-server has a valid session key. # We still fall back to SESSION_API_KEY for compatibility with cloud-mode # deployments and older agent-server versions that only set the bare name. -session_key = ( - os.environ.get("OH_SESSION_API_KEYS_0") - or os.environ.get("SESSION_API_KEY", "") +session_key = os.environ.get("OH_SESSION_API_KEYS_0") or os.environ.get( + "SESSION_API_KEY", "" ) model_profile = os.environ.get("AUTOMATION_MODEL") or None automation_user_id = os.environ.get("AUTOMATION_USER_ID") or None @@ -100,10 +99,7 @@ if IS_LOCAL_MODE: # Local mode: AGENT_SERVER_URL required print(f" AGENT_SERVER_URL: {'OK' if agent_server_url else 'MISSING'}") - print( - f" OH_SESSION_API_KEYS_0: " - f"{'OK' if session_key else 'NONE (may fail auth)'}" - ) + print(f" OH_SESSION_API_KEYS_0: {'OK' if session_key else 'NONE (may fail auth)'}") if not agent_server_url: print("FAIL: AGENT_SERVER_URL not set for local mode", file=sys.stderr) sys.exit(1) @@ -130,6 +126,7 @@ # SDK imports (before workspace context so import errors are caught) from openhands.sdk import Conversation, RemoteConversation +from openhands.tools.preset import TaskOutcome try: from openhands.sdk.mcp.config import coerce_mcp_config as _coerce_mcp_config @@ -395,7 +392,12 @@ def _build_conversation_title(event_context) -> str | None: # Get default agent with tools and condenser (CLI mode to disable browser) print("\n=== AGENT ===") - agent = get_default_agent(llm=llm, cli_mode=True) + # Keep finish-tool schema wiring in sync with presets/prompt/sdk_main.py. + agent = get_default_agent( + llm=llm, + cli_mode=True, + finish_tool_response_schema=TaskOutcome, + ) # Add MCP config and agent_context using model_copy if configured # (Plugin MCP configs will be merged when plugins are loaded) diff --git a/openhands/automation/presets/prompt/sdk_main.py b/openhands/automation/presets/prompt/sdk_main.py index c0014af5..fca5b00a 100644 --- a/openhands/automation/presets/prompt/sdk_main.py +++ b/openhands/automation/presets/prompt/sdk_main.py @@ -89,9 +89,8 @@ # may be missing here even when the agent-server has a valid session key. # We still fall back to SESSION_API_KEY for compatibility with cloud-mode # deployments and older agent-server versions that only set the bare name. -session_key = ( - os.environ.get("OH_SESSION_API_KEYS_0") - or os.environ.get("SESSION_API_KEY", "") +session_key = os.environ.get("OH_SESSION_API_KEYS_0") or os.environ.get( + "SESSION_API_KEY", "" ) model_profile = os.environ.get("AUTOMATION_MODEL") or None automation_user_id = os.environ.get("AUTOMATION_USER_ID") or None @@ -103,10 +102,7 @@ if IS_LOCAL_MODE: # Local mode: AGENT_SERVER_URL required print(f" AGENT_SERVER_URL: {'OK' if agent_server_url else 'MISSING'}") - print( - f" OH_SESSION_API_KEYS_0: " - f"{'OK' if session_key else 'NONE (may fail auth)'}" - ) + print(f" OH_SESSION_API_KEYS_0: {'OK' if session_key else 'NONE (may fail auth)'}") if not agent_server_url: print("FAIL: AGENT_SERVER_URL not set for local mode", file=sys.stderr) sys.exit(1) @@ -134,6 +130,7 @@ # SDK imports (before workspace context so import errors are caught) from openhands.sdk import Conversation, RemoteConversation +from openhands.tools.preset import TaskOutcome try: from openhands.sdk.mcp.config import coerce_mcp_config as _coerce_mcp_config @@ -364,7 +361,12 @@ def _build_conversation_title(event_context) -> str | None: # Get default agent with tools and condenser (CLI mode to disable browser) print("\n=== AGENT ===") - agent = get_default_agent(llm=llm, cli_mode=True) + # Keep finish-tool schema wiring in sync with presets/plugin/sdk_main.py. + agent = get_default_agent( + llm=llm, + cli_mode=True, + finish_tool_response_schema=TaskOutcome, + ) # Add MCP config and agent_context using model_copy if configured agent_updates = {} diff --git a/openhands/automation/router.py b/openhands/automation/router.py index 4dc8f7ac..df30a687 100644 --- a/openhands/automation/router.py +++ b/openhands/automation/router.py @@ -52,6 +52,9 @@ get_api_key_for_automation_run, ) from openhands.automation.utils.callback_error import format_callback_error +from openhands.automation.utils.conversation_outcome import ( + fetch_latest_finish_tool_response_for_run, +) from openhands.automation.utils.model_profiles import resolve_model_profile_for_user from openhands.automation.utils.run import create_pending_run, record_first_run_outcome from openhands.automation.utils.run_status_detail import ( @@ -492,6 +495,15 @@ async def complete_run( ) elif body.status == "COMPLETED": values["status_detail"] = None + if body.conversation_id: + finish_tool_response = await fetch_latest_finish_tool_response_for_run( + run, body.conversation_id + ) + if finish_tool_response is not None: + values["run_metadata"] = { + **(run.run_metadata or {}), + "finish_tool_response": finish_tool_response, + } stmt = ( update(AutomationRun) diff --git a/openhands/automation/schemas.py b/openhands/automation/schemas.py index fe517f9a..6bf2e378 100644 --- a/openhands/automation/schemas.py +++ b/openhands/automation/schemas.py @@ -737,6 +737,7 @@ class AutomationRunResponse(BaseModel): timeout_at: UtcDatetime | None sandbox_id: str | None bash_command_id: str | None = None + run_metadata: dict[str, Any] | None = None created_at: UtcDatetime started_at: UtcDatetime | None completed_at: UtcDatetime | None diff --git a/openhands/automation/utils/conversation_outcome.py b/openhands/automation/utils/conversation_outcome.py new file mode 100644 index 00000000..515c3604 --- /dev/null +++ b/openhands/automation/utils/conversation_outcome.py @@ -0,0 +1,121 @@ +"""Helpers for reading raw FinishTool responses from conversations.""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +import httpx + +from openhands.automation.backends import get_backend +from openhands.automation.config import get_config +from openhands.automation.models import AutomationRun +from openhands.automation.utils.sandbox import get_sandbox_agent_url + + +logger = logging.getLogger(__name__) + +ACTION_EVENT_KIND = "openhands.sdk.event.llm_convertible.action.ActionEvent" +FINISH_TOOL_NAME = "finish" + + +def finish_tool_response_from_event(event: dict[str, Any]) -> Any | None: + """Return the raw JSON-decoded arguments from a FinishTool action event.""" + if event.get("tool_name") != FINISH_TOOL_NAME: + return None + + tool_call = event.get("tool_call") + if not isinstance(tool_call, dict): + return None + + raw_arguments = tool_call.get("arguments") + if not isinstance(raw_arguments, str): + return None + + try: + return json.loads(raw_arguments) + except json.JSONDecodeError: + logger.warning("Could not decode finish tool arguments as JSON") + return raw_arguments + + +def latest_finish_tool_response_from_events( + events: list[dict[str, Any]], +) -> Any | None: + """Return the latest finish action response from newest-first events.""" + for event in events: + if event.get("tool_name") == FINISH_TOOL_NAME: + return finish_tool_response_from_event(event) + return None + + +async def fetch_latest_finish_tool_response( + client: httpx.AsyncClient, + agent_url: str, + session_key: str, + conversation_id: str, +) -> Any | None: + """Fetch recent conversation actions and return the latest finish response.""" + response = await client.get( + f"{agent_url.rstrip('/')}/api/conversations/{conversation_id}/events/search", + params={ + "kind": ACTION_EVENT_KIND, + "sort_order": "TIMESTAMP_DESC", + "limit": 100, + }, + headers={"X-Session-API-Key": session_key}, + timeout=30.0, + ) + response.raise_for_status() + page = response.json() + items = page.get("items") if isinstance(page, dict) else None + if not isinstance(items, list): + return None + return latest_finish_tool_response_from_events(items) + + +async def fetch_latest_finish_tool_response_for_run( + run: AutomationRun, + conversation_id: str, +) -> Any | None: + """Best-effort lookup of the latest raw FinishTool response for a run.""" + try: + backend = get_backend(run) + async with httpx.AsyncClient(timeout=60.0) as client: + if backend.is_local_mode: + ctx = await backend.get_execution_context(client) + return await fetch_latest_finish_tool_response( + client, + ctx.agent_url, + ctx.session_key, + conversation_id, + ) + + if not run.sandbox_id: + return None + + api_key = await backend.get_api_key() + result = await get_sandbox_agent_url( + client, + get_config().service.openhands_api_base_url, + api_key, + run.sandbox_id, + ) + if result is None: + return None + agent_url, session_key = result + return await fetch_latest_finish_tool_response( + client, + agent_url, + session_key, + conversation_id, + ) + except Exception as exc: + logger.warning( + "Could not fetch finish tool response for run %s conversation %s: %s", + run.id, + conversation_id, + exc, + ) + return None diff --git a/tests/test_conversation_outcome.py b/tests/test_conversation_outcome.py new file mode 100644 index 00000000..0d3f480f --- /dev/null +++ b/tests/test_conversation_outcome.py @@ -0,0 +1,214 @@ +import json +from types import SimpleNamespace +from typing import cast + +import httpx +import pytest + +from openhands.automation.models import AutomationRun +from openhands.automation.utils import conversation_outcome as outcome_module +from openhands.automation.utils.conversation_outcome import ( + ACTION_EVENT_KIND, + fetch_latest_finish_tool_response, + finish_tool_response_from_event, + latest_finish_tool_response_from_events, +) + + +def _finish_event(arguments: dict | str, *, timestamp: str = "2026-08-12T21:00:00Z"): + raw_arguments = arguments if isinstance(arguments, str) else json.dumps(arguments) + return { + "kind": "ActionEvent", + "timestamp": timestamp, + "tool_name": "finish", + "tool_call": { + "id": "call_1", + "name": "finish", + "arguments": raw_arguments, + "origin": "completion", + }, + } + + +def test_finish_tool_response_from_event_returns_raw_arguments(): + arguments = { + "message": "Done", + "summary": "finishing task", + "status": "partial_success", + "outcome_summary": "Completed the main work but missed one item.", + "blockers": [ + { + "type": "external_service", + "message": "The reporting API timed out.", + "recoverable": True, + } + ], + "confidence": 0.8, + "needs_user_action": True, + "terminal_reason": "finish_action", + } + + assert finish_tool_response_from_event(_finish_event(arguments)) == arguments + + +def test_finish_tool_response_keeps_legacy_finish_arguments(): + arguments = {"message": "Done", "summary": "finishing task"} + + assert finish_tool_response_from_event(_finish_event(arguments)) == arguments + + +def test_finish_tool_response_keeps_invalid_json_as_raw_string(): + assert finish_tool_response_from_event(_finish_event("not-json")) == "not-json" + + +def test_latest_finish_tool_response_uses_latest_finish_event_only(): + non_finish_event = { + "tool_name": "think", + "tool_call": {"arguments": json.dumps({"thought": "newer"})}, + } + structured_event = _finish_event( + { + "message": "Done", + "status": "success", + "outcome_summary": "Everything completed.", + }, + timestamp="2026-08-12T21:01:00Z", + ) + + assert latest_finish_tool_response_from_events( + [non_finish_event, structured_event] + ) == { + "message": "Done", + "status": "success", + "outcome_summary": "Everything completed.", + } + + +def test_latest_finish_tool_response_does_not_fall_back_past_latest_finish(): + legacy_event = _finish_event({"message": "Legacy"}) + older_structured_event = _finish_event( + { + "message": "Done", + "status": "success", + "outcome_summary": "Everything completed.", + }, + timestamp="2026-08-12T20:59:00Z", + ) + + assert latest_finish_tool_response_from_events( + [legacy_event, older_structured_event] + ) == {"message": "Legacy"} + + +@pytest.mark.asyncio +async def test_fetch_latest_finish_tool_response_queries_conversation_events(): + event = _finish_event( + { + "message": "Done", + "status": "success", + "outcome_summary": "Everything completed.", + } + ) + + async def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/conversations/conv-123/events/search" + assert request.url.params["kind"] == ACTION_EVENT_KIND + assert request.url.params["sort_order"] == "TIMESTAMP_DESC" + assert request.url.params["limit"] == "100" + assert request.headers["X-Session-API-Key"] == "session-key" + return httpx.Response(200, json={"items": [event]}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + response = await fetch_latest_finish_tool_response( + client, + "https://agent.example.com", + "session-key", + "conv-123", + ) + + assert response == { + "message": "Done", + "status": "success", + "outcome_summary": "Everything completed.", + } + + +@pytest.mark.asyncio +async def test_fetch_latest_finish_tool_response_for_run_uses_local_context( + monkeypatch, +): + calls = {} + + class FakeBackend: + is_local_mode = True + + async def get_execution_context(self, client): + calls["context_client"] = client + return SimpleNamespace( + agent_url="https://local-agent.example.com", + session_key="local-session-key", + ) + + async def fake_fetch(client, agent_url, session_key, conversation_id): + calls["fetch"] = (client, agent_url, session_key, conversation_id) + return {"status": "success"} + + monkeypatch.setattr(outcome_module, "get_backend", lambda run: FakeBackend()) + monkeypatch.setattr(outcome_module, "fetch_latest_finish_tool_response", fake_fetch) + + run = cast(AutomationRun, SimpleNamespace(id="run-1", sandbox_id=None)) + + assert await outcome_module.fetch_latest_finish_tool_response_for_run( + run, "conv-1" + ) == {"status": "success"} + assert calls["fetch"] == ( + calls["context_client"], + "https://local-agent.example.com", + "local-session-key", + "conv-1", + ) + + +@pytest.mark.asyncio +async def test_fetch_latest_finish_tool_response_for_run_uses_remote_sandbox( + monkeypatch, +): + calls = {} + + class FakeBackend: + is_local_mode = False + + async def get_api_key(self): + calls["api_key_requested"] = True + return "sandbox-api-key" + + async def fake_get_sandbox_agent_url(client, api_url, api_key, sandbox_id): + calls["sandbox_lookup"] = (client, api_url, api_key, sandbox_id) + return "https://sandbox-agent.example.com", "sandbox-session-key" + + async def fake_fetch(client, agent_url, session_key, conversation_id): + calls["fetch"] = (client, agent_url, session_key, conversation_id) + return {"status": "partial_success"} + + monkeypatch.setattr(outcome_module, "get_backend", lambda run: FakeBackend()) + monkeypatch.setattr( + outcome_module, "get_sandbox_agent_url", fake_get_sandbox_agent_url + ) + monkeypatch.setattr(outcome_module, "fetch_latest_finish_tool_response", fake_fetch) + + run = cast(AutomationRun, SimpleNamespace(id="run-2", sandbox_id="sandbox-123")) + + assert await outcome_module.fetch_latest_finish_tool_response_for_run( + run, "conv-2" + ) == {"status": "partial_success"} + assert calls["api_key_requested"] is True + lookup_client, api_url, api_key, sandbox_id = calls["sandbox_lookup"] + assert api_url + assert api_key == "sandbox-api-key" + assert sandbox_id == "sandbox-123" + assert calls["fetch"] == ( + lookup_client, + "https://sandbox-agent.example.com", + "sandbox-session-key", + "conv-2", + ) diff --git a/tests/test_preset_router.py b/tests/test_preset_router.py index 64109b82..90c96e77 100644 --- a/tests/test_preset_router.py +++ b/tests/test_preset_router.py @@ -162,6 +162,18 @@ def test_plugin_setup_sh_fetches_sdk_version_from_api(self): "— do not hardcode the version" ) + @pytest.mark.parametrize("preset_name", ["prompt", "plugin"]) + def test_preset_finish_tool_uses_task_outcome_schema(self, preset_name): + """Preset agents attach TaskOutcome structured output to FinishTool.""" + sdk_main_path = PRESETS_DIR / preset_name / "sdk_main.py" + content = sdk_main_path.read_text() + + assert "from openhands.sdk import Conversation, RemoteConversation" in content + assert "from openhands.tools.preset import TaskOutcome" in content + assert "class TaskOutcome" not in content + assert "finish_tool_response_schema=TaskOutcome" in content + assert 'Tool(name="FinishTool"' not in content + class TestPresetEntrypoint: def test_get_preset_entrypoint_posix(self, monkeypatch): @@ -431,7 +443,7 @@ class TestReplacePromptInTarball: def test_replaces_prompt_and_preserves_sibling_files(self): """The prompt is swapped while every other file is left byte-for-byte intact.""" - # Arrange — a plugin preset tarball carries main.py, setup.sh, prompt.txt, + # Arrange — a plugin preset tarball carries generated code, prompt, # plugins_config.json and repos_config.json; all but the prompt must survive. original = _generate_plugin_tarball( [PluginSource(source="github:owner/repo")], @@ -460,7 +472,12 @@ def _read(tarball_bytes): new_files, new_setup_mode = _read(updated) assert new_files["prompt.txt"].decode() == "New prompt" - for name in ("main.py", "setup.sh", "plugins_config.json", "repos_config.json"): + for name in ( + "main.py", + "setup.sh", + "plugins_config.json", + "repos_config.json", + ): assert new_files[name] == old_files[name] assert new_setup_mode & 0o100 # setup.sh stays executable diff --git a/tests/test_router.py b/tests/test_router.py index 2ed9c16c..d416901b 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -1932,6 +1932,69 @@ async def test_complete_run_saves_conversation_id_for_completed_runs( assert run.conversation_id == "conv-completed-123" assert run.status == AutomationRunStatus.COMPLETED + async def test_complete_run_stores_finish_tool_response_metadata( + self, async_client, async_session, monkeypatch + ): + """Complete endpoint stores the raw latest FinishTool response.""" + from openhands.automation.models import AutomationRun, AutomationRunStatus + + automation = Automation( + user_id=TEST_USER_ID, + org_id=TEST_ORG_ID, + name="Test Automation", + trigger={"type": "cron", "schedule": "0 9 * * *", "timezone": "UTC"}, + tarball_path="s3://bucket/code.tar.gz", + entrypoint="uv run script.py", + ) + async_session.add(automation) + await async_session.commit() + + run = AutomationRun( + automation_id=automation.id, + status=AutomationRunStatus.RUNNING, + run_metadata={"existing": "value"}, + ) + async_session.add(run) + await async_session.commit() + + async def fake_fetch_latest_finish_tool_response_for_run( + callback_run, conversation_id + ): + assert callback_run.id == run.id + assert conversation_id == "conv-outcome-123" + return { + "status": "success", + "outcome_summary": "Completed all requested work.", + "confidence": 0.95, + "terminal_reason": "finish_action", + } + + monkeypatch.setattr( + "openhands.automation.router.fetch_latest_finish_tool_response_for_run", + fake_fetch_latest_finish_tool_response_for_run, + ) + + response = await async_client.post( + f"/api/automation/v1/runs/{run.id}/complete", + json={"status": "COMPLETED", "conversation_id": "conv-outcome-123"}, + ) + + assert response.status_code == 200 + finish_tool_response = response.json()["run_metadata"]["finish_tool_response"] + assert finish_tool_response == { + "status": "success", + "outcome_summary": "Completed all requested work.", + "confidence": 0.95, + "terminal_reason": "finish_action", + } + + await async_session.refresh(run) + assert run.run_metadata is not None + assert run.run_metadata["existing"] == "value" + assert run.run_metadata["finish_tool_response"]["status"] == "success" + + assert run.status == AutomationRunStatus.COMPLETED + async def test_complete_run_saves_conversation_id_for_failed_runs( self, async_client, async_session ): diff --git a/tests/test_schemas.py b/tests/test_schemas.py index a525d161..f221f4a6 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -112,6 +112,7 @@ def _make_run(self, **overrides: Any) -> AutomationRunResponse: timeout_at=None, sandbox_id=None, bash_command_id=None, + run_metadata=None, created_at=_NAIVE, started_at=_NAIVE, completed_at=_NAIVE,