diff --git a/frontend/src/api/conversation-service/v1-conversation-service.types.ts b/frontend/src/api/conversation-service/v1-conversation-service.types.ts index bbab7aab9..beaf2b9a1 100644 --- a/frontend/src/api/conversation-service/v1-conversation-service.types.ts +++ b/frontend/src/api/conversation-service/v1-conversation-service.types.ts @@ -53,6 +53,14 @@ export interface V1SendMessageRequest { content: V1MessageContent[]; } +type ObservabilityMetadataValue = + | string + | number + | boolean + | string[] + | number[] + | boolean[]; + export interface V1AppConversationStartRequest { sandbox_id?: string | null; initial_message?: V1SendMessageRequest | null; @@ -67,6 +75,9 @@ export interface V1AppConversationStartRequest { pr_number?: number[]; parent_conversation_id?: string | null; agent_type?: "default" | "plan"; + observability_metadata?: Record | null; + observability_tags?: string[] | null; + observability_span_name?: string | null; plugins?: PluginSpec[] | null; // Plugins to load when starting the conversation } diff --git a/openhands/app_server/app_conversation/app_conversation_models.py b/openhands/app_server/app_conversation/app_conversation_models.py index ccf6ca0c4..4257ca92a 100644 --- a/openhands/app_server/app_conversation/app_conversation_models.py +++ b/openhands/app_server/app_conversation/app_conversation_models.py @@ -21,6 +21,11 @@ # Import from new location and re-export for backward compatibility from openhands.app_server.settings.settings_models import SandboxGroupingStrategy from openhands.sdk.conversation import ConversationExecutionStatus +from openhands.sdk.conversation.types import ( + ConversationObservabilityMetadata, + ConversationObservabilitySpanName, + ConversationObservabilityTags, +) from openhands.sdk.llm import MetricsSnapshot from openhands.sdk.plugin import PluginSource from openhands.sdk.profiles import LaunchedAgentProfile @@ -249,6 +254,25 @@ class AppConversationStartRequest(OpenHandsModel): agent_type: AgentType = Field(default=AgentType.DEFAULT) public: bool | None = None + observability_metadata: ConversationObservabilityMetadata | None = Field( + default=None, + description=( + 'Trace-level metadata to attach to observability backends. Values must ' + 'be scalars or homogeneous scalar lists supported by OpenTelemetry.' + ), + ) + observability_tags: ConversationObservabilityTags | None = Field( + default=None, + description='Tags to attach to the conversation root observability span.', + ) + observability_span_name: ConversationObservabilitySpanName | None = Field( + default=None, + description=( + 'Optional named child span to emit under the conversation root. Use ' + 'stable, low-cardinality names because observability backends may use ' + 'span names for grouping or signal routing.' + ), + ) # Plugin parameters - for loading remote plugins into the conversation plugins: list[PluginSpec] | None = Field( diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.py index 089cde6db..fabc07c09 100644 --- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py +++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py @@ -505,6 +505,9 @@ async def _start_app_conversation( plugins=request.plugins, api_secrets=request.secrets, agent_profile_id=request.agent_profile_id, + request_observability_metadata=request.observability_metadata, + request_observability_tags=request.observability_tags, + request_observability_span_name=request.observability_span_name, ) ) @@ -1658,6 +1661,15 @@ def _extend_observability_metadata( continue target[key] = value + @staticmethod + def _extend_observability_tags(target: list[str], tags: Sequence[str]) -> None: + seen = set(target) + for tag in tags: + if tag in seen: + continue + target.append(tag) + seen.add(tag) + @staticmethod def _apply_server_agent_overrides( agent: Agent, @@ -1903,6 +1915,9 @@ async def _build_start_conversation_request_for_user( plugins: list[PluginSpec] | None = None, api_secrets: dict[str, SecretStr] | None = None, agent_profile_id: str | None = None, + request_observability_metadata: Mapping[str, Any] | None = None, + request_observability_tags: Sequence[str] | None = None, + request_observability_span_name: str | None = None, ) -> StartConversationRequest: """Build a complete StartConversationRequest for a user. @@ -1935,6 +1950,12 @@ async def _build_start_conversation_request_for_user( agent_profile_id: One-off Agent Profile override for this conversation only (cloud-only; does not change the member's active pointer). ``None`` uses the ambient active profile. + request_observability_metadata: Optional caller-provided trace metadata to + merge with app-server conversation metadata. + request_observability_tags: Optional caller-provided tags to append to the + conversation root observability span. + request_observability_span_name: Optional named child span to emit + under the conversation root. """ # Conversation start builds the agent, so it consumes the RESOLVED # (effective launch) view; plain settings reads/round-trips elsewhere @@ -1988,6 +2009,9 @@ async def _build_start_conversation_request_for_user( registered_marketplaces=registered_marketplaces, api_secrets=api_secrets, agent_profile_id=agent_profile_id, + request_observability_metadata=request_observability_metadata, + request_observability_tags=request_observability_tags, + request_observability_span_name=request_observability_span_name, ) if remote_workspace: acp_request = await self._load_skills_onto_request( @@ -2202,6 +2226,14 @@ async def _build_start_conversation_request_for_user( self._extend_observability_metadata( observability_metadata, resolved_observability_metadata ) + if request_observability_metadata: + self._extend_observability_metadata( + observability_metadata, request_observability_metadata + ) + if request_observability_tags: + self._extend_observability_tags( + observability_tags, request_observability_tags + ) create_kwargs: dict[str, Any] = {'agent': agent, 'user_id': laminar_user_id} title_llm_profile = _resolve_title_llm_profile(user) if title_llm_profile: @@ -2210,6 +2242,8 @@ async def _build_start_conversation_request_for_user( create_kwargs['observability_metadata'] = observability_metadata if observability_tags: create_kwargs['observability_tags'] = observability_tags + if request_observability_span_name: + create_kwargs['observability_span_name'] = request_observability_span_name request = conv_settings.create_request( StartConversationRequest, **create_kwargs ) @@ -2302,6 +2336,9 @@ async def _build_acp_start_conversation_request( registered_marketplaces: list[MarketplaceRegistration] | None = None, api_secrets: dict[str, SecretStr] | None = None, agent_profile_id: str | None = None, + request_observability_metadata: Mapping[str, Any] | None = None, + request_observability_tags: Sequence[str] | None = None, + request_observability_span_name: str | None = None, ) -> StartConversationRequest: """Build a StartConversationRequest for ACP agent conversations. @@ -2334,6 +2371,12 @@ async def _build_acp_start_conversation_request( agent_profile_id: One-off Agent Profile override for this conversation only (cloud-only; does not change the member's active pointer). ``None`` uses the ambient active profile. + request_observability_metadata: Optional caller-provided trace metadata to + merge with app-server conversation metadata. + request_observability_tags: Optional caller-provided tags to append to the + conversation root observability span. + request_observability_span_name: Optional named child span to emit + under the conversation root. """ user = await self.user_context.get_user_info( resolve_agent_profile=True, @@ -2484,6 +2527,14 @@ async def _build_acp_start_conversation_request( self._extend_observability_metadata( observability_metadata, resolved_observability_metadata ) + if request_observability_metadata: + self._extend_observability_metadata( + observability_metadata, request_observability_metadata + ) + if request_observability_tags: + self._extend_observability_tags( + observability_tags, request_observability_tags + ) create_kwargs: dict[str, Any] = { 'agent': acp_agent, 'user_id': laminar_user_id, @@ -2496,6 +2547,8 @@ async def _build_acp_start_conversation_request( create_kwargs['observability_metadata'] = observability_metadata if observability_tags: create_kwargs['observability_tags'] = observability_tags + if request_observability_span_name: + create_kwargs['observability_span_name'] = request_observability_span_name return conv_settings.create_request(StartConversationRequest, **create_kwargs) async def _process_pending_messages( diff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.py index 90f934c2b..7a9df1f2d 100644 --- a/tests/unit/app_server/test_live_status_app_conversation_service.py +++ b/tests/unit/app_server/test_live_status_app_conversation_service.py @@ -12,7 +12,7 @@ from uuid import uuid4 import pytest -from pydantic import SecretStr +from pydantic import SecretStr, ValidationError from openhands.agent_server.models import ( SendMessageRequest, @@ -1151,6 +1151,43 @@ def test_build_observability_context_includes_repository(self): assert 'branch:main' in tags assert 'git_provider:github' in tags + def test_app_conversation_start_request_accepts_observability_fields(self): + request = AppConversationStartRequest( + observability_span_name='mySpanName', + observability_tags=['wb-rubric', 'recall'], + observability_metadata={ + 'evaluation': 'wb', + 'attempt': 1, + 'replay': False, + }, + ) + + assert request.observability_span_name == 'mySpanName' + assert request.observability_tags == ['wb-rubric', 'recall'] + assert request.observability_metadata == { + 'evaluation': 'wb', + 'attempt': 1, + 'replay': False, + } + + @pytest.mark.parametrize( + 'kwargs', + [ + {'observability_metadata': {'scores': [1, 1.5]}}, + {'observability_metadata': {'nested': {'a': 1}}}, + {'observability_metadata': {'': 'x'}}, + {'observability_tags': ['ok', '']}, + {'observability_tags': 'not-a-list'}, + {'observability_span_name': 'bad name!'}, + {'observability_span_name': 'x' * 129}, + ], + ) + def test_app_conversation_start_request_rejects_invalid_observability_fields( + self, kwargs + ): + with pytest.raises(ValidationError): + AppConversationStartRequest(**kwargs) + def test_apply_server_overrides_adds_repo_metadata(self): llm = LLM(model='openhands/gpt-4', api_key='k', usage_id='agent') agent = Agent(llm=llm, tools=[]) @@ -1475,6 +1512,59 @@ async def test_build_request_populates_observability_metadata(self, _mock_tools) 'git_provider': 'github', } + @patch( + 'openhands.app_server.app_conversation.live_status_app_conversation_service.get_default_tools', + return_value=[], + ) + @pytest.mark.asyncio + async def test_build_request_forwards_api_observability_fields(self, _mock_tools): + self.mock_user_context.get_user_info.return_value = self.mock_user + self.service._setup_secrets_for_git_providers = AsyncMock(return_value={}) + self.service._configure_llm_and_mcp = AsyncMock( + return_value=(LLM(model='gpt-4', api_key=SecretStr('k')), {}) + ) + + result = await self.service._build_start_conversation_request_for_user( + sandbox=self.mock_sandbox, + conversation_id=uuid4(), + initial_message=None, + system_message_suffix=None, + git_provider=ProviderType.GITHUB, + working_dir='/test/dir', + remote_workspace=None, + selected_repository='test/repo', + selected_branch='feature-x', + request_observability_span_name='mySpanName', + request_observability_tags=['wb-rubric', 'repo:test/repo'], + request_observability_metadata={ + 'evaluation': 'wb', + 'attempt': 1, + 'repo_name': 'caller/repo', + }, + ) + + assert result.observability_span_name == 'mySpanName' + assert result.observability_tags == [ + 'app:openhands', + 'agent_kind:openhands', + 'repo:test/repo', + 'branch:feature-x', + 'git_provider:github', + 'wb-rubric', + ] + assert result.observability_metadata == { + 'app': 'openhands', + 'conversation_id': str(result.conversation_id), + 'agent_kind': 'openhands', + 'repo_name': 'test/repo', + 'selected_branch': 'feature-x', + 'repo': 'test/repo', + 'branch': 'feature-x', + 'git_provider': 'github', + 'evaluation': 'wb', + 'attempt': 1, + } + @patch( 'openhands.app_server.app_conversation.live_status_app_conversation_service.get_default_tools', return_value=[], @@ -1643,6 +1733,54 @@ async def test_build_request_routes_acp_user_to_observability_metadata(self): 'commit': 'def456sha', } + @patch( + 'openhands.app_server.app_conversation.live_status_app_conversation_service.get_default_tools', + return_value=[], + ) + @pytest.mark.asyncio + async def test_build_request_forwards_observability_to_acp_builder( + self, _mock_tools + ): + from openhands.sdk.settings import ACPAgentSettings + + self.mock_user.agent_settings = ACPAgentSettings( + acp_server='claude-code', + llm=LLM(model='claude-sonnet-4-5', api_key=None), + agent_context=None, + ) + self.mock_user_context.get_user_info.return_value = self.mock_user + self.service._setup_secrets_for_git_providers = AsyncMock(return_value={}) + self.service._configure_llm_and_mcp = AsyncMock( + return_value=(LLM(model='gpt-4', api_key=SecretStr('k')), {}) + ) + self.service._resolve_registered_marketplaces = AsyncMock(return_value=None) + sentinel = Mock(spec=StartConversationRequest) + self.service._build_acp_start_conversation_request = AsyncMock( + return_value=sentinel + ) + + result = await self.service._build_start_conversation_request_for_user( + sandbox=self.mock_sandbox, + conversation_id=uuid4(), + initial_message=None, + system_message_suffix=None, + git_provider=ProviderType.GITHUB, + working_dir='/test/dir', + remote_workspace=None, + selected_repository='test/repo', + selected_branch='feature-x', + request_observability_span_name='mySpanName', + request_observability_tags=['wb-rubric'], + request_observability_metadata={'evaluation': 'wb'}, + ) + + assert result is sentinel + self.service._build_acp_start_conversation_request.assert_called_once() + kwargs = self.service._build_acp_start_conversation_request.call_args.kwargs + assert kwargs['request_observability_metadata'] == {'evaluation': 'wb'} + assert kwargs['request_observability_tags'] == ['wb-rubric'] + assert kwargs['request_observability_span_name'] == 'mySpanName' + @patch( 'openhands.app_server.app_conversation.live_status_app_conversation_service.get_default_tools', return_value=[], @@ -2527,6 +2665,86 @@ async def mock_run_setup_scripts( ) assert saved_info.id == conversation_id + @patch( + 'openhands.app_server.app_conversation.live_status_app_conversation_service.AsyncRemoteWorkspace' + ) + @patch( + 'openhands.app_server.app_conversation.live_status_app_conversation_service.ConversationInfo' + ) + @pytest.mark.asyncio + async def test_start_app_conversation_forwards_observability_to_builder( + self, mock_conversation_info_class, mock_remote_workspace_class + ): + conversation_id = uuid4() + self.mock_user_context.get_user_id = AsyncMock(return_value='test_user_123') + self.mock_user_context.get_user_info = AsyncMock(return_value=self.mock_user) + + mock_sandbox_spec = Mock(spec=SandboxSpecInfo) + mock_sandbox_spec.working_dir = '/test/workspace' + self.mock_sandbox.sandbox_spec_id = str(uuid4()) + self.mock_sandbox.id = str(uuid4()) + self.mock_sandbox.session_api_key = 'test_session_key' + self.mock_sandbox.exposed_urls = [ + ExposedUrl(name=AGENT_SERVER, url='http://agent-server:8000', port=60000) + ] + self.mock_sandbox_service.get_sandbox = AsyncMock( + return_value=self.mock_sandbox + ) + self.mock_sandbox_spec_service.get_sandbox_spec = AsyncMock( + return_value=mock_sandbox_spec + ) + mock_remote_workspace_class.return_value = Mock() + + async def mock_wait_for_sandbox(task): + task.sandbox_id = self.mock_sandbox.id + yield task + + async def mock_run_setup_scripts( + task, sandbox, workspace, agent_server_url, conversation_id + ): + yield task + + self.service._wait_for_sandbox_start = mock_wait_for_sandbox + self.service.run_setup_scripts = mock_run_setup_scripts + + mock_agent = Mock(spec=Agent) + mock_agent.llm = Mock(spec=LLM) + mock_agent.llm.model = 'gpt-4' + mock_start_request = Mock(spec=StartConversationRequest) + mock_start_request.agent = mock_agent + mock_start_request.model_dump.return_value = {'test': 'data'} + self.service._build_start_conversation_request_for_user = AsyncMock( + return_value=mock_start_request + ) + + mock_conversation_info = Mock() + mock_conversation_info.id = conversation_id + mock_conversation_info_class.model_validate.return_value = ( + mock_conversation_info + ) + mock_response = Mock() + mock_response.json.return_value = {'id': str(conversation_id)} + mock_response.raise_for_status = Mock() + self.mock_httpx_client.post = AsyncMock(return_value=mock_response) + self.mock_event_callback_service.save_event_callback = AsyncMock() + + request = AppConversationStartRequest( + observability_metadata={'evaluation': 'wb'}, + observability_tags=['wb-rubric'], + observability_span_name='mySpanName', + ) + + async for _ in self.service._start_app_conversation(request): + pass + + self.service._build_start_conversation_request_for_user.assert_called_once() + kwargs = ( + self.service._build_start_conversation_request_for_user.call_args.kwargs + ) + assert kwargs['request_observability_metadata'] == {'evaluation': 'wb'} + assert kwargs['request_observability_tags'] == ['wb-rubric'] + assert kwargs['request_observability_span_name'] == 'mySpanName' + @patch( 'openhands.app_server.app_conversation.live_status_app_conversation_service.AsyncRemoteWorkspace' ) @@ -4351,6 +4569,9 @@ def _call_build( selected_branch=None, remote_workspace=None, registered_marketplaces=None, + request_observability_metadata=None, + request_observability_tags=None, + request_observability_span_name=None, ): """Wire user_context and call _build_acp_start_conversation_request.""" service.user_context.get_user_info = AsyncMock(return_value=user) @@ -4369,6 +4590,9 @@ def _call_build( remote_workspace=remote_workspace, registered_marketplaces=registered_marketplaces, plugins=None, + request_observability_metadata=request_observability_metadata, + request_observability_tags=request_observability_tags, + request_observability_span_name=request_observability_span_name, ) @pytest.mark.asyncio @@ -4620,6 +4844,48 @@ async def test_observability_metadata_populated(self, service, tmp_path): 'git_provider': 'github', } + @pytest.mark.asyncio + async def test_forwards_api_observability_fields(self, service, tmp_path): + user = self._make_acp_user() + + request = await self._call_build( + service, + user, + tmp_path, + git_provider=ProviderType.GITHUB, + selected_repository='test/repo', + selected_branch='feature-x', + request_observability_span_name='mySpanName', + request_observability_tags=['wb-rubric', 'repo:test/repo'], + request_observability_metadata={ + 'evaluation': 'wb', + 'attempt': 1, + 'repo_name': 'caller/repo', + }, + ) + + assert request.observability_span_name == 'mySpanName' + assert request.observability_tags == [ + 'app:openhands', + 'agent_kind:acp', + 'repo:test/repo', + 'branch:feature-x', + 'git_provider:github', + 'wb-rubric', + ] + assert request.observability_metadata == { + 'app': 'openhands', + 'conversation_id': str(request.conversation_id), + 'agent_kind': 'acp', + 'repo_name': 'test/repo', + 'selected_branch': 'feature-x', + 'repo': 'test/repo', + 'branch': 'feature-x', + 'git_provider': 'github', + 'evaluation': 'wb', + 'attempt': 1, + } + @pytest.mark.asyncio async def test_observability_metadata_includes_commit(self, service, tmp_path): """The post-clone HEAD sha is resolved from the workspace and added,