diff --git a/.pr/gpt-5-nano-integration-results.md b/.pr/gpt-5-nano-integration-results.md deleted file mode 100644 index 3cda307865..0000000000 --- a/.pr/gpt-5-nano-integration-results.md +++ /dev/null @@ -1,74 +0,0 @@ -# GPT-5-nano integration results - -Tested PR head `cf0fd2e11e3e26ded2ed1eb31fc9178c567d6ba6` on 2026-08-18 with `litellm_proxy/openai/gpt-5-nano`, `reasoning_effort=high`, and `https://llm-proxy.eval.all-hands.dev`. - -No `b*` behavior tests were run. - -## Result summary - -| Test | Result | -| --- | --- | -| Focused unit suite, `tests/sdk/event/test_events_to_messages.py` | 23 passed | -| New reasoning-item regression against `origin/main` | Failed as expected: the combined message had `responses_reasoning_item=None` | -| `t*` integration suite | 8 passed, 1 failed | -| `c*` condenser suite | 2 passed, 2 failed, 1 skipped | -| Isolated rerun of `c02` and `c05` | `c05` passed; `c02` failed again | -| Purpose-built parallel-tool replay probe | Passed | - -## Integration suite details - -### `t*` - -Passed: `t01`, `t02`, `t03`, `t04`, `t06`, `t07`, `t08`, and `t09`. - -`t05_simple_browsing` failed twice, including an isolated retry. Chromium launched and the agent navigated the test site, but GPT-5-nano stopped after saying it would fetch the answer instead of reporting it. This is model behavior and does not exercise action batching. - -### `c*` - -- `c01_thinking_block_condenser`: skipped as designed because GPT-5-nano produces Responses API reasoning items rather than Anthropic thinking blocks. -- `c03_delayed_condensation`: passed with five condensations. -- `c04_token_condenser`: passed. -- `c05_size_condenser`: failed initially because the model emitted only one tool call and stopped before enough events existed; it passed on an isolated rerun, confirming model-dependent flakiness. -- `c02_hard_context_reset`: failed twice because GPT-5-nano answered calculation requests directly instead of creating enough tool-loop events for the second condensation to become a normal condensation. - -The `c02` and initial `c05` failures did not produce parallel sibling `ActionEvent`s, so they did not exercise the code changed by this PR. - -## Direct parallel-tool replay validation - -A first live attempt asked GPT-5-nano to issue two calls to the same terminal tool in parallel. The model instead emitted them in two separate LLM responses, confirming that a generic integration task does not reliably cover this regression. - -A second probe exposed two distinct independent tools, `get_alpha` and `get_beta`, and required both before a final response. GPT-5-nano then produced: - -1. two `ActionEvent`s with the same `llm_response_id`; -2. a Responses API reasoning item only on the first action; -3. a recombined assistant message containing both tool calls in order; -4. a reasoning item exactly equal to the first action's item; and -5. a successful follow-up Responses API turn ending with `alpha beta verified`. - -This exercises the PR's changed path end to end: the reasoning item is retained on the recombined message and accepted when the tool-call batch is sent back to GPT-5-nano. - -## Commands - -```bash -uv run --frozen pytest tests/sdk/event/test_events_to_messages.py - -LLM_API_KEY=... \ -LLM_BASE_URL=https://llm-proxy.eval.all-hands.dev \ -IN_DOCKER=true \ -uv run --frozen python tests/integration/run_infer.py \ - --llm-config '{"model":"litellm_proxy/openai/gpt-5-nano","reasoning_effort":"high"}' \ - --num-workers 4 \ - --test-type integration - -LLM_API_KEY=... \ -LLM_BASE_URL=https://llm-proxy.eval.all-hands.dev \ -IN_DOCKER=true \ -uv run --frozen python tests/integration/run_infer.py \ - --llm-config '{"model":"litellm_proxy/openai/gpt-5-nano","reasoning_effort":"high"}' \ - --num-workers 4 \ - --test-type condenser -``` - -## Assessment - -The focused regression and the live parallel-tool replay both validate the fix. The remaining integration failures are explained by GPT-5-nano task compliance and did not execute the changed parallel-action reconstruction path. diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 32f9e30f59..95bfdbb085 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -2399,8 +2399,15 @@ def _generate_title_traced( llm: LLM | None, max_length: int, on_error: Callable[[Exception], None] | None = None, + prompt: str | None = None, ) -> str: - return generate_title_from_message(message, llm, max_length, on_error=on_error) + return generate_title_from_message( + message, + llm, + max_length, + on_error=on_error, + prompt=prompt, + ) @dataclass @@ -2446,6 +2453,7 @@ async def _generate_and_save() -> None: title_llm, 50, _on_title_error, + self.service.stored.prompt, ) if title and self.service.stored.title is None: self.service.stored.title = title diff --git a/openhands-sdk/openhands/sdk/conversation/request.py b/openhands-sdk/openhands/sdk/conversation/request.py index b99722b176..78020e4b1c 100644 --- a/openhands-sdk/openhands/sdk/conversation/request.py +++ b/openhands-sdk/openhands/sdk/conversation/request.py @@ -277,6 +277,17 @@ class ConversationConfig(BaseModel): "the agent's LLM." ), ) + prompt: str | None = Field( + default=None, + max_length=2000, + description=( + "Optional prompt that replaces the default title-generation user " + "message. Use {conversation_content} to place the first user " + "message and {max_length} to place the title length limit. If the " + "conversation placeholder is omitted, the message is appended " + "automatically. Empty or unset values use the default prompt." + ), + ) class StartConversationRequest(ConversationConfig): diff --git a/openhands-sdk/openhands/sdk/conversation/title_utils.py b/openhands-sdk/openhands/sdk/conversation/title_utils.py index 4e98dd015e..b321354357 100644 --- a/openhands-sdk/openhands/sdk/conversation/title_utils.py +++ b/openhands-sdk/openhands/sdk/conversation/title_utils.py @@ -1,6 +1,7 @@ """Utility functions for generating conversation titles.""" from collections.abc import Callable, Sequence +from typing import Final from openhands.sdk.event import MessageEvent from openhands.sdk.event.base import Event @@ -28,6 +29,9 @@ {"emoji": "♻️", "name": "refactor", "description": "Code refactoring"}, ] +CONVERSATION_CONTENT_PLACEHOLDER: Final[str] = "{conversation_content}" +MAX_LENGTH_PLACEHOLDER: Final[str] = "{max_length}" + def extract_message_text(event: MessageEvent) -> str | None: """Extract plain-text content from a message event.""" @@ -64,6 +68,7 @@ def generate_title_with_llm( llm: LLM, max_length: int = 50, on_error: Callable[[Exception], None] | None = None, + prompt: str | None = None, ) -> str | None: """Generate a conversation title using LLM. @@ -74,6 +79,10 @@ def generate_title_with_llm( on_error: Optional callback invoked with the exception when the LLM call fails. Title generation still falls back (returns None); the callback lets callers surface the otherwise-swallowed error. + prompt: Optional user-message prompt override. The + ``{conversation_content}`` and ``{max_length}`` placeholders are + replaced when present. If the conversation placeholder is omitted, + the message content is appended automatically. Returns: Generated title, or None if LLM fails or returns empty response. @@ -88,6 +97,23 @@ def generate_title_with_llm( f"{c['emoji']} {c['name']}: {c['description']}" for c in categories ) + template = (prompt or "").strip() + if template: + user_prompt = template.replace( + CONVERSATION_CONTENT_PLACEHOLDER, truncated_message + ).replace(MAX_LENGTH_PLACEHOLDER, str(max_length)) + if CONVERSATION_CONTENT_PLACEHOLDER not in template: + user_prompt = f"{user_prompt}\n\nConversation content:\n{truncated_message}" + else: + user_prompt = ( + f"Generate a title (maximum {max_length} characters) " + f"for a conversation that starts with this message:\n\n" + f"{truncated_message}." + "Also make sure to include ONE most relevant emoji at " + "the start of the title." + f" Choose the emoji from this list:{emojis_descriptions} " + ) + try: # Create messages for the LLM to generate a title messages = [ @@ -111,18 +137,7 @@ def generate_title_with_llm( ), Message( role="user", - content=[ - TextContent( - text=( - f"Generate a title (maximum {max_length} characters) " - f"for a conversation that starts with this message:\n\n" - f"{truncated_message}." - "Also make sure to include ONE most relevant emoji at " - "the start of the title." - f" Choose the emoji from this list:{emojis_descriptions} " - ) - ) - ], + content=[TextContent(text=user_prompt)], ), ] @@ -178,6 +193,7 @@ def generate_title_from_message( llm: LLM | None = None, max_length: int = 50, on_error: Callable[[Exception], None] | None = None, + prompt: str | None = None, ) -> str: """Generate a title from an already-extracted user message.""" # Skip the ACP sentinel LLM — it has no credentials and cannot be @@ -187,7 +203,11 @@ def generate_title_from_message( if llm_to_use: llm_title = generate_title_with_llm( - message, llm_to_use, max_length, on_error=on_error + message, + llm_to_use, + max_length, + on_error=on_error, + prompt=prompt, ) if llm_title: return llm_title diff --git a/tests/agent_server/test_conversation_service.py b/tests/agent_server/test_conversation_service.py index 4fb3b79784..792b307257 100644 --- a/tests/agent_server/test_conversation_service.py +++ b/tests/agent_server/test_conversation_service.py @@ -3004,6 +3004,7 @@ def _make_service( self, title: str | None = None, title_llm_profile: str | None = None, + prompt: str | None = None, llm_model: str = "gpt-4o", llm_usage_id: str = "test-llm", ) -> AsyncMock: @@ -3016,6 +3017,7 @@ def _make_service( metrics=None, title=title, title_llm_profile=title_llm_profile, + prompt=prompt, ) service = AsyncMock(spec=EventService) service.stored = stored @@ -3063,6 +3065,21 @@ async def test_autotitle_sets_title_on_first_user_message(self): assert service.stored.title == "✨ Generated Title" service.save_meta.assert_called_once() + @pytest.mark.asyncio + async def test_autotitle_passes_custom_prompt_to_title_generation(self): + service = self._make_service(prompt="Use {conversation_content} as the title.") + + with patch( + self._GENERATE_TITLE_PATH, return_value="Fix the login bug" + ) as mock_generate_title: + subscriber = AutoTitleSubscriber(service=service) + await subscriber(self._user_message_event()) + await self._drain_title_task(lambda: service.stored.title is not None) + + assert mock_generate_title.call_args.kwargs["prompt"] == ( + "Use {conversation_content} as the title." + ) + @pytest.mark.asyncio async def test_autotitle_skips_non_user_events(self): """Non-user events do not trigger title generation. diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index 74f55baac5..8c0b304a40 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -1864,6 +1864,21 @@ async def test_save_meta_preserves_updated_at(self, event_service, tmp_path): loaded = StoredConversation.model_validate_json(meta_file.read_text()) assert loaded.updated_at == original_updated_at + @pytest.mark.asyncio + async def test_save_meta_round_trips_title_prompt(self, event_service, tmp_path): + event_service.stored.prompt = ( + "Create a concise title for {conversation_content}." + ) + event_service.conversations_dir = tmp_path + conv_dir = tmp_path / event_service.stored.id.hex + conv_dir.mkdir(parents=True, exist_ok=True) + + await event_service.save_meta() + + meta_file = conv_dir / "meta.json" + loaded = StoredConversation.model_validate_json(meta_file.read_text()) + assert loaded.prompt == ("Create a concise title for {conversation_content}.") + @pytest.mark.asyncio async def test_save_meta_round_trips_agent_definition_mcp_secrets( self, sample_stored_conversation, tmp_path diff --git a/tests/sdk/conversation/test_generate_title.py b/tests/sdk/conversation/test_generate_title.py index 858404dd40..2085a3f08e 100644 --- a/tests/sdk/conversation/test_generate_title.py +++ b/tests/sdk/conversation/test_generate_title.py @@ -67,6 +67,13 @@ def create_mock_llm_response(content: str) -> LLMResponse: ) +def get_completion_user_prompt(mock_completion: MagicMock) -> str: + messages = mock_completion.call_args.args[0] + content = messages[1].content[0] + assert isinstance(content, TextContent) + return content.text + + @patch("openhands.sdk.llm.llm.LLM.completion") def test_generate_title_without_llm_uses_agent_llm(mock_completion): """Without an explicit LLM, generate_title falls back to the agent's LLM. @@ -141,6 +148,56 @@ def test_generate_title_with_llm_invokes_on_error(mock_completion): assert str(seen[0]) == "model does not exist" +@patch("openhands.sdk.llm.llm.LLM.completion") +def test_generate_title_with_llm_renders_custom_prompt_placeholders(mock_completion): + custom_llm = LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="test") + mock_completion.return_value = create_mock_llm_response("Fix Login") + + result = generate_title_with_llm( + "Fix the login bug", + custom_llm, + max_length=32, + prompt=( + "Return a title under {max_length} characters for: {conversation_content}" + ), + ) + + assert result == "Fix Login" + assert get_completion_user_prompt(mock_completion) == ( + "Return a title under 32 characters for: Fix the login bug" + ) + + +@patch("openhands.sdk.llm.llm.LLM.completion") +def test_generate_title_with_llm_appends_content_to_custom_prompt(mock_completion): + custom_llm = LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="test") + mock_completion.return_value = create_mock_llm_response("Fix Login") + + generate_title_with_llm( + "Fix the login bug", + custom_llm, + prompt="Use sentence case without an emoji.", + ) + + assert get_completion_user_prompt(mock_completion) == ( + "Use sentence case without an emoji.\n\n" + "Conversation content:\nFix the login bug" + ) + + +@pytest.mark.parametrize("prompt", [None, "", " "]) +@patch("openhands.sdk.llm.llm.LLM.completion") +def test_generate_title_with_llm_uses_default_for_blank_prompt(mock_completion, prompt): + custom_llm = LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="test") + mock_completion.return_value = create_mock_llm_response("🐛 Fix Login") + + generate_title_with_llm("Fix the login bug", custom_llm, prompt=prompt) + + user_prompt = get_completion_user_prompt(mock_completion) + assert user_prompt.startswith("Generate a title (maximum 50 characters)") + assert "🐛 bugfix: Bug fixes" in user_prompt + + @patch("openhands.sdk.llm.llm.LLM.completion") def test_generate_title_truncation_respects_max_length(mock_completion): """When LLM fails, truncation fallback respects max_length."""