From 37495c5fcbcc03f396b5343045f9d2d83781f39f Mon Sep 17 00:00:00 2001 From: Devin Date: Thu, 20 Aug 2026 19:29:31 -0400 Subject: [PATCH 1/7] feat(sdk): support custom title generation prompts Co-authored-by: openhands --- .../agent_server/conversation_service.py | 10 +++- .../openhands/sdk/conversation/request.py | 11 ++++ .../openhands/sdk/conversation/title_utils.py | 46 +++++++++++---- .../agent_server/test_conversation_service.py | 19 ++++++ tests/agent_server/test_event_service.py | 19 ++++++ tests/sdk/conversation/test_generate_title.py | 59 +++++++++++++++++++ 6 files changed, 150 insertions(+), 14 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 32f9e30f59..d0f7bf9c35 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -2399,8 +2399,15 @@ def _generate_title_traced( llm: LLM | None, max_length: int, on_error: Callable[[Exception], None] | None = None, + title_generation_prompt: str | None = None, ) -> str: - return generate_title_from_message(message, llm, max_length, on_error=on_error) + return generate_title_from_message( + message, + llm, + max_length, + on_error=on_error, + title_generation_prompt=title_generation_prompt, + ) @dataclass @@ -2446,6 +2453,7 @@ async def _generate_and_save() -> None: title_llm, 50, _on_title_error, + self.service.stored.title_generation_prompt, ) if title and self.service.stored.title is None: self.service.stored.title = title diff --git a/openhands-sdk/openhands/sdk/conversation/request.py b/openhands-sdk/openhands/sdk/conversation/request.py index b99722b176..b608bd9cd3 100644 --- a/openhands-sdk/openhands/sdk/conversation/request.py +++ b/openhands-sdk/openhands/sdk/conversation/request.py @@ -277,6 +277,17 @@ class ConversationConfig(BaseModel): "the agent's LLM." ), ) + title_generation_prompt: str | None = Field( + default=None, + max_length=2000, + description=( + "Optional prompt that replaces the default title-generation user " + "message. Use {conversation_content} to place the first user " + "message and {max_length} to place the title length limit. If the " + "conversation placeholder is omitted, the message is appended " + "automatically. Empty or unset values use the default prompt." + ), + ) class StartConversationRequest(ConversationConfig): diff --git a/openhands-sdk/openhands/sdk/conversation/title_utils.py b/openhands-sdk/openhands/sdk/conversation/title_utils.py index 4e98dd015e..43f74c482c 100644 --- a/openhands-sdk/openhands/sdk/conversation/title_utils.py +++ b/openhands-sdk/openhands/sdk/conversation/title_utils.py @@ -28,6 +28,9 @@ {"emoji": "♻️", "name": "refactor", "description": "Code refactoring"}, ] +CONVERSATION_CONTENT_PLACEHOLDER = "{conversation_content}" +MAX_LENGTH_PLACEHOLDER = "{max_length}" + def extract_message_text(event: MessageEvent) -> str | None: """Extract plain-text content from a message event.""" @@ -64,6 +67,7 @@ def generate_title_with_llm( llm: LLM, max_length: int = 50, on_error: Callable[[Exception], None] | None = None, + title_generation_prompt: str | None = None, ) -> str | None: """Generate a conversation title using LLM. @@ -74,6 +78,10 @@ def generate_title_with_llm( on_error: Optional callback invoked with the exception when the LLM call fails. Title generation still falls back (returns None); the callback lets callers surface the otherwise-swallowed error. + title_generation_prompt: Optional user-message prompt override. The + ``{conversation_content}`` and ``{max_length}`` placeholders are + replaced when present. If the conversation placeholder is omitted, + the message content is appended automatically. Returns: Generated title, or None if LLM fails or returns empty response. @@ -88,6 +96,24 @@ def generate_title_with_llm( f"{c['emoji']} {c['name']}: {c['description']}" for c in categories ) + if title_generation_prompt and title_generation_prompt.strip(): + prompt = title_generation_prompt.strip() + includes_conversation = CONVERSATION_CONTENT_PLACEHOLDER in prompt + user_prompt = prompt.replace( + CONVERSATION_CONTENT_PLACEHOLDER, truncated_message + ).replace(MAX_LENGTH_PLACEHOLDER, str(max_length)) + if not includes_conversation: + user_prompt = f"{user_prompt}\n\nConversation content:\n{truncated_message}" + else: + user_prompt = ( + f"Generate a title (maximum {max_length} characters) " + f"for a conversation that starts with this message:\n\n" + f"{truncated_message}." + "Also make sure to include ONE most relevant emoji at " + "the start of the title." + f" Choose the emoji from this list:{emojis_descriptions} " + ) + try: # Create messages for the LLM to generate a title messages = [ @@ -111,18 +137,7 @@ def generate_title_with_llm( ), Message( role="user", - content=[ - TextContent( - text=( - f"Generate a title (maximum {max_length} characters) " - f"for a conversation that starts with this message:\n\n" - f"{truncated_message}." - "Also make sure to include ONE most relevant emoji at " - "the start of the title." - f" Choose the emoji from this list:{emojis_descriptions} " - ) - ) - ], + content=[TextContent(text=user_prompt)], ), ] @@ -178,6 +193,7 @@ def generate_title_from_message( llm: LLM | None = None, max_length: int = 50, on_error: Callable[[Exception], None] | None = None, + title_generation_prompt: str | None = None, ) -> str: """Generate a title from an already-extracted user message.""" # Skip the ACP sentinel LLM — it has no credentials and cannot be @@ -187,7 +203,11 @@ def generate_title_from_message( if llm_to_use: llm_title = generate_title_with_llm( - message, llm_to_use, max_length, on_error=on_error + message, + llm_to_use, + max_length, + on_error=on_error, + title_generation_prompt=title_generation_prompt, ) if llm_title: return llm_title diff --git a/tests/agent_server/test_conversation_service.py b/tests/agent_server/test_conversation_service.py index 4fb3b79784..994297190e 100644 --- a/tests/agent_server/test_conversation_service.py +++ b/tests/agent_server/test_conversation_service.py @@ -3004,6 +3004,7 @@ def _make_service( self, title: str | None = None, title_llm_profile: str | None = None, + title_generation_prompt: str | None = None, llm_model: str = "gpt-4o", llm_usage_id: str = "test-llm", ) -> AsyncMock: @@ -3016,6 +3017,7 @@ def _make_service( metrics=None, title=title, title_llm_profile=title_llm_profile, + title_generation_prompt=title_generation_prompt, ) service = AsyncMock(spec=EventService) service.stored = stored @@ -3063,6 +3065,23 @@ async def test_autotitle_sets_title_on_first_user_message(self): assert service.stored.title == "✨ Generated Title" service.save_meta.assert_called_once() + @pytest.mark.asyncio + async def test_autotitle_passes_custom_prompt_to_title_generation(self): + service = self._make_service( + title_generation_prompt="Use {conversation_content} as the title." + ) + + with patch( + self._GENERATE_TITLE_PATH, return_value="Fix the login bug" + ) as mock_generate_title: + subscriber = AutoTitleSubscriber(service=service) + await subscriber(self._user_message_event()) + await self._drain_title_task(lambda: service.stored.title is not None) + + assert mock_generate_title.call_args.kwargs["title_generation_prompt"] == ( + "Use {conversation_content} as the title." + ) + @pytest.mark.asyncio async def test_autotitle_skips_non_user_events(self): """Non-user events do not trigger title generation. diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index 74f55baac5..ee1c21c298 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -1864,6 +1864,25 @@ async def test_save_meta_preserves_updated_at(self, event_service, tmp_path): loaded = StoredConversation.model_validate_json(meta_file.read_text()) assert loaded.updated_at == original_updated_at + @pytest.mark.asyncio + async def test_save_meta_round_trips_title_generation_prompt( + self, event_service, tmp_path + ): + event_service.stored.title_generation_prompt = ( + "Create a concise title for {conversation_content}." + ) + event_service.conversations_dir = tmp_path + conv_dir = tmp_path / event_service.stored.id.hex + conv_dir.mkdir(parents=True, exist_ok=True) + + await event_service.save_meta() + + meta_file = conv_dir / "meta.json" + loaded = StoredConversation.model_validate_json(meta_file.read_text()) + assert loaded.title_generation_prompt == ( + "Create a concise title for {conversation_content}." + ) + @pytest.mark.asyncio async def test_save_meta_round_trips_agent_definition_mcp_secrets( self, sample_stored_conversation, tmp_path diff --git a/tests/sdk/conversation/test_generate_title.py b/tests/sdk/conversation/test_generate_title.py index 858404dd40..c0cf3767ab 100644 --- a/tests/sdk/conversation/test_generate_title.py +++ b/tests/sdk/conversation/test_generate_title.py @@ -67,6 +67,13 @@ def create_mock_llm_response(content: str) -> LLMResponse: ) +def get_completion_user_prompt(mock_completion: MagicMock) -> str: + messages = mock_completion.call_args.args[0] + content = messages[1].content[0] + assert isinstance(content, TextContent) + return content.text + + @patch("openhands.sdk.llm.llm.LLM.completion") def test_generate_title_without_llm_uses_agent_llm(mock_completion): """Without an explicit LLM, generate_title falls back to the agent's LLM. @@ -141,6 +148,58 @@ def test_generate_title_with_llm_invokes_on_error(mock_completion): assert str(seen[0]) == "model does not exist" +@patch("openhands.sdk.llm.llm.LLM.completion") +def test_generate_title_with_llm_renders_custom_prompt_placeholders(mock_completion): + custom_llm = LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="test") + mock_completion.return_value = create_mock_llm_response("Fix Login") + + result = generate_title_with_llm( + "Fix the login bug", + custom_llm, + max_length=32, + title_generation_prompt=( + "Return a title under {max_length} characters for: {conversation_content}" + ), + ) + + assert result == "Fix Login" + assert get_completion_user_prompt(mock_completion) == ( + "Return a title under 32 characters for: Fix the login bug" + ) + + +@patch("openhands.sdk.llm.llm.LLM.completion") +def test_generate_title_with_llm_appends_content_to_custom_prompt(mock_completion): + custom_llm = LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="test") + mock_completion.return_value = create_mock_llm_response("Fix Login") + + generate_title_with_llm( + "Fix the login bug", + custom_llm, + title_generation_prompt="Use sentence case without an emoji.", + ) + + assert get_completion_user_prompt(mock_completion) == ( + "Use sentence case without an emoji.\n\n" + "Conversation content:\nFix the login bug" + ) + + +@pytest.mark.parametrize("prompt", [None, "", " "]) +@patch("openhands.sdk.llm.llm.LLM.completion") +def test_generate_title_with_llm_uses_default_for_blank_prompt(mock_completion, prompt): + custom_llm = LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="test") + mock_completion.return_value = create_mock_llm_response("🐛 Fix Login") + + generate_title_with_llm( + "Fix the login bug", custom_llm, title_generation_prompt=prompt + ) + + user_prompt = get_completion_user_prompt(mock_completion) + assert user_prompt.startswith("Generate a title (maximum 50 characters)") + assert "🐛 bugfix: Bug fixes" in user_prompt + + @patch("openhands.sdk.llm.llm.LLM.completion") def test_generate_title_truncation_respects_max_length(mock_completion): """When LLM fails, truncation fallback respects max_length.""" From 26b6b881bcd3ca40ddd3932a0f0d7b33d0f4a608 Mon Sep 17 00:00:00 2001 From: Devin Date: Fri, 21 Aug 2026 09:06:41 -0400 Subject: [PATCH 2/7] Update openhands-sdk/openhands/sdk/conversation/title_utils.py Co-authored-by: Vasco Schiavo <115561717+VascoSch92@users.noreply.github.com> --- openhands-sdk/openhands/sdk/conversation/title_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openhands-sdk/openhands/sdk/conversation/title_utils.py b/openhands-sdk/openhands/sdk/conversation/title_utils.py index 43f74c482c..b17839a0d4 100644 --- a/openhands-sdk/openhands/sdk/conversation/title_utils.py +++ b/openhands-sdk/openhands/sdk/conversation/title_utils.py @@ -67,7 +67,7 @@ def generate_title_with_llm( llm: LLM, max_length: int = 50, on_error: Callable[[Exception], None] | None = None, - title_generation_prompt: str | None = None, + prompt: str | None = None, ) -> str | None: """Generate a conversation title using LLM. From 3ff38dc3accfe51c3483f8698ad1adc8720ddc28 Mon Sep 17 00:00:00 2001 From: Devin Date: Fri, 21 Aug 2026 09:08:19 -0400 Subject: [PATCH 3/7] Update openhands-sdk/openhands/sdk/conversation/request.py Co-authored-by: Vasco Schiavo <115561717+VascoSch92@users.noreply.github.com> --- openhands-sdk/openhands/sdk/conversation/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openhands-sdk/openhands/sdk/conversation/request.py b/openhands-sdk/openhands/sdk/conversation/request.py index b608bd9cd3..78020e4b1c 100644 --- a/openhands-sdk/openhands/sdk/conversation/request.py +++ b/openhands-sdk/openhands/sdk/conversation/request.py @@ -277,7 +277,7 @@ class ConversationConfig(BaseModel): "the agent's LLM." ), ) - title_generation_prompt: str | None = Field( + prompt: str | None = Field( default=None, max_length=2000, description=( From c1dfa5a5e47c60d12e5a4557b0925e8ff4505ded Mon Sep 17 00:00:00 2001 From: Devin Date: Fri, 21 Aug 2026 09:08:26 -0400 Subject: [PATCH 4/7] Update openhands-agent-server/openhands/agent_server/conversation_service.py Co-authored-by: Vasco Schiavo <115561717+VascoSch92@users.noreply.github.com> --- .../openhands/agent_server/conversation_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index d0f7bf9c35..b9da8d13bb 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -2399,7 +2399,7 @@ def _generate_title_traced( llm: LLM | None, max_length: int, on_error: Callable[[Exception], None] | None = None, - title_generation_prompt: str | None = None, + prompt: str | None = None, ) -> str: return generate_title_from_message( message, From e326b3c80e61b0b1de4f259eb277a4043a7bd3b8 Mon Sep 17 00:00:00 2001 From: Devin Date: Fri, 21 Aug 2026 09:08:45 -0400 Subject: [PATCH 5/7] Update openhands-sdk/openhands/sdk/conversation/title_utils.py Co-authored-by: Vasco Schiavo <115561717+VascoSch92@users.noreply.github.com> --- openhands-sdk/openhands/sdk/conversation/title_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openhands-sdk/openhands/sdk/conversation/title_utils.py b/openhands-sdk/openhands/sdk/conversation/title_utils.py index b17839a0d4..804a860d2d 100644 --- a/openhands-sdk/openhands/sdk/conversation/title_utils.py +++ b/openhands-sdk/openhands/sdk/conversation/title_utils.py @@ -193,7 +193,7 @@ def generate_title_from_message( llm: LLM | None = None, max_length: int = 50, on_error: Callable[[Exception], None] | None = None, - title_generation_prompt: str | None = None, + prompt: str | None = None, ) -> str: """Generate a title from an already-extracted user message.""" # Skip the ACP sentinel LLM — it has no credentials and cannot be From b3a9b6316252f647a255b79c1361012c69577278 Mon Sep 17 00:00:00 2001 From: Devin Date: Fri, 21 Aug 2026 09:43:56 -0400 Subject: [PATCH 6/7] fix(sdk): complete title prompt rename Co-authored-by: openhands --- .../agent_server/conversation_service.py | 4 ++-- .../openhands/sdk/conversation/title_utils.py | 18 +++++++++--------- .../agent_server/test_conversation_service.py | 10 ++++------ tests/agent_server/test_event_service.py | 10 +++------- tests/sdk/conversation/test_generate_title.py | 8 +++----- 5 files changed, 21 insertions(+), 29 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index b9da8d13bb..95bfdbb085 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -2406,7 +2406,7 @@ def _generate_title_traced( llm, max_length, on_error=on_error, - title_generation_prompt=title_generation_prompt, + prompt=prompt, ) @@ -2453,7 +2453,7 @@ async def _generate_and_save() -> None: title_llm, 50, _on_title_error, - self.service.stored.title_generation_prompt, + self.service.stored.prompt, ) if title and self.service.stored.title is None: self.service.stored.title = title diff --git a/openhands-sdk/openhands/sdk/conversation/title_utils.py b/openhands-sdk/openhands/sdk/conversation/title_utils.py index 804a860d2d..b321354357 100644 --- a/openhands-sdk/openhands/sdk/conversation/title_utils.py +++ b/openhands-sdk/openhands/sdk/conversation/title_utils.py @@ -1,6 +1,7 @@ """Utility functions for generating conversation titles.""" from collections.abc import Callable, Sequence +from typing import Final from openhands.sdk.event import MessageEvent from openhands.sdk.event.base import Event @@ -28,8 +29,8 @@ {"emoji": "♻️", "name": "refactor", "description": "Code refactoring"}, ] -CONVERSATION_CONTENT_PLACEHOLDER = "{conversation_content}" -MAX_LENGTH_PLACEHOLDER = "{max_length}" +CONVERSATION_CONTENT_PLACEHOLDER: Final[str] = "{conversation_content}" +MAX_LENGTH_PLACEHOLDER: Final[str] = "{max_length}" def extract_message_text(event: MessageEvent) -> str | None: @@ -78,7 +79,7 @@ def generate_title_with_llm( on_error: Optional callback invoked with the exception when the LLM call fails. Title generation still falls back (returns None); the callback lets callers surface the otherwise-swallowed error. - title_generation_prompt: Optional user-message prompt override. The + prompt: Optional user-message prompt override. The ``{conversation_content}`` and ``{max_length}`` placeholders are replaced when present. If the conversation placeholder is omitted, the message content is appended automatically. @@ -96,13 +97,12 @@ def generate_title_with_llm( f"{c['emoji']} {c['name']}: {c['description']}" for c in categories ) - if title_generation_prompt and title_generation_prompt.strip(): - prompt = title_generation_prompt.strip() - includes_conversation = CONVERSATION_CONTENT_PLACEHOLDER in prompt - user_prompt = prompt.replace( + template = (prompt or "").strip() + if template: + user_prompt = template.replace( CONVERSATION_CONTENT_PLACEHOLDER, truncated_message ).replace(MAX_LENGTH_PLACEHOLDER, str(max_length)) - if not includes_conversation: + if CONVERSATION_CONTENT_PLACEHOLDER not in template: user_prompt = f"{user_prompt}\n\nConversation content:\n{truncated_message}" else: user_prompt = ( @@ -207,7 +207,7 @@ def generate_title_from_message( llm_to_use, max_length, on_error=on_error, - title_generation_prompt=title_generation_prompt, + prompt=prompt, ) if llm_title: return llm_title diff --git a/tests/agent_server/test_conversation_service.py b/tests/agent_server/test_conversation_service.py index 994297190e..792b307257 100644 --- a/tests/agent_server/test_conversation_service.py +++ b/tests/agent_server/test_conversation_service.py @@ -3004,7 +3004,7 @@ def _make_service( self, title: str | None = None, title_llm_profile: str | None = None, - title_generation_prompt: str | None = None, + prompt: str | None = None, llm_model: str = "gpt-4o", llm_usage_id: str = "test-llm", ) -> AsyncMock: @@ -3017,7 +3017,7 @@ def _make_service( metrics=None, title=title, title_llm_profile=title_llm_profile, - title_generation_prompt=title_generation_prompt, + prompt=prompt, ) service = AsyncMock(spec=EventService) service.stored = stored @@ -3067,9 +3067,7 @@ async def test_autotitle_sets_title_on_first_user_message(self): @pytest.mark.asyncio async def test_autotitle_passes_custom_prompt_to_title_generation(self): - service = self._make_service( - title_generation_prompt="Use {conversation_content} as the title." - ) + service = self._make_service(prompt="Use {conversation_content} as the title.") with patch( self._GENERATE_TITLE_PATH, return_value="Fix the login bug" @@ -3078,7 +3076,7 @@ async def test_autotitle_passes_custom_prompt_to_title_generation(self): await subscriber(self._user_message_event()) await self._drain_title_task(lambda: service.stored.title is not None) - assert mock_generate_title.call_args.kwargs["title_generation_prompt"] == ( + assert mock_generate_title.call_args.kwargs["prompt"] == ( "Use {conversation_content} as the title." ) diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index ee1c21c298..8c0b304a40 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -1865,10 +1865,8 @@ async def test_save_meta_preserves_updated_at(self, event_service, tmp_path): assert loaded.updated_at == original_updated_at @pytest.mark.asyncio - async def test_save_meta_round_trips_title_generation_prompt( - self, event_service, tmp_path - ): - event_service.stored.title_generation_prompt = ( + async def test_save_meta_round_trips_title_prompt(self, event_service, tmp_path): + event_service.stored.prompt = ( "Create a concise title for {conversation_content}." ) event_service.conversations_dir = tmp_path @@ -1879,9 +1877,7 @@ async def test_save_meta_round_trips_title_generation_prompt( meta_file = conv_dir / "meta.json" loaded = StoredConversation.model_validate_json(meta_file.read_text()) - assert loaded.title_generation_prompt == ( - "Create a concise title for {conversation_content}." - ) + assert loaded.prompt == ("Create a concise title for {conversation_content}.") @pytest.mark.asyncio async def test_save_meta_round_trips_agent_definition_mcp_secrets( diff --git a/tests/sdk/conversation/test_generate_title.py b/tests/sdk/conversation/test_generate_title.py index c0cf3767ab..2085a3f08e 100644 --- a/tests/sdk/conversation/test_generate_title.py +++ b/tests/sdk/conversation/test_generate_title.py @@ -157,7 +157,7 @@ def test_generate_title_with_llm_renders_custom_prompt_placeholders(mock_complet "Fix the login bug", custom_llm, max_length=32, - title_generation_prompt=( + prompt=( "Return a title under {max_length} characters for: {conversation_content}" ), ) @@ -176,7 +176,7 @@ def test_generate_title_with_llm_appends_content_to_custom_prompt(mock_completio generate_title_with_llm( "Fix the login bug", custom_llm, - title_generation_prompt="Use sentence case without an emoji.", + prompt="Use sentence case without an emoji.", ) assert get_completion_user_prompt(mock_completion) == ( @@ -191,9 +191,7 @@ def test_generate_title_with_llm_uses_default_for_blank_prompt(mock_completion, custom_llm = LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="test") mock_completion.return_value = create_mock_llm_response("🐛 Fix Login") - generate_title_with_llm( - "Fix the login bug", custom_llm, title_generation_prompt=prompt - ) + generate_title_with_llm("Fix the login bug", custom_llm, prompt=prompt) user_prompt = get_completion_user_prompt(mock_completion) assert user_prompt.startswith("Generate a title (maximum 50 characters)") From 4afe9a979fda23b125388e898d8c3f7895f2696e Mon Sep 17 00:00:00 2001 From: allhands-bot Date: Fri, 21 Aug 2026 16:31:19 +0000 Subject: [PATCH 7/7] chore: Remove PR-only artifacts [automated] --- .pr/gpt-5-nano-integration-results.md | 74 --------------------------- 1 file changed, 74 deletions(-) delete mode 100644 .pr/gpt-5-nano-integration-results.md diff --git a/.pr/gpt-5-nano-integration-results.md b/.pr/gpt-5-nano-integration-results.md deleted file mode 100644 index 3cda307865..0000000000 --- a/.pr/gpt-5-nano-integration-results.md +++ /dev/null @@ -1,74 +0,0 @@ -# GPT-5-nano integration results - -Tested PR head `cf0fd2e11e3e26ded2ed1eb31fc9178c567d6ba6` on 2026-08-18 with `litellm_proxy/openai/gpt-5-nano`, `reasoning_effort=high`, and `https://llm-proxy.eval.all-hands.dev`. - -No `b*` behavior tests were run. - -## Result summary - -| Test | Result | -| --- | --- | -| Focused unit suite, `tests/sdk/event/test_events_to_messages.py` | 23 passed | -| New reasoning-item regression against `origin/main` | Failed as expected: the combined message had `responses_reasoning_item=None` | -| `t*` integration suite | 8 passed, 1 failed | -| `c*` condenser suite | 2 passed, 2 failed, 1 skipped | -| Isolated rerun of `c02` and `c05` | `c05` passed; `c02` failed again | -| Purpose-built parallel-tool replay probe | Passed | - -## Integration suite details - -### `t*` - -Passed: `t01`, `t02`, `t03`, `t04`, `t06`, `t07`, `t08`, and `t09`. - -`t05_simple_browsing` failed twice, including an isolated retry. Chromium launched and the agent navigated the test site, but GPT-5-nano stopped after saying it would fetch the answer instead of reporting it. This is model behavior and does not exercise action batching. - -### `c*` - -- `c01_thinking_block_condenser`: skipped as designed because GPT-5-nano produces Responses API reasoning items rather than Anthropic thinking blocks. -- `c03_delayed_condensation`: passed with five condensations. -- `c04_token_condenser`: passed. -- `c05_size_condenser`: failed initially because the model emitted only one tool call and stopped before enough events existed; it passed on an isolated rerun, confirming model-dependent flakiness. -- `c02_hard_context_reset`: failed twice because GPT-5-nano answered calculation requests directly instead of creating enough tool-loop events for the second condensation to become a normal condensation. - -The `c02` and initial `c05` failures did not produce parallel sibling `ActionEvent`s, so they did not exercise the code changed by this PR. - -## Direct parallel-tool replay validation - -A first live attempt asked GPT-5-nano to issue two calls to the same terminal tool in parallel. The model instead emitted them in two separate LLM responses, confirming that a generic integration task does not reliably cover this regression. - -A second probe exposed two distinct independent tools, `get_alpha` and `get_beta`, and required both before a final response. GPT-5-nano then produced: - -1. two `ActionEvent`s with the same `llm_response_id`; -2. a Responses API reasoning item only on the first action; -3. a recombined assistant message containing both tool calls in order; -4. a reasoning item exactly equal to the first action's item; and -5. a successful follow-up Responses API turn ending with `alpha beta verified`. - -This exercises the PR's changed path end to end: the reasoning item is retained on the recombined message and accepted when the tool-call batch is sent back to GPT-5-nano. - -## Commands - -```bash -uv run --frozen pytest tests/sdk/event/test_events_to_messages.py - -LLM_API_KEY=... \ -LLM_BASE_URL=https://llm-proxy.eval.all-hands.dev \ -IN_DOCKER=true \ -uv run --frozen python tests/integration/run_infer.py \ - --llm-config '{"model":"litellm_proxy/openai/gpt-5-nano","reasoning_effort":"high"}' \ - --num-workers 4 \ - --test-type integration - -LLM_API_KEY=... \ -LLM_BASE_URL=https://llm-proxy.eval.all-hands.dev \ -IN_DOCKER=true \ -uv run --frozen python tests/integration/run_infer.py \ - --llm-config '{"model":"litellm_proxy/openai/gpt-5-nano","reasoning_effort":"high"}' \ - --num-workers 4 \ - --test-type condenser -``` - -## Assessment - -The focused regression and the live parallel-tool replay both validate the fix. The remaining integration failures are explained by GPT-5-nano task compliance and did not execute the changed parallel-action reconstruction path.