Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs-website/docs/pipeline-components/agents-1/agent.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ The `exit_reason` output tells you why the agent stopped, which makes it easy to

- `"text"`: the model returned a complete reply with no tool calls.
- `"length"`: the model reached its output-token limit. `last_message` may contain a partial response.
- `"content_filter"`: a content filter stopped the model response. `last_message` may contain a partial response. See [Content moderation by the provider](../../concepts/guardrails.mdx#content-moderation-by-the-provider).
- `"content_filter"`: a content filter stopped the model response, or the model returned a refusal in `last_message.meta["refusal"]` without tool calls. The original refusal and provider metadata are preserved; `last_message.text` may be absent. See [Content moderation by the provider](../../concepts/guardrails.mdx#content-moderation-by-the-provider).
- the name of the tool that satisfied a tool exit condition. Its result is the last tool-result `ChatMessage` in `messages` whose `tool_call_result.origin.tool_name` matches `exit_reason`; read it from `tool_call_result.result`. When the model calls several tools in one reply, this may not be `last_message`. To make the Agent finish only once a tool exit condition is met, use an `on_exit` hook that sends it back when it stops on `"text"`, as shown in [Requiring a tool call before the Agent finishes](./hooks.mdx#requiring-a-tool-call-before-the-agent-finishes).
- `"max_agent_steps"`: the agent reached `max_agent_steps` before meeting an exit condition.
- a custom reason set by a hook through the `stop_run` state key, such as `"token_budget_exceeded"` from the [`TokenBudgetHook`](./token-budget.mdx).
Expand Down
8 changes: 6 additions & 2 deletions haystack/components/agents/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,9 @@ def _get_model_exit_reason(messages: list[ChatMessage]) -> str | None:
Return the exit reason for a terminal assistant reply without tool calls.

Incomplete generation reasons take precedence over text so callers can distinguish a partial response from a
complete answer. An empty response without a recognized terminal reason does not trigger an exit, preserving the
Agent's recovery behavior for malformed tool calls that a Chat Generator discarded.
complete answer. Refusal metadata also ends the run with the content_filter reason. An empty response without a
recognized terminal reason does not trigger an exit, preserving the Agent's recovery behavior for malformed tool
calls that a Chat Generator discarded.
"""
# If the messages list is empty or the last message has tool calls, don't exit.
if not messages or any(message.tool_call for message in messages):
Expand All @@ -188,6 +189,9 @@ def _get_model_exit_reason(messages: list[ChatMessage]) -> str | None:
if last.meta.get("finish_reason") == _EXIT_REASON_CONTENT_FILTER:
return _EXIT_REASON_CONTENT_FILTER

if isinstance(refusal := last.meta.get("refusal"), str) and refusal.strip():
return _EXIT_REASON_CONTENT_FILTER

# If the last message has text, exit with the text reason.
if last.text:
return _EXIT_REASON_TEXT
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
---
fixes:
- |
Agents now stop on refusal-only replies instead of repeating model requests
until the step limit. The original refusal metadata is preserved.
upgrade:
- |
Replies with a non-empty string in ``ChatMessage.meta["refusal"]`` and no
tool calls now produce ``exit_reason="content_filter"``, including runs without
tools that previously reported ``"text"``. Route these refusals through the
existing ``content_filter`` branch rather than the normal text-answer branch.
40 changes: 40 additions & 0 deletions test/components/agents/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
from haystack.dataclasses.streaming_chunk import StreamingChunk
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.hooks import hook
from haystack.hooks.structured_output import StructuredOutputHook
from haystack.tools import ComponentTool, SearchableToolset, Tool, warm_up_tools, warm_up_tools_async
from haystack.tools.toolset import Toolset
from haystack.utils import Secret
Expand Down Expand Up @@ -720,6 +721,14 @@ class TestGetModelExitReason:
(ChatMessage.from_assistant("Partial answer.", meta={"finish_reason": "length"}), "length"),
(ChatMessage.from_assistant("", meta={"finish_reason": "content_filter"}), "content_filter"),
(ChatMessage.from_assistant("Partial answer.", meta={"finish_reason": "content_filter"}), "content_filter"),
(
ChatMessage.from_assistant(meta={"refusal": "I cannot answer.", "finish_reason": "stop"}),
"content_filter",
),
(ChatMessage.from_assistant("I cannot answer.", meta={"refusal": "I cannot answer."}), "content_filter"),
(ChatMessage.from_assistant(meta={"refusal": "I cannot answer.", "finish_reason": "length"}), "length"),
(ChatMessage.from_assistant(meta={"refusal": " "}), None),
(ChatMessage.from_assistant(meta={"refusal": {"message": "not a refusal string"}}), None),
(ChatMessage.from_assistant(""), None),
(ChatMessage.from_user("Not an assistant reply."), None),
],
Expand All @@ -730,6 +739,13 @@ def test_classifies_tool_call_free_model_replies(self, message, expected):
def test_empty_message_batch_is_not_an_exit(self):
assert _get_model_exit_reason([]) is None

def test_refusal_metadata_does_not_override_tool_calls(self):
reply = ChatMessage.from_assistant(
tool_calls=[ToolCall(tool_name="weather", arguments={"city": "Berlin"}, id="call-1")],
meta={"refusal": "I cannot answer."},
)
assert _get_model_exit_reason([reply]) is None


class TestAgentRun:
def test_agent_with_no_tools(self):
Expand Down Expand Up @@ -1266,6 +1282,30 @@ def test_incomplete_model_reply_without_tools_preserves_reason(self, finish_reas
assert result["step_count"] == 1
assert result["exit_reason"] == finish_reason

@pytest.mark.asyncio
@pytest.mark.parametrize("async_mode", [False, True])
@pytest.mark.parametrize("with_tools", [False, True])
@pytest.mark.parametrize("structured_output", [False, True])
async def test_refusal_stops_without_retrying(self, weather_tool, async_mode, with_tools, structured_output):
refusal = ChatMessage.from_assistant(meta={"refusal": "I cannot answer.", "finish_reason": "stop"})
agent = Agent(
chat_generator=MockChatGenerator([refusal, "This reply must not be requested."]),
tools=[weather_tool] if with_tools else [],
state_schema={"structured_output": {"type": dict}} if structured_output else None,
hooks={"on_exit": [StructuredOutputHook(schema={"type": "object"})]} if structured_output else None,
)
messages = [ChatMessage.from_user("A synthetic request.")]

result = await agent.run_async(messages) if async_mode else agent.run(messages)

assert result["step_count"] == 1
assert result["exit_reason"] == "content_filter"
assert result["last_message"].text is None
assert result["last_message"].meta["refusal"] == "I cannot answer."
assert result["last_message"].meta["finish_reason"] == "stop"
if structured_output:
assert result["structured_output"] is None

def test_exits_on_empty_assistant_message_truncated_by_output_limit(self, monkeypatch, weather_tool):
monkeypatch.setenv("OPENAI_API_KEY", "fake-key")
agent = Agent(chat_generator=OpenAIChatGenerator(), tools=[weather_tool], exit_conditions=["text"])
Expand Down
14 changes: 9 additions & 5 deletions test/components/agents/test_agent_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -843,17 +843,21 @@ async def require_save_async(state: State) -> None:
assert result["tool_call_counts"]["save"] == 1

@pytest.mark.asyncio
async def test_incomplete_generation_can_recover(self):
@pytest.mark.parametrize(
"reply",
[
ChatMessage.from_assistant("Partial", meta={"finish_reason": "content_filter"}),
ChatMessage.from_assistant(meta={"refusal": "I cannot answer.", "finish_reason": "stop"}),
],
)
async def test_incomplete_generation_can_recover(self, reply):
agent = _agent(
MockChatGenerator(),
state_schema={"seen_reasons": {"type": list[str]}},
hooks={"on_exit": [request_safe_answer]},
)
agent.chat_generator.run_async = AsyncMock(
side_effect=[
{"replies": [ChatMessage.from_assistant("Partial", meta={"finish_reason": "content_filter"})]},
{"replies": [ChatMessage.from_assistant("Safe recovered answer")]},
]
side_effect=[{"replies": [reply]}, {"replies": [ChatMessage.from_assistant("Safe recovered answer")]}]
)

result = await agent.run_async(messages=[ChatMessage.from_user("hi")])
Expand Down