mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-20 15:33:54 +00:00
fix(agent): treat Codex incomplete content filter as refusal
Map Codex Responses status=incomplete with incomplete_details.reason=content_filter to finish_reason=content_filter so the existing refusal/fallback path runs instead of burning incomplete continuation attempts.
This commit is contained in:
parent
704bbcca80
commit
fe1ab949fd
6 changed files with 152 additions and 10 deletions
|
|
@ -1118,6 +1118,22 @@ def _normalize_codex_response(
|
|||
differs from the one that minted the encrypted_content blob and drop
|
||||
the item instead of triggering HTTP 400 invalid_encrypted_content.
|
||||
"""
|
||||
response_status = getattr(response, "status", None)
|
||||
if isinstance(response_status, str):
|
||||
response_status = response_status.strip().lower()
|
||||
else:
|
||||
response_status = None
|
||||
|
||||
incomplete_details = getattr(response, "incomplete_details", None)
|
||||
incomplete_reason = ""
|
||||
if isinstance(incomplete_details, dict):
|
||||
incomplete_reason = str(incomplete_details.get("reason") or "").strip().lower()
|
||||
elif incomplete_details is not None:
|
||||
incomplete_reason = str(getattr(incomplete_details, "reason", "") or "").strip().lower()
|
||||
response_incomplete_content_filter = (
|
||||
response_status == "incomplete" and incomplete_reason == "content_filter"
|
||||
)
|
||||
|
||||
output = getattr(response, "output", None)
|
||||
if not isinstance(output, list) or not output:
|
||||
# The Codex backend can return empty output when the answer was
|
||||
|
|
@ -1134,15 +1150,18 @@ def _normalize_codex_response(
|
|||
content=[SimpleNamespace(type="output_text", text=out_text.strip())],
|
||||
)]
|
||||
response.output = output
|
||||
elif response_incomplete_content_filter:
|
||||
# This is a deterministic provider safety block, not a partial
|
||||
# answer. Synthesize an empty message so finish_reason below becomes
|
||||
# content_filter and the conversation loop can fallback/surface it
|
||||
# instead of burning three continuation attempts.
|
||||
output = [SimpleNamespace(
|
||||
type="message", role="assistant", status="completed", content=[]
|
||||
)]
|
||||
response.output = output
|
||||
else:
|
||||
raise RuntimeError("Responses API returned no output items")
|
||||
|
||||
response_status = getattr(response, "status", None)
|
||||
if isinstance(response_status, str):
|
||||
response_status = response_status.strip().lower()
|
||||
else:
|
||||
response_status = None
|
||||
|
||||
if response_status in {"failed", "cancelled"}:
|
||||
error_obj = getattr(response, "error", None)
|
||||
error_msg = _format_responses_error(error_obj, response_status)
|
||||
|
|
@ -1411,6 +1430,8 @@ def _normalize_codex_response(
|
|||
|
||||
if tool_calls:
|
||||
finish_reason = "tool_calls"
|
||||
elif response_incomplete_content_filter:
|
||||
finish_reason = "content_filter"
|
||||
elif leaked_tool_call_text:
|
||||
finish_reason = "incomplete"
|
||||
elif saw_streaming_or_item_incomplete:
|
||||
|
|
|
|||
|
|
@ -1642,12 +1642,16 @@ def run_conversation(
|
|||
# Check finish_reason before proceeding
|
||||
if agent.api_mode == "codex_responses":
|
||||
status = getattr(response, "status", None)
|
||||
if isinstance(status, str):
|
||||
status = status.strip().lower()
|
||||
incomplete_details = getattr(response, "incomplete_details", None)
|
||||
incomplete_reason = None
|
||||
if isinstance(incomplete_details, dict):
|
||||
incomplete_reason = incomplete_details.get("reason")
|
||||
else:
|
||||
incomplete_reason = getattr(incomplete_details, "reason", None)
|
||||
if incomplete_reason is not None:
|
||||
incomplete_reason = str(incomplete_reason).strip().lower()
|
||||
if status == "incomplete" and incomplete_reason in {"max_output_tokens", "length"}:
|
||||
# Responses API max-output exhaustion is a normal
|
||||
# Codex incomplete turn. Let the Codex-specific
|
||||
|
|
@ -1657,6 +1661,8 @@ def run_conversation(
|
|||
# emits "Response truncated due to output length
|
||||
# limit" and stops gateway turns.
|
||||
finish_reason = "incomplete"
|
||||
elif status == "incomplete" and incomplete_reason == "content_filter":
|
||||
finish_reason = "content_filter"
|
||||
else:
|
||||
finish_reason = "stop"
|
||||
elif agent.api_mode == "anthropic_messages":
|
||||
|
|
|
|||
|
|
@ -435,15 +435,27 @@ class ResponsesApiTransport(ProviderTransport):
|
|||
def validate_response(self, response: Any) -> bool:
|
||||
"""Check Codex Responses API response has valid output structure.
|
||||
|
||||
Returns True only if response.output is a non-empty list.
|
||||
Does NOT check output_text fallback — the caller handles that
|
||||
with diagnostic logging for stream backfill recovery.
|
||||
Returns True only if response.output is a non-empty list. Also treats
|
||||
terminal content-filter incomplete responses as valid: the Responses API
|
||||
may return status=incomplete with incomplete_details.reason='content_filter'
|
||||
and no output items. That is a provider refusal signal, not a malformed
|
||||
response, and must reach normalization so the agent loop can use the
|
||||
content-policy / fallback path instead of invalid-response retries.
|
||||
|
||||
Does NOT check output_text fallback — the caller handles that with
|
||||
diagnostic logging for stream backfill recovery.
|
||||
"""
|
||||
if response is None:
|
||||
return False
|
||||
output = getattr(response, "output", None)
|
||||
if not isinstance(output, list) or not output:
|
||||
return False
|
||||
status = str(getattr(response, "status", "") or "").strip().lower()
|
||||
incomplete_details = getattr(response, "incomplete_details", None)
|
||||
if isinstance(incomplete_details, dict):
|
||||
reason = str(incomplete_details.get("reason") or "").strip().lower()
|
||||
else:
|
||||
reason = str(getattr(incomplete_details, "reason", "") or "").strip().lower()
|
||||
return status == "incomplete" and reason == "content_filter"
|
||||
return True
|
||||
|
||||
def preflight_kwargs(
|
||||
|
|
|
|||
|
|
@ -79,6 +79,21 @@ def test_normalize_codex_response_treats_summary_only_reasoning_as_incomplete():
|
|||
assert assistant_message.codex_reasoning_items is None
|
||||
|
||||
|
||||
def test_normalize_codex_response_maps_incomplete_content_filter_to_refusal():
|
||||
response = SimpleNamespace(
|
||||
status="incomplete",
|
||||
incomplete_details=SimpleNamespace(reason="content_filter"),
|
||||
output=[],
|
||||
output_text="",
|
||||
)
|
||||
|
||||
assistant_message, finish_reason = _normalize_codex_response(response)
|
||||
|
||||
assert finish_reason == "content_filter"
|
||||
assert assistant_message.content == ""
|
||||
assert response.output
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server-side built-in tool calls (xAI native web_search, code interpreter,
|
||||
# etc.) come back as discrete ``*_call`` output items that xAI's
|
||||
|
|
|
|||
|
|
@ -673,6 +673,34 @@ class TestCodexValidateResponse:
|
|||
r = SimpleNamespace(output=None, output_text="Some text")
|
||||
assert transport.validate_response(r) is False
|
||||
|
||||
def test_empty_output_content_filter_incomplete_is_valid(self, transport):
|
||||
r = SimpleNamespace(
|
||||
status="incomplete",
|
||||
incomplete_details=SimpleNamespace(reason="content_filter"),
|
||||
output=[],
|
||||
output_text="",
|
||||
)
|
||||
assert transport.validate_response(r) is True
|
||||
|
||||
def test_empty_output_content_filter_dict_incomplete_is_valid(self, transport):
|
||||
r = SimpleNamespace(
|
||||
status=" incomplete ",
|
||||
incomplete_details={"reason": " content_filter "},
|
||||
output=[],
|
||||
output_text="",
|
||||
)
|
||||
assert transport.validate_response(r) is True
|
||||
|
||||
@pytest.mark.parametrize("reason", ["max_output_tokens", "length", "", None])
|
||||
def test_empty_output_other_incomplete_reasons_remain_invalid(self, transport, reason):
|
||||
r = SimpleNamespace(
|
||||
status="incomplete",
|
||||
incomplete_details=SimpleNamespace(reason=reason),
|
||||
output=[],
|
||||
output_text="",
|
||||
)
|
||||
assert transport.validate_response(r) is False
|
||||
|
||||
|
||||
class TestCodexMapFinishReason:
|
||||
|
||||
|
|
|
|||
|
|
@ -4149,6 +4149,66 @@ class TestRunConversation:
|
|||
assert result["final_response"] == "Final answer"
|
||||
assert result["completed"] is True
|
||||
|
||||
def test_codex_content_filter_incomplete_routes_to_policy_fallback(self, agent):
|
||||
self._setup_agent(agent)
|
||||
agent.api_mode = "codex_responses"
|
||||
agent.provider = "openai-codex"
|
||||
agent.base_url = "https://chatgpt.com/backend-api/codex"
|
||||
agent._base_url_lower = agent.base_url.lower()
|
||||
agent._base_url_hostname = "chatgpt.com"
|
||||
agent.model = "gpt-5.5"
|
||||
agent._fallback_chain = [
|
||||
{"provider": "openrouter", "model": "anthropic/claude-sonnet-4.7"},
|
||||
]
|
||||
agent._fallback_index = 0
|
||||
|
||||
content_filter_response = SimpleNamespace(
|
||||
status="incomplete",
|
||||
incomplete_details=SimpleNamespace(reason="content_filter"),
|
||||
output=[],
|
||||
output_text="",
|
||||
model="gpt-5.5",
|
||||
usage=None,
|
||||
)
|
||||
fallback_response = SimpleNamespace(
|
||||
status="completed",
|
||||
incomplete_details=None,
|
||||
output=[
|
||||
SimpleNamespace(
|
||||
type="message",
|
||||
status="completed",
|
||||
content=[SimpleNamespace(type="output_text", text="Recovered on fallback")],
|
||||
)
|
||||
],
|
||||
model="fallback/model",
|
||||
usage=None,
|
||||
)
|
||||
hook_events = []
|
||||
|
||||
def _fake_activate(reason=None):
|
||||
agent._fallback_index = len(agent._fallback_chain)
|
||||
return True
|
||||
|
||||
with (
|
||||
patch.object(agent, "_create_request_openai_client", return_value=MagicMock()),
|
||||
patch.object(agent, "_close_request_openai_client"),
|
||||
patch.object(agent, "_run_codex_stream", side_effect=[content_filter_response, fallback_response]) as mock_run_codex_stream,
|
||||
patch.object(agent, "_try_activate_fallback", side_effect=_fake_activate) as mock_try_activate_fallback,
|
||||
patch.object(agent, "_invoke_api_request_error_hook", side_effect=lambda **kw: hook_events.append(kw)),
|
||||
patch.object(agent, "_persist_session"),
|
||||
patch.object(agent, "_save_trajectory"),
|
||||
patch.object(agent, "_cleanup_task_resources"),
|
||||
):
|
||||
result = agent.run_conversation("summarize this large Slack thread")
|
||||
|
||||
assert result["final_response"] == "Recovered on fallback"
|
||||
assert result["completed"] is True
|
||||
mock_try_activate_fallback.assert_called_once_with()
|
||||
assert mock_run_codex_stream.call_count == 2
|
||||
assert hook_events[0]["error_type"] == "ContentPolicyBlocked"
|
||||
assert hook_events[0]["retryable"] is False
|
||||
assert hook_events[0]["reason"] == FailoverReason.content_policy_blocked.value
|
||||
|
||||
def test_ollama_small_runtime_context_fails_before_api_call(self, agent, caplog):
|
||||
self._setup_agent(agent)
|
||||
agent.model = "qwen3.5:9b"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue