diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 9eb3fdad4b04..86d26690b7fd 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -6308,7 +6308,28 @@ def run_conversation( ". No fallback providers configured.") ) - final_response = "(empty)" + # Deliver a labeled reasoning excerpt instead of a bare + # "(empty)" when the model DID think but never produced + # visible text. This is delivery-only: the persisted + # assistant message above keeps the "(empty)" sentinel + # (its replay semantics prevent empty-response loops), + # and raw chain-of-thought is never promoted to a normal + # answer earlier in the ladder — prefill continuation, + # empty-content retries, and provider fallback all run + # first. Only at this terminal, where the alternative is + # returning nothing, is showing the model's own reasoning + # (clearly labeled as such) strictly more useful. + # Idea credit: PR #48795 (@ligl0325). + if reasoning_text: + final_response = ( + "⚠️ The model produced only internal reasoning and " + "no final answer, despite retries" + + (" and fallback" if agent._fallback_chain else "") + + ". Its last reasoning, which may contain the " + "answer:\n\n" + reasoning_preview + ) + else: + final_response = "(empty)" break # Reset retry counter/signature on successful content diff --git a/tests/run_agent/test_empty_terminal_reasoning_surface.py b/tests/run_agent/test_empty_terminal_reasoning_surface.py new file mode 100644 index 000000000000..7cfd1aea02c9 --- /dev/null +++ b/tests/run_agent/test_empty_terminal_reasoning_surface.py @@ -0,0 +1,161 @@ +"""Tests for the empty-terminal reasoning surface. + +When the empty-response ladder is fully exhausted (prefill continuation, +empty-content retries, provider fallback) and the model produced structured +reasoning but no visible text, the DELIVERED final_response is a clearly +labeled reasoning excerpt instead of a bare "(empty)" — the reasoning often +contains the actual answer. Idea credit: PR #48795 (@ligl0325). + +Invariants pinned here: +- The persisted assistant message keeps the "(empty)" sentinel and the + ``_empty_terminal_sentinel`` marker (replay semantics unchanged). +- Raw reasoning is NEVER promoted earlier in the ladder — a reasoning-only + response still goes through prefill continuation first. +- A truly empty exhaustion (no reasoning either) still returns "(empty)". +""" + +from __future__ import annotations + +import sys +import types +from types import SimpleNamespace + +# Stub optional heavy imports so run_agent imports cleanly in isolation. +sys.modules.setdefault("fire", types.SimpleNamespace(Fire=lambda *a, **k: None)) +sys.modules.setdefault("firecrawl", types.SimpleNamespace(Firecrawl=object)) +sys.modules.setdefault("fal_client", types.SimpleNamespace()) + + +def _build_agent(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / ".env").write_text("", encoding="utf-8") + (tmp_path / "config.yaml").write_text("{}\n", encoding="utf-8") + from run_agent import AIAgent + + agent = AIAgent( + model="test-model", + api_key="sk-dummy", + base_url="https://example.invalid/v1", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + platform="cli", + ) + # Route through the non-streaming _interruptible_api_call path so the + # monkeypatched fake responses are what the loop consumes. + agent._disable_streaming = True + return agent + + +def _reasoning_only_response(): + return SimpleNamespace( + choices=[SimpleNamespace( + message=SimpleNamespace( + content="", + reasoning="The answer is 42 because of the calculation above.", + reasoning_content=None, + reasoning_details=None, + tool_calls=None, + ), + finish_reason="stop", + )], + usage=None, + model="test-model", + ) + + +def _truly_empty_response(): + return SimpleNamespace( + choices=[SimpleNamespace( + message=SimpleNamespace( + content="", + reasoning=None, + reasoning_content=None, + reasoning_details=None, + tool_calls=None, + ), + finish_reason="stop", + )], + usage=None, + model="test-model", + ) + + +def test_exhausted_reasoning_only_delivers_labeled_excerpt(tmp_path, monkeypatch): + """After the full ladder is exhausted on reasoning-only responses, the + delivered text is the labeled excerpt — not a bare '(empty)' — while the + transcript keeps its existing sentinel-scaffolding semantics.""" + agent = _build_agent(tmp_path, monkeypatch) + monkeypatch.setattr( + agent, "_interruptible_api_call", + lambda api_kwargs: _reasoning_only_response(), + ) + + result = agent.run_conversation("what is the answer?") + + final = result["final_response"] + assert "(empty)" != final + assert "only internal reasoning" in final + assert "The answer is 42" in final + + # Persistence semantics unchanged: the delivered excerpt is + # delivery-only. The turn finalizer strips the "(empty)" terminal + # sentinel from the transcript tail (replay safety, existing design), + # and the labeled excerpt must never be persisted as assistant content. + assert not any( + m.get("role") == "assistant" + and "only internal reasoning" in (m.get("content") or "") + for m in result["messages"] + ) + + +def test_exhausted_truly_empty_keeps_existing_behavior(tmp_path, monkeypatch): + """No reasoning anywhere → behavior unchanged from main: the '(empty)' + terminal (possibly rewritten by the downstream turn-completion explainer) + is delivered, and no reasoning excerpt appears.""" + agent = _build_agent(tmp_path, monkeypatch) + monkeypatch.setattr( + agent, "_interruptible_api_call", + lambda api_kwargs: _truly_empty_response(), + ) + + result = agent.run_conversation("hello?") + + final = result["final_response"] + # Either the raw sentinel (explainer off) or the explainer's rewrite — + # never the reasoning-excerpt frame, which requires reasoning to exist. + assert final == "(empty)" or final.startswith("⚠️ No reply:") + assert "only internal reasoning" not in final + + +def test_reasoning_never_promoted_before_ladder_exhaustion(tmp_path, monkeypatch): + """A reasoning-only response must first go through prefill continuation — + if the model then produces real text, THAT is the answer, and no labeled + reasoning excerpt appears.""" + agent = _build_agent(tmp_path, monkeypatch) + responses = [ + _reasoning_only_response(), + SimpleNamespace( + choices=[SimpleNamespace( + message=SimpleNamespace( + content="42.", + reasoning=None, + reasoning_content=None, + reasoning_details=None, + tool_calls=None, + ), + finish_reason="stop", + )], + usage=None, + model="test-model", + ), + ] + monkeypatch.setattr( + agent, "_interruptible_api_call", + lambda api_kwargs: responses.pop(0), + ) + + result = agent.run_conversation("what is the answer?") + + assert result["final_response"] == "42." + assert "only internal reasoning" not in result["final_response"]