mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-27 17:58:07 +00:00
feat(agent): surface a labeled reasoning excerpt at the empty-response terminal
When the empty-response ladder is fully exhausted (thinking-prefill continuation, empty-content retries, provider fallback) and the model produced structured reasoning but never any visible text, deliver a clearly labeled excerpt of that reasoning instead of a bare '(empty)' — the reasoning frequently contains the actual answer. Delivery-only by design: raw chain-of-thought is never promoted to a normal answer earlier in the ladder (prefill continuation still gets first crack, retries and fallback still run), transcript persistence semantics are untouched (the '(empty)' sentinel scaffolding keeps its replay-safety behavior), and a truly empty exhaustion still returns the existing terminal. Idea credit: PR #48795 (@ligl0325) proposed falling back to reasoning_content on empty content; this lands the safe kernel of that idea at the one point in the ladder where it is strictly an improvement.
This commit is contained in:
parent
a751924c04
commit
214ae7b77c
2 changed files with 183 additions and 1 deletions
|
|
@ -6308,7 +6308,28 @@ def run_conversation(
|
|||
". No fallback providers configured.")
|
||||
)
|
||||
|
||||
final_response = "(empty)"
|
||||
# Deliver a labeled reasoning excerpt instead of a bare
|
||||
# "(empty)" when the model DID think but never produced
|
||||
# visible text. This is delivery-only: the persisted
|
||||
# assistant message above keeps the "(empty)" sentinel
|
||||
# (its replay semantics prevent empty-response loops),
|
||||
# and raw chain-of-thought is never promoted to a normal
|
||||
# answer earlier in the ladder — prefill continuation,
|
||||
# empty-content retries, and provider fallback all run
|
||||
# first. Only at this terminal, where the alternative is
|
||||
# returning nothing, is showing the model's own reasoning
|
||||
# (clearly labeled as such) strictly more useful.
|
||||
# Idea credit: PR #48795 (@ligl0325).
|
||||
if reasoning_text:
|
||||
final_response = (
|
||||
"⚠️ The model produced only internal reasoning and "
|
||||
"no final answer, despite retries"
|
||||
+ (" and fallback" if agent._fallback_chain else "")
|
||||
+ ". Its last reasoning, which may contain the "
|
||||
"answer:\n\n" + reasoning_preview
|
||||
)
|
||||
else:
|
||||
final_response = "(empty)"
|
||||
break
|
||||
|
||||
# Reset retry counter/signature on successful content
|
||||
|
|
|
|||
161
tests/run_agent/test_empty_terminal_reasoning_surface.py
Normal file
161
tests/run_agent/test_empty_terminal_reasoning_surface.py
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
"""Tests for the empty-terminal reasoning surface.
|
||||
|
||||
When the empty-response ladder is fully exhausted (prefill continuation,
|
||||
empty-content retries, provider fallback) and the model produced structured
|
||||
reasoning but no visible text, the DELIVERED final_response is a clearly
|
||||
labeled reasoning excerpt instead of a bare "(empty)" — the reasoning often
|
||||
contains the actual answer. Idea credit: PR #48795 (@ligl0325).
|
||||
|
||||
Invariants pinned here:
|
||||
- The persisted assistant message keeps the "(empty)" sentinel and the
|
||||
``_empty_terminal_sentinel`` marker (replay semantics unchanged).
|
||||
- Raw reasoning is NEVER promoted earlier in the ladder — a reasoning-only
|
||||
response still goes through prefill continuation first.
|
||||
- A truly empty exhaustion (no reasoning either) still returns "(empty)".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
|
||||
# Stub optional heavy imports so run_agent imports cleanly in isolation.
|
||||
sys.modules.setdefault("fire", types.SimpleNamespace(Fire=lambda *a, **k: None))
|
||||
sys.modules.setdefault("firecrawl", types.SimpleNamespace(Firecrawl=object))
|
||||
sys.modules.setdefault("fal_client", types.SimpleNamespace())
|
||||
|
||||
|
||||
def _build_agent(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
(tmp_path / "config.yaml").write_text("{}\n", encoding="utf-8")
|
||||
from run_agent import AIAgent
|
||||
|
||||
agent = AIAgent(
|
||||
model="test-model",
|
||||
api_key="sk-dummy",
|
||||
base_url="https://example.invalid/v1",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
platform="cli",
|
||||
)
|
||||
# Route through the non-streaming _interruptible_api_call path so the
|
||||
# monkeypatched fake responses are what the loop consumes.
|
||||
agent._disable_streaming = True
|
||||
return agent
|
||||
|
||||
|
||||
def _reasoning_only_response():
|
||||
return SimpleNamespace(
|
||||
choices=[SimpleNamespace(
|
||||
message=SimpleNamespace(
|
||||
content="",
|
||||
reasoning="The answer is 42 because of the calculation above.",
|
||||
reasoning_content=None,
|
||||
reasoning_details=None,
|
||||
tool_calls=None,
|
||||
),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
usage=None,
|
||||
model="test-model",
|
||||
)
|
||||
|
||||
|
||||
def _truly_empty_response():
|
||||
return SimpleNamespace(
|
||||
choices=[SimpleNamespace(
|
||||
message=SimpleNamespace(
|
||||
content="",
|
||||
reasoning=None,
|
||||
reasoning_content=None,
|
||||
reasoning_details=None,
|
||||
tool_calls=None,
|
||||
),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
usage=None,
|
||||
model="test-model",
|
||||
)
|
||||
|
||||
|
||||
def test_exhausted_reasoning_only_delivers_labeled_excerpt(tmp_path, monkeypatch):
|
||||
"""After the full ladder is exhausted on reasoning-only responses, the
|
||||
delivered text is the labeled excerpt — not a bare '(empty)' — while the
|
||||
transcript keeps its existing sentinel-scaffolding semantics."""
|
||||
agent = _build_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
agent, "_interruptible_api_call",
|
||||
lambda api_kwargs: _reasoning_only_response(),
|
||||
)
|
||||
|
||||
result = agent.run_conversation("what is the answer?")
|
||||
|
||||
final = result["final_response"]
|
||||
assert "(empty)" != final
|
||||
assert "only internal reasoning" in final
|
||||
assert "The answer is 42" in final
|
||||
|
||||
# Persistence semantics unchanged: the delivered excerpt is
|
||||
# delivery-only. The turn finalizer strips the "(empty)" terminal
|
||||
# sentinel from the transcript tail (replay safety, existing design),
|
||||
# and the labeled excerpt must never be persisted as assistant content.
|
||||
assert not any(
|
||||
m.get("role") == "assistant"
|
||||
and "only internal reasoning" in (m.get("content") or "")
|
||||
for m in result["messages"]
|
||||
)
|
||||
|
||||
|
||||
def test_exhausted_truly_empty_keeps_existing_behavior(tmp_path, monkeypatch):
|
||||
"""No reasoning anywhere → behavior unchanged from main: the '(empty)'
|
||||
terminal (possibly rewritten by the downstream turn-completion explainer)
|
||||
is delivered, and no reasoning excerpt appears."""
|
||||
agent = _build_agent(tmp_path, monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
agent, "_interruptible_api_call",
|
||||
lambda api_kwargs: _truly_empty_response(),
|
||||
)
|
||||
|
||||
result = agent.run_conversation("hello?")
|
||||
|
||||
final = result["final_response"]
|
||||
# Either the raw sentinel (explainer off) or the explainer's rewrite —
|
||||
# never the reasoning-excerpt frame, which requires reasoning to exist.
|
||||
assert final == "(empty)" or final.startswith("⚠️ No reply:")
|
||||
assert "only internal reasoning" not in final
|
||||
|
||||
|
||||
def test_reasoning_never_promoted_before_ladder_exhaustion(tmp_path, monkeypatch):
|
||||
"""A reasoning-only response must first go through prefill continuation —
|
||||
if the model then produces real text, THAT is the answer, and no labeled
|
||||
reasoning excerpt appears."""
|
||||
agent = _build_agent(tmp_path, monkeypatch)
|
||||
responses = [
|
||||
_reasoning_only_response(),
|
||||
SimpleNamespace(
|
||||
choices=[SimpleNamespace(
|
||||
message=SimpleNamespace(
|
||||
content="42.",
|
||||
reasoning=None,
|
||||
reasoning_content=None,
|
||||
reasoning_details=None,
|
||||
tool_calls=None,
|
||||
),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
usage=None,
|
||||
model="test-model",
|
||||
),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
agent, "_interruptible_api_call",
|
||||
lambda api_kwargs: responses.pop(0),
|
||||
)
|
||||
|
||||
result = agent.run_conversation("what is the answer?")
|
||||
|
||||
assert result["final_response"] == "42."
|
||||
assert "only internal reasoning" not in result["final_response"]
|
||||
Loading…
Add table
Add a link
Reference in a new issue