fix(agent): never replay chain-of-thought in the active-turn redirect checkpoint

A /steer redirect during a thinking phase serialized the streamed
reasoning into the persisted assistant checkpoint ('Reasoning shown
before the interruption: ...'). An assistant turn exposing its own
chain-of-thought reads to Anthropic's output classifier as
reasoning-injection/prefill jailbreak, so every subsequent call on the
session deterministically returned 'Provider returned an empty
response' — and because the checkpoint is persisted and replayed, no
retry, nudge, or empty-recovery branch could ever escape it. Four
sessions were permanently bricked this way in the week of Jul 21-27
(42+ blocked calls; every reasoning-free checkpoint that week was
untouched — same mechanism as the prefill.json incident).

Class fix: streamed reasoning is now display-only state. The
_current_streamed_reasoning_text accumulator is removed entirely
(producer in _fire_reasoning_delta, resets, and init), so no future
path can serialize CoT into replayable content. The checkpoint keeps
only the visible response text; the model regenerates its reasoning on
the retried turn. Invariant documented in _apply_active_turn_redirect.

Regression tests: CoT never appears in either checkpoint shape,
reasoning-only interrupts produce a bare checkpoint, reasoning deltas
stay display-only.
This commit is contained in:
teknium1 2026-07-27 14:57:11 -07:00 committed by Teknium
parent 96db67b849
commit cf0c42fa0b
5 changed files with 79 additions and 37 deletions

View file

@ -958,12 +958,6 @@ def init_agent(
agent._stream_writer_tls = threading.local()
agent._stream_writer_dropped = 0
# Displayed reasoning text streamed during the current model response,
# captured only when a surface consumed it via a reasoning callback. Used
# by active-turn redirect to checkpoint what the user actually saw without
# ever persisting hidden provider reasoning.
agent._current_streamed_reasoning_text = ""
# Optional current-turn user-message override used when the API-facing
# user message intentionally differs from the persisted transcript
# (e.g. CLI voice mode adds a temporary prefix for the live call only).

View file

@ -121,22 +121,31 @@ def _apply_active_turn_redirect(agent: Any, messages: List[Dict[str, Any]], text
Incomplete provider reasoning blocks are not valid replay items (Anthropic
signs them; Responses reasoning items require their following output).
Preserve only what Hermes actually displayed, demoted to ordinary text,
then add the correction as a real user message. This keeps role alternation
Preserve only the *visible* response text, demoted to ordinary text, then
add the correction as a real user message. This keeps role alternation
valid and leaves every previously cached message byte-for-byte unchanged.
INVARIANT raw chain-of-thought must never be serialized into replayable
message content. Streamed reasoning is display-only state: it may be shown
live, but it does not re-enter the transcript as assistant (or user) text.
An assistant turn whose content inlines its own chain-of-thought reads to
Anthropic's output classifier as reasoning-injection/prefill jailbreak,
and because the poisoned checkpoint is persisted and replayed on every
subsequent call, the session dies permanently with deterministic
"Provider returned an empty response" storms that no retry, nudge, or
empty-recovery branch can escape (July 2026: four sessions bricked this
way; every reasoning-free checkpoint that week was untouched same
mechanism as the ~/.hermes/prefill.json incident, 20/20 blocked with
assistant-exposed CoT vs 0/20 without). The interrupted reasoning was
incomplete by definition; the model regenerates it on the retried turn.
If a future path needs to preserve interrupted thinking, carry it in a
provider-gated reasoning *field*, never in content.
"""
reasoning = str(
getattr(agent, "_current_streamed_reasoning_text", "") or ""
).strip()
visible = agent._strip_think_blocks(
getattr(agent, "_current_streamed_assistant_text", "") or ""
).strip()
checkpoint_parts = ["[This response was interrupted by a user correction.]"]
if reasoning:
checkpoint_parts.extend(
["Reasoning shown before the interruption:", reasoning]
)
if visible:
checkpoint_parts.extend(
["Visible response before the interruption:", visible]
@ -159,7 +168,6 @@ def _apply_active_turn_redirect(agent: Any, messages: List[Dict[str, Any]], text
messages.append({"role": "user", "content": text})
agent._current_streamed_assistant_text = ""
agent._current_streamed_reasoning_text = ""
agent._stream_needs_break = True

View file

@ -5324,7 +5324,6 @@ class AIAgent:
pass
self._record_streamed_assistant_text(tail)
self._current_streamed_assistant_text = ""
self._current_streamed_reasoning_text = ""
def _record_streamed_assistant_text(self, text: str) -> None:
"""Accumulate visible assistant text emitted through stream callbacks."""
@ -5665,15 +5664,6 @@ class AIAgent:
cb(text)
except Exception:
pass
else:
# Only checkpoint reasoning that a surface actually displayed.
# show_reasoning=false leaves the callback unset, so hidden
# provider thinking never becomes visible transcript content.
if isinstance(text, str) and text:
self._current_streamed_reasoning_text = (
getattr(self, "_current_streamed_reasoning_text", "")
+ text
)
def _fire_tool_gen_started(self, tool_name: str) -> None:
"""Notify display layer that the model is generating tool call arguments.

View file

@ -4894,7 +4894,8 @@ class TestRunConversation:
}
def test_redirect_during_thinking_retries_same_turn_with_context(self, agent):
"""A corrective follow-up keeps displayed reasoning and does not end the turn."""
"""A corrective follow-up does not end the turn, and displayed reasoning
never re-enters the transcript (classifier-poisoning guard)."""
self._setup_agent(agent)
agent.reasoning_callback = lambda _text: None
final = _mock_response(content="Using Postgres instead.", finish_reason="stop")
@ -4936,7 +4937,11 @@ class TestRunConversation:
]
checkpoint = replay[-2]["content"]
assert "interrupted by a user correction" in checkpoint
assert "I should implement this with SQLite." in checkpoint
# Displayed chain-of-thought must NOT be replayed: an assistant turn
# inlining its own reasoning trips Anthropic's output classifier and
# bricks the session with deterministic empty responses (July 2026).
assert "I should implement this with SQLite." not in checkpoint
assert "Reasoning shown before the interruption" not in checkpoint
assert replay[-1]["content"] == "No, use Postgres instead."
assert agent._pending_redirect is None
assert any(
@ -5018,7 +5023,10 @@ class TestRunConversation:
assert results["result"]["completed"] is True
assert results["result"]["final_response"] == "Corrected answer."
checkpoint = results["result"]["messages"][-3]
assert "Following the original approach." in checkpoint["content"]
assert "interrupted by a user correction" in checkpoint["content"]
# Displayed reasoning is display-only — replaying it as assistant
# content trips Anthropic's output classifier (July 2026 brickings).
assert "Following the original approach." not in checkpoint["content"]
assert results["result"]["messages"][-2]["content"] == (
"Use the corrected approach."
)

View file

@ -35,7 +35,6 @@ def _bare_agent() -> AIAgent:
agent._active_children_lock = threading.Lock()
agent._tool_worker_threads = None
agent._tool_worker_threads_lock = None
agent._current_streamed_reasoning_text = ""
agent._current_streamed_assistant_text = ""
agent._stream_needs_break = False
agent._strip_think_blocks = lambda content: content
@ -125,14 +124,20 @@ class TestActiveTurnRedirect:
assert agent.redirect("too late") is False
assert agent._pending_redirect is None
def test_hidden_reasoning_is_not_checkpointed(self):
def test_reasoning_deltas_are_display_only(self):
"""Streamed reasoning must never accumulate into replayable transcript
state an assistant checkpoint that inlines chain-of-thought trips
Anthropic's output classifier and permanently bricks the session
(deterministic empty-response storms on every replay)."""
agent = _bare_agent()
agent.reasoning_callback = None
agent._current_streamed_reasoning_text = ""
seen = []
agent.reasoning_callback = seen.append
agent._fire_reasoning_delta("private provider thinking")
agent._fire_reasoning_delta("visible provider thinking")
assert agent._current_streamed_reasoning_text == ""
# Displayed to the surface, but never checkpointed anywhere.
assert seen == ["visible provider thinking"]
assert not getattr(agent, "_current_streamed_reasoning_text", "")
def test_response_completion_before_redirect_lock_rejects_correction(self):
agent = _bare_agent()
@ -227,7 +232,6 @@ class TestActiveTurnRedirectCheckpoint:
from agent.conversation_loop import _apply_active_turn_redirect
agent = _bare_agent()
agent._current_streamed_reasoning_text = "Shown reasoning."
agent._current_streamed_assistant_text = "Visible draft."
messages = [
{"role": "user", "content": "start"},
@ -240,10 +244,48 @@ class TestActiveTurnRedirectCheckpoint:
assert messages[-1]["role"] == "user"
assert messages[-1]["content"].endswith("Use Postgres instead.")
assert sum(1 for m in messages if m["role"] == "assistant") == 1
assert "Shown reasoning." in messages[-1]["content"]
assert "Visible draft." in messages[-1]["content"]
assert "Context from the interrupted assistant response" in messages[-1]["content"]
def test_checkpoint_never_replays_chain_of_thought(self):
"""Raw CoT serialized into checkpoint content reads to Anthropic's
output classifier as reasoning-injection; because the checkpoint is
persisted and replayed on every later call, one redirect during a
thinking phase permanently bricked sessions with deterministic
empty-response storms (July 2026). Reasoning must never appear in
replayable content in either the assistant-checkpoint or the
merged-user-correction shape."""
from agent.conversation_loop import _apply_active_turn_redirect
for tail_role in ("user", "assistant"):
agent = _bare_agent()
# Simulate a surface having displayed reasoning this turn.
agent._current_streamed_reasoning_text = "SECRET chain of thought."
agent._current_streamed_assistant_text = "Visible draft."
messages = [{"role": "user", "content": "start"}]
if tail_role == "assistant":
messages.append({"role": "assistant", "content": "committed"})
_apply_active_turn_redirect(agent, messages, "Change course.")
serialized = "".join(str(m.get("content", "")) for m in messages)
assert "SECRET chain of thought." not in serialized
assert "Reasoning shown before the interruption" not in serialized
assert "Visible draft." in serialized
def test_checkpoint_omits_reasoning_label_when_nothing_visible(self):
from agent.conversation_loop import _apply_active_turn_redirect
agent = _bare_agent()
agent._current_streamed_reasoning_text = "thinking only, no text yet"
messages = [{"role": "user", "content": "start"}]
_apply_active_turn_redirect(agent, messages, "New direction.")
checkpoint = messages[-2]["content"]
assert checkpoint == "[This response was interrupted by a user correction.]"
assert messages[-1]["content"] == "New direction."
class TestSteerInjection:
def test_appends_to_last_tool_result(self):