diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 3425f0970f5..7c85ae8ff54 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -4944,12 +4944,17 @@ def run_conversation( getattr(agent, "_verification_stop_nudges", 0) + 1 ) final_msg["finish_reason"] = "verification_required" + final_msg["_verification_stop_synthetic"] = True messages.append(final_msg) # Keep the attempted final answer in model history so the # synthetic user nudge preserves role alternation, but do # not surface it to the user as an interim answer. The # whole point of this guard is to prevent premature - # "done" claims before checks run. + # "done" claims before checks run. Both the attempted + # answer and the nudge are flagged synthetic so neither + # persists — otherwise the resumed transcript keeps a + # premature "done" with the nudge stripped, producing an + # assistant→assistant adjacency. (#55733) messages.append({ "role": "user", "content": _verify_nudge, @@ -4998,9 +5003,11 @@ def run_conversation( if _verify_nudge2: agent._pre_verify_nudges = _attempt + 1 final_msg["finish_reason"] = "verify_hook_continue" + final_msg["_pre_verify_synthetic"] = True # Same alternation contract as verify-on-stop: keep the # attempted answer in history, follow it with a synthetic - # user nudge, and don't surface the premature answer. + # user nudge, and don't surface the premature answer. Both + # are flagged synthetic so neither persists. (#55733) messages.append(final_msg) messages.append({ "role": "user", diff --git a/run_agent.py b/run_agent.py index 18e9f8e0c40..9b8078a5147 100644 --- a/run_agent.py +++ b/run_agent.py @@ -225,6 +225,13 @@ _EPHEMERAL_SCAFFOLDING_FLAGS = ( "_empty_recovery_synthetic", "_empty_terminal_sentinel", "_thinking_prefill", + # verify-on-stop and pre_verify nudges append a synthetic assistant + # "done" plus a synthetic user nudge to keep the agent going one more + # turn before it can claim completion. Those messages exist only to + # drive the verification loop; persisting them poisons the resumed + # transcript and breaks prompt-prefix cache reuse on later turns. (#55733) + "_verification_stop_synthetic", + "_pre_verify_synthetic", ) diff --git a/tests/agent/test_verification_stop_caching.py b/tests/agent/test_verification_stop_caching.py new file mode 100644 index 00000000000..41fee3b3374 --- /dev/null +++ b/tests/agent/test_verification_stop_caching.py @@ -0,0 +1,110 @@ +"""Verification-loop synthetic scaffolding must never reach durable session state. + +verify_on_stop / pre_verify append a synthetic assistant "done" plus a synthetic +user nudge to keep the agent going one more turn before it can claim completion. +These messages exist only to drive the loop; persisting them poisons the resumed +transcript and breaks prompt-prefix cache reuse on later turns (#55733). + +Both persistence sinks (SQLite flush + JSON snapshot) route through the single +``_is_ephemeral_scaffolding`` chokepoint, which is driven by +``_EPHEMERAL_SCAFFOLDING_FLAGS``. These tests assert that the verification-loop +flags are registered there and that both sinks drop the flagged messages while +keeping the real conversation. +""" + +import json +import sys +from unittest.mock import MagicMock + +import pytest + + +def _fresh_run_agent(hermes_home): + for mod in list(sys.modules): + if mod == "run_agent" or mod.startswith("agent.") or mod.startswith("tools.") or mod.startswith("hermes_"): + del sys.modules[mod] + import run_agent # noqa: F401 + return sys.modules["run_agent"] + + +def test_verification_flags_registered_as_ephemeral(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + ra = _fresh_run_agent(tmp_path) + + assert "_verification_stop_synthetic" in ra._EPHEMERAL_SCAFFOLDING_FLAGS + assert "_pre_verify_synthetic" in ra._EPHEMERAL_SCAFFOLDING_FLAGS + + # The central classifier drives both persistence sinks. + assert ra._is_ephemeral_scaffolding( + {"role": "assistant", "content": "done", "_verification_stop_synthetic": True} + ) + assert ra._is_ephemeral_scaffolding( + {"role": "user", "content": "[System: run tests]", "_pre_verify_synthetic": True} + ) + # Real messages are not scaffolding. + assert not ra._is_ephemeral_scaffolding({"role": "user", "content": "hi"}) + + +def _make_agent(ra, session_id, tmp_path): + agent = ra.AIAgent( + session_id=session_id, + api_key="test-key", + base_url="http://127.0.0.1:8000/v1", + provider="openai-compat", + model="test-model", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + agent._session_db = MagicMock() + agent._session_db_created = True + agent._session_json_enabled = True + agent.logs_dir = tmp_path / "logs" + agent.logs_dir.mkdir(parents=True, exist_ok=True) + return agent + + +def test_db_flush_drops_verification_scaffolding(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + ra = _fresh_run_agent(tmp_path) + agent = _make_agent(ra, "sess_db", tmp_path) + + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "premature done", "_verification_stop_synthetic": True}, + {"role": "user", "content": "[System: run tests]", "_verification_stop_synthetic": True}, + {"role": "assistant", "content": "verified and clean"}, + ] + + agent._flush_messages_to_session_db(messages, conversation_history=[]) + + persisted = [ + kwargs.get("content") + for _args, kwargs in agent._session_db.append_message.call_args_list + ] + assert "hi" in persisted + assert "verified and clean" in persisted + assert "premature done" not in persisted + assert "[System: run tests]" not in persisted + + +def test_json_log_drops_verification_scaffolding(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + ra = _fresh_run_agent(tmp_path) + agent = _make_agent(ra, "sess_json", tmp_path) + + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "premature done", "_pre_verify_synthetic": True}, + {"role": "user", "content": "[System: run tests]", "_pre_verify_synthetic": True}, + {"role": "assistant", "content": "verified and clean"}, + ] + + agent._save_session_log(messages) + + log_file = agent.logs_dir / "session_sess_json.json" + assert log_file.exists() + data = json.loads(log_file.read_text(encoding="utf-8")) + contents = [m.get("content") for m in data["messages"]] + assert contents == ["hi", "verified and clean"] + assert all(not m.get("_pre_verify_synthetic") for m in data["messages"])