diff --git a/tests/hermes_cli/test_goals.py b/tests/hermes_cli/test_goals.py index d89bf3656c0..5b96ce832ba 100644 --- a/tests/hermes_cli/test_goals.py +++ b/tests/hermes_cli/test_goals.py @@ -438,9 +438,12 @@ class TestJudgeParseFailureAutoPause: "agent.auxiliary_client.call_llm", side_effect=RuntimeError("connection reset"), ): - verdict, _, parse_failed, _wd, _tf = goals.judge_goal("goal", "response") + verdict, _, parse_failed, _wd, transport_failed = goals.judge_goal( + "goal", "response" + ) assert verdict == "continue" assert parse_failed is False + assert transport_failed is True def test_empty_judge_reply_flagged_as_parse_failure(self): """End-to-end: judge returns empty content → parse_failed=True.""" @@ -507,24 +510,44 @@ class TestJudgeParseFailureAutoPause: assert d["should_continue"] is True assert mgr.state.consecutive_parse_failures == 0 - def test_parse_failure_counter_not_incremented_by_api_errors(self, hermes_home): - """API/transport errors must NOT count toward the auto-pause threshold.""" + def test_transport_failures_do_not_increment_parse_counter(self, hermes_home): + """Transport failures use their own counter and a good reply resets both.""" from hermes_cli import goals from hermes_cli.goals import GoalManager mgr = GoalManager(session_id="parse-fail-sid-3", default_max_turns=20) mgr.set("goal") + assert mgr.state is not None with patch.object( - goals, "judge_goal", return_value=("continue", "judge error: RuntimeError", False, None, False) - ): - for _ in range(5): + goals, + "judge_goal", + return_value=( + "continue", + "judge error: RuntimeError", + False, + None, + True, + ), + ): + for _ in range(2): d = mgr.evaluate_after_turn("still going") assert d["should_continue"] is True assert mgr.state.consecutive_parse_failures == 0 - assert mgr.state.consecutive_transport_failures == 0 + assert mgr.state.consecutive_transport_failures == 2 assert mgr.state.status == "active" + with patch.object( + goals, + "judge_goal", + return_value=("continue", "making progress", False, None, False), + ): + d = mgr.evaluate_after_turn("recovered") + + assert d["should_continue"] is True + assert mgr.state.consecutive_parse_failures == 0 + assert mgr.state.consecutive_transport_failures == 0 + def test_consecutive_parse_failures_persists_across_goalmanager_reloads( self, hermes_home ): diff --git a/tests/hermes_cli/test_kanban_goal_mode.py b/tests/hermes_cli/test_kanban_goal_mode.py index 026c317b616..08ddb174541 100644 --- a/tests/hermes_cli/test_kanban_goal_mode.py +++ b/tests/hermes_cli/test_kanban_goal_mode.py @@ -181,8 +181,8 @@ def _patch_judge(monkeypatch, verdicts): def _fake_judge(goal, response, subgoals=None, background_processes=None, **_kw): v = seq.pop(0) if seq else "done" - # 4-tuple contract: (verdict, reason, parse_failed, wait_directive) - return v, f"scripted:{v}", False, None + # 5-tuple contract: verdict, reason, parse failure, wait, transport failure. + return v, f"scripted:{v}", False, None, False monkeypatch.setattr(goals, "judge_goal", _fake_judge) diff --git a/tests/tools/test_kanban_tools.py b/tests/tools/test_kanban_tools.py index e9ea181ec2a..c535aa59a63 100644 --- a/tests/tools/test_kanban_tools.py +++ b/tests/tools/test_kanban_tools.py @@ -649,8 +649,8 @@ def test_complete_goal_mode_rejected_by_judge(monkeypatch, tmp_path): # judge is reachable, so force the availability probe True as well. def mock_judge_goal(goal, last_response, *, timeout=30.0, subgoals=None): # Match the real judge_goal contract: - # (verdict, reason, parse_failed, wait_directive) - return "continue", "missing verification evidence", False, None + # (verdict, reason, parse_failed, wait_directive, transport_failed) + return "continue", "missing verification evidence", False, None, False monkeypatch.setattr("tools.kanban_tools.judge_goal", mock_judge_goal) monkeypatch.setattr("tools.kanban_tools._goal_judge_available", lambda: True) diff --git a/tools/kanban_tools.py b/tools/kanban_tools.py index 51fac3f8eaa..5535c01e2e6 100644 --- a/tools/kanban_tools.py +++ b/tools/kanban_tools.py @@ -603,11 +603,11 @@ def _handle_complete(args: dict, **kw) -> str: reason = "" try: # judge_goal returns (verdict, reason, parse_failed, - # wait_directive) — see hermes_cli/goals.py. Unpacking - # fewer raises ValueError, which the defensive handler - # below swallows, leaving verdict="done" and silently - # disabling the gate. - verdict, reason, _, _ = judge_goal( + # wait_directive, transport_failed) — see + # hermes_cli/goals.py. Unpacking fewer raises ValueError, + # which the defensive handler below swallows, leaving + # verdict="done" and silently disabling the gate. + verdict, reason, _, _, _ = judge_goal( goal=f"{task.title}\n\n{task.body or ''}".strip(), last_response=(summary or result or "").strip(), )