fix(goals): update judge consumers for transport result

This commit is contained in:
João Vitor Cunha 2026-07-15 10:51:52 -03:00 committed by Teknium
parent 3b4f96ce94
commit d401fd7251
4 changed files with 39 additions and 16 deletions

View file

@ -438,9 +438,12 @@ class TestJudgeParseFailureAutoPause:
"agent.auxiliary_client.call_llm",
side_effect=RuntimeError("connection reset"),
):
verdict, _, parse_failed, _wd, _tf = goals.judge_goal("goal", "response")
verdict, _, parse_failed, _wd, transport_failed = goals.judge_goal(
"goal", "response"
)
assert verdict == "continue"
assert parse_failed is False
assert transport_failed is True
def test_empty_judge_reply_flagged_as_parse_failure(self):
"""End-to-end: judge returns empty content → parse_failed=True."""
@ -507,24 +510,44 @@ class TestJudgeParseFailureAutoPause:
assert d["should_continue"] is True
assert mgr.state.consecutive_parse_failures == 0
def test_parse_failure_counter_not_incremented_by_api_errors(self, hermes_home):
"""API/transport errors must NOT count toward the auto-pause threshold."""
def test_transport_failures_do_not_increment_parse_counter(self, hermes_home):
"""Transport failures use their own counter and a good reply resets both."""
from hermes_cli import goals
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="parse-fail-sid-3", default_max_turns=20)
mgr.set("goal")
assert mgr.state is not None
with patch.object(
goals, "judge_goal", return_value=("continue", "judge error: RuntimeError", False, None, False)
):
for _ in range(5):
goals,
"judge_goal",
return_value=(
"continue",
"judge error: RuntimeError",
False,
None,
True,
),
):
for _ in range(2):
d = mgr.evaluate_after_turn("still going")
assert d["should_continue"] is True
assert mgr.state.consecutive_parse_failures == 0
assert mgr.state.consecutive_transport_failures == 0
assert mgr.state.consecutive_transport_failures == 2
assert mgr.state.status == "active"
with patch.object(
goals,
"judge_goal",
return_value=("continue", "making progress", False, None, False),
):
d = mgr.evaluate_after_turn("recovered")
assert d["should_continue"] is True
assert mgr.state.consecutive_parse_failures == 0
assert mgr.state.consecutive_transport_failures == 0
def test_consecutive_parse_failures_persists_across_goalmanager_reloads(
self, hermes_home
):

View file

@ -181,8 +181,8 @@ def _patch_judge(monkeypatch, verdicts):
def _fake_judge(goal, response, subgoals=None, background_processes=None, **_kw):
v = seq.pop(0) if seq else "done"
# 4-tuple contract: (verdict, reason, parse_failed, wait_directive)
return v, f"scripted:{v}", False, None
# 5-tuple contract: verdict, reason, parse failure, wait, transport failure.
return v, f"scripted:{v}", False, None, False
monkeypatch.setattr(goals, "judge_goal", _fake_judge)

View file

@ -649,8 +649,8 @@ def test_complete_goal_mode_rejected_by_judge(monkeypatch, tmp_path):
# judge is reachable, so force the availability probe True as well.
def mock_judge_goal(goal, last_response, *, timeout=30.0, subgoals=None):
# Match the real judge_goal contract:
# (verdict, reason, parse_failed, wait_directive)
return "continue", "missing verification evidence", False, None
# (verdict, reason, parse_failed, wait_directive, transport_failed)
return "continue", "missing verification evidence", False, None, False
monkeypatch.setattr("tools.kanban_tools.judge_goal", mock_judge_goal)
monkeypatch.setattr("tools.kanban_tools._goal_judge_available", lambda: True)

View file

@ -603,11 +603,11 @@ def _handle_complete(args: dict, **kw) -> str:
reason = ""
try:
# judge_goal returns (verdict, reason, parse_failed,
# wait_directive) — see hermes_cli/goals.py. Unpacking
# fewer raises ValueError, which the defensive handler
# below swallows, leaving verdict="done" and silently
# disabling the gate.
verdict, reason, _, _ = judge_goal(
# wait_directive, transport_failed) — see
# hermes_cli/goals.py. Unpacking fewer raises ValueError,
# which the defensive handler below swallows, leaving
# verdict="done" and silently disabling the gate.
verdict, reason, _, _, _ = judge_goal(
goal=f"{task.title}\n\n{task.body or ''}".strip(),
last_response=(summary or result or "").strip(),
)