mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(goals): update judge consumers for transport result
This commit is contained in:
parent
3b4f96ce94
commit
d401fd7251
4 changed files with 39 additions and 16 deletions
|
|
@ -438,9 +438,12 @@ class TestJudgeParseFailureAutoPause:
|
|||
"agent.auxiliary_client.call_llm",
|
||||
side_effect=RuntimeError("connection reset"),
|
||||
):
|
||||
verdict, _, parse_failed, _wd, _tf = goals.judge_goal("goal", "response")
|
||||
verdict, _, parse_failed, _wd, transport_failed = goals.judge_goal(
|
||||
"goal", "response"
|
||||
)
|
||||
assert verdict == "continue"
|
||||
assert parse_failed is False
|
||||
assert transport_failed is True
|
||||
|
||||
def test_empty_judge_reply_flagged_as_parse_failure(self):
|
||||
"""End-to-end: judge returns empty content → parse_failed=True."""
|
||||
|
|
@ -507,24 +510,44 @@ class TestJudgeParseFailureAutoPause:
|
|||
assert d["should_continue"] is True
|
||||
assert mgr.state.consecutive_parse_failures == 0
|
||||
|
||||
def test_parse_failure_counter_not_incremented_by_api_errors(self, hermes_home):
|
||||
"""API/transport errors must NOT count toward the auto-pause threshold."""
|
||||
def test_transport_failures_do_not_increment_parse_counter(self, hermes_home):
|
||||
"""Transport failures use their own counter and a good reply resets both."""
|
||||
from hermes_cli import goals
|
||||
from hermes_cli.goals import GoalManager
|
||||
|
||||
mgr = GoalManager(session_id="parse-fail-sid-3", default_max_turns=20)
|
||||
mgr.set("goal")
|
||||
assert mgr.state is not None
|
||||
|
||||
with patch.object(
|
||||
goals, "judge_goal", return_value=("continue", "judge error: RuntimeError", False, None, False)
|
||||
):
|
||||
for _ in range(5):
|
||||
goals,
|
||||
"judge_goal",
|
||||
return_value=(
|
||||
"continue",
|
||||
"judge error: RuntimeError",
|
||||
False,
|
||||
None,
|
||||
True,
|
||||
),
|
||||
):
|
||||
for _ in range(2):
|
||||
d = mgr.evaluate_after_turn("still going")
|
||||
assert d["should_continue"] is True
|
||||
assert mgr.state.consecutive_parse_failures == 0
|
||||
assert mgr.state.consecutive_transport_failures == 0
|
||||
assert mgr.state.consecutive_transport_failures == 2
|
||||
assert mgr.state.status == "active"
|
||||
|
||||
with patch.object(
|
||||
goals,
|
||||
"judge_goal",
|
||||
return_value=("continue", "making progress", False, None, False),
|
||||
):
|
||||
d = mgr.evaluate_after_turn("recovered")
|
||||
|
||||
assert d["should_continue"] is True
|
||||
assert mgr.state.consecutive_parse_failures == 0
|
||||
assert mgr.state.consecutive_transport_failures == 0
|
||||
|
||||
def test_consecutive_parse_failures_persists_across_goalmanager_reloads(
|
||||
self, hermes_home
|
||||
):
|
||||
|
|
|
|||
|
|
@ -181,8 +181,8 @@ def _patch_judge(monkeypatch, verdicts):
|
|||
|
||||
def _fake_judge(goal, response, subgoals=None, background_processes=None, **_kw):
|
||||
v = seq.pop(0) if seq else "done"
|
||||
# 4-tuple contract: (verdict, reason, parse_failed, wait_directive)
|
||||
return v, f"scripted:{v}", False, None
|
||||
# 5-tuple contract: verdict, reason, parse failure, wait, transport failure.
|
||||
return v, f"scripted:{v}", False, None, False
|
||||
|
||||
monkeypatch.setattr(goals, "judge_goal", _fake_judge)
|
||||
|
||||
|
|
|
|||
|
|
@ -649,8 +649,8 @@ def test_complete_goal_mode_rejected_by_judge(monkeypatch, tmp_path):
|
|||
# judge is reachable, so force the availability probe True as well.
|
||||
def mock_judge_goal(goal, last_response, *, timeout=30.0, subgoals=None):
|
||||
# Match the real judge_goal contract:
|
||||
# (verdict, reason, parse_failed, wait_directive)
|
||||
return "continue", "missing verification evidence", False, None
|
||||
# (verdict, reason, parse_failed, wait_directive, transport_failed)
|
||||
return "continue", "missing verification evidence", False, None, False
|
||||
|
||||
monkeypatch.setattr("tools.kanban_tools.judge_goal", mock_judge_goal)
|
||||
monkeypatch.setattr("tools.kanban_tools._goal_judge_available", lambda: True)
|
||||
|
|
|
|||
|
|
@ -603,11 +603,11 @@ def _handle_complete(args: dict, **kw) -> str:
|
|||
reason = ""
|
||||
try:
|
||||
# judge_goal returns (verdict, reason, parse_failed,
|
||||
# wait_directive) — see hermes_cli/goals.py. Unpacking
|
||||
# fewer raises ValueError, which the defensive handler
|
||||
# below swallows, leaving verdict="done" and silently
|
||||
# disabling the gate.
|
||||
verdict, reason, _, _ = judge_goal(
|
||||
# wait_directive, transport_failed) — see
|
||||
# hermes_cli/goals.py. Unpacking fewer raises ValueError,
|
||||
# which the defensive handler below swallows, leaving
|
||||
# verdict="done" and silently disabling the gate.
|
||||
verdict, reason, _, _, _ = judge_goal(
|
||||
goal=f"{task.title}\n\n{task.body or ''}".strip(),
|
||||
last_response=(summary or result or "").strip(),
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue