mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
feat(agent): detect cyclic tool-call loops in tool guardrails
Port from google-gemini/gemini-cli#28429: the LoopDetectionService there gained detection of alternating/cyclic tool-call execution patterns (A->B->A->B and longer cycles) that per-call repetition counters structurally cannot see, because every call differs from its predecessor. Adapted to Hermes' ToolCallGuardrailController: - Track the per-turn sequence of exact tool-call signatures (success or failure alike) and detect the trailing k-call window (k=2..5) repeating consecutively. - warn_after.cycle (default 3 full repetitions) injects the standard soft warning; hard_stop_after.cycle (default 5) halts the turn when hard_stop_enabled is on, via the existing warn/halt plumbing (zero runtime changes). - Length-1 cycles are deliberately excluded: pure self-repeats are already covered by exact_failure / idempotent_no_progress, and successful self-repeats of mutating tools (e.g. polling a background process) are legitimate. - More specific failure/no-progress warnings take precedence over the cycle warning; cycle halt takes precedence over everything. Validation: 30/30 tests in tests/agent/test_tool_guardrails.py + tests/run_agent/test_tool_call_guardrail_runtime.py; E2E via real DEFAULT_CONFIG -> from_mapping -> controller -> append_toolguard_guidance.
This commit is contained in:
parent
3651627d88
commit
f21f388697
4 changed files with 239 additions and 2 deletions
|
|
@ -38,6 +38,14 @@ IDEMPOTENT_TOOL_NAMES = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
# Cyclic tool-call loop detection bounds. Cycles of length 1 (the same call
|
||||
# repeated back-to-back) are intentionally excluded: pure self-repeats are
|
||||
# already covered by the exact-failure and idempotent-no-progress guards, and
|
||||
# successful self-repeats of mutating tools (e.g. polling a background process)
|
||||
# are legitimate. Ported/adapted from google-gemini/gemini-cli#28429.
|
||||
CYCLE_MIN_LENGTH = 2
|
||||
CYCLE_MAX_LENGTH = 5
|
||||
|
||||
MUTATING_TOOL_NAMES = frozenset(
|
||||
{
|
||||
"terminal",
|
||||
|
|
@ -77,6 +85,8 @@ class ToolCallGuardrailConfig:
|
|||
same_tool_failure_halt_after: int = 8
|
||||
no_progress_warn_after: int = 2
|
||||
no_progress_block_after: int = 5
|
||||
cycle_warn_after: int = 3
|
||||
cycle_block_after: int = 5
|
||||
idempotent_tools: frozenset[str] = field(default_factory=lambda: IDEMPOTENT_TOOL_NAMES)
|
||||
mutating_tools: frozenset[str] = field(default_factory=lambda: MUTATING_TOOL_NAMES)
|
||||
|
||||
|
|
@ -121,6 +131,14 @@ class ToolCallGuardrailConfig:
|
|||
hard_stop_after.get("idempotent_no_progress", data.get("no_progress_block_after")),
|
||||
defaults.no_progress_block_after,
|
||||
),
|
||||
cycle_warn_after=_positive_int(
|
||||
warn_after.get("cycle", data.get("cycle_warn_after")),
|
||||
defaults.cycle_warn_after,
|
||||
),
|
||||
cycle_block_after=_positive_int(
|
||||
hard_stop_after.get("cycle", data.get("cycle_block_after")),
|
||||
defaults.cycle_block_after,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -232,6 +250,7 @@ class ToolCallGuardrailController:
|
|||
self._exact_failure_counts: dict[ToolCallSignature, int] = {}
|
||||
self._same_tool_failure_counts: dict[str, int] = {}
|
||||
self._no_progress: dict[ToolCallSignature, tuple[str, int]] = {}
|
||||
self._call_sequence: list[tuple[str, str]] = []
|
||||
self._halt_decision: ToolGuardrailDecision | None = None
|
||||
|
||||
@property
|
||||
|
|
@ -295,6 +314,94 @@ class ToolCallGuardrailController:
|
|||
if failed is None:
|
||||
failed, _ = classify_tool_failure(tool_name, result)
|
||||
|
||||
cycle_decision = self._record_and_check_cycle(tool_name, signature)
|
||||
if cycle_decision is not None and cycle_decision.should_halt:
|
||||
self._halt_decision = cycle_decision
|
||||
return cycle_decision
|
||||
|
||||
decision = self._check_repetition(tool_name, signature, result, failed)
|
||||
if decision.action == "allow" and cycle_decision is not None:
|
||||
return cycle_decision
|
||||
return decision
|
||||
|
||||
def _record_and_check_cycle(
|
||||
self, tool_name: str, signature: ToolCallSignature
|
||||
) -> ToolGuardrailDecision | None:
|
||||
"""Detect repeating tool-call cycles of length 2..CYCLE_MAX_LENGTH.
|
||||
|
||||
Tracks the per-turn sequence of exact call signatures (success or
|
||||
failure alike) and looks for the trailing k-call window repeating
|
||||
consecutively — the A→B→A→B and A→B→C→A→B→C patterns that the
|
||||
per-signature counters structurally cannot see because every
|
||||
individual signature differs from its predecessor.
|
||||
Adapted from gemini-cli's LoopDetectionService (google-gemini/gemini-cli#28429).
|
||||
"""
|
||||
key = f"{signature.tool_name}\x00{signature.args_hash}"
|
||||
self._call_sequence.append((key, signature.tool_name))
|
||||
max_needed = CYCLE_MAX_LENGTH * max(
|
||||
self.config.cycle_warn_after, self.config.cycle_block_after
|
||||
)
|
||||
if len(self._call_sequence) > max_needed:
|
||||
del self._call_sequence[: len(self._call_sequence) - max_needed]
|
||||
|
||||
keys = [entry[0] for entry in self._call_sequence]
|
||||
n = len(keys)
|
||||
min_reps = min(self.config.cycle_warn_after, self.config.cycle_block_after)
|
||||
best: tuple[int, int] | None = None # (repetitions, cycle_length)
|
||||
for k in range(CYCLE_MIN_LENGTH, CYCLE_MAX_LENGTH + 1):
|
||||
if n < k * 2:
|
||||
break
|
||||
cycle = keys[-k:]
|
||||
if len(set(cycle)) < 2:
|
||||
continue # uniform run: covered by the exact-repeat guards
|
||||
reps = 1
|
||||
i = n - k
|
||||
while i - k >= 0 and keys[i - k : i] == cycle:
|
||||
reps += 1
|
||||
i -= k
|
||||
if reps >= min_reps and (best is None or reps > best[0]):
|
||||
best = (reps, k)
|
||||
|
||||
if best is None:
|
||||
return None
|
||||
reps, k = best
|
||||
cycle_desc = " -> ".join(name for _, name in self._call_sequence[-k:])
|
||||
if self.config.hard_stop_enabled and reps >= self.config.cycle_block_after:
|
||||
return ToolGuardrailDecision(
|
||||
action="halt",
|
||||
code="tool_call_cycle_halt",
|
||||
message=(
|
||||
f"Stopped {tool_name}: the tool-call cycle [{cycle_desc}] has "
|
||||
f"repeated {reps} times in a row with identical arguments. "
|
||||
"This is a loop; change strategy instead of repeating the cycle."
|
||||
),
|
||||
tool_name=tool_name,
|
||||
count=reps,
|
||||
signature=signature,
|
||||
)
|
||||
if self.config.warnings_enabled and reps >= self.config.cycle_warn_after:
|
||||
return ToolGuardrailDecision(
|
||||
action="warn",
|
||||
code="tool_call_cycle_warning",
|
||||
message=(
|
||||
f"The tool-call cycle [{cycle_desc}] has repeated {reps} times "
|
||||
"in a row with identical arguments. If you are intentionally "
|
||||
"polling, proceed deliberately; otherwise this looks like a "
|
||||
"loop — change strategy instead of repeating the cycle."
|
||||
),
|
||||
tool_name=tool_name,
|
||||
count=reps,
|
||||
signature=signature,
|
||||
)
|
||||
return None
|
||||
|
||||
def _check_repetition(
|
||||
self,
|
||||
tool_name: str,
|
||||
signature: ToolCallSignature,
|
||||
result: str | None,
|
||||
failed: bool,
|
||||
) -> ToolGuardrailDecision:
|
||||
if failed:
|
||||
exact_count = self._exact_failure_counts.get(signature, 0) + 1
|
||||
self._exact_failure_counts[signature] = exact_count
|
||||
|
|
|
|||
|
|
@ -1388,11 +1388,13 @@ DEFAULT_CONFIG = {
|
|||
"exact_failure": 2,
|
||||
"same_tool_failure": 3,
|
||||
"idempotent_no_progress": 2,
|
||||
"cycle": 3,
|
||||
},
|
||||
"hard_stop_after": {
|
||||
"exact_failure": 5,
|
||||
"same_tool_failure": 8,
|
||||
"idempotent_no_progress": 5,
|
||||
"cycle": 5,
|
||||
},
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ def test_default_config_is_soft_warning_only_with_hard_stop_disabled():
|
|||
assert cfg.exact_failure_block_after == 5
|
||||
assert cfg.same_tool_failure_halt_after == 8
|
||||
assert cfg.no_progress_block_after == 5
|
||||
assert cfg.cycle_warn_after == 3
|
||||
assert cfg.cycle_block_after == 5
|
||||
|
||||
|
||||
def test_config_parses_nested_warn_and_hard_stop_thresholds():
|
||||
|
|
@ -55,11 +57,13 @@ def test_config_parses_nested_warn_and_hard_stop_thresholds():
|
|||
"exact_failure": 3,
|
||||
"same_tool_failure": 4,
|
||||
"idempotent_no_progress": 5,
|
||||
"cycle": 6,
|
||||
},
|
||||
"hard_stop_after": {
|
||||
"exact_failure": 6,
|
||||
"same_tool_failure": 7,
|
||||
"idempotent_no_progress": 8,
|
||||
"cycle": 9,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
|
@ -72,6 +76,8 @@ def test_config_parses_nested_warn_and_hard_stop_thresholds():
|
|||
assert cfg.exact_failure_block_after == 6
|
||||
assert cfg.same_tool_failure_halt_after == 7
|
||||
assert cfg.no_progress_block_after == 8
|
||||
assert cfg.cycle_warn_after == 6
|
||||
assert cfg.cycle_block_after == 9
|
||||
|
||||
|
||||
def test_default_repeated_identical_failed_call_warns_without_blocking():
|
||||
|
|
@ -229,8 +235,15 @@ def test_hard_stop_enabled_blocks_idempotent_no_progress_future_repeat():
|
|||
|
||||
|
||||
def test_mutating_or_unknown_tools_are_not_blocked_for_repeated_identical_success_output_by_default():
|
||||
# cycle thresholds raised: alternating two identical calls IS a legitimate
|
||||
# length-2 cycle; this test isolates the no_progress behavior only.
|
||||
controller = ToolCallGuardrailController(
|
||||
ToolCallGuardrailConfig(no_progress_warn_after=2, no_progress_block_after=2)
|
||||
ToolCallGuardrailConfig(
|
||||
no_progress_warn_after=2,
|
||||
no_progress_block_after=2,
|
||||
cycle_warn_after=99,
|
||||
cycle_block_after=99,
|
||||
)
|
||||
)
|
||||
|
||||
for _ in range(3):
|
||||
|
|
@ -277,3 +290,114 @@ def test_after_call_survives_lone_surrogates_in_result_and_args():
|
|||
controller.after_call("web_search", {"query": dirty}, '{"error":"\ud835 boom"}', failed=True)
|
||||
controller.after_call("web_search", {"query": dirty}, '{"error":"\ud835 boom"}', failed=True)
|
||||
assert controller.before_call("web_search", {"query": dirty}).action == "block"
|
||||
|
||||
|
||||
# ── Cyclic tool-call loop detection (ported from google-gemini/gemini-cli#28429) ──
|
||||
|
||||
|
||||
def _cycle(controller, calls):
|
||||
"""Feed (tool_name, args) pairs through after_call; return the decisions.
|
||||
|
||||
Results are unique per call so the idempotent no-progress guard stays
|
||||
quiet and only cycle behavior is exercised.
|
||||
"""
|
||||
return [
|
||||
controller.after_call(name, args, f"ok-{i}", failed=False)
|
||||
for i, (name, args) in enumerate(calls)
|
||||
]
|
||||
|
||||
|
||||
def test_alternating_two_call_cycle_warns_by_default():
|
||||
controller = ToolCallGuardrailController()
|
||||
a = ("read_file", {"path": "/tmp/loop_a.txt"})
|
||||
b = ("browser_click", {"ref": "@e5"})
|
||||
|
||||
decisions = _cycle(controller, [a, b] * 3)
|
||||
|
||||
assert [d.action for d in decisions[:-1]] == ["allow"] * 5
|
||||
last = decisions[-1]
|
||||
assert last.action == "warn"
|
||||
assert last.code == "tool_call_cycle_warning"
|
||||
assert last.count == 3
|
||||
assert "read_file -> browser_click" in last.message
|
||||
assert controller.halt_decision is None
|
||||
|
||||
|
||||
def test_three_call_cycle_warns_by_default():
|
||||
controller = ToolCallGuardrailController()
|
||||
seq = [
|
||||
("read_file", {"path": "/tmp/a"}),
|
||||
("read_file", {"path": "/tmp/b"}),
|
||||
("terminal", {"command": "ls /tmp"}),
|
||||
]
|
||||
|
||||
decisions = _cycle(controller, seq * 3)
|
||||
|
||||
assert decisions[-1].action == "warn"
|
||||
assert decisions[-1].code == "tool_call_cycle_warning"
|
||||
assert decisions[-1].count == 3
|
||||
|
||||
|
||||
def test_cycle_halts_with_hard_stop_enabled():
|
||||
controller = ToolCallGuardrailController(
|
||||
ToolCallGuardrailConfig(hard_stop_enabled=True, cycle_warn_after=2, cycle_block_after=3)
|
||||
)
|
||||
a = ("web_search", {"query": "alpha"})
|
||||
b = ("web_search", {"query": "beta"})
|
||||
|
||||
decisions = _cycle(controller, [a, b] * 3)
|
||||
|
||||
assert decisions[-1].action == "halt"
|
||||
assert decisions[-1].code == "tool_call_cycle_halt"
|
||||
assert decisions[-1].count == 3
|
||||
assert controller.halt_decision is decisions[-1]
|
||||
|
||||
|
||||
def test_broken_pattern_does_not_trip_cycle_detector():
|
||||
controller = ToolCallGuardrailController()
|
||||
a = ("read_file", {"path": "/tmp/loop_a.txt"})
|
||||
b = ("read_file", {"path": "/tmp/loop_b.txt"})
|
||||
c = ("read_file", {"path": "/tmp/loop_c.txt"})
|
||||
|
||||
# A B A B, then break the pattern with C, then A B A B again
|
||||
decisions = _cycle(controller, [a, b, a, b, c, a, b, a, b])
|
||||
|
||||
assert all(d.action == "allow" for d in decisions)
|
||||
|
||||
|
||||
def test_uniform_self_repeats_do_not_trip_cycle_detector():
|
||||
# Back-to-back repeats of one call (e.g. polling a background process) are
|
||||
# deliberately out of scope for the cycle detector.
|
||||
controller = ToolCallGuardrailController()
|
||||
args = {"action": "poll", "session_id": "watch_1"}
|
||||
|
||||
decisions = _cycle(controller, [("process", args)] * 12)
|
||||
|
||||
assert all(d.action == "allow" for d in decisions)
|
||||
|
||||
|
||||
def test_more_specific_failure_warning_wins_over_cycle_warning():
|
||||
controller = ToolCallGuardrailController()
|
||||
a = ("web_search", {"query": "same"})
|
||||
b = ("read_file", {"path": "/tmp/x"})
|
||||
|
||||
for _ in range(3):
|
||||
controller.after_call(a[0], a[1], '{"error":"boom"}', failed=True)
|
||||
controller.after_call(b[0], b[1], "ok", failed=False)
|
||||
decision = controller.after_call(a[0], a[1], '{"error":"boom"}', failed=True)
|
||||
|
||||
# cycle reps hit 3 here, but the exact-failure warning is more specific
|
||||
assert decision.action == "warn"
|
||||
assert decision.code == "repeated_exact_failure_warning"
|
||||
|
||||
|
||||
def test_reset_for_turn_clears_cycle_state():
|
||||
controller = ToolCallGuardrailController()
|
||||
a = ("read_file", {"path": "/tmp/loop_a.txt"})
|
||||
b = ("browser_click", {"ref": "@e5"})
|
||||
|
||||
_cycle(controller, [a, b, a, b])
|
||||
controller.reset_for_turn()
|
||||
decisions = _cycle(controller, [a, b])
|
||||
|
||||
assert all(d.action == "allow" for d in decisions)
|
||||
|
|
|
|||
|
|
@ -1410,7 +1410,7 @@ agent:
|
|||
|
||||
## Tool-Loop Guardrails
|
||||
|
||||
Hermes detects when the agent is stuck in an unproductive tool-calling loop — the same tool call failing repeatedly, the same tool failing over and over, or an idempotent call returning the same result with no progress. By default it injects a **warning** into the tool result so the model self-corrects; it does not hard-stop, since a person watching the CLI/TUI can intervene.
|
||||
Hermes detects when the agent is stuck in an unproductive tool-calling loop — the same tool call failing repeatedly, the same tool failing over and over, an idempotent call returning the same result with no progress, or a repeating multi-call cycle (e.g. alternating A→B→A→B between two identical calls). By default it injects a **warning** into the tool result so the model self-corrects; it does not hard-stop, since a person watching the CLI/TUI can intervene.
|
||||
|
||||
For unattended gateway / server deployments, enable hard stops so a stuck agent is circuit-broken instead of burning the iteration budget:
|
||||
|
||||
|
|
@ -1422,12 +1422,16 @@ tool_loop_guardrails:
|
|||
exact_failure: 2 # identical failing call repeated N times
|
||||
same_tool_failure: 3 # same tool failing N times (different args)
|
||||
idempotent_no_progress: 2 # same result, no progress, N times
|
||||
cycle: 3 # a repeating cycle of 2-5 identical calls, N full repetitions
|
||||
hard_stop_after:
|
||||
exact_failure: 5
|
||||
same_tool_failure: 8
|
||||
idempotent_no_progress: 5
|
||||
cycle: 5
|
||||
```
|
||||
|
||||
The `cycle` detector catches loops the per-call counters structurally cannot see: the agent bouncing between two or more calls with identical arguments (A→B→A→B... or A→B→C→A→B→C...), success or failure alike. Back-to-back repeats of a single call are deliberately excluded — those are covered by `exact_failure` / `idempotent_no_progress`, and successful self-repeats (e.g. polling a background process) are legitimate.
|
||||
|
||||
`hard_stop_enabled` defaults to `false` because interactive sessions have a human in the loop. In unattended deployments (gateway, cron, kanban workers) set it to `true` so repeated failures are blocked rather than only warned. See also [Docker / unattended deployments](docker.md).
|
||||
|
||||
## TTS Configuration
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue