mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
feat(gateway): session activity watchdog, stall notify, compress timeout (#72424)
Three mechanisms to detect and notify when gateway sessions stall silently: 1. Mid-turn activity heartbeats stamped to SessionDB so hermes sessions list and hermes status show progress during long turns without new message rows. 2. Stall watchdog: when a busy session has pending inbound and the shared activity clock is idle past agent.session_stall_timeout (default 300), log a WARNING and notify the user once to try /new. Notify-only; does not kill the turn. 3. Compaction timeout: fenceless compress_context callers get a progress-aware host budget (compression.context_timeout_seconds default 120 idle, compression.context_total_ceiling_seconds default 600 ceiling). On timeout, cancel via commit fence, skip compaction without dropping messages, and continue the turn. Closes #72016 (slices 1-3; slice 4 cumulative SSE stream-retry deadline remains a follow-up). Cherry-picked from PR #72424 by @fangliquanflq.
This commit is contained in:
parent
5fde131eb2
commit
cfb206fe2e
24 changed files with 2638 additions and 107 deletions
|
|
@ -127,7 +127,7 @@ def test_compression_activity_heartbeat_touches_agent_during_long_compress(tmp_p
|
|||
agent = _build_agent_with_db(db, session_id)
|
||||
agent._compression_activity_heartbeat_interval = 0.1
|
||||
touch_calls: list[str] = []
|
||||
agent._touch_activity = lambda desc: touch_calls.append(desc)
|
||||
agent._touch_activity = lambda desc, **_kw: touch_calls.append(desc)
|
||||
|
||||
def _slow_compress(*_a, **_kw):
|
||||
_wait_for_touch(touch_calls, "context compression in progress")
|
||||
|
|
@ -156,7 +156,7 @@ def test_compression_activity_heartbeat_stops_on_compress_exception(tmp_path: Pa
|
|||
agent = _build_agent_with_db(db, session_id)
|
||||
agent._compression_activity_heartbeat_interval = 0.1
|
||||
touch_calls: list[str] = []
|
||||
agent._touch_activity = lambda desc: touch_calls.append(desc)
|
||||
agent._touch_activity = lambda desc, **_kw: touch_calls.append(desc)
|
||||
|
||||
def _failing_compress(*_a, **_kw):
|
||||
_wait_for_touch(touch_calls, "context compression in progress")
|
||||
|
|
@ -182,7 +182,7 @@ def test_compression_activity_heartbeat_ignores_touch_errors(tmp_path: Path) ->
|
|||
|
||||
agent = _build_agent_with_db(db, session_id)
|
||||
agent._compression_activity_heartbeat_interval = 0.1
|
||||
agent._touch_activity = lambda _desc: (_ for _ in ()).throw(RuntimeError("touch boom"))
|
||||
agent._touch_activity = lambda _desc, **_kw: (_ for _ in ()).throw(RuntimeError("touch boom"))
|
||||
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
|
||||
|
||||
compressed, _sp = agent._compress_context(messages, "sys", approx_tokens=120_000)
|
||||
|
|
@ -207,7 +207,7 @@ def test_compression_activity_heartbeat_strict_signature_fallback_releases_lock(
|
|||
agent = _build_agent_with_db(db, session_id)
|
||||
agent._compression_activity_heartbeat_interval = "not-a-number"
|
||||
touch_calls: list[str] = []
|
||||
agent._touch_activity = lambda desc: touch_calls.append(desc)
|
||||
agent._touch_activity = lambda desc, **_kw: touch_calls.append(desc)
|
||||
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
|
||||
|
||||
strict_calls: list[int | None] = []
|
||||
|
|
@ -240,7 +240,13 @@ def test_compression_activity_heartbeat_nonfinite_interval_falls_back(tmp_path:
|
|||
|
||||
agent = _build_agent_with_db(db, session_id)
|
||||
touch_calls: list[str] = []
|
||||
agent._touch_activity = lambda desc: touch_calls.append(desc)
|
||||
touch_provenances: list = []
|
||||
|
||||
def _capture(desc, *, provenance=None):
|
||||
touch_calls.append(desc)
|
||||
touch_provenances.append(provenance)
|
||||
|
||||
agent._touch_activity = _capture
|
||||
|
||||
heartbeat = _CompressionActivityHeartbeat(agent, interval_seconds=float("inf"))
|
||||
|
||||
|
|
@ -248,7 +254,104 @@ def test_compression_activity_heartbeat_nonfinite_interval_falls_back(tmp_path:
|
|||
heartbeat.start()
|
||||
heartbeat.stop()
|
||||
assert touch_calls == ["context compression started", "context compression completed"]
|
||||
from agent.session_activity import ActivityProvenance
|
||||
|
||||
assert touch_provenances == [
|
||||
ActivityProvenance.AGENT_COMPRESSION,
|
||||
ActivityProvenance.AGENT_COMPRESSION,
|
||||
]
|
||||
|
||||
|
||||
def test_compression_heartbeat_does_not_clobber_timeout_provenance() -> None:
|
||||
"""Detached heartbeat/stop must not overwrite a host timeout stamp."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.conversation_compression import _CompressionActivityHeartbeat
|
||||
from agent.session_activity import ActivityProvenance
|
||||
|
||||
agent = SimpleNamespace(
|
||||
_last_activity_provenance=ActivityProvenance.AGENT_COMPRESSION_TIMEOUT,
|
||||
_last_activity_desc="context compression timed out",
|
||||
touches=[],
|
||||
)
|
||||
|
||||
def _touch(desc, *, provenance=None):
|
||||
agent.touches.append((desc, provenance))
|
||||
agent._last_activity_provenance = provenance
|
||||
agent._last_activity_desc = desc
|
||||
|
||||
agent._touch_activity = _touch
|
||||
|
||||
hb = _CompressionActivityHeartbeat(agent, interval_seconds=60.0)
|
||||
hb._touch("context compression in progress")
|
||||
hb.stop("context compression completed")
|
||||
|
||||
assert agent.touches == []
|
||||
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION_TIMEOUT
|
||||
assert agent._last_activity_desc == "context compression timed out"
|
||||
|
||||
|
||||
def test_compression_heartbeat_does_not_clobber_cooldown_provenance() -> None:
|
||||
"""Cooldown/abort stamps must also survive a late heartbeat stop."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.conversation_compression import _CompressionActivityHeartbeat
|
||||
from agent.session_activity import ActivityProvenance
|
||||
|
||||
agent = SimpleNamespace(
|
||||
_last_activity_provenance=ActivityProvenance.AGENT_COMPRESSION_COOLDOWN,
|
||||
_last_activity_desc="compression blocked (cooldown: 30s remaining)",
|
||||
touches=[],
|
||||
)
|
||||
|
||||
def _touch(desc, *, provenance=None):
|
||||
agent.touches.append((desc, provenance))
|
||||
agent._last_activity_provenance = provenance
|
||||
agent._last_activity_desc = desc
|
||||
|
||||
agent._touch_activity = _touch
|
||||
|
||||
hb = _CompressionActivityHeartbeat(agent, interval_seconds=60.0)
|
||||
hb._touch("context compression in progress")
|
||||
hb.stop("context compression failed")
|
||||
|
||||
assert agent.touches == []
|
||||
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION_COOLDOWN
|
||||
|
||||
|
||||
def test_compression_heartbeat_start_republishes_after_terminal_provenance() -> None:
|
||||
"""A new compression episode may overwrite a prior timeout/cooldown stamp."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.conversation_compression import _CompressionActivityHeartbeat
|
||||
from agent.session_activity import ActivityProvenance
|
||||
|
||||
agent = SimpleNamespace(
|
||||
_last_activity_provenance=ActivityProvenance.AGENT_COMPRESSION_TIMEOUT,
|
||||
_last_activity_desc="context compression timed out",
|
||||
touches=[],
|
||||
)
|
||||
|
||||
def _touch(desc, *, provenance=None):
|
||||
agent.touches.append((desc, provenance))
|
||||
agent._last_activity_provenance = provenance
|
||||
agent._last_activity_desc = desc
|
||||
|
||||
agent._touch_activity = _touch
|
||||
|
||||
hb = _CompressionActivityHeartbeat(agent, interval_seconds=60.0)
|
||||
hb.start()
|
||||
hb.stop()
|
||||
|
||||
assert agent.touches[0] == (
|
||||
"context compression started",
|
||||
ActivityProvenance.AGENT_COMPRESSION,
|
||||
)
|
||||
assert agent.touches[-1] == (
|
||||
"context compression completed",
|
||||
ActivityProvenance.AGENT_COMPRESSION,
|
||||
)
|
||||
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION
|
||||
|
||||
|
||||
def test_concurrent_compression_does_not_fork_session(tmp_path: Path) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue