mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-31 19:16:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat(gateway): session activity watchdog, stall notify, compress timeout (#72424)

Browse source 
Three mechanisms to detect and notify when gateway sessions stall silently:

1. Mid-turn activity heartbeats stamped to SessionDB so hermes sessions list
   and hermes status show progress during long turns without new message rows.

2. Stall watchdog: when a busy session has pending inbound and the shared
   activity clock is idle past agent.session_stall_timeout (default 300),
   log a WARNING and notify the user once to try /new. Notify-only; does
   not kill the turn.

3. Compaction timeout: fenceless compress_context callers get a progress-aware
   host budget (compression.context_timeout_seconds default 120 idle,
   compression.context_total_ceiling_seconds default 600 ceiling). On timeout,
   cancel via commit fence, skip compaction without dropping messages, and
   continue the turn.

Closes #72016 (slices 1-3; slice 4 cumulative SSE stream-retry deadline
remains a follow-up).

Cherry-picked from PR #72424 by @fangliquanflq.
This commit is contained in:
fangliquanflq 2026-07-27 23:16:24 +05:00 • committed by kshitij
parent 5fde131eb2
commit cfb206fe2e
24 changed files with 2638 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

394
tests/agent/test_compress_context_progress_timeout.py Normal file
View file

@ -0,0 +1,394 @@
"""Progress-aware timeout around in-agent compress_context (#72016).
In-loop / preflight / manual ``/compress`` paths historically waited on
``compress_context`` with no host-level inactivity budget. Gateway session
hygiene already had a progress-aware wait; these tests pin the same contract
for the owned wrapper used when callers do not pass a ``commit_fence``.
"""
from __future__ import annotations
import threading
import time
from unittest.mock import MagicMock
import pytest
from agent.conversation_compression import (
CompressionCommitFence,
resolve_context_compression_timeouts,
run_compress_context_with_progress_timeout,
)
class TestResolveContextCompressionTimeouts:
def test_defaults_when_empty_cfg(self):
idle, ceiling = resolve_context_compression_timeouts({})
assert idle == 120.0
assert ceiling == 600.0
def test_zero_idle_disables_wrapper(self):
idle, ceiling = resolve_context_compression_timeouts(
{"context_timeout_seconds": 0}
)
assert idle == 0.0
assert ceiling == 600.0
def test_ceiling_clamped_to_idle(self):
idle, ceiling = resolve_context_compression_timeouts(
{
"context_timeout_seconds": 90,
"context_total_ceiling_seconds": 30,
}
)
assert idle == 90.0
assert ceiling == 90.0
class TestRunCompressContextWithProgressTimeout:
def test_silent_worker_times_out_and_preserves_messages(self):
original = [{"role": "user", "content": "keep-me"}]
started = threading.Event()
release = threading.Event()
commit_attempted = threading.Event()
def worker(fence: CompressionCommitFence):
started.set()
assert release.wait(timeout=2)
if not fence.begin_commit():
return ([{"role": "assistant", "content": "should-not-land"}], "x")
try:
commit_attempted.set()
return ([{"role": "assistant", "content": "too-late"}], "x")
finally:
fence.finish_commit()
warnings = []
result_msgs, result_prompt = run_compress_context_with_progress_timeout(
worker=worker,
messages=original,
system_prompt_fallback="fallback-prompt",
idle_timeout_seconds=0.05,
total_ceiling_seconds=0.2,
on_timeout=lambda idle, waited, since: warnings.append(
(idle, waited, since)
),
)
assert started.wait(timeout=1)
# Give the waiter time to cancel before releasing the worker.
time.sleep(0.15)
release.set()
# Worker may still be winding down; fence cancel must have won.
deadline = time.time() + 1.0
while time.time() < deadline and not commit_attempted.is_set():
time.sleep(0.01)
assert result_msgs is original
assert result_prompt == "fallback-prompt"
assert warnings, "timeout callback should fire"
assert not commit_attempted.is_set(), (
"cancelled fence must block late session mutation"
)
def test_progress_extends_idle_budget_until_success(self):
original = [{"role": "user", "content": "a"}]
compressed = [{"role": "user", "content": "summarized"}]
fence_holder: dict = {}
def worker(fence: CompressionCommitFence):
fence_holder["fence"] = fence
# Keep ticking within each idle window so the waiter extends.
for _ in range(6):
time.sleep(0.04)
fence.touch_progress()
if not fence.begin_commit():
return (original, "aborted")
try:
return (compressed, "ok-prompt")
finally:
fence.finish_commit()
result_msgs, result_prompt = run_compress_context_with_progress_timeout(
worker=worker,
messages=original,
system_prompt_fallback="fallback",
idle_timeout_seconds=0.1,
total_ceiling_seconds=1.0,
)
assert result_msgs == compressed
assert result_prompt == "ok-prompt"
assert "fence" in fence_holder
def test_commit_started_before_timeout_returns_worker_result(self):
original = [{"role": "user", "content": "a"}]
compressed = [{"role": "assistant", "content": "done"}]
entered = threading.Event()
def worker(fence: CompressionCommitFence):
assert fence.begin_commit()
entered.set()
try:
time.sleep(0.2)
return (compressed, "committed")
finally:
fence.finish_commit()
result_msgs, result_prompt = run_compress_context_with_progress_timeout(
worker=worker,
messages=original,
system_prompt_fallback="fallback",
idle_timeout_seconds=0.05,
total_ceiling_seconds=0.05,
)
assert entered.wait(timeout=1)
assert result_msgs == compressed
assert result_prompt == "committed"
def test_rejects_non_positive_idle(self):
with pytest.raises(ValueError):
run_compress_context_with_progress_timeout(
worker=lambda fence: ([], ""),
messages=[],
system_prompt_fallback="",
idle_timeout_seconds=0,
total_ceiling_seconds=1,
)
def test_propagates_conversation_context_into_worker(self):
from agent.portal_tags import (
get_conversation_context,
reset_conversation_context,
set_conversation_context,
)
seen = {}
token = set_conversation_context("conv-timeout-ctx")
try:
def worker(fence: CompressionCommitFence):
seen["ctx"] = get_conversation_context()
if not fence.begin_commit():
return ([], "")
try:
return ([{"role": "user", "content": "ok"}], "p")
finally:
fence.finish_commit()
msgs, prompt = run_compress_context_with_progress_timeout(
worker=worker,
messages=[{"role": "user", "content": "x"}],
system_prompt_fallback="fallback",
idle_timeout_seconds=1.0,
total_ceiling_seconds=2.0,
)
finally:
reset_conversation_context(token)
assert seen.get("ctx") == "conv-timeout-ctx"
assert prompt == "p"
assert msgs[0]["content"] == "ok"
def test_runs_worker_off_caller_thread(self):
"""Mirror gateway run_in_executor: compress work must leave the caller thread."""
caller = threading.current_thread().ident
seen = {}
def worker(fence: CompressionCommitFence):
seen["worker"] = threading.current_thread().ident
if not fence.begin_commit():
return ([], "")
try:
return ([{"role": "user", "content": "ok"}], "p")
finally:
fence.finish_commit()
msgs, prompt = run_compress_context_with_progress_timeout(
worker=worker,
messages=[],
system_prompt_fallback="",
idle_timeout_seconds=1.0,
total_ceiling_seconds=1.0,
)
assert seen.get("worker") is not None
assert seen["worker"] != caller
assert prompt == "p"
assert msgs[0]["content"] == "ok"
def test_reuses_module_shared_executor(self):
from tools.daemon_pool import DaemonThreadPoolExecutor
from agent import conversation_compression as mod
first = mod._get_compress_timeout_executor()
second = mod._get_compress_timeout_executor()
assert first is second
assert isinstance(first, DaemonThreadPoolExecutor)
class TestCompressContextForwarderOwnsTimeout:
"""AIAgent._compress_context wraps when no caller fence is supplied."""
def test_owned_timeout_skips_hung_compress(self, monkeypatch):
from run_agent import AIAgent
agent = object.__new__(AIAgent)
agent.session_id = "s1"
agent._cached_system_prompt = "sys"
agent._emit_warning = MagicMock()
agent._touch_activity = MagicMock()
agent._build_system_prompt = MagicMock(return_value="sys")
agent._conversation_root_id = MagicMock(return_value=None)
agent.context_compressor = MagicMock()
agent.context_compressor._consecutive_timeout_failures = 0
agent.context_compressor._record_compression_failure_cooldown = MagicMock()
hang = threading.Event()
calls = {"n": 0}
def fake_compress(agent_obj, messages, system_message, **kwargs):
calls["n"] += 1
fence = kwargs.get("commit_fence")
assert fence is not None
hang.wait(timeout=2)
if not fence.begin_commit():
return messages, "sys"
try:
return ([{"role": "assistant", "content": "nope"}], "sys")
finally:
fence.finish_commit()
monkeypatch.setattr(
"agent.conversation_compression.compress_context",
fake_compress,
)
monkeypatch.setattr(
"agent.conversation_compression.resolve_context_compression_timeouts",
lambda compression_cfg=None: (0.05, 0.2),
)
monkeypatch.setattr(
"agent.portal_tags.get_conversation_context",
lambda: object(),
)
original = [{"role": "user", "content": "stay"}]
out_msgs, out_prompt = AIAgent._compress_context(
agent, original, "sys"
)
hang.set()
assert out_msgs is original
assert out_prompt == "sys"
assert calls["n"] == 1
agent._emit_warning.assert_called_once()
assert agent.context_compressor._consecutive_timeout_failures == 1
agent.context_compressor._record_compression_failure_cooldown.assert_called_once()
cooldown_args = (
agent.context_compressor._record_compression_failure_cooldown.call_args[0]
)
assert cooldown_args[0] == 60.0
assert "host compress_context timeout" in cooldown_args[1]
from agent.session_activity import ActivityProvenance
agent._touch_activity.assert_called_with(
"context compression timed out",
provenance=ActivityProvenance.AGENT_COMPRESSION_TIMEOUT,
)
def test_fallback_prompt_resolved_lazily_on_timeout(self, monkeypatch):
"""Eager prompt rebuild must not run before compression starts."""
from run_agent import AIAgent
agent = object.__new__(AIAgent)
agent.session_id = "s1"
agent._cached_system_prompt = None
agent._emit_warning = MagicMock()
agent._touch_activity = MagicMock()
agent._conversation_root_id = MagicMock(return_value=None)
agent.context_compressor = MagicMock()
agent.context_compressor._consecutive_timeout_failures = 0
agent.context_compressor._record_compression_failure_cooldown = MagicMock()
builds = {"n": 0}
def boom_build(*_a, **_kw):
builds["n"] += 1
raise RuntimeError("prompt rebuild boom")
agent._build_system_prompt = boom_build
hang = threading.Event()
def fake_compress(agent_obj, messages, system_message, **kwargs):
hang.wait(timeout=2)
fence = kwargs.get("commit_fence")
if fence is not None and not fence.begin_commit():
return messages, "sys"
return messages, "sys"
monkeypatch.setattr(
"agent.conversation_compression.compress_context",
fake_compress,
)
monkeypatch.setattr(
"agent.conversation_compression.resolve_context_compression_timeouts",
lambda compression_cfg=None: (0.05, 0.2),
)
monkeypatch.setattr(
"agent.portal_tags.get_conversation_context",
lambda: object(),
)
original = [{"role": "user", "content": "stay"}]
out_msgs, out_prompt = AIAgent._compress_context(
agent, original, "sys"
)
hang.set()
assert out_msgs is original
assert out_prompt == "sys"
# Fallback rebuild runs only on the timeout return path.
assert builds["n"] == 1
agent._emit_warning.assert_called_once()
def test_caller_fence_bypasses_owned_wrapper(self, monkeypatch):
from run_agent import AIAgent
agent = object.__new__(AIAgent)
agent.session_id = "s1"
agent._cached_system_prompt = "sys"
agent._conversation_root_id = MagicMock(return_value=None)
seen = {}
def fake_compress(agent_obj, messages, system_message, **kwargs):
seen["fence"] = kwargs.get("commit_fence")
return ([{"role": "assistant", "content": "ok"}], "sys")
monkeypatch.setattr(
"agent.conversation_compression.compress_context",
fake_compress,
)
# If the owned wrapper ran, this would raise — prove we never call it.
monkeypatch.setattr(
"agent.conversation_compression.run_compress_context_with_progress_timeout",
lambda **kwargs: (_ for _ in ()).throw(
AssertionError("owned wrapper must not run")
),
)
monkeypatch.setattr(
"agent.portal_tags.get_conversation_context",
lambda: object(),
)
fence = CompressionCommitFence()
msgs, prompt = AIAgent._compress_context(
agent,
[{"role": "user", "content": "x"}],
"sys",
commit_fence=fence,
)
assert seen["fence"] is fence
assert prompt == "sys"
assert msgs[0]["content"] == "ok"

113
tests/agent/test_compression_concurrent_fork.py
View file

@ -127,7 +127,7 @@ def test_compression_activity_heartbeat_touches_agent_during_long_compress(tmp_p
agent = _build_agent_with_db(db, session_id)
agent._compression_activity_heartbeat_interval = 0.1
touch_calls: list[str] = []
agent._touch_activity = lambda desc: touch_calls.append(desc)
agent._touch_activity = lambda desc, **_kw: touch_calls.append(desc)
def _slow_compress(*_a, **_kw):
_wait_for_touch(touch_calls, "context compression in progress")
@ -156,7 +156,7 @@ def test_compression_activity_heartbeat_stops_on_compress_exception(tmp_path: Pa
agent = _build_agent_with_db(db, session_id)
agent._compression_activity_heartbeat_interval = 0.1
touch_calls: list[str] = []
agent._touch_activity = lambda desc: touch_calls.append(desc)
agent._touch_activity = lambda desc, **_kw: touch_calls.append(desc)
def _failing_compress(*_a, **_kw):
_wait_for_touch(touch_calls, "context compression in progress")
@ -182,7 +182,7 @@ def test_compression_activity_heartbeat_ignores_touch_errors(tmp_path: Path) ->
agent = _build_agent_with_db(db, session_id)
agent._compression_activity_heartbeat_interval = 0.1
agent._touch_activity = lambda _desc: (_ for _ in ()).throw(RuntimeError("touch boom"))
agent._touch_activity = lambda _desc, **_kw: (_ for _ in ()).throw(RuntimeError("touch boom"))
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
compressed, _sp = agent._compress_context(messages, "sys", approx_tokens=120_000)
@ -207,7 +207,7 @@ def test_compression_activity_heartbeat_strict_signature_fallback_releases_lock(
agent = _build_agent_with_db(db, session_id)
agent._compression_activity_heartbeat_interval = "not-a-number"
touch_calls: list[str] = []
agent._touch_activity = lambda desc: touch_calls.append(desc)
agent._touch_activity = lambda desc, **_kw: touch_calls.append(desc)
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
strict_calls: list[int | None] = []
@ -240,7 +240,13 @@ def test_compression_activity_heartbeat_nonfinite_interval_falls_back(tmp_path:
agent = _build_agent_with_db(db, session_id)
touch_calls: list[str] = []
agent._touch_activity = lambda desc: touch_calls.append(desc)
touch_provenances: list = []
def _capture(desc, *, provenance=None):
touch_calls.append(desc)
touch_provenances.append(provenance)
agent._touch_activity = _capture
heartbeat = _CompressionActivityHeartbeat(agent, interval_seconds=float("inf"))
@ -248,7 +254,104 @@ def test_compression_activity_heartbeat_nonfinite_interval_falls_back(tmp_path:
heartbeat.start()
heartbeat.stop()
assert touch_calls == ["context compression started", "context compression completed"]
from agent.session_activity import ActivityProvenance
assert touch_provenances == [
ActivityProvenance.AGENT_COMPRESSION,
ActivityProvenance.AGENT_COMPRESSION,
]
def test_compression_heartbeat_does_not_clobber_timeout_provenance() -> None:
"""Detached heartbeat/stop must not overwrite a host timeout stamp."""
from types import SimpleNamespace
from agent.conversation_compression import _CompressionActivityHeartbeat
from agent.session_activity import ActivityProvenance
agent = SimpleNamespace(
_last_activity_provenance=ActivityProvenance.AGENT_COMPRESSION_TIMEOUT,
_last_activity_desc="context compression timed out",
touches=[],
)
def _touch(desc, *, provenance=None):
agent.touches.append((desc, provenance))
agent._last_activity_provenance = provenance
agent._last_activity_desc = desc
agent._touch_activity = _touch
hb = _CompressionActivityHeartbeat(agent, interval_seconds=60.0)
hb._touch("context compression in progress")
hb.stop("context compression completed")
assert agent.touches == []
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION_TIMEOUT
assert agent._last_activity_desc == "context compression timed out"
def test_compression_heartbeat_does_not_clobber_cooldown_provenance() -> None:
"""Cooldown/abort stamps must also survive a late heartbeat stop."""
from types import SimpleNamespace
from agent.conversation_compression import _CompressionActivityHeartbeat
from agent.session_activity import ActivityProvenance
agent = SimpleNamespace(
_last_activity_provenance=ActivityProvenance.AGENT_COMPRESSION_COOLDOWN,
_last_activity_desc="compression blocked (cooldown: 30s remaining)",
touches=[],
)
def _touch(desc, *, provenance=None):
agent.touches.append((desc, provenance))
agent._last_activity_provenance = provenance
agent._last_activity_desc = desc
agent._touch_activity = _touch
hb = _CompressionActivityHeartbeat(agent, interval_seconds=60.0)
hb._touch("context compression in progress")
hb.stop("context compression failed")
assert agent.touches == []
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION_COOLDOWN
def test_compression_heartbeat_start_republishes_after_terminal_provenance() -> None:
"""A new compression episode may overwrite a prior timeout/cooldown stamp."""
from types import SimpleNamespace
from agent.conversation_compression import _CompressionActivityHeartbeat
from agent.session_activity import ActivityProvenance
agent = SimpleNamespace(
_last_activity_provenance=ActivityProvenance.AGENT_COMPRESSION_TIMEOUT,
_last_activity_desc="context compression timed out",
touches=[],
)
def _touch(desc, *, provenance=None):
agent.touches.append((desc, provenance))
agent._last_activity_provenance = provenance
agent._last_activity_desc = desc
agent._touch_activity = _touch
hb = _CompressionActivityHeartbeat(agent, interval_seconds=60.0)
hb.start()
hb.stop()
assert agent.touches[0] == (
"context compression started",
ActivityProvenance.AGENT_COMPRESSION,
)
assert agent.touches[-1] == (
"context compression completed",
ActivityProvenance.AGENT_COMPRESSION,
)
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION
def test_concurrent_compression_does_not_fork_session(tmp_path: Path) -> None:

82
tests/agent/test_session_activity.py Normal file
View file

@ -0,0 +1,82 @@
"""Unit tests for the shared session activity observation contract."""
from agent.session_activity import (
ActivityProvenance,
bound_activity_description,
build_activity_snapshot,
normalize_activity_provenance,
)
def test_bound_activity_description_truncates():
long = "x" * 200
out = bound_activity_description(long)
assert len(out) == 120
assert out.endswith("…")
def test_normalize_activity_provenance_defaults_to_unknown():
assert normalize_activity_provenance(None) is ActivityProvenance.UNKNOWN
assert normalize_activity_provenance("") is ActivityProvenance.UNKNOWN
assert normalize_activity_provenance("not-a-real-source") is ActivityProvenance.UNKNOWN
assert normalize_activity_provenance("agent.activity") is ActivityProvenance.UNKNOWN
assert (
normalize_activity_provenance(ActivityProvenance.AGENT_COMPRESSION)
is ActivityProvenance.AGENT_COMPRESSION
)
assert (
normalize_activity_provenance("agent.compression_timeout")
is ActivityProvenance.AGENT_COMPRESSION_TIMEOUT
)
def test_build_activity_snapshot_includes_compat_aliases():
snap = build_activity_snapshot(
last_activity_at=100.0,
last_activity_description="starting API call #1",
last_activity_provenance=ActivityProvenance.UNKNOWN,
now=110.0,
extra={"api_call_count": 1},
)
assert snap["last_activity_at"] == 100.0
assert snap["last_activity_description"] == "starting API call #1"
assert snap["last_activity_provenance"] == "unknown"
assert snap["seconds_since_activity"] == 10.0
assert snap["last_activity_ts"] == 100.0
assert snap["last_activity_desc"] == "starting API call #1"
assert snap["description"] == "starting API call #1"
assert snap["api_call_count"] == 1
assert "phase" not in snap
assert "last_progress_at" not in snap
def test_build_activity_snapshot_maps_missing_provenance_to_unknown():
snap = build_activity_snapshot(
last_activity_at=1.0,
last_activity_description="starting new turn (cached)",
last_activity_provenance=None,
now=2.0,
)
assert snap["last_activity_provenance"] == "unknown"
def test_build_activity_snapshot_preserves_compression_transition_provenances():
"""Compaction / timeout / cooldown share the observation source (#72424)."""
for provenance, desc in (
(ActivityProvenance.AGENT_COMPRESSION, "context compression in progress"),
(ActivityProvenance.AGENT_COMPRESSION_TIMEOUT, "context compression timed out"),
(
ActivityProvenance.AGENT_COMPRESSION_COOLDOWN,
"compression blocked (cooldown: 30s remaining)",
),
):
snap = build_activity_snapshot(
last_activity_at=50.0,
last_activity_description=desc,
last_activity_provenance=provenance,
now=55.0,
)
assert snap["last_activity_provenance"] == provenance.value
assert snap["provenance"] == provenance.value
assert snap["last_activity_description"] == desc
assert snap["seconds_since_activity"] == 5.0

28
tests/gateway/test_agent_cache.py
View file

@ -1598,10 +1598,13 @@ class TestCachedAgentInactivityReset:
"""
def _fake_agent(self, stale_seconds: float = 1800.0):
from agent.session_activity import ActivityProvenance
m = MagicMock()
m._last_activity_ts = _FAKE_NOW - stale_seconds
m._api_call_count = 10
m._last_activity_desc = "previous turn activity"
m._last_activity_provenance = ActivityProvenance.AGENT_COMPRESSION
return m
def test_fresh_turn_resets_idle_clock(self):
@ -1635,6 +1638,20 @@ class TestCachedAgentInactivityReset:
assert agent._last_activity_desc == "starting new turn (cached)"
def test_fresh_turn_resets_provenance(self):
"""interrupt_depth=0: provenance resets with ts/desc (#72039)."""
from agent.session_activity import ActivityProvenance
from gateway.run import GatewayRunner
agent = self._fake_agent()
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION
with patch("gateway.run.time") as mock_time:
mock_time.time.return_value = _FAKE_NOW
GatewayRunner._init_cached_agent_for_turn(agent, interrupt_depth=0)
assert agent._last_activity_provenance is ActivityProvenance.UNKNOWN
def test_interrupt_turn_preserves_idle_clock(self):
"""interrupt_depth=1: clock preserved so accumulated stuck-turn
idle time is not discarded by an interrupt-recursive re-entry (#15654)."""
@ -1663,6 +1680,17 @@ class TestCachedAgentInactivityReset:
"it describes the activity *at* _last_activity_ts"
)
def test_interrupt_turn_preserves_provenance(self):
"""interrupt_depth=1: provenance preserved with ts/desc."""
from agent.session_activity import ActivityProvenance
from gateway.run import GatewayRunner
agent = self._fake_agent(stale_seconds=1200.0)
GatewayRunner._init_cached_agent_for_turn(agent, interrupt_depth=1)
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION
def test_deep_interrupt_recursion_preserves_idle_clock(self):
"""interrupt_depth=MAX-1: clock still preserved at any non-zero depth."""
from gateway.run import GatewayRunner

4
tests/gateway/test_cached_agent_max_iterations.py
View file

@ -21,6 +21,7 @@ import time
from types import SimpleNamespace
from agent.iteration_budget import IterationBudget
from agent.session_activity import ActivityProvenance
def _make_cached_agent(max_iterations: int) -> SimpleNamespace:
@ -32,6 +33,7 @@ def _make_cached_agent(max_iterations: int) -> SimpleNamespace:
return SimpleNamespace(
_last_activity_ts=time.time() - 1000,
_last_activity_desc="previous turn",
_last_activity_provenance=ActivityProvenance.AGENT_COMPRESSION,
_api_call_count=42,
_last_flushed_db_idx=5,
max_iterations=max_iterations,
@ -53,6 +55,7 @@ def test_init_cached_agent_for_turn_does_not_touch_max_iterations():
# Per-turn state was reset...
assert agent._api_call_count == 0
assert agent._last_activity_desc == "starting new turn (cached)"
assert agent._last_activity_provenance is ActivityProvenance.UNKNOWN
assert agent._last_flushed_db_idx == 0
# ...but the iteration budget was NOT changed by the helper itself.
assert agent.max_iterations == 90
@ -67,6 +70,7 @@ def test_init_cached_agent_preserves_max_iterations_on_interrupt_depth():
# Activity timestamps preserved for the inactivity watchdog (#15654)...
assert agent._last_activity_desc == "previous turn"
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION
# ...and max_iterations untouched.
assert agent.max_iterations == 200

4
tests/gateway/test_config_env_bridge_authority.py
View file

@ -44,6 +44,7 @@ def _run_gateway_import(hermes_home: Path, initial_env: dict[str, str]) -> dict[
"HERMES_MAX_ITERATIONS",
"HERMES_AGENT_TIMEOUT",
"HERMES_AGENT_TIMEOUT_WARNING",
"HERMES_SESSION_STALL_TIMEOUT",
"HERMES_GATEWAY_BUSY_INPUT_MODE",
"HERMES_GATEWAY_BUSY_TEXT_MODE",
"HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT",
@ -126,16 +127,19 @@ def test_config_gateway_timeout_wins_over_stale_env(hermes_home: Path) -> None:
_write_config(hermes_home, agent_cfg={
"gateway_timeout": 1800,
"gateway_timeout_warning": 900,
"session_stall_timeout": 300,
})
_write_env(hermes_home, {
"HERMES_AGENT_TIMEOUT": "60",
"HERMES_AGENT_TIMEOUT_WARNING": "30",
"HERMES_SESSION_STALL_TIMEOUT": "15",
})
env = _run_gateway_import(hermes_home, initial_env={})
assert env.get("HERMES_AGENT_TIMEOUT") == "1800"
assert env.get("HERMES_AGENT_TIMEOUT_WARNING") == "900"
assert env.get("HERMES_SESSION_STALL_TIMEOUT") == "300"
def test_config_display_busy_input_mode_wins_over_stale_env(hermes_home: Path) -> None:

462
tests/gateway/test_session_stall_watchdog.py Normal file
View file

@ -0,0 +1,462 @@
"""Tests for gateway session stall watchdog (#72016 item 2)."""
from __future__ import annotations
import asyncio
import time
from types import SimpleNamespace
import pytest
from agent.session_activity import ActivityProvenance, build_activity_snapshot
from gateway.run import GatewayRunner, _AGENT_PENDING_SENTINEL
from gateway.session_stall import (
format_session_stall_notification,
resolve_session_idle_seconds_from_activity,
should_clear_session_stall_notification,
should_emit_session_stall_notification,
)
class _FakeAdapter:
def __init__(self):
self._pending_messages = {}
self.sent = []
async def send(self, chat_id, content, metadata=None):
self.sent.append(
{"chat_id": chat_id, "content": content, "metadata": metadata}
)
class _FakeAgent:
"""Exposes the shared #72039 activity snapshot as the sole progress source."""
def __init__(
self,
last_activity_ts: float,
*,
description: str = "api call",
provenance: ActivityProvenance = ActivityProvenance.UNKNOWN,
):
self._last_activity_ts = last_activity_ts
self._last_activity_desc = description
self._last_activity_provenance = provenance
def get_activity_summary(self):
return build_activity_snapshot(
last_activity_at=self._last_activity_ts,
last_activity_description=self._last_activity_desc,
last_activity_provenance=self._last_activity_provenance,
)
class _AgentWithoutSummary:
"""Agent with a raw clock but no shared summary consumer API."""
def __init__(self, last_activity_ts: float):
self._last_activity_ts = last_activity_ts
def test_should_emit_requires_pending_and_idle():
assert should_emit_session_stall_notification(
timeout_seconds=300,
idle_seconds=400,
has_pending_inbound=True,
already_notified=False,
)
assert not should_emit_session_stall_notification(
timeout_seconds=300,
idle_seconds=400,
has_pending_inbound=False,
already_notified=False,
)
assert not should_emit_session_stall_notification(
timeout_seconds=300,
idle_seconds=100,
has_pending_inbound=True,
already_notified=False,
)
assert not should_emit_session_stall_notification(
timeout_seconds=0,
idle_seconds=9999,
has_pending_inbound=True,
already_notified=False,
)
assert not should_emit_session_stall_notification(
timeout_seconds=300,
idle_seconds=400,
has_pending_inbound=True,
already_notified=True,
)
def test_should_clear_when_pending_gone_or_activity_resumes():
assert should_clear_session_stall_notification(
timeout_seconds=300,
idle_seconds=400,
has_pending_inbound=False,
)
assert should_clear_session_stall_notification(
timeout_seconds=300,
idle_seconds=10,
has_pending_inbound=True,
)
assert not should_clear_session_stall_notification(
timeout_seconds=300,
idle_seconds=400,
has_pending_inbound=True,
)
def test_should_clear_holds_latch_when_idle_unknown():
assert not should_clear_session_stall_notification(
timeout_seconds=300,
idle_seconds=None,
has_pending_inbound=True,
)
def test_format_session_stall_notification_minutes():
msg = format_session_stall_notification(125)
assert "2 min ago" in msg
assert "/new" in msg
assert format_session_stall_notification(30).count("1 min ago") == 1
def test_resolve_idle_uses_shared_activity_snapshot_only():
now = 1_000_000.0
snap = build_activity_snapshot(
last_activity_at=now - 120,
last_activity_description="tool: terminal",
last_activity_provenance=ActivityProvenance.UNKNOWN,
now=now,
)
assert resolve_session_idle_seconds_from_activity(snap, now=now) == 120.0
assert resolve_session_idle_seconds_from_activity(None, now=now) is None
assert resolve_session_idle_seconds_from_activity({}, now=now) is None
def test_resolve_idle_prefers_seconds_since_activity_field():
idle = resolve_session_idle_seconds_from_activity(
{
"seconds_since_activity": 42.5,
"last_activity_at": 1.0, # must be ignored when seconds present
},
now=999.0,
)
assert idle == 42.5
def _runner_for_stall(adapter: _FakeAdapter) -> GatewayRunner:
r = GatewayRunner.__new__(GatewayRunner)
r._running = True
r.adapters = {"fake": adapter}
r._profile_adapters = {}
r._running_agents = {}
r._running_agents_ts = {}
r._queued_events = {}
r._session_stall_notified = {}
r._thread_metadata_for_source = lambda source, *a, **k: {
"thread_id": getattr(source, "thread_id", None)
}
return r
def _pending_event(chat_id: str = "chat-1", thread_id: str | None = None):
source = SimpleNamespace(chat_id=chat_id, thread_id=thread_id, platform=None)
return SimpleNamespace(text="follow-up", source=source, timestamp=time.time())
@pytest.mark.asyncio
async def test_check_session_stalls_notifies_once(monkeypatch):
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
monkeypatch.setenv("HERMES_SESSION_STALL_TIMEOUT", "60")
session_key = "agent:main:telegram:dm:1"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
sent = await runner._check_session_stalls(60)
assert sent == 1
assert len(adapter.sent) == 1
assert "/new" in adapter.sent[0]["content"]
assert runner._session_stall_notified.get(session_key) is True
# Second pass must not spam.
sent2 = await runner._check_session_stalls(60)
assert sent2 == 0
assert len(adapter.sent) == 1
@pytest.mark.asyncio
async def test_check_session_stalls_skips_fresh_activity():
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:2"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 5)
sent = await runner._check_session_stalls(60)
assert sent == 0
assert adapter.sent == []
@pytest.mark.asyncio
async def test_check_session_stalls_skips_without_pending():
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:3"
runner._running_agents[session_key] = _FakeAgent(time.time() - 999)
sent = await runner._check_session_stalls(60)
assert sent == 0
assert adapter.sent == []
@pytest.mark.asyncio
async def test_check_session_stalls_clears_latch_when_pending_drains():
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:4"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 200)
assert await runner._check_session_stalls(60) == 1
assert session_key in runner._session_stall_notified
adapter._pending_messages.clear()
assert await runner._check_session_stalls(60) == 0
assert session_key not in runner._session_stall_notified
@pytest.mark.asyncio
async def test_check_session_stalls_skips_pending_sentinel_without_activity():
"""Pending construction has no shared activity snapshot — no parallel clocks."""
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:5"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _AGENT_PENDING_SENTINEL
runner._running_agents_ts[session_key] = time.time() - 90
sent = await runner._check_session_stalls(60)
assert sent == 0
assert adapter.sent == []
@pytest.mark.asyncio
async def test_check_session_stalls_ignores_raw_clock_without_summary():
"""Do not fall back to agent._last_activity_ts outside get_activity_summary()."""
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:raw"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _AgentWithoutSummary(time.time() - 999)
runner._running_agents_ts[session_key] = time.time() - 999
sent = await runner._check_session_stalls(60)
assert sent == 0
assert adapter.sent == []
@pytest.mark.asyncio
async def test_session_stall_watcher_disabled_when_timeout_zero(monkeypatch):
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
monkeypatch.setenv("HERMES_SESSION_STALL_TIMEOUT", "0")
session_key = "agent:main:telegram:dm:6"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 999)
task = asyncio.create_task(runner._session_stall_watcher(interval=1))
await asyncio.sleep(0.05)
# Force one check path via timeout read; watcher should no-op when 0.
assert runner._session_stall_timeout_seconds() == 0.0
runner._running = False
await asyncio.wait_for(task, timeout=2)
assert adapter.sent == []
@pytest.mark.asyncio
async def test_check_session_stalls_queued_events_overflow_notifies():
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:overflow"
event = _pending_event()
runner._queued_events[session_key] = [event]
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
runner._adapter_for_source = lambda source: adapter
assert await runner._check_session_stalls(60) == 1
assert adapter.sent and "/new" in adapter.sent[0]["content"]
@pytest.mark.asyncio
async def test_check_session_stalls_scans_profile_adapters():
adapter = _FakeAdapter()
runner = _runner_for_stall(_FakeAdapter()) # empty primary path unused
runner.adapters = {}
runner._profile_adapters = {"coder": {"fake": adapter}}
session_key = "agent:coder:telegram:dm:1"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
assert await runner._check_session_stalls(60) == 1
assert len(adapter.sent) == 1
@pytest.mark.asyncio
async def test_check_session_stalls_logs_compression_provenance(caplog):
"""Stale compression stamps still stall, but provenance stays visible."""
import logging
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:compress"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(
time.time() - 120,
description="compressing context",
provenance=ActivityProvenance.AGENT_COMPRESSION,
)
with caplog.at_level(logging.WARNING):
assert await runner._check_session_stalls(60) == 1
assert any("agent.compression" in r.message for r in caplog.records)
assert any("compressing context" in r.message for r in caplog.records)
@pytest.mark.asyncio
async def test_check_session_stalls_skips_active_compression_heartbeat():
"""Fresh agent.compression heartbeats are progress, not a silent stall."""
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:compacting"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(
time.time() - 5,
description="context compression in progress",
provenance=ActivityProvenance.AGENT_COMPRESSION,
)
assert await runner._check_session_stalls(60) == 0
assert adapter.sent == []
assert session_key not in runner._session_stall_notified
@pytest.mark.asyncio
async def test_check_session_stalls_does_not_renotify_after_summary_gap():
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:gap"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
assert await runner._check_session_stalls(60) == 1
# Transient observation gap (no summary API).
runner._running_agents[session_key] = _AgentWithoutSummary(time.time() - 999)
assert await runner._check_session_stalls(60) == 0
assert runner._session_stall_notified.get(session_key) is True
# Stale progress returns — must not spam again in the same episode.
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
assert await runner._check_session_stalls(60) == 0
assert len(adapter.sent) == 1
@pytest.mark.asyncio
async def test_check_session_stalls_retries_after_send_failure():
class _FailThenOk(_FakeAdapter):
def __init__(self):
super().__init__()
self.calls = 0
async def send(self, chat_id, content, metadata=None):
self.calls += 1
if self.calls == 1:
raise RuntimeError("boom")
await super().send(chat_id, content, metadata=metadata)
adapter = _FailThenOk()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:retry"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
assert await runner._check_session_stalls(60) == 0
assert session_key not in runner._session_stall_notified
assert await runner._check_session_stalls(60) == 1
assert runner._session_stall_notified.get(session_key) is True
assert len(adapter.sent) == 1
@pytest.mark.asyncio
async def test_check_session_stalls_retries_after_soft_send_failure():
class _SoftFailThenOk(_FakeAdapter):
def __init__(self):
super().__init__()
self.calls = 0
async def send(self, chat_id, content, metadata=None):
from gateway.platforms.base import SendResult
self.calls += 1
if self.calls == 1:
return SendResult(success=False, error="chat not found")
await super().send(chat_id, content, metadata=metadata)
return SendResult(success=True)
adapter = _SoftFailThenOk()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:soft"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
assert await runner._check_session_stalls(60) == 0
assert session_key not in runner._session_stall_notified
assert await runner._check_session_stalls(60) == 1
assert runner._session_stall_notified.get(session_key) is True
assert len(adapter.sent) == 1
@pytest.mark.asyncio
async def test_check_session_stalls_renotifies_after_resume_then_restall():
adapter = _FakeAdapter()
runner = _runner_for_stall(adapter)
session_key = "agent:main:telegram:dm:episode"
adapter._pending_messages[session_key] = _pending_event()
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
assert await runner._check_session_stalls(60) == 1
# Activity resumes (still pending) — clears latch for a new episode.
runner._running_agents[session_key] = _FakeAgent(time.time() - 5)
assert await runner._check_session_stalls(60) == 0
assert session_key not in runner._session_stall_notified
# Stall again — second episode may notify once more.
runner._running_agents[session_key] = _FakeAgent(time.time() - 120)
assert await runner._check_session_stalls(60) == 1
assert len(adapter.sent) == 2
def test_resolve_idle_rejects_nonfinite_seconds_since_activity():
now = 1_000_000.0
idle = resolve_session_idle_seconds_from_activity(
{
"seconds_since_activity": float("nan"),
"last_activity_at": now - 15,
},
now=now,
)
assert idle == 15.0
def test_session_stall_timeout_in_default_config():
from hermes_cli.config import DEFAULT_CONFIG
timeout = DEFAULT_CONFIG["agent"]["session_stall_timeout"]
assert isinstance(timeout, (int, float))
assert timeout > 0 # enabled by default; 0 would disable the watchdog

40
tests/hermes_cli/test_status.py
View file

@ -352,3 +352,43 @@ class TestShowStatusXaiOAuth:
assert "xAI OAuth" in out
assert "not logged in (run: hermes auth add xai-oauth)" in out
def test_show_status_reports_gateway_session_last_activity(monkeypatch, capsys, tmp_path):
"""hermes status should surface freshest gateway last_active (#72016)."""
from hermes_cli import status as status_mod
import hermes_cli.auth as auth_mod
import hermes_cli.gateway as gateway_mod
import hermes_state
import time
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setattr(status_mod, "get_env_path", lambda: tmp_path / ".env", raising=False)
monkeypatch.setattr(status_mod, "get_hermes_home", lambda: tmp_path, raising=False)
monkeypatch.setattr(status_mod, "load_config", lambda: {"model": "gpt-5.4"}, raising=False)
monkeypatch.setattr(status_mod, "resolve_requested_provider", lambda requested=None: "openai-codex", raising=False)
monkeypatch.setattr(status_mod, "resolve_provider", lambda requested=None, **kwargs: "openai-codex", raising=False)
monkeypatch.setattr(status_mod, "provider_label", lambda provider: "OpenAI Codex", raising=False)
monkeypatch.setattr(auth_mod, "get_nous_auth_status", lambda: {}, raising=False)
monkeypatch.setattr(auth_mod, "get_codex_auth_status", lambda: {}, raising=False)
monkeypatch.setattr(auth_mod, "get_qwen_auth_status", lambda: {}, raising=False)
monkeypatch.setattr(auth_mod, "get_xai_oauth_auth_status", lambda: {}, raising=False)
monkeypatch.setattr(gateway_mod, "find_gateway_pids", lambda exclude_pids=None: [], raising=False)
class _FakeDB:
def list_gateway_sessions(self, active_only=True):
return [
{"id": "gw-old", "last_active": time.time() - 7200},
{"id": "gw-new", "last_active": time.time() - 90},
]
def close(self):
return None
monkeypatch.setattr(hermes_state, "SessionDB", _FakeDB)
status_mod.show_status(SimpleNamespace(all=False, deep=False))
output = capsys.readouterr().out
assert "Active: 2 session(s)" in output
assert "Last activity:" in output
assert "1m ago" in output

10
tests/run_agent/test_codex_app_server_compaction.py
View file

@ -69,10 +69,12 @@ class DummyAgent:
self.events = []
self.built_prompts = []
self.touch_calls = []
self.touch_provenances = []
self._compression_activity_heartbeat_interval = 0.1
def _touch_activity(self, desc):
def _touch_activity(self, desc, *, provenance=None):
self.touch_calls.append(desc)
self.touch_provenances.append(provenance)
def _emit_status(self, message):
self.statuses.append(message)
@ -136,6 +138,12 @@ def test_codex_app_server_compaction_heartbeat_refreshes_activity_while_waiting(
assert "context compression started" in agent.touch_calls
assert "context compression in progress" in agent.touch_calls
assert agent.touch_calls[-1] == "context compression completed"
from agent.session_activity import ActivityProvenance
assert agent.touch_provenances
assert all(
p is ActivityProvenance.AGENT_COMPRESSION for p in agent.touch_provenances
)
def test_codex_app_server_manual_compression_routes_to_codex_thread():

263
tests/run_agent/test_session_activity_persist.py Normal file
View file

@ -0,0 +1,263 @@
"""Durable session activity projection from AIAgent._touch_activity (#72016)."""
from types import SimpleNamespace
from unittest.mock import MagicMock
import run_agent
from agent.session_activity import ActivityProvenance
def _agent_with_db(session_id: str = "sess-1"):
agent = SimpleNamespace(
session_id=session_id,
_session_db=MagicMock(),
_last_activity_ts=0.0,
_last_activity_desc="",
_last_activity_provenance=ActivityProvenance.UNKNOWN,
_session_activity_last_persist_mono=0.0,
_current_tool=None,
_api_call_count=0,
max_iterations=10,
iteration_budget=SimpleNamespace(used=0, max_total=10),
)
agent._touch_activity = run_agent.AIAgent._touch_activity.__get__(agent, SimpleNamespace)
agent._persist_session_activity_if_due = (
run_agent.AIAgent._persist_session_activity_if_due.__get__(agent, SimpleNamespace)
)
agent._reset_activity_labels_after_turn = (
run_agent.AIAgent._reset_activity_labels_after_turn.__get__(agent, SimpleNamespace)
)
agent.get_activity_summary = run_agent.AIAgent.get_activity_summary.__get__(
agent, SimpleNamespace
)
return agent
def test_touch_activity_persists_session_activity_once_per_minute(monkeypatch):
agent = _agent_with_db()
mono = {"t": 1000.0}
monkeypatch.setattr(run_agent.time, "time", lambda: 1_700_000_000.0)
monkeypatch.setattr(run_agent.time, "monotonic", lambda: mono["t"])
monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False)
agent._touch_activity("starting API call #1")
agent._session_db.touch_session_activity.assert_called_once_with(
"sess-1",
1_700_000_000.0,
description="starting API call #1",
provenance=ActivityProvenance.UNKNOWN,
)
agent._session_db.touch_session_activity.reset_mock()
mono["t"] = 1030.0 # within 60s window
agent._touch_activity("receiving stream response")
agent._session_db.touch_session_activity.assert_not_called()
mono["t"] = 1061.0
agent._touch_activity("API call #1 completed")
agent._session_db.touch_session_activity.assert_called_once_with(
"sess-1",
1_700_000_000.0,
description="API call #1 completed",
provenance=ActivityProvenance.UNKNOWN,
)
def test_touch_activity_skips_persist_without_session_db(monkeypatch):
agent = _agent_with_db()
agent._session_db = None
monkeypatch.setattr(run_agent.time, "time", lambda: 1.0)
monkeypatch.setattr(run_agent.time, "monotonic", lambda: 1.0)
monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False)
agent._touch_activity("starting API call #1")
assert agent._last_activity_desc == "starting API call #1"
assert agent._last_activity_provenance is ActivityProvenance.UNKNOWN
def test_touch_activity_accepts_named_provenance(monkeypatch):
agent = _agent_with_db()
monkeypatch.setattr(run_agent.time, "time", lambda: 1_700_000_000.0)
monkeypatch.setattr(run_agent.time, "monotonic", lambda: 1000.0)
monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False)
agent._touch_activity(
"compressing context",
provenance=ActivityProvenance.AGENT_COMPRESSION,
)
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION
agent._session_db.touch_session_activity.assert_called_once_with(
"sess-1",
1_700_000_000.0,
description="compressing context",
provenance=ActivityProvenance.AGENT_COMPRESSION,
)
agent._session_db.touch_session_activity.reset_mock()
agent._session_activity_last_persist_mono = 0.0
agent._touch_activity("starting API call #1")
assert agent._last_activity_provenance is ActivityProvenance.UNKNOWN
agent._session_db.touch_session_activity.assert_called_once_with(
"sess-1",
1_700_000_000.0,
description="starting API call #1",
provenance=ActivityProvenance.UNKNOWN,
)
def test_touch_activity_persist_errors_are_swallowed(monkeypatch):
agent = _agent_with_db()
agent._session_db.touch_session_activity.side_effect = RuntimeError("db locked")
monkeypatch.setattr(run_agent.time, "time", lambda: 1.0)
monkeypatch.setattr(run_agent.time, "monotonic", lambda: 1.0)
monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False)
agent._touch_activity("tool completed: terminal (1.0s)")
assert agent._last_activity_desc == "tool completed: terminal (1.0s)"
def test_get_activity_summary_exposes_shared_activity_contract(monkeypatch):
agent = _agent_with_db()
monkeypatch.setattr(run_agent.time, "time", lambda: 1_700_000_010.0)
monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False)
agent._last_activity_ts = 1_700_000_000.0
agent._last_activity_desc = "executing tool: terminal"
agent._last_activity_provenance = ActivityProvenance.UNKNOWN
summary = agent.get_activity_summary()
assert summary["last_activity_at"] == 1_700_000_000.0
assert summary["last_activity_description"] == "executing tool: terminal"
assert summary["last_activity_provenance"] == "unknown"
assert summary["seconds_since_activity"] == 10.0
assert summary["last_activity_ts"] == 1_700_000_000.0
assert summary["last_activity_desc"] == "executing tool: terminal"
assert "phase" not in summary
assert "last_progress_at" not in summary
def test_reset_activity_labels_after_turn_keeps_ts_and_clears_labels():
"""Turn-end cleanup must not bump ts (watchdog continuity) but must
clear mid-turn description/provenance and force a durable label clear.
"""
agent = _agent_with_db()
agent._last_activity_ts = 1_700_000_000.0
agent._last_activity_desc = "compressing context"
agent._last_activity_provenance = ActivityProvenance.AGENT_COMPRESSION
# Still inside the 60s persist window from a prior heartbeat — label
# clear must bypass that rate limit via clear_session_activity_labels.
agent._session_activity_last_persist_mono = 1_000.0
agent._reset_activity_labels_after_turn()
assert agent._last_activity_ts == 1_700_000_000.0
assert agent._last_activity_desc == ""
assert agent._last_activity_provenance is ActivityProvenance.UNKNOWN
agent._session_db.clear_session_activity_labels.assert_called_once_with("sess-1")
agent._session_db.touch_session_activity.assert_not_called()
def test_reset_activity_labels_after_turn_skips_db_without_session():
agent = _agent_with_db()
agent.session_id = None
agent._last_activity_ts = 42.0
agent._last_activity_desc = "executing tool: terminal"
agent._last_activity_provenance = ActivityProvenance.AGENT_COMPRESSION
agent._reset_activity_labels_after_turn()
assert agent._last_activity_ts == 42.0
assert agent._last_activity_desc == ""
assert agent._last_activity_provenance is ActivityProvenance.UNKNOWN
agent._session_db.clear_session_activity_labels.assert_not_called()
def test_reset_activity_labels_after_turn_swallows_db_errors():
agent = _agent_with_db()
agent._last_activity_ts = 99.0
agent._last_activity_desc = "starting API call #1"
agent._last_activity_provenance = ActivityProvenance.UNKNOWN
agent._session_db.clear_session_activity_labels.side_effect = RuntimeError("db locked")
agent._reset_activity_labels_after_turn()
assert agent._last_activity_ts == 99.0
assert agent._last_activity_desc == ""
assert agent._last_activity_provenance is ActivityProvenance.UNKNOWN
def test_warn_context_overflow_blocked_stamps_compression_cooldown(monkeypatch):
agent = _agent_with_db()
agent._last_ctx_overflow_warn = None
agent._emit_warning = MagicMock()
agent._touch_activity = run_agent.AIAgent._touch_activity.__get__(
agent, SimpleNamespace
)
monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False)
monkeypatch.setattr(run_agent.time, "time", lambda: 1_700_000_100.0)
monkeypatch.setattr(run_agent.time, "monotonic", lambda: 2000.0)
run_agent.AIAgent._warn_context_overflow_blocked(
agent, "cooldown: 30s remaining", 80_000, 40_000
)
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION_COOLDOWN
assert "compression blocked" in agent._last_activity_desc
agent._emit_warning.assert_called_once()
# Deduped re-entry must not re-touch or re-emit.
agent._session_db.touch_session_activity.reset_mock()
prev_desc = agent._last_activity_desc
run_agent.AIAgent._warn_context_overflow_blocked(
agent, "cooldown: 29s remaining", 80_000, 40_000
)
assert agent._last_activity_desc == prev_desc
agent._emit_warning.assert_called_once()
def test_warn_context_overflow_blocked_stamps_cooldown_for_ineffective(monkeypatch):
agent = _agent_with_db()
agent._last_ctx_overflow_warn = None
agent._emit_warning = MagicMock()
agent._touch_activity = run_agent.AIAgent._touch_activity.__get__(
agent, SimpleNamespace
)
monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False)
monkeypatch.setattr(run_agent.time, "time", lambda: 1_700_000_100.0)
monkeypatch.setattr(run_agent.time, "monotonic", lambda: 2000.0)
run_agent.AIAgent._warn_context_overflow_blocked(
agent, "ineffective: last pass saved 0 tokens", 80_000, 40_000
)
assert agent._last_activity_provenance is ActivityProvenance.AGENT_COMPRESSION_COOLDOWN
agent._emit_warning.assert_called_once()
def test_compression_transition_provenances_surface_in_activity_summary(monkeypatch):
"""Compaction / timeout / cooldown publish through get_activity_summary."""
agent = _agent_with_db()
monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False)
monkeypatch.setattr(run_agent.time, "time", lambda: 1_700_000_200.0)
monkeypatch.setattr(run_agent.time, "monotonic", lambda: 3000.0)
transitions = (
(
ActivityProvenance.AGENT_COMPRESSION,
"context compression in progress",
),
(
ActivityProvenance.AGENT_COMPRESSION_TIMEOUT,
"context compression timed out",
),
(
ActivityProvenance.AGENT_COMPRESSION_COOLDOWN,
"compression blocked (cooldown: 30s remaining)",
),
)
for provenance, desc in transitions:
agent._touch_activity(desc, provenance=provenance)
summary = agent.get_activity_summary()
assert summary["last_activity_provenance"] == provenance.value
assert summary["provenance"] == provenance.value
assert summary["last_activity_description"] == desc
assert summary["last_activity_desc"] == desc

105
tests/test_hermes_state.py
View file

@ -8,6 +8,7 @@ from unittest import mock
import pytest
import hermes_state
from agent.session_activity import ActivityProvenance
from hermes_state import SCHEMA_SQL, SCHEMA_VERSION, SessionDB
@ -4674,6 +4675,110 @@ class TestListSessionsRich:
# No messages, so last_active falls back to started_at
assert sessions[0]["last_active"] == sessions[0]["started_at"]
def test_last_active_prefers_session_activity_heartbeat(self, db):
"""Mid-turn agent heartbeats must advance last_active without new messages (#72016)."""
db.create_session("s1", "cli")
db.append_message("s1", "user", "hello")
with db._lock:
db._conn.execute(
"UPDATE messages SET timestamp=? WHERE session_id=? AND role=?",
(1_700_000_000.0, "s1", "user"),
)
db._conn.commit()
before = db.list_sessions_rich()[0]["last_active"]
heartbeat = 1_700_000_500.0
db.touch_session_activity(
"s1",
heartbeat,
description="starting API call #1",
provenance=ActivityProvenance.UNKNOWN,
)
after = db.list_sessions_rich()[0]["last_active"]
assert after == heartbeat
assert after > before
row = db.get_session("s1")
assert row["last_activity_at"] == heartbeat
assert row["last_activity_description"] == "starting API call #1"
assert row["last_activity_provenance"] == "unknown"
activity = db.get_session_activity("s1")
assert activity["last_activity_at"] == heartbeat
assert activity["last_activity_description"] == "starting API call #1"
assert "phase" not in activity
# Never move last_activity_at backwards.
db.touch_session_activity("s1", heartbeat - 100, description="ignored")
assert db.get_session("s1")["last_activity_at"] == heartbeat
assert db.get_session("s1")["last_activity_description"] == "starting API call #1"
def test_clear_session_activity_labels_keeps_timestamp(self, db):
"""Turn-end label clear must wipe desc/provenance without moving ts."""
db.create_session("s1", "cli")
heartbeat = 1_700_000_500.0
db.touch_session_activity(
"s1",
heartbeat,
description="compressing context",
provenance=ActivityProvenance.AGENT_COMPRESSION,
)
row = db.get_session("s1")
assert row["last_activity_at"] == heartbeat
assert row["last_activity_description"] == "compressing context"
assert row["last_activity_provenance"] == "agent.compression"
db.clear_session_activity_labels("s1")
row = db.get_session("s1")
assert row["last_activity_at"] == heartbeat
assert row["last_activity_description"] == ""
assert row["last_activity_provenance"] == "unknown"
activity = db.get_session_activity("s1")
assert activity["last_activity_at"] == heartbeat
assert activity["last_activity_description"] == ""
assert activity["last_activity_provenance"] == "unknown"
def test_last_active_uses_newer_message_over_stale_heartbeat(self, db):
"""Rate-limited heartbeats can lag message writes; last_active must take max."""
db.create_session("s1", "cli")
db.append_message("s1", "user", "hello")
with db._lock:
db._conn.execute(
"UPDATE messages SET timestamp=? WHERE session_id=?",
(1_700_000_800.0, "s1"),
)
db._conn.commit()
db.touch_session_activity("s1", 1_700_000_500.0, description="api") # older than message
assert db.list_sessions_rich()[0]["last_active"] == 1_700_000_800.0
def test_list_gateway_sessions_last_active_uses_activity_heartbeat(self, db):
db.create_session(
"gw-1",
"telegram",
session_key="agent:main:telegram:dm:c1",
chat_id="c1",
chat_type="dm",
)
db.append_message("gw-1", "user", "ping")
with db._lock:
db._conn.execute(
"UPDATE messages SET timestamp=? WHERE session_id=?",
(1_700_000_000.0, "gw-1"),
)
db._conn.commit()
heartbeat = 1_700_000_900.0
db.touch_session_activity(
"gw-1",
heartbeat,
description="compressing context",
)
rows = db.list_gateway_sessions(active_only=True)
assert len(rows) == 1
assert rows[0]["last_active"] == heartbeat
activity = db.get_session_activity("gw-1")
assert activity["last_activity_description"] == "compressing context"
def test_order_by_last_active_surfaces_recently_touched_older_session_first(self, db):
t0 = 1709500000.0
db.create_session("old", "cli")