feat(gateway): session activity watchdog, stall notify, compress timeout (#72424)

Three mechanisms to detect and notify when gateway sessions stall silently:

1. Mid-turn activity heartbeats stamped to SessionDB so hermes sessions list
   and hermes status show progress during long turns without new message rows.

2. Stall watchdog: when a busy session has pending inbound and the shared
   activity clock is idle past agent.session_stall_timeout (default 300),
   log a WARNING and notify the user once to try /new. Notify-only; does
   not kill the turn.

3. Compaction timeout: fenceless compress_context callers get a progress-aware
   host budget (compression.context_timeout_seconds default 120 idle,
   compression.context_total_ceiling_seconds default 600 ceiling). On timeout,
   cancel via commit fence, skip compaction without dropping messages, and
   continue the turn.

Closes #72016 (slices 1-3; slice 4 cumulative SSE stream-retry deadline
remains a follow-up).

Cherry-picked from PR #72424 by @fangliquanflq.
This commit is contained in:
fangliquanflq 2026-07-27 23:16:24 +05:00 • committed by kshitij
parent 5fde131eb2
commit cfb206fe2e
24 changed files with 2638 additions and 107 deletions

View file

@ -951,6 +951,12 @@ DEFAULT_CONFIG = {
# tools or receiving API responses. Only fires when the agent has
# been completely idle for this duration. 0 = unlimited.
"gateway_timeout": 1800,
# Session stall watchdog (seconds): when a gateway session has a
# pending inbound message AND the running agent has not updated its
# activity clock for this long, log a WARNING and notify the user to
# try /new. Distinct from gateway_timeout (which kills the turn) and
# gateway_notify_interval ("still working" heartbeats). 0 = disable.
"session_stall_timeout": 300,
# Graceful drain timeout for gateway stop/restart (seconds).
# The gateway stops accepting new work, waits for running agents
# to finish, then interrupts any remaining runs after the timeout.
@ -1501,6 +1507,16 @@ DEFAULT_CONFIG = {
# while tokens are still moving — bounds a degenerate
# trickle stream. Clamped to >= hygiene_timeout_seconds.
"hygiene_failure_cooldown_seconds": 300, # skip repeated failed hygiene attempts for this session
"context_timeout_seconds": 120, # inactivity budget for in-agent compress_context
# (conversation loop, /compress, preflight, etc.).
# Same progress-aware semantics as hygiene_timeout_seconds:
# streamed summary tokens extend the wait; only a silent
# worker is cut off. 0 = disable the owned wrapper
# (callers that already pass commit_fence, e.g. gateway
# hygiene, never use this path).
"context_total_ceiling_seconds": 600, # absolute cap on in-agent compress_context wait
# even while tokens are still moving. Clamped to
# >= context_timeout_seconds when the idle budget is > 0.
"protect_first_n": 3, # non-system head messages always preserved
# verbatim, in ADDITION to the system prompt
# (which is always implicitly protected). Set to

View file

@ -236,6 +236,7 @@ def _config_overrides(config: dict) -> dict[str, str]:
interesting_paths = [
("agent", "max_turns"),
("agent", "gateway_timeout"),
("agent", "session_stall_timeout"),
("agent", "tool_use_enforcement"),
("terminal", "backend"),
("terminal", "docker_image"),

View file

@ -60,6 +60,27 @@ def _format_iso_timestamp(value) -> str:
return parsed.astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
def _format_relative_ts(ts: float) -> str:
"""Format an epoch timestamp as a short relative age for status output."""
if not ts:
return "?"
import time as _time
from datetime import datetime
delta = _time.time() - float(ts)
if delta < 60:
return "just now"
if delta < 3600:
return f"{int(delta / 60)}m ago"
if delta < 86400:
return f"{int(delta / 3600)}h ago"
if delta < 172800:
return "yesterday"
if delta < 604800:
return f"{int(delta / 86400)}d ago"
return datetime.fromtimestamp(ts).strftime("%Y-%m-%d")
def _configured_model_label(config: dict) -> str:
"""Return the configured default model from config.yaml."""
model_cfg = config.get("model")
@ -550,20 +571,29 @@ def show_status(args):
# Gateway session count: state.db is the source of truth (#9006);
# fall back to sessions.json for pre-migration installs.
_session_count = None
_gateway_rows = []
try:
from hermes_state import SessionDB
_db = SessionDB()
try:
_lister = getattr(_db, "list_gateway_sessions", None)
if callable(_lister):
_session_count = len(_lister(active_only=True))
_gateway_rows = _lister(active_only=True) or []
_session_count = len(_gateway_rows)
finally:
_db.close()
except Exception:
_session_count = None
_gateway_rows = []
if _session_count is not None and _session_count > 0:
print(f" Active: {_session_count} session(s)")
freshest = max(
(float(r.get("last_active") or 0) for r in _gateway_rows),
default=0.0,
)
if freshest > 0:
print(f" Last activity:{_format_relative_ts(freshest):>13}")
else:
sessions_file = get_hermes_home() / "sessions" / "sessions.json"
if sessions_file.exists():