mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-10-20 05:15:31 +00:00
feat(gateway): session activity watchdog, stall notify, compress timeout (#72424)
Three mechanisms to detect and notify when gateway sessions stall silently: 1. Mid-turn activity heartbeats stamped to SessionDB so hermes sessions list and hermes status show progress during long turns without new message rows. 2. Stall watchdog: when a busy session has pending inbound and the shared activity clock is idle past agent.session_stall_timeout (default 300), log a WARNING and notify the user once to try /new. Notify-only; does not kill the turn. 3. Compaction timeout: fenceless compress_context callers get a progress-aware host budget (compression.context_timeout_seconds default 120 idle, compression.context_total_ceiling_seconds default 600 ceiling). On timeout, cancel via commit fence, skip compaction without dropping messages, and continue the turn. Closes #72016 (slices 1-3; slice 4 cumulative SSE stream-retry deadline remains a follow-up). Cherry-picked from PR #72424 by @fangliquanflq.
This commit is contained in:
parent
5fde131eb2
commit
cfb206fe2e
24 changed files with 2638 additions and 107 deletions
|
|
@ -951,6 +951,12 @@ DEFAULT_CONFIG = {
|
|||
# tools or receiving API responses. Only fires when the agent has
|
||||
# been completely idle for this duration. 0 = unlimited.
|
||||
"gateway_timeout": 1800,
|
||||
# Session stall watchdog (seconds): when a gateway session has a
|
||||
# pending inbound message AND the running agent has not updated its
|
||||
# activity clock for this long, log a WARNING and notify the user to
|
||||
# try /new. Distinct from gateway_timeout (which kills the turn) and
|
||||
# gateway_notify_interval ("still working" heartbeats). 0 = disable.
|
||||
"session_stall_timeout": 300,
|
||||
# Graceful drain timeout for gateway stop/restart (seconds).
|
||||
# The gateway stops accepting new work, waits for running agents
|
||||
# to finish, then interrupts any remaining runs after the timeout.
|
||||
|
|
@ -1501,6 +1507,16 @@ DEFAULT_CONFIG = {
|
|||
# while tokens are still moving — bounds a degenerate
|
||||
# trickle stream. Clamped to >= hygiene_timeout_seconds.
|
||||
"hygiene_failure_cooldown_seconds": 300, # skip repeated failed hygiene attempts for this session
|
||||
"context_timeout_seconds": 120, # inactivity budget for in-agent compress_context
|
||||
# (conversation loop, /compress, preflight, etc.).
|
||||
# Same progress-aware semantics as hygiene_timeout_seconds:
|
||||
# streamed summary tokens extend the wait; only a silent
|
||||
# worker is cut off. 0 = disable the owned wrapper
|
||||
# (callers that already pass commit_fence, e.g. gateway
|
||||
# hygiene, never use this path).
|
||||
"context_total_ceiling_seconds": 600, # absolute cap on in-agent compress_context wait
|
||||
# even while tokens are still moving. Clamped to
|
||||
# >= context_timeout_seconds when the idle budget is > 0.
|
||||
"protect_first_n": 3, # non-system head messages always preserved
|
||||
# verbatim, in ADDITION to the system prompt
|
||||
# (which is always implicitly protected). Set to
|
||||
|
|
|
|||
|
|
@ -236,6 +236,7 @@ def _config_overrides(config: dict) -> dict[str, str]:
|
|||
interesting_paths = [
|
||||
("agent", "max_turns"),
|
||||
("agent", "gateway_timeout"),
|
||||
("agent", "session_stall_timeout"),
|
||||
("agent", "tool_use_enforcement"),
|
||||
("terminal", "backend"),
|
||||
("terminal", "docker_image"),
|
||||
|
|
|
|||
|
|
@ -60,6 +60,27 @@ def _format_iso_timestamp(value) -> str:
|
|||
return parsed.astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
|
||||
|
||||
|
||||
def _format_relative_ts(ts: float) -> str:
|
||||
"""Format an epoch timestamp as a short relative age for status output."""
|
||||
if not ts:
|
||||
return "?"
|
||||
import time as _time
|
||||
from datetime import datetime
|
||||
|
||||
delta = _time.time() - float(ts)
|
||||
if delta < 60:
|
||||
return "just now"
|
||||
if delta < 3600:
|
||||
return f"{int(delta / 60)}m ago"
|
||||
if delta < 86400:
|
||||
return f"{int(delta / 3600)}h ago"
|
||||
if delta < 172800:
|
||||
return "yesterday"
|
||||
if delta < 604800:
|
||||
return f"{int(delta / 86400)}d ago"
|
||||
return datetime.fromtimestamp(ts).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def _configured_model_label(config: dict) -> str:
|
||||
"""Return the configured default model from config.yaml."""
|
||||
model_cfg = config.get("model")
|
||||
|
|
@ -550,20 +571,29 @@ def show_status(args):
|
|||
# Gateway session count: state.db is the source of truth (#9006);
|
||||
# fall back to sessions.json for pre-migration installs.
|
||||
_session_count = None
|
||||
_gateway_rows = []
|
||||
try:
|
||||
from hermes_state import SessionDB
|
||||
_db = SessionDB()
|
||||
try:
|
||||
_lister = getattr(_db, "list_gateway_sessions", None)
|
||||
if callable(_lister):
|
||||
_session_count = len(_lister(active_only=True))
|
||||
_gateway_rows = _lister(active_only=True) or []
|
||||
_session_count = len(_gateway_rows)
|
||||
finally:
|
||||
_db.close()
|
||||
except Exception:
|
||||
_session_count = None
|
||||
_gateway_rows = []
|
||||
|
||||
if _session_count is not None and _session_count > 0:
|
||||
print(f" Active: {_session_count} session(s)")
|
||||
freshest = max(
|
||||
(float(r.get("last_active") or 0) for r in _gateway_rows),
|
||||
default=0.0,
|
||||
)
|
||||
if freshest > 0:
|
||||
print(f" Last activity:{_format_relative_ts(freshest):>13}")
|
||||
else:
|
||||
sessions_file = get_hermes_home() / "sessions" / "sessions.json"
|
||||
if sessions_file.exists():
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue