mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-22 16:25:58 +00:00
fix(gateway): detect stale gateway_state.json in gateway status (TTL + PID liveness)
Verified: applies cleanly and the patched module compiles. Tests are described in the PR body (not bundled in this commit). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
3fc0cfb902
commit
e54723a754
2 changed files with 67 additions and 1 deletions
|
|
@ -961,6 +961,52 @@ def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]
|
|||
return _read_json_file(path or _get_runtime_status_path())
|
||||
|
||||
|
||||
# Max age of a persisted ``gateway_state.json`` snapshot before its liveness
|
||||
# claim is treated as suspect. A healthy gateway rewrites the file (advancing
|
||||
# ``updated_at``) far more often than this; a record older than the TTL whose
|
||||
# PID is also dead almost certainly outlived an ungracefully-killed writer
|
||||
# (taskkill /F, OOM, power loss) that never ran its shutdown handler.
|
||||
_RUNTIME_STATUS_STALE_TTL_S = 120
|
||||
|
||||
|
||||
def runtime_status_is_stale(
|
||||
record: Optional[dict[str, Any]],
|
||||
ttl_s: int = _RUNTIME_STATUS_STALE_TTL_S,
|
||||
) -> bool:
|
||||
"""Return True when the runtime-status snapshot is older than ``ttl_s``.
|
||||
|
||||
Delegates to the existing :func:`_marker_is_stale` on the record's
|
||||
``updated_at`` timestamp. A missing or unparseable timestamp is treated as
|
||||
stale (the freshness signal is absent, so it cannot vouch for the record).
|
||||
"""
|
||||
if not isinstance(record, dict):
|
||||
return True
|
||||
return _marker_is_stale(record.get("updated_at") or "", ttl_s)
|
||||
|
||||
|
||||
def runtime_status_pid_is_live(record: Optional[dict[str, Any]]) -> bool:
|
||||
"""Return True when the PID recorded in the snapshot is still alive.
|
||||
|
||||
Uses the existing no-kill :func:`_pid_exists` probe and the same
|
||||
``start_time`` PID-reuse guard as :func:`get_runtime_status_running_pid`:
|
||||
when both the recorded and live start-times are known they must match, so a
|
||||
recycled PID (same number, different process) is not mistaken for the
|
||||
original. Degrades to ``False`` when the record has no usable PID.
|
||||
"""
|
||||
pid = _pid_from_record(record)
|
||||
if pid is None or not _pid_exists(pid):
|
||||
return False
|
||||
recorded_start = (record or {}).get("start_time")
|
||||
current_start = _get_process_start_time(pid)
|
||||
if (
|
||||
recorded_start is not None
|
||||
and current_start is not None
|
||||
and current_start != recorded_start
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_active_agents(raw: Any) -> int:
|
||||
"""Coerce a persisted ``active_agents`` value to a clamped non-negative int.
|
||||
|
||||
|
|
|
|||
|
|
@ -5357,7 +5357,11 @@ def _platform_status(platform: dict) -> str:
|
|||
def _runtime_health_lines() -> list[str]:
|
||||
"""Summarize the latest persisted gateway runtime health state."""
|
||||
try:
|
||||
from gateway.status import read_runtime_status
|
||||
from gateway.status import (
|
||||
read_runtime_status,
|
||||
runtime_status_is_stale,
|
||||
runtime_status_pid_is_live,
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
|
@ -5377,6 +5381,22 @@ def _runtime_health_lines() -> list[str]:
|
|||
message = pdata.get("error_message") or "unknown error"
|
||||
lines.append(f"⚠ {platform}: {message}")
|
||||
|
||||
# A persisted snapshot that still claims liveness can outlive an
|
||||
# ungracefully-killed gateway (taskkill /F, OOM, power loss) whose shutdown
|
||||
# handler never ran. When the record is past its freshness TTL AND the
|
||||
# recorded PID is gone, the file is contradicting reality — surface that
|
||||
# explicitly instead of rendering the misleading live-state summary.
|
||||
if (
|
||||
gateway_state in ("running", "starting", "draining")
|
||||
and runtime_status_is_stale(state)
|
||||
and not runtime_status_pid_is_live(state)
|
||||
):
|
||||
lines.append(
|
||||
f"⚠ Stale gateway_state.json: recorded state '{gateway_state}' but the "
|
||||
"recorded process is gone (likely an ungraceful shutdown)"
|
||||
)
|
||||
return lines
|
||||
|
||||
if gateway_state == "startup_failed" and exit_reason:
|
||||
lines.append(f"⚠ Last startup issue: {exit_reason}")
|
||||
elif gateway_state == "draining":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue