fix(gateway): detect stale gateway_state.json in gateway status (TTL + PID liveness)

Verified: applies cleanly and the patched module compiles. Tests are
described in the PR body (not bundled in this commit).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Cossackx 2026-06-23 08:30:04 -04:00 committed by Teknium
parent 3fc0cfb902
commit e54723a754
2 changed files with 67 additions and 1 deletions

View file

@ -961,6 +961,52 @@ def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]
return _read_json_file(path or _get_runtime_status_path())
# Max age of a persisted ``gateway_state.json`` snapshot before its liveness
# claim is treated as suspect. A healthy gateway rewrites the file (advancing
# ``updated_at``) far more often than this; a record older than the TTL whose
# PID is also dead almost certainly outlived an ungracefully-killed writer
# (taskkill /F, OOM, power loss) that never ran its shutdown handler.
_RUNTIME_STATUS_STALE_TTL_S = 120
def runtime_status_is_stale(
record: Optional[dict[str, Any]],
ttl_s: int = _RUNTIME_STATUS_STALE_TTL_S,
) -> bool:
"""Return True when the runtime-status snapshot is older than ``ttl_s``.
Delegates to the existing :func:`_marker_is_stale` on the record's
``updated_at`` timestamp. A missing or unparseable timestamp is treated as
stale (the freshness signal is absent, so it cannot vouch for the record).
"""
if not isinstance(record, dict):
return True
return _marker_is_stale(record.get("updated_at") or "", ttl_s)
def runtime_status_pid_is_live(record: Optional[dict[str, Any]]) -> bool:
"""Return True when the PID recorded in the snapshot is still alive.
Uses the existing no-kill :func:`_pid_exists` probe and the same
``start_time`` PID-reuse guard as :func:`get_runtime_status_running_pid`:
when both the recorded and live start-times are known they must match, so a
recycled PID (same number, different process) is not mistaken for the
original. Degrades to ``False`` when the record has no usable PID.
"""
pid = _pid_from_record(record)
if pid is None or not _pid_exists(pid):
return False
recorded_start = (record or {}).get("start_time")
current_start = _get_process_start_time(pid)
if (
recorded_start is not None
and current_start is not None
and current_start != recorded_start
):
return False
return True
def parse_active_agents(raw: Any) -> int:
"""Coerce a persisted ``active_agents`` value to a clamped non-negative int.

View file

@ -5357,7 +5357,11 @@ def _platform_status(platform: dict) -> str:
def _runtime_health_lines() -> list[str]:
"""Summarize the latest persisted gateway runtime health state."""
try:
from gateway.status import read_runtime_status
from gateway.status import (
read_runtime_status,
runtime_status_is_stale,
runtime_status_pid_is_live,
)
except Exception:
return []
@ -5377,6 +5381,22 @@ def _runtime_health_lines() -> list[str]:
message = pdata.get("error_message") or "unknown error"
lines.append(f"{platform}: {message}")
# A persisted snapshot that still claims liveness can outlive an
# ungracefully-killed gateway (taskkill /F, OOM, power loss) whose shutdown
# handler never ran. When the record is past its freshness TTL AND the
# recorded PID is gone, the file is contradicting reality — surface that
# explicitly instead of rendering the misleading live-state summary.
if (
gateway_state in ("running", "starting", "draining")
and runtime_status_is_stale(state)
and not runtime_status_pid_is_live(state)
):
lines.append(
f"⚠ Stale gateway_state.json: recorded state '{gateway_state}' but the "
"recorded process is gone (likely an ungraceful shutdown)"
)
return lines
if gateway_state == "startup_failed" and exit_reason:
lines.append(f"⚠ Last startup issue: {exit_reason}")
elif gateway_state == "draining":