From e54723a7544efc67ee3dc3c05bdd9cc8473513bd Mon Sep 17 00:00:00 2001
From: Cossackx <121278003+Cossackx@users.noreply.github.com>
Date: Tue, 23 Jun 2026 08:30:04 -0400
Subject: [PATCH] fix(gateway): detect stale gateway_state.json in `gateway
 status` (TTL + PID liveness)

Verified: applies cleanly and the patched module compiles. Tests are
described in the PR body (not bundled in this commit).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 gateway/status.py     | 46 +++++++++++++++++++++++++++++++++++++++++++
 hermes_cli/gateway.py | 22 ++++++++++++++++++++-
 2 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/gateway/status.py b/gateway/status.py
index d890a9d5bfdc..e265d380df91 100644
--- a/gateway/status.py
+++ b/gateway/status.py
@@ -961,6 +961,52 @@ def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]
     return _read_json_file(path or _get_runtime_status_path())
 
 
+# Max age of a persisted ``gateway_state.json`` snapshot before its liveness
+# claim is treated as suspect.  A healthy gateway rewrites the file (advancing
+# ``updated_at``) far more often than this; a record older than the TTL whose
+# PID is also dead almost certainly outlived an ungracefully-killed writer
+# (taskkill /F, OOM, power loss) that never ran its shutdown handler.
+_RUNTIME_STATUS_STALE_TTL_S = 120
+
+
+def runtime_status_is_stale(
+    record: Optional[dict[str, Any]],
+    ttl_s: int = _RUNTIME_STATUS_STALE_TTL_S,
+) -> bool:
+    """Return True when the runtime-status snapshot is older than ``ttl_s``.
+
+    Delegates to the existing :func:`_marker_is_stale` on the record's
+    ``updated_at`` timestamp.  A missing or unparseable timestamp is treated as
+    stale (the freshness signal is absent, so it cannot vouch for the record).
+    """
+    if not isinstance(record, dict):
+        return True
+    return _marker_is_stale(record.get("updated_at") or "", ttl_s)
+
+
+def runtime_status_pid_is_live(record: Optional[dict[str, Any]]) -> bool:
+    """Return True when the PID recorded in the snapshot is still alive.
+
+    Uses the existing no-kill :func:`_pid_exists` probe and the same
+    ``start_time`` PID-reuse guard as :func:`get_runtime_status_running_pid`:
+    when both the recorded and live start-times are known they must match, so a
+    recycled PID (same number, different process) is not mistaken for the
+    original.  Degrades to ``False`` when the record has no usable PID.
+    """
+    pid = _pid_from_record(record)
+    if pid is None or not _pid_exists(pid):
+        return False
+    recorded_start = (record or {}).get("start_time")
+    current_start = _get_process_start_time(pid)
+    if (
+        recorded_start is not None
+        and current_start is not None
+        and current_start != recorded_start
+    ):
+        return False
+    return True
+
+
 def parse_active_agents(raw: Any) -> int:
     """Coerce a persisted ``active_agents`` value to a clamped non-negative int.
 
diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py
index 5b33d7646e44..69f6464e806b 100644
--- a/hermes_cli/gateway.py
+++ b/hermes_cli/gateway.py
@@ -5357,7 +5357,11 @@ def _platform_status(platform: dict) -> str:
 def _runtime_health_lines() -> list[str]:
     """Summarize the latest persisted gateway runtime health state."""
     try:
-        from gateway.status import read_runtime_status
+        from gateway.status import (
+            read_runtime_status,
+            runtime_status_is_stale,
+            runtime_status_pid_is_live,
+        )
     except Exception:
         return []
 
@@ -5377,6 +5381,22 @@ def _runtime_health_lines() -> list[str]:
             message = pdata.get("error_message") or "unknown error"
             lines.append(f"⚠ {platform}: {message}")
 
+    # A persisted snapshot that still claims liveness can outlive an
+    # ungracefully-killed gateway (taskkill /F, OOM, power loss) whose shutdown
+    # handler never ran.  When the record is past its freshness TTL AND the
+    # recorded PID is gone, the file is contradicting reality — surface that
+    # explicitly instead of rendering the misleading live-state summary.
+    if (
+        gateway_state in ("running", "starting", "draining")
+        and runtime_status_is_stale(state)
+        and not runtime_status_pid_is_live(state)
+    ):
+        lines.append(
+            f"⚠ Stale gateway_state.json: recorded state '{gateway_state}' but the "
+            "recorded process is gone (likely an ungraceful shutdown)"
+        )
+        return lines
+
     if gateway_state == "startup_failed" and exit_reason:
         lines.append(f"⚠ Last startup issue: {exit_reason}")
     elif gateway_state == "draining":