feat(status): expose nous_session_valid on /api/status for hosted-agent self-heal

A hosted agent whose Nous bootstrap session dies terminally (invalid_grant /
quarantine) looks HEALTHY to every liveness/connectivity probe — the machine,
relay ws, and dashboard all stay up — yet every inference turn hard-fails with
a provider-auth error until a human re-logs-in. Nothing currently surfaces that
condition to NAS.

Add get_nous_session_validity() (valid|terminal|unknown), classified from local
auth-store state (no working token required), and report it on the public
/api/status payload. NAS's 2-min health sweep reads it and re-mints the
bootstrap session in place on 'terminal'.

Anti-flap: only a terminal failure (relogin_required / persisted quarantine
marker with tokens cleared) maps to 'terminal'; transient/mid-rotation blips and
merely-expiring tokens report 'unknown' so a healthy box never triggers a
spurious re-mint.

Part of the hosted-agent bootstrap-session self-heal (NAS side reads this field).
This commit is contained in:
Ben 2026-07-07 13:47:02 +10:00 committed by Teknium
parent 182256206a
commit 5eac665252
3 changed files with 189 additions and 0 deletions

View file

@ -5968,6 +5968,75 @@ def _compute_nous_auth_status() -> Dict[str, Any]:
return _snapshot_nous_pool_status()
# Enum values reported on the dashboard /api/status as ``nous_session_valid``.
# NAS's health sweep re-mints the bootstrap session ONLY on "terminal"; "valid"
# and "unknown" are no-ops. Keep this set small and stable — NAS parses it with
# a permissive schema, so new members are non-breaking but should stay rare.
NOUS_SESSION_VALID = "valid"
NOUS_SESSION_TERMINAL = "terminal"
NOUS_SESSION_UNKNOWN = "unknown"
def get_nous_session_validity() -> str:
"""Classify the Nous bootstrap session for the dashboard /api/status probe.
Returns one of:
- ``"valid"`` a usable Nous credential is present (login healthy).
- ``"terminal"`` the Nous session has taken a terminal auth failure
(invalid_grant / quarantined / relogin required). This is the sole
signal NAS acts on to re-mint a hosted-agent bootstrap session.
- ``"unknown"`` indeterminate (no Nous provider state, or a transient/
non-terminal error). Never triggers a re-mint.
Determinable with NO working token it reads local auth-store state only,
which is exactly the condition a dead hosted box is in.
ANTI-FLAP CONTRACT: only a *terminal* failure maps to "terminal". A normal
mid-rotation blip, a transient network error, or a merely-expiring token
must NOT report "terminal" (that would trigger a spurious NAS re-mint on a
healthy box). We key "terminal" on the auth layer's own terminal signal
(`relogin_required`) plus a persisted quarantine marker, never on a bare
"not logged in".
"""
# A persisted quarantine marker is the strongest, most stable terminal
# signal: the refresh path writes `last_auth_error.relogin_required=True`
# into the Nous provider state when it clears dead tokens (the exact path
# that produced the incident's "No access token found"). Read it directly
# so we report "terminal" even after the in-memory AuthError is long gone.
try:
state = get_provider_auth_state("nous")
except Exception:
state = None
if state:
last_err = state.get("last_auth_error")
if isinstance(last_err, dict) and last_err.get("relogin_required"):
# Only terminal while there is no usable credential left. If a later
# successful login repopulated tokens, the stale marker must not
# keep reporting terminal.
if not (state.get("access_token") or state.get("refresh_token")):
return NOUS_SESSION_TERMINAL
try:
status = get_nous_auth_status()
except Exception:
# Status computation itself failed — indeterminate, not terminal.
return NOUS_SESSION_UNKNOWN
if status.get("logged_in"):
return NOUS_SESSION_VALID
# Not logged in. Distinguish a terminal (relogin-required) failure from a
# transient / indeterminate one. Only the former is actionable by NAS.
if status.get("relogin_required"):
return NOUS_SESSION_TERMINAL
# No Nous provider state at all, or a non-terminal not-logged-in condition
# (e.g. a transient refresh error that did not set relogin_required). Treat
# as unknown so a healthy box mid-blip never triggers a re-mint.
return NOUS_SESSION_UNKNOWN
def get_codex_auth_status() -> Dict[str, Any]:
"""Status snapshot for Codex auth.

View file

@ -2400,6 +2400,21 @@ async def get_status(profile: Optional[str] = None):
# Module not importable yet (early startup) — leave as [].
pass
# Nous bootstrap-session validity for the NAS health sweep. A hosted
# agent whose Nous auth dies terminally (invalid_grant / quarantine)
# looks HEALTHY to every liveness/connectivity probe — the machine,
# relay, and this dashboard all stay up — yet every inference turn
# fails. This is the ONLY signal that surfaces that condition, and it
# is determinable with no working token (local auth-store state). NAS
# re-mints the bootstrap session when it reads "terminal". Best-effort:
# never let auth classification break the public liveness probe.
nous_session_valid = "unknown"
try:
from hermes_cli.auth import get_nous_session_validity
nous_session_valid = get_nous_session_validity()
except Exception:
nous_session_valid = "unknown"
# Always-public liveness + auth-gate shape. Safe for external uptime
# probes (NAS's wildcard-subdomain liveness probe), the SPA's pre-login
# bootstrap, and anyone who can curl the host — i.e. exactly the audience
@ -2422,6 +2437,7 @@ async def get_status(profile: Optional[str] = None):
"active_sessions": active_sessions,
"auth_required": auth_required,
"auth_providers": auth_providers,
"nous_session_valid": nous_session_valid,
}
# Absolute host paths, the gateway PID, and the internal gateway health

View file

@ -0,0 +1,104 @@
"""Tests for get_nous_session_validity — the /api/status classifier NAS reads
to decide whether to re-mint a hosted-agent bootstrap session.
The anti-flap contract is the load-bearing property: only a *terminal* auth
failure may report "terminal" (a spurious "terminal" triggers an unnecessary
NAS re-mint + machine restart on a healthy box). A mid-rotation blip, a
transient error, or a merely-expiring token must NOT report "terminal".
"""
import hermes_cli.auth as auth
from hermes_cli.auth import (
NOUS_SESSION_TERMINAL,
NOUS_SESSION_UNKNOWN,
NOUS_SESSION_VALID,
get_nous_session_validity,
)
def _clear_cache():
auth.invalidate_nous_auth_status_cache()
def test_valid_when_logged_in(monkeypatch):
"""A healthy login → 'valid'."""
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
"access_token": "at", "refresh_token": "rt",
})
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True})
assert get_nous_session_validity() == NOUS_SESSION_VALID
def test_terminal_on_persisted_quarantine_marker(monkeypatch):
"""A persisted last_auth_error.relogin_required with tokens cleared →
'terminal'. This is the exact on-disk state the incident produced."""
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
# tokens cleared by the quarantine path
"last_auth_error": {"relogin_required": True, "code": "invalid_grant"},
})
# status would also say not-logged-in, but the marker short-circuits first
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": False})
assert get_nous_session_validity() == NOUS_SESSION_TERMINAL
def test_terminal_on_relogin_required_status(monkeypatch):
"""Not logged in + relogin_required from the live status → 'terminal'."""
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
"refresh_token": "rt", # present, but status resolution fails terminally
})
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {
"logged_in": False, "relogin_required": True, "error_code": "invalid_grant",
})
assert get_nous_session_validity() == NOUS_SESSION_TERMINAL
def test_unknown_when_no_provider_state(monkeypatch):
"""No Nous provider state at all → 'unknown' (never terminal)."""
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: None)
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": False})
assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN
def test_anti_flap_transient_not_logged_in_is_unknown(monkeypatch):
"""ANTI-FLAP: not-logged-in WITHOUT relogin_required (a transient/network
blip) must be 'unknown', NOT 'terminal' otherwise a healthy box mid-blip
triggers a spurious re-mint."""
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
"access_token": "at", "refresh_token": "rt",
})
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {
"logged_in": False, "error": "connection reset", # no relogin_required
})
assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN
def test_stale_quarantine_marker_ignored_after_relogin(monkeypatch):
"""ANTI-FLAP: a leftover last_auth_error marker must NOT report 'terminal'
once a subsequent login has repopulated tokens."""
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
"access_token": "new-at", "refresh_token": "new-rt",
"last_auth_error": {"relogin_required": True, "code": "invalid_grant"},
})
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True})
assert get_nous_session_validity() == NOUS_SESSION_VALID
def test_status_exception_is_unknown_not_terminal(monkeypatch):
"""If status computation itself throws, that's indeterminate → 'unknown'."""
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {"refresh_token": "rt"})
def _boom():
raise RuntimeError("boom")
monkeypatch.setattr(auth, "get_nous_auth_status", _boom)
assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN
def test_provider_state_exception_falls_through_to_status(monkeypatch):
"""If reading provider state throws, fall through to status (don't crash)."""
def _boom(p):
raise RuntimeError("disk error")
monkeypatch.setattr(auth, "get_provider_auth_state", _boom)
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True})
assert get_nous_session_validity() == NOUS_SESSION_VALID