mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-08 13:12:08 +00:00
feat(status): expose nous_session_valid on /api/status for hosted-agent self-heal
A hosted agent whose Nous bootstrap session dies terminally (invalid_grant / quarantine) looks HEALTHY to every liveness/connectivity probe — the machine, relay ws, and dashboard all stay up — yet every inference turn hard-fails with a provider-auth error until a human re-logs-in. Nothing currently surfaces that condition to NAS. Add get_nous_session_validity() (valid|terminal|unknown), classified from local auth-store state (no working token required), and report it on the public /api/status payload. NAS's 2-min health sweep reads it and re-mints the bootstrap session in place on 'terminal'. Anti-flap: only a terminal failure (relogin_required / persisted quarantine marker with tokens cleared) maps to 'terminal'; transient/mid-rotation blips and merely-expiring tokens report 'unknown' so a healthy box never triggers a spurious re-mint. Part of the hosted-agent bootstrap-session self-heal (NAS side reads this field).
This commit is contained in:
parent
182256206a
commit
5eac665252
3 changed files with 189 additions and 0 deletions
|
|
@ -5968,6 +5968,75 @@ def _compute_nous_auth_status() -> Dict[str, Any]:
|
|||
return _snapshot_nous_pool_status()
|
||||
|
||||
|
||||
# Enum values reported on the dashboard /api/status as ``nous_session_valid``.
|
||||
# NAS's health sweep re-mints the bootstrap session ONLY on "terminal"; "valid"
|
||||
# and "unknown" are no-ops. Keep this set small and stable — NAS parses it with
|
||||
# a permissive schema, so new members are non-breaking but should stay rare.
|
||||
NOUS_SESSION_VALID = "valid"
|
||||
NOUS_SESSION_TERMINAL = "terminal"
|
||||
NOUS_SESSION_UNKNOWN = "unknown"
|
||||
|
||||
|
||||
def get_nous_session_validity() -> str:
|
||||
"""Classify the Nous bootstrap session for the dashboard /api/status probe.
|
||||
|
||||
Returns one of:
|
||||
- ``"valid"`` — a usable Nous credential is present (login healthy).
|
||||
- ``"terminal"`` — the Nous session has taken a terminal auth failure
|
||||
(invalid_grant / quarantined / relogin required). This is the sole
|
||||
signal NAS acts on to re-mint a hosted-agent bootstrap session.
|
||||
- ``"unknown"`` — indeterminate (no Nous provider state, or a transient/
|
||||
non-terminal error). Never triggers a re-mint.
|
||||
|
||||
Determinable with NO working token — it reads local auth-store state only,
|
||||
which is exactly the condition a dead hosted box is in.
|
||||
|
||||
ANTI-FLAP CONTRACT: only a *terminal* failure maps to "terminal". A normal
|
||||
mid-rotation blip, a transient network error, or a merely-expiring token
|
||||
must NOT report "terminal" (that would trigger a spurious NAS re-mint on a
|
||||
healthy box). We key "terminal" on the auth layer's own terminal signal
|
||||
(`relogin_required`) plus a persisted quarantine marker, never on a bare
|
||||
"not logged in".
|
||||
"""
|
||||
# A persisted quarantine marker is the strongest, most stable terminal
|
||||
# signal: the refresh path writes `last_auth_error.relogin_required=True`
|
||||
# into the Nous provider state when it clears dead tokens (the exact path
|
||||
# that produced the incident's "No access token found"). Read it directly
|
||||
# so we report "terminal" even after the in-memory AuthError is long gone.
|
||||
try:
|
||||
state = get_provider_auth_state("nous")
|
||||
except Exception:
|
||||
state = None
|
||||
|
||||
if state:
|
||||
last_err = state.get("last_auth_error")
|
||||
if isinstance(last_err, dict) and last_err.get("relogin_required"):
|
||||
# Only terminal while there is no usable credential left. If a later
|
||||
# successful login repopulated tokens, the stale marker must not
|
||||
# keep reporting terminal.
|
||||
if not (state.get("access_token") or state.get("refresh_token")):
|
||||
return NOUS_SESSION_TERMINAL
|
||||
|
||||
try:
|
||||
status = get_nous_auth_status()
|
||||
except Exception:
|
||||
# Status computation itself failed — indeterminate, not terminal.
|
||||
return NOUS_SESSION_UNKNOWN
|
||||
|
||||
if status.get("logged_in"):
|
||||
return NOUS_SESSION_VALID
|
||||
|
||||
# Not logged in. Distinguish a terminal (relogin-required) failure from a
|
||||
# transient / indeterminate one. Only the former is actionable by NAS.
|
||||
if status.get("relogin_required"):
|
||||
return NOUS_SESSION_TERMINAL
|
||||
|
||||
# No Nous provider state at all, or a non-terminal not-logged-in condition
|
||||
# (e.g. a transient refresh error that did not set relogin_required). Treat
|
||||
# as unknown so a healthy box mid-blip never triggers a re-mint.
|
||||
return NOUS_SESSION_UNKNOWN
|
||||
|
||||
|
||||
def get_codex_auth_status() -> Dict[str, Any]:
|
||||
"""Status snapshot for Codex auth.
|
||||
|
||||
|
|
|
|||
|
|
@ -2400,6 +2400,21 @@ async def get_status(profile: Optional[str] = None):
|
|||
# Module not importable yet (early startup) — leave as [].
|
||||
pass
|
||||
|
||||
# Nous bootstrap-session validity for the NAS health sweep. A hosted
|
||||
# agent whose Nous auth dies terminally (invalid_grant / quarantine)
|
||||
# looks HEALTHY to every liveness/connectivity probe — the machine,
|
||||
# relay, and this dashboard all stay up — yet every inference turn
|
||||
# fails. This is the ONLY signal that surfaces that condition, and it
|
||||
# is determinable with no working token (local auth-store state). NAS
|
||||
# re-mints the bootstrap session when it reads "terminal". Best-effort:
|
||||
# never let auth classification break the public liveness probe.
|
||||
nous_session_valid = "unknown"
|
||||
try:
|
||||
from hermes_cli.auth import get_nous_session_validity
|
||||
nous_session_valid = get_nous_session_validity()
|
||||
except Exception:
|
||||
nous_session_valid = "unknown"
|
||||
|
||||
# Always-public liveness + auth-gate shape. Safe for external uptime
|
||||
# probes (NAS's wildcard-subdomain liveness probe), the SPA's pre-login
|
||||
# bootstrap, and anyone who can curl the host — i.e. exactly the audience
|
||||
|
|
@ -2422,6 +2437,7 @@ async def get_status(profile: Optional[str] = None):
|
|||
"active_sessions": active_sessions,
|
||||
"auth_required": auth_required,
|
||||
"auth_providers": auth_providers,
|
||||
"nous_session_valid": nous_session_valid,
|
||||
}
|
||||
|
||||
# Absolute host paths, the gateway PID, and the internal gateway health
|
||||
|
|
|
|||
104
tests/hermes_cli/test_nous_session_validity.py
Normal file
104
tests/hermes_cli/test_nous_session_validity.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""Tests for get_nous_session_validity — the /api/status classifier NAS reads
|
||||
to decide whether to re-mint a hosted-agent bootstrap session.
|
||||
|
||||
The anti-flap contract is the load-bearing property: only a *terminal* auth
|
||||
failure may report "terminal" (a spurious "terminal" triggers an unnecessary
|
||||
NAS re-mint + machine restart on a healthy box). A mid-rotation blip, a
|
||||
transient error, or a merely-expiring token must NOT report "terminal".
|
||||
"""
|
||||
|
||||
import hermes_cli.auth as auth
|
||||
from hermes_cli.auth import (
|
||||
NOUS_SESSION_TERMINAL,
|
||||
NOUS_SESSION_UNKNOWN,
|
||||
NOUS_SESSION_VALID,
|
||||
get_nous_session_validity,
|
||||
)
|
||||
|
||||
|
||||
def _clear_cache():
|
||||
auth.invalidate_nous_auth_status_cache()
|
||||
|
||||
|
||||
def test_valid_when_logged_in(monkeypatch):
|
||||
"""A healthy login → 'valid'."""
|
||||
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
|
||||
"access_token": "at", "refresh_token": "rt",
|
||||
})
|
||||
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True})
|
||||
assert get_nous_session_validity() == NOUS_SESSION_VALID
|
||||
|
||||
|
||||
def test_terminal_on_persisted_quarantine_marker(monkeypatch):
|
||||
"""A persisted last_auth_error.relogin_required with tokens cleared →
|
||||
'terminal'. This is the exact on-disk state the incident produced."""
|
||||
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
|
||||
# tokens cleared by the quarantine path
|
||||
"last_auth_error": {"relogin_required": True, "code": "invalid_grant"},
|
||||
})
|
||||
# status would also say not-logged-in, but the marker short-circuits first
|
||||
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": False})
|
||||
assert get_nous_session_validity() == NOUS_SESSION_TERMINAL
|
||||
|
||||
|
||||
def test_terminal_on_relogin_required_status(monkeypatch):
|
||||
"""Not logged in + relogin_required from the live status → 'terminal'."""
|
||||
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
|
||||
"refresh_token": "rt", # present, but status resolution fails terminally
|
||||
})
|
||||
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {
|
||||
"logged_in": False, "relogin_required": True, "error_code": "invalid_grant",
|
||||
})
|
||||
assert get_nous_session_validity() == NOUS_SESSION_TERMINAL
|
||||
|
||||
|
||||
def test_unknown_when_no_provider_state(monkeypatch):
|
||||
"""No Nous provider state at all → 'unknown' (never terminal)."""
|
||||
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: None)
|
||||
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": False})
|
||||
assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN
|
||||
|
||||
|
||||
def test_anti_flap_transient_not_logged_in_is_unknown(monkeypatch):
|
||||
"""ANTI-FLAP: not-logged-in WITHOUT relogin_required (a transient/network
|
||||
blip) must be 'unknown', NOT 'terminal' — otherwise a healthy box mid-blip
|
||||
triggers a spurious re-mint."""
|
||||
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
|
||||
"access_token": "at", "refresh_token": "rt",
|
||||
})
|
||||
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {
|
||||
"logged_in": False, "error": "connection reset", # no relogin_required
|
||||
})
|
||||
assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN
|
||||
|
||||
|
||||
def test_stale_quarantine_marker_ignored_after_relogin(monkeypatch):
|
||||
"""ANTI-FLAP: a leftover last_auth_error marker must NOT report 'terminal'
|
||||
once a subsequent login has repopulated tokens."""
|
||||
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {
|
||||
"access_token": "new-at", "refresh_token": "new-rt",
|
||||
"last_auth_error": {"relogin_required": True, "code": "invalid_grant"},
|
||||
})
|
||||
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True})
|
||||
assert get_nous_session_validity() == NOUS_SESSION_VALID
|
||||
|
||||
|
||||
def test_status_exception_is_unknown_not_terminal(monkeypatch):
|
||||
"""If status computation itself throws, that's indeterminate → 'unknown'."""
|
||||
monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {"refresh_token": "rt"})
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(auth, "get_nous_auth_status", _boom)
|
||||
assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN
|
||||
|
||||
|
||||
def test_provider_state_exception_falls_through_to_status(monkeypatch):
|
||||
"""If reading provider state throws, fall through to status (don't crash)."""
|
||||
def _boom(p):
|
||||
raise RuntimeError("disk error")
|
||||
|
||||
monkeypatch.setattr(auth, "get_provider_auth_state", _boom)
|
||||
monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True})
|
||||
assert get_nous_session_validity() == NOUS_SESSION_VALID
|
||||
Loading…
Add table
Add a link
Reference in a new issue