diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index b499963ea7e..8ac31020def 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -5968,6 +5968,75 @@ def _compute_nous_auth_status() -> Dict[str, Any]: return _snapshot_nous_pool_status() +# Enum values reported on the dashboard /api/status as ``nous_session_valid``. +# NAS's health sweep re-mints the bootstrap session ONLY on "terminal"; "valid" +# and "unknown" are no-ops. Keep this set small and stable — NAS parses it with +# a permissive schema, so new members are non-breaking but should stay rare. +NOUS_SESSION_VALID = "valid" +NOUS_SESSION_TERMINAL = "terminal" +NOUS_SESSION_UNKNOWN = "unknown" + + +def get_nous_session_validity() -> str: + """Classify the Nous bootstrap session for the dashboard /api/status probe. + + Returns one of: + - ``"valid"`` — a usable Nous credential is present (login healthy). + - ``"terminal"`` — the Nous session has taken a terminal auth failure + (invalid_grant / quarantined / relogin required). This is the sole + signal NAS acts on to re-mint a hosted-agent bootstrap session. + - ``"unknown"`` — indeterminate (no Nous provider state, or a transient/ + non-terminal error). Never triggers a re-mint. + + Determinable with NO working token — it reads local auth-store state only, + which is exactly the condition a dead hosted box is in. + + ANTI-FLAP CONTRACT: only a *terminal* failure maps to "terminal". A normal + mid-rotation blip, a transient network error, or a merely-expiring token + must NOT report "terminal" (that would trigger a spurious NAS re-mint on a + healthy box). We key "terminal" on the auth layer's own terminal signal + (`relogin_required`) plus a persisted quarantine marker, never on a bare + "not logged in". + """ + # A persisted quarantine marker is the strongest, most stable terminal + # signal: the refresh path writes `last_auth_error.relogin_required=True` + # into the Nous provider state when it clears dead tokens (the exact path + # that produced the incident's "No access token found"). Read it directly + # so we report "terminal" even after the in-memory AuthError is long gone. + try: + state = get_provider_auth_state("nous") + except Exception: + state = None + + if state: + last_err = state.get("last_auth_error") + if isinstance(last_err, dict) and last_err.get("relogin_required"): + # Only terminal while there is no usable credential left. If a later + # successful login repopulated tokens, the stale marker must not + # keep reporting terminal. + if not (state.get("access_token") or state.get("refresh_token")): + return NOUS_SESSION_TERMINAL + + try: + status = get_nous_auth_status() + except Exception: + # Status computation itself failed — indeterminate, not terminal. + return NOUS_SESSION_UNKNOWN + + if status.get("logged_in"): + return NOUS_SESSION_VALID + + # Not logged in. Distinguish a terminal (relogin-required) failure from a + # transient / indeterminate one. Only the former is actionable by NAS. + if status.get("relogin_required"): + return NOUS_SESSION_TERMINAL + + # No Nous provider state at all, or a non-terminal not-logged-in condition + # (e.g. a transient refresh error that did not set relogin_required). Treat + # as unknown so a healthy box mid-blip never triggers a re-mint. + return NOUS_SESSION_UNKNOWN + + def get_codex_auth_status() -> Dict[str, Any]: """Status snapshot for Codex auth. diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index a5292224cd5..3da2c370b8a 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -2400,6 +2400,21 @@ async def get_status(profile: Optional[str] = None): # Module not importable yet (early startup) — leave as []. pass + # Nous bootstrap-session validity for the NAS health sweep. A hosted + # agent whose Nous auth dies terminally (invalid_grant / quarantine) + # looks HEALTHY to every liveness/connectivity probe — the machine, + # relay, and this dashboard all stay up — yet every inference turn + # fails. This is the ONLY signal that surfaces that condition, and it + # is determinable with no working token (local auth-store state). NAS + # re-mints the bootstrap session when it reads "terminal". Best-effort: + # never let auth classification break the public liveness probe. + nous_session_valid = "unknown" + try: + from hermes_cli.auth import get_nous_session_validity + nous_session_valid = get_nous_session_validity() + except Exception: + nous_session_valid = "unknown" + # Always-public liveness + auth-gate shape. Safe for external uptime # probes (NAS's wildcard-subdomain liveness probe), the SPA's pre-login # bootstrap, and anyone who can curl the host — i.e. exactly the audience @@ -2422,6 +2437,7 @@ async def get_status(profile: Optional[str] = None): "active_sessions": active_sessions, "auth_required": auth_required, "auth_providers": auth_providers, + "nous_session_valid": nous_session_valid, } # Absolute host paths, the gateway PID, and the internal gateway health diff --git a/tests/hermes_cli/test_nous_session_validity.py b/tests/hermes_cli/test_nous_session_validity.py new file mode 100644 index 00000000000..02673d6ad78 --- /dev/null +++ b/tests/hermes_cli/test_nous_session_validity.py @@ -0,0 +1,104 @@ +"""Tests for get_nous_session_validity — the /api/status classifier NAS reads +to decide whether to re-mint a hosted-agent bootstrap session. + +The anti-flap contract is the load-bearing property: only a *terminal* auth +failure may report "terminal" (a spurious "terminal" triggers an unnecessary +NAS re-mint + machine restart on a healthy box). A mid-rotation blip, a +transient error, or a merely-expiring token must NOT report "terminal". +""" + +import hermes_cli.auth as auth +from hermes_cli.auth import ( + NOUS_SESSION_TERMINAL, + NOUS_SESSION_UNKNOWN, + NOUS_SESSION_VALID, + get_nous_session_validity, +) + + +def _clear_cache(): + auth.invalidate_nous_auth_status_cache() + + +def test_valid_when_logged_in(monkeypatch): + """A healthy login → 'valid'.""" + monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: { + "access_token": "at", "refresh_token": "rt", + }) + monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True}) + assert get_nous_session_validity() == NOUS_SESSION_VALID + + +def test_terminal_on_persisted_quarantine_marker(monkeypatch): + """A persisted last_auth_error.relogin_required with tokens cleared → + 'terminal'. This is the exact on-disk state the incident produced.""" + monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: { + # tokens cleared by the quarantine path + "last_auth_error": {"relogin_required": True, "code": "invalid_grant"}, + }) + # status would also say not-logged-in, but the marker short-circuits first + monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": False}) + assert get_nous_session_validity() == NOUS_SESSION_TERMINAL + + +def test_terminal_on_relogin_required_status(monkeypatch): + """Not logged in + relogin_required from the live status → 'terminal'.""" + monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: { + "refresh_token": "rt", # present, but status resolution fails terminally + }) + monkeypatch.setattr(auth, "get_nous_auth_status", lambda: { + "logged_in": False, "relogin_required": True, "error_code": "invalid_grant", + }) + assert get_nous_session_validity() == NOUS_SESSION_TERMINAL + + +def test_unknown_when_no_provider_state(monkeypatch): + """No Nous provider state at all → 'unknown' (never terminal).""" + monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: None) + monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": False}) + assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN + + +def test_anti_flap_transient_not_logged_in_is_unknown(monkeypatch): + """ANTI-FLAP: not-logged-in WITHOUT relogin_required (a transient/network + blip) must be 'unknown', NOT 'terminal' — otherwise a healthy box mid-blip + triggers a spurious re-mint.""" + monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: { + "access_token": "at", "refresh_token": "rt", + }) + monkeypatch.setattr(auth, "get_nous_auth_status", lambda: { + "logged_in": False, "error": "connection reset", # no relogin_required + }) + assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN + + +def test_stale_quarantine_marker_ignored_after_relogin(monkeypatch): + """ANTI-FLAP: a leftover last_auth_error marker must NOT report 'terminal' + once a subsequent login has repopulated tokens.""" + monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: { + "access_token": "new-at", "refresh_token": "new-rt", + "last_auth_error": {"relogin_required": True, "code": "invalid_grant"}, + }) + monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True}) + assert get_nous_session_validity() == NOUS_SESSION_VALID + + +def test_status_exception_is_unknown_not_terminal(monkeypatch): + """If status computation itself throws, that's indeterminate → 'unknown'.""" + monkeypatch.setattr(auth, "get_provider_auth_state", lambda p: {"refresh_token": "rt"}) + + def _boom(): + raise RuntimeError("boom") + + monkeypatch.setattr(auth, "get_nous_auth_status", _boom) + assert get_nous_session_validity() == NOUS_SESSION_UNKNOWN + + +def test_provider_state_exception_falls_through_to_status(monkeypatch): + """If reading provider state throws, fall through to status (don't crash).""" + def _boom(p): + raise RuntimeError("disk error") + + monkeypatch.setattr(auth, "get_provider_auth_state", _boom) + monkeypatch.setattr(auth, "get_nous_auth_status", lambda: {"logged_in": True}) + assert get_nous_session_validity() == NOUS_SESSION_VALID