feat(docker): re-seed a terminally-dead Nous bootstrap session on boot (#59983)

The stage2-hook auth.json seed is first-boot-only ([ ! -f auth.json ]) to avoid
clobbering rotated refresh tokens on restart. That guard means a container whose
Nous bootstrap session took a terminal invalid_grant (tokens cleared,
providers.nous.last_auth_error.relogin_required stamped) cannot recover from a
restart — it stays unauthenticated until the credential is replaced.

Add a self-heal path: an orchestrator that manages the container supplies a
freshly-issued session via HERMES_AUTH_JSON_REBOOTSTRAP (distinct from the
create-only *_BOOTSTRAP var). On boot, scripts/docker_rebootstrap_nous_session.py
swaps ONLY the providers.nous entry, and ONLY when the on-disk entry is provably
terminal (quarantine marker + no usable tokens). Healthy/rotating/absent/
unparseable auth.json is always a no-op, so the env is safe to leave set across
restarts and never clobbers a good token. Pure stdlib, runs as its own
subprocess, always exits 0 so a re-seed error never fails the boot.

Reuses the same terminal predicate as get_nous_session_validity() so we re-seed
only a session that is genuinely dead.

docker/stage2-hook.sh

@@ -443,6 +443,31 @@ if [ ! -f "$HERMES_HOME/auth.json" ] && [ -n "${HERMES_AUTH_JSON_BOOTSTRAP:-}" ]
     fi
 fi
 
+# auth.json: re-seed a TERMINALLY-DEAD Nous bootstrap session (self-heal).
+#
+# The [ ! -f ] guard above deliberately refuses to clobber an existing
+# auth.json, so a container whose Nous bootstrap session took a terminal
+# invalid_grant (tokens cleared, providers.nous.last_auth_error.relogin_required
+# stamped) can NOT recover from a plain restart — it stays unauthenticated until
+# the credential is replaced. An orchestrator that manages the container can
+# supply a freshly-issued session via HERMES_AUTH_JSON_REBOOTSTRAP (distinct
+# from the create-only *_BOOTSTRAP var); this helper swaps ONLY the
+# providers.nous entry, and ONLY when the on-disk entry is provably terminal.
+# Every other case (healthy, rotating, absent, or unparseable auth.json) is a
+# no-op, so it is safe to leave the env set across restarts and never risks
+# clobbering a good/rotated token. Runs as its own stdlib-only subprocess (no
+# app imports) and always exits 0.
+if [ -f "$HERMES_HOME/auth.json" ] && [ -n "${HERMES_AUTH_JSON_REBOOTSTRAP:-}" ]; then
+    if refuse_symlinked_path "reseed" "$HERMES_HOME/auth.json"; then
+        :
+    else
+        s6-setuidgid hermes "$INSTALL_DIR/.venv/bin/python" \
+            "$INSTALL_DIR/scripts/docker_rebootstrap_nous_session.py" \
+            "$HERMES_HOME/auth.json" \
+            || echo "[stage2] Warning: docker_rebootstrap_nous_session.py failed; continuing"
+    fi
+fi
+
 # gateway_state.json: declare the gateway's INITIAL supervised state on a
 # fresh volume. Same first-boot-only env-seed pattern as auth.json above.
 #

scripts/docker_rebootstrap_nous_session.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""Boot-time re-seed of a terminally-dead Nous bootstrap session.
+
+Background
+----------
+A Nous bootstrap session (client_id ``hermes-cli-vps``) can take a terminal
+``invalid_grant`` and be quarantined locally — the refresh path clears the dead
+tokens from ``auth.json`` and stamps
+``providers.nous.last_auth_error.relogin_required = true``. From then on every
+inference turn hard-fails with a provider-auth error until the credential is
+replaced, even though the gateway and dashboard otherwise look healthy.
+
+``stage2-hook.sh`` seeds ``auth.json`` from ``HERMES_AUTH_JSON_BOOTSTRAP`` only
+on a *blank* volume (``[ ! -f auth.json ]``) — that guard is load-bearing: it
+stops a container restart from clobbering a healthy, rotated refresh token. So a
+plain restart with a fresh seed env can NOT recover a container whose volume
+already has an auth.json.
+
+This script is the narrow, safe exception. An orchestrator that manages the
+container can supply a freshly-issued bootstrap session via
+``HERMES_AUTH_JSON_REBOOTSTRAP`` (plus a restart). On boot we re-seed the Nous
+provider entry from that env **only when the on-disk Nous entry is provably
+terminal** (the quarantine marker above with no usable tokens left). Every other
+case is a no-op, so we never clobber a healthy or merely-rotating session.
+
+Design constraints
+------------------
+- Pure stdlib, no hermes_cli imports: runs early in the boot hook, before the
+  app venv/modules are guaranteed importable, as its own subprocess.
+- Surgical: replaces ONLY ``providers.nous`` in the existing auth.json, leaving
+  every other provider, the version, and any other top-level state untouched.
+- Fail-safe: any parse/IO error leaves auth.json exactly as-is and exits 0 (a
+  failed re-seed must never take the container further down than it already is).
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import Any, Optional
+
+# Env var the orchestrator sets to the re-seed payload. Deliberately DISTINCT
+# from HERMES_AUTH_JSON_BOOTSTRAP (create-only, blank-volume seed) so the two
+# paths can never be confused: BOOTSTRAP seeds a fresh volume; REBOOTSTRAP
+# overwrites a terminally-dead Nous entry on an existing volume.
+REBOOTSTRAP_ENV = "HERMES_AUTH_JSON_REBOOTSTRAP"
+
+
+def _nous_entry_is_terminal(nous_state: Any) -> bool:
+    """True iff the on-disk Nous provider entry is in the terminal/quarantined
+    state AND holds no usable credential.
+
+    Mirrors the ``terminal`` predicate in ``hermes_cli.auth.get_nous_session_validity``:
+    a persisted ``last_auth_error.relogin_required`` with the token material
+    already cleared. Keeping this in lockstep is what guarantees we only re-seed
+    a session that is genuinely dead.
+    """
+    if not isinstance(nous_state, dict):
+        return False
+    last_err = nous_state.get("last_auth_error")
+    if not (isinstance(last_err, dict) and last_err.get("relogin_required")):
+        return False
+    # Only terminal while there is no usable credential left. If a live token is
+    # somehow present, treat it as healthy and do NOT clobber it.
+    if nous_state.get("access_token") or nous_state.get("refresh_token"):
+        return False
+    return True
+
+
+def _extract_nous_from_seed(seed_raw: str) -> Optional[dict]:
+    """Pull the ``providers.nous`` block out of a HERMES_AUTH_JSON_REBOOTSTRAP
+    payload. The payload is a full auth.json document (same shape as
+    HERMES_AUTH_JSON_BOOTSTRAP). Returns None if it can't be parsed or carries no
+    nous entry — caller treats None as "nothing to do"."""
+    try:
+        seed = json.loads(seed_raw)
+    except (ValueError, TypeError):
+        return None
+    if not isinstance(seed, dict):
+        return None
+    providers = seed.get("providers")
+    if not isinstance(providers, dict):
+        return None
+    nous = providers.get("nous")
+    if not isinstance(nous, dict) or not nous:
+        return None
+    return nous
+
+
+def reseed_if_terminal(auth_path: str, seed_raw: str) -> str:
+    """Core logic. Returns a short status string for logging/testing:
+
+      - "no_seed"          — seed env empty/absent
+      - "bad_seed"         — seed present but unparseable / no nous entry
+      - "no_auth_file"     — auth.json absent (blank volume → let the normal
+                             HERMES_AUTH_JSON_BOOTSTRAP path handle it)
+      - "auth_unreadable"  — auth.json present but unparseable (leave as-is)
+      - "not_terminal"     — on-disk nous entry is healthy/absent → no-op
+      - "reseeded"         — nous entry was terminal; replaced from seed
+    """
+    if not seed_raw:
+        return "no_seed"
+
+    seed_nous = _extract_nous_from_seed(seed_raw)
+    if seed_nous is None:
+        return "bad_seed"
+
+    if not os.path.exists(auth_path):
+        # Blank volume — this is the normal first-boot case, not a re-seed.
+        return "no_auth_file"
+
+    try:
+        with open(auth_path, "r", encoding="utf-8") as fh:
+            store = json.load(fh)
+    except (OSError, ValueError):
+        # Corrupt/unreadable auth.json: do NOT overwrite blindly. A separate
+        # concern; leave it for the operator / other recovery paths.
+        return "auth_unreadable"
+
+    if not isinstance(store, dict):
+        return "auth_unreadable"
+
+    providers = store.get("providers")
+    if not isinstance(providers, dict):
+        providers = {}
+        store["providers"] = providers
+
+    if not _nous_entry_is_terminal(providers.get("nous")):
+        # Healthy, rotating, or absent nous entry — the load-bearing guard.
+        # Never clobber a good session; this is what makes the re-seed safe to
+        # push on every restart.
+        return "not_terminal"
+
+    # Surgical replacement: swap ONLY providers.nous, preserve everything else.
+    providers["nous"] = seed_nous
+
+    tmp_path = f"{auth_path}.rebootstrap.tmp"
+    with open(tmp_path, "w", encoding="utf-8") as fh:
+        json.dump(store, fh)
+    os.replace(tmp_path, auth_path)
+    try:
+        os.chmod(auth_path, 0o600)
+    except OSError:
+        pass
+    return "reseeded"
+
+
+def main() -> int:
+    auth_path = sys.argv[1] if len(sys.argv) > 1 else ""
+    if not auth_path:
+        home = os.environ.get("HERMES_HOME", "")
+        auth_path = os.path.join(home, "auth.json") if home else "auth.json"
+    seed_raw = os.environ.get(REBOOTSTRAP_ENV, "")
+
+    try:
+        result = reseed_if_terminal(auth_path, seed_raw)
+    except Exception as exc:  # never let a re-seed error fail the boot
+        print(f"[rebootstrap] error (ignored): {exc!r}", file=sys.stderr)
+        return 0
+
+    if result == "reseeded":
+        print("[rebootstrap] Nous bootstrap session was terminal; re-seeded auth.json from "
+              f"{REBOOTSTRAP_ENV}")
+    else:
+        # Quiet by default for the common no-op cases; still emit a breadcrumb.
+        print(f"[rebootstrap] no-op ({result})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/tools/test_docker_rebootstrap_nous_session.py

@@ -0,0 +1,140 @@
+"""Unit tests for scripts/docker_rebootstrap_nous_session.py.
+
+The boot-time re-seed is the load-bearing "does not clobber a healthy session"
+guard: it must overwrite the on-disk Nous provider entry ONLY when that entry is
+provably terminal (quarantine marker + no usable tokens), and no-op in every
+other case. These are pure-stdlib tmp_path tests (no container build).
+"""
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+# Import the stdlib-only boot helper by path (it lives under scripts/, not an
+# installed package) — mirrors the repo's other scripts/-helper tests.
+_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "docker_rebootstrap_nous_session.py"
+_spec = importlib.util.spec_from_file_location("docker_rebootstrap_nous_session", _SCRIPT)
+mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(mod)
+
+
+def _terminal_nous_state():
+    """On-disk shape after a terminal quarantine: tokens cleared, marker set."""
+    return {
+        "portal_base_url": "https://portal.example.com",
+        "client_id": "hermes-cli-vps",
+        "last_auth_error": {
+            "provider": "nous",
+            "code": "invalid_grant",
+            "relogin_required": True,
+        },
+    }
+
+
+def _healthy_nous_state():
+    return {
+        "portal_base_url": "https://portal.example.com",
+        "client_id": "hermes-cli-vps",
+        "access_token": "live-at",
+        "refresh_token": "live-rt",
+    }
+
+
+def _write_auth(tmp_path: Path, providers: dict) -> str:
+    p = tmp_path / "auth.json"
+    p.write_text(json.dumps({"version": 1, "providers": providers}))
+    return str(p)
+
+
+_FRESH_SEED = json.dumps({
+    "version": 1,
+    "providers": {
+        "nous": {
+            "portal_base_url": "https://portal.example.com",
+            "client_id": "hermes-cli-vps",
+            "access_token": "FRESH-at",
+            "refresh_token": "FRESH-rt",
+        }
+    },
+})
+
+
+def test_reseeds_terminal_entry(tmp_path):
+    """Terminal on-disk entry + valid seed → providers.nous replaced."""
+    auth = _write_auth(tmp_path, {"nous": _terminal_nous_state()})
+    result = mod.reseed_if_terminal(auth, _FRESH_SEED)
+    assert result == "reseeded"
+    store = json.loads(Path(auth).read_text())
+    assert store["providers"]["nous"]["refresh_token"] == "FRESH-rt"
+    assert "last_auth_error" not in store["providers"]["nous"]
+
+
+def test_does_not_clobber_healthy_entry(tmp_path):
+    """LOAD-BEARING: a healthy (live-token) entry must never be overwritten."""
+    auth = _write_auth(tmp_path, {"nous": _healthy_nous_state()})
+    result = mod.reseed_if_terminal(auth, _FRESH_SEED)
+    assert result == "not_terminal"
+    store = json.loads(Path(auth).read_text())
+    # Untouched — still the live tokens, not the seed.
+    assert store["providers"]["nous"]["refresh_token"] == "live-rt"
+
+
+def test_marker_but_live_token_is_not_terminal(tmp_path):
+    """Stale marker + a live token present → NOT terminal (don't clobber)."""
+    state = _terminal_nous_state()
+    state["refresh_token"] = "somehow-live"
+    auth = _write_auth(tmp_path, {"nous": state})
+    assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "not_terminal"
+
+
+def test_preserves_other_providers(tmp_path):
+    """Re-seed swaps ONLY providers.nous; other providers survive intact."""
+    auth = _write_auth(tmp_path, {
+        "nous": _terminal_nous_state(),
+        "openai-codex": {"tokens": {"access_token": "codex-at"}},
+    })
+    assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "reseeded"
+    store = json.loads(Path(auth).read_text())
+    assert store["providers"]["openai-codex"]["tokens"]["access_token"] == "codex-at"
+    assert store["providers"]["nous"]["refresh_token"] == "FRESH-rt"
+
+
+def test_no_seed_is_noop(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": _terminal_nous_state()})
+    assert mod.reseed_if_terminal(auth, "") == "no_seed"
+
+
+def test_bad_seed_is_noop(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": _terminal_nous_state()})
+    assert mod.reseed_if_terminal(auth, "}{not json") == "bad_seed"
+    # Original terminal entry left untouched.
+    store = json.loads(Path(auth).read_text())
+    assert store["providers"]["nous"]["last_auth_error"]["relogin_required"] is True
+
+
+def test_seed_without_nous_entry_is_noop(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": _terminal_nous_state()})
+    seed = json.dumps({"version": 1, "providers": {"openai-codex": {}}})
+    assert mod.reseed_if_terminal(auth, seed) == "bad_seed"
+
+
+def test_absent_auth_file_defers_to_bootstrap(tmp_path):
+    """No auth.json → blank volume; the normal *_BOOTSTRAP path handles it."""
+    auth = str(tmp_path / "auth.json")
+    assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "no_auth_file"
+
+
+def test_unreadable_auth_file_is_left_alone(tmp_path):
+    p = tmp_path / "auth.json"
+    p.write_text("}{ corrupt")
+    assert mod.reseed_if_terminal(str(p), _FRESH_SEED) == "auth_unreadable"
+    # Not overwritten.
+    assert p.read_text() == "}{ corrupt"
+
+
+def test_terminal_entry_missing_marker_is_not_terminal(tmp_path):
+    """No last_auth_error at all (e.g. a merely-expired but not-quarantined
+    entry) → not terminal, no re-seed."""
+    auth = _write_auth(tmp_path, {"nous": {"client_id": "hermes-cli-vps"}})
+    assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "not_terminal"