From 536ffedbf4704f220fc184cc3dc1ef0bd5f91fb7 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Tue, 7 Jul 2026 16:57:23 +1000 Subject: [PATCH] feat(docker): re-seed a terminally-dead Nous bootstrap session on boot (#59983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stage2-hook auth.json seed is first-boot-only ([ ! -f auth.json ]) to avoid clobbering rotated refresh tokens on restart. That guard means a container whose Nous bootstrap session took a terminal invalid_grant (tokens cleared, providers.nous.last_auth_error.relogin_required stamped) cannot recover from a restart — it stays unauthenticated until the credential is replaced. Add a self-heal path: an orchestrator that manages the container supplies a freshly-issued session via HERMES_AUTH_JSON_REBOOTSTRAP (distinct from the create-only *_BOOTSTRAP var). On boot, scripts/docker_rebootstrap_nous_session.py swaps ONLY the providers.nous entry, and ONLY when the on-disk entry is provably terminal (quarantine marker + no usable tokens). Healthy/rotating/absent/ unparseable auth.json is always a no-op, so the env is safe to leave set across restarts and never clobbers a good token. Pure stdlib, runs as its own subprocess, always exits 0 so a re-seed error never fails the boot. Reuses the same terminal predicate as get_nous_session_validity() so we re-seed only a session that is genuinely dead. --- docker/stage2-hook.sh | 25 +++ scripts/docker_rebootstrap_nous_session.py | 171 ++++++++++++++++++ .../test_docker_rebootstrap_nous_session.py | 140 ++++++++++++++ 3 files changed, 336 insertions(+) create mode 100644 scripts/docker_rebootstrap_nous_session.py create mode 100644 tests/tools/test_docker_rebootstrap_nous_session.py diff --git a/docker/stage2-hook.sh b/docker/stage2-hook.sh index b73afdd3772..e6d1af35298 100755 --- a/docker/stage2-hook.sh +++ b/docker/stage2-hook.sh @@ -443,6 +443,31 @@ if [ ! -f "$HERMES_HOME/auth.json" ] && [ -n "${HERMES_AUTH_JSON_BOOTSTRAP:-}" ] fi fi +# auth.json: re-seed a TERMINALLY-DEAD Nous bootstrap session (self-heal). +# +# The [ ! -f ] guard above deliberately refuses to clobber an existing +# auth.json, so a container whose Nous bootstrap session took a terminal +# invalid_grant (tokens cleared, providers.nous.last_auth_error.relogin_required +# stamped) can NOT recover from a plain restart — it stays unauthenticated until +# the credential is replaced. An orchestrator that manages the container can +# supply a freshly-issued session via HERMES_AUTH_JSON_REBOOTSTRAP (distinct +# from the create-only *_BOOTSTRAP var); this helper swaps ONLY the +# providers.nous entry, and ONLY when the on-disk entry is provably terminal. +# Every other case (healthy, rotating, absent, or unparseable auth.json) is a +# no-op, so it is safe to leave the env set across restarts and never risks +# clobbering a good/rotated token. Runs as its own stdlib-only subprocess (no +# app imports) and always exits 0. +if [ -f "$HERMES_HOME/auth.json" ] && [ -n "${HERMES_AUTH_JSON_REBOOTSTRAP:-}" ]; then + if refuse_symlinked_path "reseed" "$HERMES_HOME/auth.json"; then + : + else + s6-setuidgid hermes "$INSTALL_DIR/.venv/bin/python" \ + "$INSTALL_DIR/scripts/docker_rebootstrap_nous_session.py" \ + "$HERMES_HOME/auth.json" \ + || echo "[stage2] Warning: docker_rebootstrap_nous_session.py failed; continuing" + fi +fi + # gateway_state.json: declare the gateway's INITIAL supervised state on a # fresh volume. Same first-boot-only env-seed pattern as auth.json above. # diff --git a/scripts/docker_rebootstrap_nous_session.py b/scripts/docker_rebootstrap_nous_session.py new file mode 100644 index 00000000000..a28a52e3c9b --- /dev/null +++ b/scripts/docker_rebootstrap_nous_session.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Boot-time re-seed of a terminally-dead Nous bootstrap session. + +Background +---------- +A Nous bootstrap session (client_id ``hermes-cli-vps``) can take a terminal +``invalid_grant`` and be quarantined locally — the refresh path clears the dead +tokens from ``auth.json`` and stamps +``providers.nous.last_auth_error.relogin_required = true``. From then on every +inference turn hard-fails with a provider-auth error until the credential is +replaced, even though the gateway and dashboard otherwise look healthy. + +``stage2-hook.sh`` seeds ``auth.json`` from ``HERMES_AUTH_JSON_BOOTSTRAP`` only +on a *blank* volume (``[ ! -f auth.json ]``) — that guard is load-bearing: it +stops a container restart from clobbering a healthy, rotated refresh token. So a +plain restart with a fresh seed env can NOT recover a container whose volume +already has an auth.json. + +This script is the narrow, safe exception. An orchestrator that manages the +container can supply a freshly-issued bootstrap session via +``HERMES_AUTH_JSON_REBOOTSTRAP`` (plus a restart). On boot we re-seed the Nous +provider entry from that env **only when the on-disk Nous entry is provably +terminal** (the quarantine marker above with no usable tokens left). Every other +case is a no-op, so we never clobber a healthy or merely-rotating session. + +Design constraints +------------------ +- Pure stdlib, no hermes_cli imports: runs early in the boot hook, before the + app venv/modules are guaranteed importable, as its own subprocess. +- Surgical: replaces ONLY ``providers.nous`` in the existing auth.json, leaving + every other provider, the version, and any other top-level state untouched. +- Fail-safe: any parse/IO error leaves auth.json exactly as-is and exits 0 (a + failed re-seed must never take the container further down than it already is). +""" +from __future__ import annotations + +import json +import os +import sys +from typing import Any, Optional + +# Env var the orchestrator sets to the re-seed payload. Deliberately DISTINCT +# from HERMES_AUTH_JSON_BOOTSTRAP (create-only, blank-volume seed) so the two +# paths can never be confused: BOOTSTRAP seeds a fresh volume; REBOOTSTRAP +# overwrites a terminally-dead Nous entry on an existing volume. +REBOOTSTRAP_ENV = "HERMES_AUTH_JSON_REBOOTSTRAP" + + +def _nous_entry_is_terminal(nous_state: Any) -> bool: + """True iff the on-disk Nous provider entry is in the terminal/quarantined + state AND holds no usable credential. + + Mirrors the ``terminal`` predicate in ``hermes_cli.auth.get_nous_session_validity``: + a persisted ``last_auth_error.relogin_required`` with the token material + already cleared. Keeping this in lockstep is what guarantees we only re-seed + a session that is genuinely dead. + """ + if not isinstance(nous_state, dict): + return False + last_err = nous_state.get("last_auth_error") + if not (isinstance(last_err, dict) and last_err.get("relogin_required")): + return False + # Only terminal while there is no usable credential left. If a live token is + # somehow present, treat it as healthy and do NOT clobber it. + if nous_state.get("access_token") or nous_state.get("refresh_token"): + return False + return True + + +def _extract_nous_from_seed(seed_raw: str) -> Optional[dict]: + """Pull the ``providers.nous`` block out of a HERMES_AUTH_JSON_REBOOTSTRAP + payload. The payload is a full auth.json document (same shape as + HERMES_AUTH_JSON_BOOTSTRAP). Returns None if it can't be parsed or carries no + nous entry — caller treats None as "nothing to do".""" + try: + seed = json.loads(seed_raw) + except (ValueError, TypeError): + return None + if not isinstance(seed, dict): + return None + providers = seed.get("providers") + if not isinstance(providers, dict): + return None + nous = providers.get("nous") + if not isinstance(nous, dict) or not nous: + return None + return nous + + +def reseed_if_terminal(auth_path: str, seed_raw: str) -> str: + """Core logic. Returns a short status string for logging/testing: + + - "no_seed" — seed env empty/absent + - "bad_seed" — seed present but unparseable / no nous entry + - "no_auth_file" — auth.json absent (blank volume → let the normal + HERMES_AUTH_JSON_BOOTSTRAP path handle it) + - "auth_unreadable" — auth.json present but unparseable (leave as-is) + - "not_terminal" — on-disk nous entry is healthy/absent → no-op + - "reseeded" — nous entry was terminal; replaced from seed + """ + if not seed_raw: + return "no_seed" + + seed_nous = _extract_nous_from_seed(seed_raw) + if seed_nous is None: + return "bad_seed" + + if not os.path.exists(auth_path): + # Blank volume — this is the normal first-boot case, not a re-seed. + return "no_auth_file" + + try: + with open(auth_path, "r", encoding="utf-8") as fh: + store = json.load(fh) + except (OSError, ValueError): + # Corrupt/unreadable auth.json: do NOT overwrite blindly. A separate + # concern; leave it for the operator / other recovery paths. + return "auth_unreadable" + + if not isinstance(store, dict): + return "auth_unreadable" + + providers = store.get("providers") + if not isinstance(providers, dict): + providers = {} + store["providers"] = providers + + if not _nous_entry_is_terminal(providers.get("nous")): + # Healthy, rotating, or absent nous entry — the load-bearing guard. + # Never clobber a good session; this is what makes the re-seed safe to + # push on every restart. + return "not_terminal" + + # Surgical replacement: swap ONLY providers.nous, preserve everything else. + providers["nous"] = seed_nous + + tmp_path = f"{auth_path}.rebootstrap.tmp" + with open(tmp_path, "w", encoding="utf-8") as fh: + json.dump(store, fh) + os.replace(tmp_path, auth_path) + try: + os.chmod(auth_path, 0o600) + except OSError: + pass + return "reseeded" + + +def main() -> int: + auth_path = sys.argv[1] if len(sys.argv) > 1 else "" + if not auth_path: + home = os.environ.get("HERMES_HOME", "") + auth_path = os.path.join(home, "auth.json") if home else "auth.json" + seed_raw = os.environ.get(REBOOTSTRAP_ENV, "") + + try: + result = reseed_if_terminal(auth_path, seed_raw) + except Exception as exc: # never let a re-seed error fail the boot + print(f"[rebootstrap] error (ignored): {exc!r}", file=sys.stderr) + return 0 + + if result == "reseeded": + print("[rebootstrap] Nous bootstrap session was terminal; re-seeded auth.json from " + f"{REBOOTSTRAP_ENV}") + else: + # Quiet by default for the common no-op cases; still emit a breadcrumb. + print(f"[rebootstrap] no-op ({result})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/tools/test_docker_rebootstrap_nous_session.py b/tests/tools/test_docker_rebootstrap_nous_session.py new file mode 100644 index 00000000000..a6cc02b9780 --- /dev/null +++ b/tests/tools/test_docker_rebootstrap_nous_session.py @@ -0,0 +1,140 @@ +"""Unit tests for scripts/docker_rebootstrap_nous_session.py. + +The boot-time re-seed is the load-bearing "does not clobber a healthy session" +guard: it must overwrite the on-disk Nous provider entry ONLY when that entry is +provably terminal (quarantine marker + no usable tokens), and no-op in every +other case. These are pure-stdlib tmp_path tests (no container build). +""" +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +# Import the stdlib-only boot helper by path (it lives under scripts/, not an +# installed package) — mirrors the repo's other scripts/-helper tests. +_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "docker_rebootstrap_nous_session.py" +_spec = importlib.util.spec_from_file_location("docker_rebootstrap_nous_session", _SCRIPT) +mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(mod) + + +def _terminal_nous_state(): + """On-disk shape after a terminal quarantine: tokens cleared, marker set.""" + return { + "portal_base_url": "https://portal.example.com", + "client_id": "hermes-cli-vps", + "last_auth_error": { + "provider": "nous", + "code": "invalid_grant", + "relogin_required": True, + }, + } + + +def _healthy_nous_state(): + return { + "portal_base_url": "https://portal.example.com", + "client_id": "hermes-cli-vps", + "access_token": "live-at", + "refresh_token": "live-rt", + } + + +def _write_auth(tmp_path: Path, providers: dict) -> str: + p = tmp_path / "auth.json" + p.write_text(json.dumps({"version": 1, "providers": providers})) + return str(p) + + +_FRESH_SEED = json.dumps({ + "version": 1, + "providers": { + "nous": { + "portal_base_url": "https://portal.example.com", + "client_id": "hermes-cli-vps", + "access_token": "FRESH-at", + "refresh_token": "FRESH-rt", + } + }, +}) + + +def test_reseeds_terminal_entry(tmp_path): + """Terminal on-disk entry + valid seed → providers.nous replaced.""" + auth = _write_auth(tmp_path, {"nous": _terminal_nous_state()}) + result = mod.reseed_if_terminal(auth, _FRESH_SEED) + assert result == "reseeded" + store = json.loads(Path(auth).read_text()) + assert store["providers"]["nous"]["refresh_token"] == "FRESH-rt" + assert "last_auth_error" not in store["providers"]["nous"] + + +def test_does_not_clobber_healthy_entry(tmp_path): + """LOAD-BEARING: a healthy (live-token) entry must never be overwritten.""" + auth = _write_auth(tmp_path, {"nous": _healthy_nous_state()}) + result = mod.reseed_if_terminal(auth, _FRESH_SEED) + assert result == "not_terminal" + store = json.loads(Path(auth).read_text()) + # Untouched — still the live tokens, not the seed. + assert store["providers"]["nous"]["refresh_token"] == "live-rt" + + +def test_marker_but_live_token_is_not_terminal(tmp_path): + """Stale marker + a live token present → NOT terminal (don't clobber).""" + state = _terminal_nous_state() + state["refresh_token"] = "somehow-live" + auth = _write_auth(tmp_path, {"nous": state}) + assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "not_terminal" + + +def test_preserves_other_providers(tmp_path): + """Re-seed swaps ONLY providers.nous; other providers survive intact.""" + auth = _write_auth(tmp_path, { + "nous": _terminal_nous_state(), + "openai-codex": {"tokens": {"access_token": "codex-at"}}, + }) + assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "reseeded" + store = json.loads(Path(auth).read_text()) + assert store["providers"]["openai-codex"]["tokens"]["access_token"] == "codex-at" + assert store["providers"]["nous"]["refresh_token"] == "FRESH-rt" + + +def test_no_seed_is_noop(tmp_path): + auth = _write_auth(tmp_path, {"nous": _terminal_nous_state()}) + assert mod.reseed_if_terminal(auth, "") == "no_seed" + + +def test_bad_seed_is_noop(tmp_path): + auth = _write_auth(tmp_path, {"nous": _terminal_nous_state()}) + assert mod.reseed_if_terminal(auth, "}{not json") == "bad_seed" + # Original terminal entry left untouched. + store = json.loads(Path(auth).read_text()) + assert store["providers"]["nous"]["last_auth_error"]["relogin_required"] is True + + +def test_seed_without_nous_entry_is_noop(tmp_path): + auth = _write_auth(tmp_path, {"nous": _terminal_nous_state()}) + seed = json.dumps({"version": 1, "providers": {"openai-codex": {}}}) + assert mod.reseed_if_terminal(auth, seed) == "bad_seed" + + +def test_absent_auth_file_defers_to_bootstrap(tmp_path): + """No auth.json → blank volume; the normal *_BOOTSTRAP path handles it.""" + auth = str(tmp_path / "auth.json") + assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "no_auth_file" + + +def test_unreadable_auth_file_is_left_alone(tmp_path): + p = tmp_path / "auth.json" + p.write_text("}{ corrupt") + assert mod.reseed_if_terminal(str(p), _FRESH_SEED) == "auth_unreadable" + # Not overwritten. + assert p.read_text() == "}{ corrupt" + + +def test_terminal_entry_missing_marker_is_not_terminal(tmp_path): + """No last_auth_error at all (e.g. a merely-expired but not-quarantined + entry) → not terminal, no re-seed.""" + auth = _write_auth(tmp_path, {"nous": {"client_id": "hermes-cli-vps"}}) + assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "not_terminal"