diff --git a/docker/stage2-hook.sh b/docker/stage2-hook.sh index e6d1af35298c..bd722da7319d 100755 --- a/docker/stage2-hook.sh +++ b/docker/stage2-hook.sh @@ -452,11 +452,12 @@ fi # the credential is replaced. An orchestrator that manages the container can # supply a freshly-issued session via HERMES_AUTH_JSON_REBOOTSTRAP (distinct # from the create-only *_BOOTSTRAP var); this helper swaps ONLY the -# providers.nous entry, and ONLY when the on-disk entry is provably terminal. -# Every other case (healthy, rotating, absent, or unparseable auth.json) is a -# no-op, so it is safe to leave the env set across restarts and never risks -# clobbering a good/rotated token. Runs as its own stdlib-only subprocess (no -# app imports) and always exits 0. +# providers.nous entry when the on-disk entry is provably terminal OR the +# orchestrator seed has a later obtained_at timestamp. The latter covers the +# stop/update/start sequence where NAS already revoked the still-healthy-looking +# local session. Older/incomparable seeds remain no-ops, so leaving the env set +# cannot roll a healthy rotated token backward. Runs as its own stdlib-only +# subprocess (no app imports) and always exits 0. if [ -f "$HERMES_HOME/auth.json" ] && [ -n "${HERMES_AUTH_JSON_REBOOTSTRAP:-}" ]; then if refuse_symlinked_path "reseed" "$HERMES_HOME/auth.json"; then : diff --git a/scripts/docker_rebootstrap_nous_session.py b/scripts/docker_rebootstrap_nous_session.py index a28a52e3c9bc..0c4125d64e2d 100644 --- a/scripts/docker_rebootstrap_nous_session.py +++ b/scripts/docker_rebootstrap_nous_session.py @@ -19,9 +19,11 @@ already has an auth.json. This script is the narrow, safe exception. An orchestrator that manages the container can supply a freshly-issued bootstrap session via ``HERMES_AUTH_JSON_REBOOTSTRAP`` (plus a restart). On boot we re-seed the Nous -provider entry from that env **only when the on-disk Nous entry is provably -terminal** (the quarantine marker above with no usable tokens left). Every other -case is a no-op, so we never clobber a healthy or merely-rotating session. +provider entry from that env when the on-disk entry is provably terminal, or +when the orchestrator seed's ``obtained_at`` is newer than the local session. +The latter matters because an orchestrator may revoke the previous session +before restart while its still-present local tokens look healthy. Older or +incomparable seeds remain no-ops, so a retained env cannot roll auth backward. Design constraints ------------------ @@ -37,6 +39,7 @@ from __future__ import annotations import json import os import sys +from datetime import datetime, timezone from typing import Any, Optional # Env var the orchestrator sets to the re-seed payload. Deliberately DISTINCT @@ -44,6 +47,7 @@ from typing import Any, Optional # paths can never be confused: BOOTSTRAP seeds a fresh volume; REBOOTSTRAP # overwrites a terminally-dead Nous entry on an existing volume. REBOOTSTRAP_ENV = "HERMES_AUTH_JSON_REBOOTSTRAP" +BOOTSTRAP_CLIENT_ID = "hermes-cli-vps" def _nous_entry_is_terminal(nous_state: Any) -> bool: @@ -70,8 +74,9 @@ def _nous_entry_is_terminal(nous_state: Any) -> bool: def _extract_nous_from_seed(seed_raw: str) -> Optional[dict]: """Pull the ``providers.nous`` block out of a HERMES_AUTH_JSON_REBOOTSTRAP payload. The payload is a full auth.json document (same shape as - HERMES_AUTH_JSON_BOOTSTRAP). Returns None if it can't be parsed or carries no - nous entry — caller treats None as "nothing to do".""" + HERMES_AUTH_JSON_BOOTSTRAP). Returns None unless it carries the expected VPS + bootstrap client plus non-empty access and refresh tokens — caller treats + None as "nothing to do".""" try: seed = json.loads(seed_raw) except (ValueError, TypeError): @@ -84,9 +89,54 @@ def _extract_nous_from_seed(seed_raw: str) -> Optional[dict]: nous = providers.get("nous") if not isinstance(nous, dict) or not nous: return None + if nous.get("client_id") != BOOTSTRAP_CLIENT_ID: + return None + if not ( + isinstance(nous.get("access_token"), str) + and nous["access_token"].strip() + and isinstance(nous.get("refresh_token"), str) + and nous["refresh_token"].strip() + ): + return None return nous +def _parse_timestamp(value: Any) -> Optional[datetime]: + """Parse an OAuth timestamp without guessing when either side is malformed.""" + if not isinstance(value, str) or not value.strip(): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except (ValueError, OverflowError): + return None + if parsed.tzinfo is None: + return None + try: + return parsed.astimezone(timezone.utc) + except (OverflowError, ValueError): + return None + + +def _seed_is_newer(local_nous: Any, seed_nous: dict) -> bool: + """Whether NAS supplied a bootstrap session newer than the local one. + + NAS mints the replacement before restarting the machine and revokes the + previous session. A healthy-looking local entry can therefore already be + stale. ``obtained_at`` is the server-issued ordering signal that lets boot + apply a genuinely newer replacement without allowing an old retained env + value to roll credentials back on later restarts. + """ + if not isinstance(local_nous, dict): + return False + local_obtained = _parse_timestamp(local_nous.get("obtained_at")) + seed_obtained = _parse_timestamp(seed_nous.get("obtained_at")) + return bool( + local_obtained is not None + and seed_obtained is not None + and seed_obtained > local_obtained + ) + + def reseed_if_terminal(auth_path: str, seed_raw: str) -> str: """Core logic. Returns a short status string for logging/testing: @@ -95,8 +145,9 @@ def reseed_if_terminal(auth_path: str, seed_raw: str) -> str: - "no_auth_file" — auth.json absent (blank volume → let the normal HERMES_AUTH_JSON_BOOTSTRAP path handle it) - "auth_unreadable" — auth.json present but unparseable (leave as-is) - - "not_terminal" — on-disk nous entry is healthy/absent → no-op - - "reseeded" — nous entry was terminal; replaced from seed + - "not_terminal" — local entry is healthy and at least as new → no-op + - "reseeded" — terminal entry replaced from seed + - "reseeded_newer" — healthy-but-stale entry replaced by a newer seed """ if not seed_raw: return "no_seed" @@ -125,10 +176,12 @@ def reseed_if_terminal(auth_path: str, seed_raw: str) -> str: providers = {} store["providers"] = providers - if not _nous_entry_is_terminal(providers.get("nous")): - # Healthy, rotating, or absent nous entry — the load-bearing guard. - # Never clobber a good session; this is what makes the re-seed safe to - # push on every restart. + local_nous = providers.get("nous") + terminal = _nous_entry_is_terminal(local_nous) + newer_seed = _seed_is_newer(local_nous, seed_nous) + if not terminal and not newer_seed: + # Healthy and at least as new as the seed, or incomparable. Never roll a + # session back merely because an old rebootstrap env remains configured. return "not_terminal" # Surgical replacement: swap ONLY providers.nous, preserve everything else. @@ -142,7 +195,7 @@ def reseed_if_terminal(auth_path: str, seed_raw: str) -> str: os.chmod(auth_path, 0o600) except OSError: pass - return "reseeded" + return "reseeded" if terminal else "reseeded_newer" def main() -> int: @@ -161,6 +214,9 @@ def main() -> int: if result == "reseeded": print("[rebootstrap] Nous bootstrap session was terminal; re-seeded auth.json from " f"{REBOOTSTRAP_ENV}") + elif result == "reseeded_newer": + print("[rebootstrap] Applied newer orchestrator-issued Nous bootstrap session from " + f"{REBOOTSTRAP_ENV}") else: # Quiet by default for the common no-op cases; still emit a breadcrumb. print(f"[rebootstrap] no-op ({result})") diff --git a/tests/tools/test_docker_rebootstrap_nous_session.py b/tests/tools/test_docker_rebootstrap_nous_session.py index a6cc02b97805..1f2ab6084171 100644 --- a/tests/tools/test_docker_rebootstrap_nous_session.py +++ b/tests/tools/test_docker_rebootstrap_nous_session.py @@ -1,9 +1,10 @@ """Unit tests for scripts/docker_rebootstrap_nous_session.py. The boot-time re-seed is the load-bearing "does not clobber a healthy session" -guard: it must overwrite the on-disk Nous provider entry ONLY when that entry is -provably terminal (quarantine marker + no usable tokens), and no-op in every -other case. These are pure-stdlib tmp_path tests (no container build). +guard: it may overwrite the on-disk Nous provider entry when that entry is +provably terminal (quarantine marker + no usable tokens), or when an +orchestrator seed is demonstrably newer. Older/incomparable seeds must no-op. +These are pure-stdlib tmp_path tests (no container build). """ from __future__ import annotations @@ -88,6 +89,192 @@ def test_marker_but_live_token_is_not_terminal(tmp_path): assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "not_terminal" +def test_reseeds_newer_orchestrator_session_over_healthy_stale_entry(tmp_path): + """A newer orchestrator-issued session replaces the healthy local session. + + NAS revokes the old session before restarting a hosted agent. Refusing the + re-seed merely because the local entry still has tokens leaves that revoked + session in place and guarantees ``invalid_grant`` on its next refresh. + """ + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "2026-07-14T19:00:00+00:00", + }}) + seed = json.dumps({ + "version": 1, + "providers": { + "nous": { + "portal_base_url": "https://portal.example.com", + "client_id": "hermes-cli-vps", + "access_token": "FRESH-at", + "refresh_token": "FRESH-rt", + "obtained_at": "2026-07-14T19:05:00+00:00", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "reseeded_newer" + store = json.loads(Path(auth).read_text()) + assert store["providers"]["nous"]["refresh_token"] == "FRESH-rt" + + +def test_does_not_replace_healthy_entry_with_older_seed(tmp_path): + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "2026-07-14T19:05:00+00:00", + }}) + seed = json.dumps({ + "version": 1, + "providers": { + "nous": { + "client_id": "hermes-cli-vps", + "access_token": "STALE-at", + "refresh_token": "STALE-rt", + "obtained_at": "2026-07-14T19:00:00+00:00", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "not_terminal" + store = json.loads(Path(auth).read_text()) + assert store["providers"]["nous"]["refresh_token"] == "live-rt" + + +def test_timezone_less_local_timestamp_is_incomparable(tmp_path): + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "2026-07-14T19:00:00", + }}) + seed = json.dumps({ + "providers": { + "nous": { + "client_id": "hermes-cli-vps", + "access_token": "FRESH-at", + "refresh_token": "FRESH-rt", + "obtained_at": "2026-07-14T19:05:00Z", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "not_terminal" + + +def test_malformed_timestamp_does_not_clobber_healthy_entry(tmp_path): + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "not-a-time", + }}) + seed = json.dumps({ + "providers": { + "nous": { + "client_id": "hermes-cli-vps", + "access_token": "FRESH-at", + "refresh_token": "FRESH-rt", + "obtained_at": "2026-07-14T19:05:00Z", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "not_terminal" + + +def test_newer_seed_without_tokens_does_not_clobber_healthy_entry(tmp_path): + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "2026-07-14T19:00:00Z", + }}) + seed = json.dumps({ + "providers": { + "nous": { + "client_id": "hermes-cli-vps", + "obtained_at": "2026-07-14T19:05:00Z", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "bad_seed" + store = json.loads(Path(auth).read_text()) + assert store["providers"]["nous"]["refresh_token"] == "live-rt" + + +def test_newer_seed_for_non_bootstrap_client_does_not_clobber_healthy_entry(tmp_path): + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "2026-07-14T19:00:00Z", + }}) + seed = json.dumps({ + "providers": { + "nous": { + "client_id": "hermes-cli", + "access_token": "FRESH-at", + "refresh_token": "FRESH-rt", + "obtained_at": "2026-07-14T19:05:00Z", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "bad_seed" + store = json.loads(Path(auth).read_text()) + assert store["providers"]["nous"]["refresh_token"] == "live-rt" + + +def test_timezone_less_seed_timestamp_is_incomparable(tmp_path): + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "2026-07-14T19:00:00Z", + }}) + seed = json.dumps({ + "providers": { + "nous": { + "client_id": "hermes-cli-vps", + "access_token": "FRESH-at", + "refresh_token": "FRESH-rt", + "obtained_at": "2026-07-14T19:05:00", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "not_terminal" + + +def test_extreme_timestamp_is_incomparable(tmp_path): + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "2026-07-14T19:00:00Z", + }}) + seed = json.dumps({ + "providers": { + "nous": { + "client_id": "hermes-cli-vps", + "access_token": "FRESH-at", + "refresh_token": "FRESH-rt", + "obtained_at": "0001-01-01T00:00:00+23:59", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "not_terminal" + + +def test_equal_instants_with_different_offsets_do_not_reseed(tmp_path): + auth = _write_auth(tmp_path, {"nous": { + **_healthy_nous_state(), + "obtained_at": "2026-07-14T19:00:00Z", + }}) + seed = json.dumps({ + "providers": { + "nous": { + "client_id": "hermes-cli-vps", + "access_token": "FRESH-at", + "refresh_token": "FRESH-rt", + "obtained_at": "2026-07-14T20:00:00+01:00", + } + }, + }) + + assert mod.reseed_if_terminal(auth, seed) == "not_terminal" + + def test_preserves_other_providers(tmp_path): """Re-seed swaps ONLY providers.nous; other providers survive intact.""" auth = _write_auth(tmp_path, {