fix(auth): apply newer hosted bootstrap session (#64612)

* fix(auth): apply newer hosted bootstrap session

* fix(auth): validate rebootstrap replacement seeds

docker/stage2-hook.sh

@@ -452,11 +452,12 @@ fi
 # the credential is replaced. An orchestrator that manages the container can
 # supply a freshly-issued session via HERMES_AUTH_JSON_REBOOTSTRAP (distinct
 # from the create-only *_BOOTSTRAP var); this helper swaps ONLY the
-# providers.nous entry, and ONLY when the on-disk entry is provably terminal.
-# Every other case (healthy, rotating, absent, or unparseable auth.json) is a
-# no-op, so it is safe to leave the env set across restarts and never risks
-# clobbering a good/rotated token. Runs as its own stdlib-only subprocess (no
-# app imports) and always exits 0.
+# providers.nous entry when the on-disk entry is provably terminal OR the
+# orchestrator seed has a later obtained_at timestamp. The latter covers the
+# stop/update/start sequence where NAS already revoked the still-healthy-looking
+# local session. Older/incomparable seeds remain no-ops, so leaving the env set
+# cannot roll a healthy rotated token backward. Runs as its own stdlib-only
+# subprocess (no app imports) and always exits 0.
 if [ -f "$HERMES_HOME/auth.json" ] && [ -n "${HERMES_AUTH_JSON_REBOOTSTRAP:-}" ]; then
     if refuse_symlinked_path "reseed" "$HERMES_HOME/auth.json"; then
         :

scripts/docker_rebootstrap_nous_session.py

@@ -19,9 +19,11 @@ already has an auth.json.
 This script is the narrow, safe exception. An orchestrator that manages the
 container can supply a freshly-issued bootstrap session via
 ``HERMES_AUTH_JSON_REBOOTSTRAP`` (plus a restart). On boot we re-seed the Nous
-provider entry from that env **only when the on-disk Nous entry is provably
-terminal** (the quarantine marker above with no usable tokens left). Every other
-case is a no-op, so we never clobber a healthy or merely-rotating session.
+provider entry from that env when the on-disk entry is provably terminal, or
+when the orchestrator seed's ``obtained_at`` is newer than the local session.
+The latter matters because an orchestrator may revoke the previous session
+before restart while its still-present local tokens look healthy. Older or
+incomparable seeds remain no-ops, so a retained env cannot roll auth backward.
 
 Design constraints
 ------------------
@@ -37,6 +39,7 @@ from __future__ import annotations
 import json
 import os
 import sys
+from datetime import datetime, timezone
 from typing import Any, Optional
 
 # Env var the orchestrator sets to the re-seed payload. Deliberately DISTINCT
@@ -44,6 +47,7 @@ from typing import Any, Optional
 # paths can never be confused: BOOTSTRAP seeds a fresh volume; REBOOTSTRAP
 # overwrites a terminally-dead Nous entry on an existing volume.
 REBOOTSTRAP_ENV = "HERMES_AUTH_JSON_REBOOTSTRAP"
+BOOTSTRAP_CLIENT_ID = "hermes-cli-vps"
 
 
 def _nous_entry_is_terminal(nous_state: Any) -> bool:
@@ -70,8 +74,9 @@ def _nous_entry_is_terminal(nous_state: Any) -> bool:
 def _extract_nous_from_seed(seed_raw: str) -> Optional[dict]:
     """Pull the ``providers.nous`` block out of a HERMES_AUTH_JSON_REBOOTSTRAP
     payload. The payload is a full auth.json document (same shape as
-    HERMES_AUTH_JSON_BOOTSTRAP). Returns None if it can't be parsed or carries no
-    nous entry — caller treats None as "nothing to do"."""
+    HERMES_AUTH_JSON_BOOTSTRAP). Returns None unless it carries the expected VPS
+    bootstrap client plus non-empty access and refresh tokens — caller treats
+    None as "nothing to do"."""
     try:
         seed = json.loads(seed_raw)
     except (ValueError, TypeError):
@@ -84,9 +89,54 @@ def _extract_nous_from_seed(seed_raw: str) -> Optional[dict]:
     nous = providers.get("nous")
     if not isinstance(nous, dict) or not nous:
         return None
+    if nous.get("client_id") != BOOTSTRAP_CLIENT_ID:
+        return None
+    if not (
+        isinstance(nous.get("access_token"), str)
+        and nous["access_token"].strip()
+        and isinstance(nous.get("refresh_token"), str)
+        and nous["refresh_token"].strip()
+    ):
+        return None
     return nous
 
 
+def _parse_timestamp(value: Any) -> Optional[datetime]:
+    """Parse an OAuth timestamp without guessing when either side is malformed."""
+    if not isinstance(value, str) or not value.strip():
+        return None
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except (ValueError, OverflowError):
+        return None
+    if parsed.tzinfo is None:
+        return None
+    try:
+        return parsed.astimezone(timezone.utc)
+    except (OverflowError, ValueError):
+        return None
+
+
+def _seed_is_newer(local_nous: Any, seed_nous: dict) -> bool:
+    """Whether NAS supplied a bootstrap session newer than the local one.
+
+    NAS mints the replacement before restarting the machine and revokes the
+    previous session.  A healthy-looking local entry can therefore already be
+    stale.  ``obtained_at`` is the server-issued ordering signal that lets boot
+    apply a genuinely newer replacement without allowing an old retained env
+    value to roll credentials back on later restarts.
+    """
+    if not isinstance(local_nous, dict):
+        return False
+    local_obtained = _parse_timestamp(local_nous.get("obtained_at"))
+    seed_obtained = _parse_timestamp(seed_nous.get("obtained_at"))
+    return bool(
+        local_obtained is not None
+        and seed_obtained is not None
+        and seed_obtained > local_obtained
+    )
+
+
 def reseed_if_terminal(auth_path: str, seed_raw: str) -> str:
     """Core logic. Returns a short status string for logging/testing:
 
@@ -95,8 +145,9 @@ def reseed_if_terminal(auth_path: str, seed_raw: str) -> str:
       - "no_auth_file"     — auth.json absent (blank volume → let the normal
                              HERMES_AUTH_JSON_BOOTSTRAP path handle it)
       - "auth_unreadable"  — auth.json present but unparseable (leave as-is)
-      - "not_terminal"     — on-disk nous entry is healthy/absent → no-op
-      - "reseeded"         — nous entry was terminal; replaced from seed
+      - "not_terminal"     — local entry is healthy and at least as new → no-op
+      - "reseeded"         — terminal entry replaced from seed
+      - "reseeded_newer"   — healthy-but-stale entry replaced by a newer seed
     """
     if not seed_raw:
         return "no_seed"
@@ -125,10 +176,12 @@ def reseed_if_terminal(auth_path: str, seed_raw: str) -> str:
         providers = {}
         store["providers"] = providers
 
-    if not _nous_entry_is_terminal(providers.get("nous")):
-        # Healthy, rotating, or absent nous entry — the load-bearing guard.
-        # Never clobber a good session; this is what makes the re-seed safe to
-        # push on every restart.
+    local_nous = providers.get("nous")
+    terminal = _nous_entry_is_terminal(local_nous)
+    newer_seed = _seed_is_newer(local_nous, seed_nous)
+    if not terminal and not newer_seed:
+        # Healthy and at least as new as the seed, or incomparable. Never roll a
+        # session back merely because an old rebootstrap env remains configured.
         return "not_terminal"
 
     # Surgical replacement: swap ONLY providers.nous, preserve everything else.
@@ -142,7 +195,7 @@ def reseed_if_terminal(auth_path: str, seed_raw: str) -> str:
         os.chmod(auth_path, 0o600)
     except OSError:
         pass
-    return "reseeded"
+    return "reseeded" if terminal else "reseeded_newer"
 
 
 def main() -> int:
@@ -161,6 +214,9 @@ def main() -> int:
     if result == "reseeded":
         print("[rebootstrap] Nous bootstrap session was terminal; re-seeded auth.json from "
               f"{REBOOTSTRAP_ENV}")
+    elif result == "reseeded_newer":
+        print("[rebootstrap] Applied newer orchestrator-issued Nous bootstrap session from "
+              f"{REBOOTSTRAP_ENV}")
     else:
         # Quiet by default for the common no-op cases; still emit a breadcrumb.
         print(f"[rebootstrap] no-op ({result})")

tests/tools/test_docker_rebootstrap_nous_session.py

@@ -1,9 +1,10 @@
 """Unit tests for scripts/docker_rebootstrap_nous_session.py.
 
 The boot-time re-seed is the load-bearing "does not clobber a healthy session"
-guard: it must overwrite the on-disk Nous provider entry ONLY when that entry is
-provably terminal (quarantine marker + no usable tokens), and no-op in every
-other case. These are pure-stdlib tmp_path tests (no container build).
+guard: it may overwrite the on-disk Nous provider entry when that entry is
+provably terminal (quarantine marker + no usable tokens), or when an
+orchestrator seed is demonstrably newer. Older/incomparable seeds must no-op.
+These are pure-stdlib tmp_path tests (no container build).
 """
 from __future__ import annotations
 
@@ -88,6 +89,192 @@ def test_marker_but_live_token_is_not_terminal(tmp_path):
     assert mod.reseed_if_terminal(auth, _FRESH_SEED) == "not_terminal"
 
 
+def test_reseeds_newer_orchestrator_session_over_healthy_stale_entry(tmp_path):
+    """A newer orchestrator-issued session replaces the healthy local session.
+
+    NAS revokes the old session before restarting a hosted agent.  Refusing the
+    re-seed merely because the local entry still has tokens leaves that revoked
+    session in place and guarantees ``invalid_grant`` on its next refresh.
+    """
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "2026-07-14T19:00:00+00:00",
+    }})
+    seed = json.dumps({
+        "version": 1,
+        "providers": {
+            "nous": {
+                "portal_base_url": "https://portal.example.com",
+                "client_id": "hermes-cli-vps",
+                "access_token": "FRESH-at",
+                "refresh_token": "FRESH-rt",
+                "obtained_at": "2026-07-14T19:05:00+00:00",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "reseeded_newer"
+    store = json.loads(Path(auth).read_text())
+    assert store["providers"]["nous"]["refresh_token"] == "FRESH-rt"
+
+
+def test_does_not_replace_healthy_entry_with_older_seed(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "2026-07-14T19:05:00+00:00",
+    }})
+    seed = json.dumps({
+        "version": 1,
+        "providers": {
+            "nous": {
+                "client_id": "hermes-cli-vps",
+                "access_token": "STALE-at",
+                "refresh_token": "STALE-rt",
+                "obtained_at": "2026-07-14T19:00:00+00:00",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "not_terminal"
+    store = json.loads(Path(auth).read_text())
+    assert store["providers"]["nous"]["refresh_token"] == "live-rt"
+
+
+def test_timezone_less_local_timestamp_is_incomparable(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "2026-07-14T19:00:00",
+    }})
+    seed = json.dumps({
+        "providers": {
+            "nous": {
+                "client_id": "hermes-cli-vps",
+                "access_token": "FRESH-at",
+                "refresh_token": "FRESH-rt",
+                "obtained_at": "2026-07-14T19:05:00Z",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "not_terminal"
+
+
+def test_malformed_timestamp_does_not_clobber_healthy_entry(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "not-a-time",
+    }})
+    seed = json.dumps({
+        "providers": {
+            "nous": {
+                "client_id": "hermes-cli-vps",
+                "access_token": "FRESH-at",
+                "refresh_token": "FRESH-rt",
+                "obtained_at": "2026-07-14T19:05:00Z",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "not_terminal"
+
+
+def test_newer_seed_without_tokens_does_not_clobber_healthy_entry(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "2026-07-14T19:00:00Z",
+    }})
+    seed = json.dumps({
+        "providers": {
+            "nous": {
+                "client_id": "hermes-cli-vps",
+                "obtained_at": "2026-07-14T19:05:00Z",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "bad_seed"
+    store = json.loads(Path(auth).read_text())
+    assert store["providers"]["nous"]["refresh_token"] == "live-rt"
+
+
+def test_newer_seed_for_non_bootstrap_client_does_not_clobber_healthy_entry(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "2026-07-14T19:00:00Z",
+    }})
+    seed = json.dumps({
+        "providers": {
+            "nous": {
+                "client_id": "hermes-cli",
+                "access_token": "FRESH-at",
+                "refresh_token": "FRESH-rt",
+                "obtained_at": "2026-07-14T19:05:00Z",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "bad_seed"
+    store = json.loads(Path(auth).read_text())
+    assert store["providers"]["nous"]["refresh_token"] == "live-rt"
+
+
+def test_timezone_less_seed_timestamp_is_incomparable(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "2026-07-14T19:00:00Z",
+    }})
+    seed = json.dumps({
+        "providers": {
+            "nous": {
+                "client_id": "hermes-cli-vps",
+                "access_token": "FRESH-at",
+                "refresh_token": "FRESH-rt",
+                "obtained_at": "2026-07-14T19:05:00",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "not_terminal"
+
+
+def test_extreme_timestamp_is_incomparable(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "2026-07-14T19:00:00Z",
+    }})
+    seed = json.dumps({
+        "providers": {
+            "nous": {
+                "client_id": "hermes-cli-vps",
+                "access_token": "FRESH-at",
+                "refresh_token": "FRESH-rt",
+                "obtained_at": "0001-01-01T00:00:00+23:59",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "not_terminal"
+
+
+def test_equal_instants_with_different_offsets_do_not_reseed(tmp_path):
+    auth = _write_auth(tmp_path, {"nous": {
+        **_healthy_nous_state(),
+        "obtained_at": "2026-07-14T19:00:00Z",
+    }})
+    seed = json.dumps({
+        "providers": {
+            "nous": {
+                "client_id": "hermes-cli-vps",
+                "access_token": "FRESH-at",
+                "refresh_token": "FRESH-rt",
+                "obtained_at": "2026-07-14T20:00:00+01:00",
+            }
+        },
+    })
+
+    assert mod.reseed_if_terminal(auth, seed) == "not_terminal"
+
+
 def test_preserves_other_providers(tmp_path):
     """Re-seed swaps ONLY providers.nous; other providers survive intact."""
     auth = _write_auth(tmp_path, {