fix(update): stdlib-only early recovery before hermes_cli.main imports

The hermes console entry point is hermes_cli.main:main, and main.py imports
dotenv (via env_loader) and yaml (via config) at module level. In the #57828
failure state — a failed lazy backend refresh wiping a core package's import
files while metadata survives — a normal launch crashed while importing
main.py, before _recover_from_interrupted_install() and the recovery markers
from PR #58004 could act.

- hermes_cli/_early_recovery.py: stdlib-only bootstrap repair invoked at the
  very top of main.py, before any third-party import. Probes the fragile
  core packages via real imports, force-reinstalls broken ones using the
  pyproject.toml pins, shares main.py's single-flight recovery lock, and
  never clears markers (the confirmed lifecycle stays with the full recovery
  path in main.py).
- Probe/repair tables now have one canonical home in _early_recovery, reused
  by main.py so the two layers cannot drift.
- Manual --force-reinstall fallback commands now print pinned specs via
  _lazy_refresh_repair_specs() instead of bare package names.
- tests: entry-point lifecycle coverage proving a broken dotenv import
  crashes main.py without repair and imports cleanly with it, a stdlib-only
  import guard for _early_recovery, and unit coverage for marker gating,
  lock single-flight, pinned specs, and marker preservation.

hermes_cli/_early_recovery.py

@@ -0,0 +1,226 @@
+"""Dependency-light venv recovery that runs BEFORE hermes_cli.main's imports.
+
+The ``hermes`` console entry point is ``hermes_cli.main:main``.  Importing
+``hermes_cli.main`` pulls in third-party packages at module level (``dotenv``
+via ``hermes_cli.env_loader``, ``yaml`` via ``hermes_cli.config``, ...).  In
+the exact failure state the update-recovery markers exist for — a failed lazy
+backend refresh or interrupted core install that wiped a core package's
+import files (#57828) — a normal launch crashes *while importing main.py*,
+before ``_recover_from_interrupted_install()`` can run.  The marker system is
+unreachable precisely when it is needed most.
+
+This module is deliberately **stdlib-only** so importing it can never fail on
+a corrupted venv.  ``hermes_cli.main`` imports and calls
+:func:`recover_if_needed` at the very top of its module body, before any
+third-party import.
+
+Scope: this early pass only repairs enough for ``hermes_cli.main`` to become
+importable again (force-reinstall of the known-fragile core packages, using
+the pins from pyproject.toml).  It NEVER clears the recovery markers — the
+full, confirmed marker lifecycle stays with ``_recover_from_interrupted_install()``
+in main.py, which runs right after import succeeds.
+"""
+
+from __future__ import annotations
+
+import importlib
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+# Core packages a failed lazy ``uv pip install`` is known to leave with intact
+# distribution metadata but wiped import files (#57828).  ``module`` is what we
+# probe via a real import; ``attr`` guards against an empty/stub module.
+# main.py's marker-recovery path reuses these tables — keep them here (the
+# dependency-light module) so both layers probe and repair the same set.
+LAZY_REFRESH_IMPORT_PROBES: tuple[tuple[str, str], ...] = (
+    ("yaml", "SafeDumper"),
+    ("dotenv", "load_dotenv"),
+    ("click", "Command"),
+    ("certifi", "contents"),
+    ("rich", "print"),
+    ("cryptography", "__version__"),
+    ("jwt", "encode"),
+)
+
+LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = {
+    "yaml": "PyYAML",
+    "dotenv": "python-dotenv",
+    "click": "click",
+    "certifi": "certifi",
+    "rich": "rich",
+    "cryptography": "cryptography",
+    "jwt": "PyJWT",
+}
+
+
+def _project_root() -> Path:
+    return Path(__file__).resolve().parent.parent
+
+
+def _pinned_specs(packages: list[str], project_root: Path) -> list[str]:
+    """Map bare package names to their pinned specs from pyproject.toml.
+
+    Stdlib-only (tomllib + naive requirement-head parsing — ``packaging`` may
+    itself be broken in the failure state this module exists for).  Unknown
+    packages fall back to their bare name.
+    """
+    pyproject = project_root / "pyproject.toml"
+    if not pyproject.is_file():
+        return packages
+    try:
+        import tomllib
+
+        with open(pyproject, "rb") as f:
+            raw_deps = tomllib.load(f).get("project", {}).get("dependencies", []) or []
+    except Exception:
+        return packages
+
+    name_to_spec: dict[str, str] = {}
+    for spec in raw_deps:
+        head = spec.split(";", 1)[0].strip()
+        bare = head
+        for op in ("==", ">=", "<=", "~=", ">", "<", "!="):
+            if op in bare:
+                bare = bare.split(op, 1)[0]
+                break
+        key = bare.strip().split("[", 1)[0].strip().lower()
+        if key:
+            name_to_spec[key] = head
+    return [name_to_spec.get(pkg.lower(), pkg) for pkg in packages]
+
+
+def _probe_broken_packages() -> list[str]:
+    """Import-probe the fragile core packages in THIS process.
+
+    Returns repair package names (deduped, probe order) for modules that fail
+    to import or lack their sentinel attribute.  Failed imports leave nothing
+    in ``sys.modules``, so a post-repair retry in the same process works.
+    """
+    broken: list[str] = []
+    for mod_name, attr in LAZY_REFRESH_IMPORT_PROBES:
+        try:
+            mod = importlib.import_module(mod_name)
+            if not hasattr(mod, attr):
+                raise ImportError(f"{mod_name} missing {attr}")
+        except Exception:
+            pkg = LAZY_REFRESH_REPAIR_PACKAGES.get(mod_name)
+            if pkg and pkg not in broken:
+                broken.append(pkg)
+    return broken
+
+
+def _run_repair_install(specs: list[str], project_root: Path) -> bool:
+    """ensurepip + ``pip install --force-reinstall`` the given specs.
+
+    Streams nothing to stdout (``hermes acp`` speaks JSON-RPC on stdout);
+    output is captured and replayed to stderr only on failure.  Never raises.
+    """
+    try:
+        subprocess.run(
+            [sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"],
+            cwd=project_root,
+            capture_output=True,
+        )
+    except Exception:
+        pass
+    try:
+        result = subprocess.run(
+            [sys.executable, "-m", "pip", "install", "--force-reinstall", *specs],
+            cwd=project_root,
+            capture_output=True,
+            text=True,
+        )
+    except Exception as exc:
+        print(f"  ✗ Early venv repair could not run pip: {exc}", file=sys.stderr)
+        return False
+    if result.returncode != 0:
+        tail = (result.stderr or result.stdout or "")[-2000:]
+        if tail:
+            print(tail, file=sys.stderr)
+        return False
+    return True
+
+
+def recover_if_needed(
+    project_root: Path | None = None,
+    argv: list[str] | None = None,
+) -> None:
+    """Repair wiped core packages so ``hermes_cli.main`` can import at all.
+
+    Fast path (no marker present) is two ``lstat`` calls.  Only acts when a
+    recovery marker from a prior ``hermes update`` exists AND an import probe
+    confirms a core package is actually broken.  Markers are intentionally
+    NOT cleared here — ``_recover_from_interrupted_install()`` in main.py owns
+    the confirmed marker lifecycle and runs immediately after import succeeds.
+
+    Never raises: on any failure the import of main.py proceeds and surfaces
+    the real error.
+    """
+    try:
+        args = sys.argv[1:] if argv is None else argv
+        # Same deliberately-loose match as main(): the real update flow writes
+        # and clears its own markers — a recovery install must not race it.
+        if "update" in args:
+            return
+        root = _project_root() if project_root is None else project_root
+        core_marker = root / ".update-incomplete"
+        lazy_marker = root / ".lazy-refresh-incomplete"
+        if not core_marker.exists() and not lazy_marker.exists():
+            return
+        # Managed/Docker/PyPI installs have no source tree here — the marker
+        # is not ours to act on; main.py's recovery clears it.
+        if not (root / "pyproject.toml").is_file():
+            return
+
+        broken = _probe_broken_packages()
+        if not broken:
+            # Imports are fine — main.py will load and run full recovery.
+            return
+
+        # Single-flight: share main.py's recovery lock so an early repair
+        # never races a concurrent full recovery into the same shared venv.
+        lock_path = root / ".update-incomplete.lock"
+        try:
+            fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+            os.write(fd, f"{os.getpid()}\n".encode())
+            os.close(fd)
+        except FileExistsError:
+            try:
+                if time.time() - lock_path.stat().st_mtime > 3600:
+                    lock_path.unlink()
+            except OSError:
+                pass
+            return
+        except OSError:
+            pass  # read-only fs / perms — proceed unlocked, install surfaces it
+
+        try:
+            specs = _pinned_specs(broken, root)
+            print(
+                "⚠ Core package(s) broken by an interrupted update — "
+                f"repairing before launch: {', '.join(broken)}",
+                file=sys.stderr,
+            )
+            if _run_repair_install(specs, root) and not _probe_broken_packages():
+                print("  ✓ Core packages repaired.", file=sys.stderr)
+            else:
+                print(
+                    "  ✗ Automatic repair incomplete. Recover manually with:",
+                    file=sys.stderr,
+                )
+                print(
+                    f"    {sys.executable} -m pip install --force-reinstall "
+                    + " ".join(specs),
+                    file=sys.stderr,
+                )
+        finally:
+            try:
+                lock_path.unlink()
+            except OSError:
+                pass
+    except Exception:
+        # Never block launch — the import of main.py will surface the truth.
+        pass

hermes_cli/main.py

@@ -64,6 +64,25 @@ except ModuleNotFoundError:
 import os
 import sys
 
+# Early venv self-heal — MUST run before any third-party import below.  When
+# a prior ``hermes update`` left a recovery marker and a core package's import
+# files were wiped (#57828 — failed lazy backend refresh), the module-level
+# ``from hermes_cli.env_loader import ...`` / ``from hermes_cli.config import
+# ...`` imports further down would crash before ``main()`` ever reaches
+# ``_recover_from_interrupted_install()``.  ``_early_recovery`` is stdlib-only
+# (safe to import on a corrupted venv), repairs just enough for this module to
+# finish importing, and leaves the marker lifecycle to the full recovery path.
+# The module import itself is unguarded on purpose: it lives in this same
+# package directory, so if IT can't import, nothing else in hermes_cli can
+# either. It is also the canonical home of the probe/repair tables reused by
+# the full recovery path below.
+from hermes_cli import _early_recovery as _early_recovery_mod
+
+try:
+    _early_recovery_mod.recover_if_needed()
+except Exception:
+    pass
+
 
 def _exit_after_oneshot(rc: object) -> None:
     """Exit one-shot mode without letting late native finalizers change rc.
@@ -7724,9 +7743,12 @@ def _recover_lazy_refresh_marker_locked() -> None:
             "Leaving `.lazy-refresh-incomplete` for the next launch."
         )
         print("  Recover manually with:")
+        all_specs = _lazy_refresh_repair_specs(
+            sorted(set(_LAZY_REFRESH_REPAIR_PACKAGES.values()))
+        )
         print(
             f"    {' '.join(install_prefix)} install --force-reinstall "
-            "PyYAML python-dotenv click certifi rich cryptography PyJWT"
+            + " ".join(shlex.quote(s) for s in all_specs)
         )
 
 
@@ -8283,25 +8305,16 @@ def _cleanup_quarantined_exes(scripts_dir: Path | None = None) -> None:
 
 # Import probes for venv corruption after a failed lazy ``uv pip install``.
 # Metadata can look fine while ``.py`` files were removed mid-install (#57828).
+# Canonical tables live in the stdlib-only ``_early_recovery`` module (which
+# also probes/repairs BEFORE this module's third-party imports can run) so the
+# early and full recovery layers can never drift apart.
 _LAZY_REFRESH_IMPORT_PROBES: tuple[tuple[str, str], ...] = (
-    ("yaml", "SafeDumper"),
-    ("dotenv", "load_dotenv"),
-    ("click", "Command"),
-    ("certifi", "contents"),
-    ("rich", "print"),
-    ("cryptography", "__version__"),
-    ("jwt", "encode"),
+    _early_recovery_mod.LAZY_REFRESH_IMPORT_PROBES
 )
 
-_LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = {
-    "yaml": "PyYAML",
-    "dotenv": "python-dotenv",
-    "click": "click",
-    "certifi": "certifi",
-    "rich": "rich",
-    "cryptography": "cryptography",
-    "jwt": "PyJWT",
-}
+_LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = (
+    _early_recovery_mod.LAZY_REFRESH_REPAIR_PACKAGES
+)
 
 
 def _run_package_only_install(
@@ -8508,7 +8521,9 @@ def _repair_venv_via_import_probes(
     ):
         print("  ✓ Venv repair succeeded")
         return "repaired"
-    manual = " ".join(repr(p) for p in broken)
+    manual = " ".join(
+        shlex.quote(s) for s in _lazy_refresh_repair_specs(broken)
+    )
     print("  ⚠ Venv repair incomplete. Run manually, then `hermes update`:")
     print(
         f"    {' '.join(install_cmd_prefix)} install --force-reinstall {manual}"

tests/hermes_cli/test_early_recovery.py

@@ -0,0 +1,277 @@
+"""Tests for hermes_cli._early_recovery — the dependency-light bootstrap
+repair that runs BEFORE hermes_cli.main's third-party imports (#57828 / #58004).
+
+Covers:
+- entry-point lifecycle: a broken core import (dotenv) crashes the import of
+  hermes_cli.main WITHOUT early recovery, and imports fine when recovery runs
+  first (proving main.py invokes recovery before its third-party imports)
+- recover_if_needed unit behavior: fast path, marker gating, update-argv skip,
+  lock single-flight, no marker clearing, pinned repair specs
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+from hermes_cli import _early_recovery as er
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+# ---------------------------------------------------------------------------
+# Entry-point lifecycle (subprocess, real imports)
+# ---------------------------------------------------------------------------
+
+def _make_broken_dotenv_shadow(tmp_path: Path) -> Path:
+    """A sys.path dir shadowing ``dotenv`` with the #57828 failure state:
+    distribution metadata intact, import files wiped/broken."""
+    shadow = tmp_path / "shadow"
+    shadow.mkdir()
+    (shadow / "dotenv.py").write_text(
+        "raise ImportError('import files wiped mid-install (#57828)')\n",
+        encoding="utf-8",
+    )
+    return shadow
+
+
+def _run_lifecycle_subprocess(tmp_path: Path, *, repair: bool) -> subprocess.CompletedProcess:
+    shadow = _make_broken_dotenv_shadow(tmp_path)
+    hermes_home = tmp_path / "hermes_home"
+    hermes_home.mkdir()
+    script = tmp_path / "lifecycle.py"
+    script.write_text(
+        textwrap.dedent(
+            f"""
+            import sys
+
+            shadow = {str(shadow)!r}
+            sys.path.insert(0, shadow)
+
+            # _early_recovery must be importable on the corrupted venv
+            # (stdlib-only) — this import itself is part of the contract.
+            import hermes_cli._early_recovery as er
+
+            REPAIR = {repair!r}
+
+            def recorder(*args, **kwargs):
+                print("EARLY_RECOVERY_CALLED", flush=True)
+                if REPAIR:
+                    sys.path.remove(shadow)
+                    sys.modules.pop("dotenv", None)
+
+            er.recover_if_needed = recorder
+
+            import hermes_cli.main  # noqa: F401
+            print("MAIN_IMPORTED_OK", flush=True)
+            """
+        ),
+        encoding="utf-8",
+    )
+    env = {
+        **os.environ,
+        "PYTHONPATH": str(REPO_ROOT),
+        "HERMES_HOME": str(hermes_home),
+    }
+    return subprocess.run(
+        [sys.executable, str(script)],
+        capture_output=True,
+        text=True,
+        cwd=REPO_ROOT,
+        env=env,
+        timeout=120,
+    )
+
+
+def test_broken_dotenv_crashes_main_import_without_repair(tmp_path):
+    """Negative control: the shadow really breaks importing hermes_cli.main,
+    and recovery was invoked BEFORE the crash (i.e. before third-party
+    imports) — so a real repair at that point can save the launch."""
+    result = _run_lifecycle_subprocess(tmp_path, repair=False)
+    assert result.returncode != 0
+    assert "EARLY_RECOVERY_CALLED" in result.stdout
+    assert "MAIN_IMPORTED_OK" not in result.stdout
+    assert "wiped mid-install" in result.stderr
+
+
+def test_early_recovery_runs_before_main_imports_and_saves_launch(tmp_path):
+    """When recovery repairs the broken package, hermes_cli.main imports
+    cleanly — proving the recovery hook fires before env_loader/dotenv."""
+    result = _run_lifecycle_subprocess(tmp_path, repair=True)
+    assert "EARLY_RECOVERY_CALLED" in result.stdout
+    assert "MAIN_IMPORTED_OK" in result.stdout, result.stderr
+    assert result.returncode == 0
+
+
+def test_early_recovery_module_is_stdlib_only(tmp_path):
+    """The module must import in a process where every non-stdlib import
+    fails — that is the whole point of its existence."""
+    script = tmp_path / "stdlib_only.py"
+    script.write_text(
+        textwrap.dedent(
+            """
+            import builtins
+            import sys
+
+            STDLIB = set(sys.stdlib_module_names) | {"hermes_cli"}
+            real_import = builtins.__import__
+
+            def guard(name, *args, **kwargs):
+                top = name.split(".")[0]
+                if top not in STDLIB:
+                    raise ImportError(f"non-stdlib import blocked: {name}")
+                return real_import(name, *args, **kwargs)
+
+            builtins.__import__ = guard
+            import hermes_cli._early_recovery  # noqa: F401
+            print("STDLIB_ONLY_OK")
+            """
+        ),
+        encoding="utf-8",
+    )
+    result = subprocess.run(
+        [sys.executable, str(script)],
+        capture_output=True,
+        text=True,
+        cwd=REPO_ROOT,
+        env={**os.environ, "PYTHONPATH": str(REPO_ROOT)},
+        timeout=60,
+    )
+    assert "STDLIB_ONLY_OK" in result.stdout, result.stderr
+
+
+# ---------------------------------------------------------------------------
+# recover_if_needed unit behavior
+# ---------------------------------------------------------------------------
+
+def _project(tmp_path: Path, *, pyproject: bool = True) -> Path:
+    root = tmp_path / "proj"
+    root.mkdir(exist_ok=True)
+    if pyproject:
+        (root / "pyproject.toml").write_text(
+            '[project]\nname = "x"\ndependencies = [\n'
+            '  "PyYAML==6.0.2",\n'
+            '  "python-dotenv==1.2.2",\n'
+            '  "PyJWT[crypto]==2.13.0",\n'
+            "]\n",
+            encoding="utf-8",
+        )
+    return root
+
+
+def test_fast_path_no_marker_never_probes(tmp_path, monkeypatch):
+    root = _project(tmp_path)
+    probed = []
+    monkeypatch.setattr(er, "_probe_broken_packages", lambda: probed.append(1) or [])
+    er.recover_if_needed(project_root=root, argv=[])
+    assert probed == []
+
+
+def test_update_argv_skips_recovery(tmp_path, monkeypatch):
+    root = _project(tmp_path)
+    (root / ".lazy-refresh-incomplete").write_text("x", encoding="utf-8")
+    probed = []
+    monkeypatch.setattr(er, "_probe_broken_packages", lambda: probed.append(1) or [])
+    er.recover_if_needed(project_root=root, argv=["update"])
+    assert probed == []
+
+
+def test_no_pyproject_skips_and_preserves_marker(tmp_path, monkeypatch):
+    root = _project(tmp_path, pyproject=False)
+    marker = root / ".update-incomplete"
+    marker.write_text("x", encoding="utf-8")
+    monkeypatch.setattr(er, "_probe_broken_packages", lambda: ["PyYAML"])
+    installs = []
+    monkeypatch.setattr(er, "_run_repair_install", lambda specs, r: installs.append(specs) or True)
+    er.recover_if_needed(project_root=root, argv=[])
+    assert installs == []
+    assert marker.exists()
+
+
+def test_marker_plus_broken_probe_repairs_with_pinned_specs(tmp_path, monkeypatch):
+    root = _project(tmp_path)
+    marker = root / ".lazy-refresh-incomplete"
+    marker.write_text("x", encoding="utf-8")
+
+    probe_results = iter([["PyYAML", "python-dotenv"], []])
+    monkeypatch.setattr(er, "_probe_broken_packages", lambda: next(probe_results))
+    installs = []
+    monkeypatch.setattr(
+        er, "_run_repair_install", lambda specs, r: installs.append(specs) or True
+    )
+
+    er.recover_if_needed(project_root=root, argv=[])
+
+    assert installs == [["PyYAML==6.0.2", "python-dotenv==1.2.2"]]
+    # Marker lifecycle belongs to main.py's full recovery — never cleared here.
+    assert marker.exists()
+    # Lock released for the full recovery pass.
+    assert not (root / ".update-incomplete.lock").exists()
+
+
+def test_healthy_probe_skips_install(tmp_path, monkeypatch):
+    root = _project(tmp_path)
+    (root / ".update-incomplete").write_text("x", encoding="utf-8")
+    monkeypatch.setattr(er, "_probe_broken_packages", lambda: [])
+    installs = []
+    monkeypatch.setattr(er, "_run_repair_install", lambda specs, r: installs.append(specs) or True)
+    er.recover_if_needed(project_root=root, argv=[])
+    assert installs == []
+
+
+def test_lock_held_skips_repair(tmp_path, monkeypatch):
+    root = _project(tmp_path)
+    (root / ".lazy-refresh-incomplete").write_text("x", encoding="utf-8")
+    (root / ".update-incomplete.lock").write_text("123\n", encoding="utf-8")
+    monkeypatch.setattr(er, "_probe_broken_packages", lambda: ["PyYAML"])
+    installs = []
+    monkeypatch.setattr(er, "_run_repair_install", lambda specs, r: installs.append(specs) or True)
+    er.recover_if_needed(project_root=root, argv=[])
+    assert installs == []
+    # Fresh (non-stale) lock is left for its owner.
+    assert (root / ".update-incomplete.lock").exists()
+
+
+def test_failed_repair_prints_manual_command_with_pins(tmp_path, monkeypatch, capsys):
+    root = _project(tmp_path)
+    (root / ".lazy-refresh-incomplete").write_text("x", encoding="utf-8")
+    monkeypatch.setattr(er, "_probe_broken_packages", lambda: ["PyJWT"])
+    monkeypatch.setattr(er, "_run_repair_install", lambda specs, r: False)
+    er.recover_if_needed(project_root=root, argv=[])
+    err = capsys.readouterr().err
+    assert "--force-reinstall" in err
+    assert "PyJWT[crypto]==2.13.0" in err
+
+
+def test_pinned_specs_falls_back_to_bare_names_without_pyproject(tmp_path):
+    root = _project(tmp_path, pyproject=False)
+    assert er._pinned_specs(["PyYAML", "unknown-pkg"], root) == ["PyYAML", "unknown-pkg"]
+
+
+def test_pinned_specs_strips_env_markers_and_matches_extras(tmp_path):
+    root = _project(tmp_path)
+    (root / "pyproject.toml").write_text(
+        '[project]\nname = "x"\ndependencies = [\n'
+        '  "cryptography==46.0.7; python_version >= \'3.11\'",\n'
+        '  "PyJWT[crypto]==2.13.0",\n'
+        "]\n",
+        encoding="utf-8",
+    )
+    assert er._pinned_specs(["cryptography", "PyJWT"], root) == [
+        "cryptography==46.0.7",
+        "PyJWT[crypto]==2.13.0",
+    ]
+
+
+def test_probe_tables_shared_with_main():
+    """The full recovery layer in main.py must probe/repair the same set as
+    the early layer — the tables have one canonical home."""
+    import hermes_cli.main as m
+
+    assert m._LAZY_REFRESH_IMPORT_PROBES == er.LAZY_REFRESH_IMPORT_PROBES
+    assert m._LAZY_REFRESH_REPAIR_PACKAGES == er.LAZY_REFRESH_REPAIR_PACKAGES