From 509960e8274defe2dcd3dc1c186fe8ad0af5b60c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:44:23 -0700 Subject: [PATCH] fix(update): stdlib-only early recovery before hermes_cli.main imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hermes console entry point is hermes_cli.main:main, and main.py imports dotenv (via env_loader) and yaml (via config) at module level. In the #57828 failure state — a failed lazy backend refresh wiping a core package's import files while metadata survives — a normal launch crashed while importing main.py, before _recover_from_interrupted_install() and the recovery markers from PR #58004 could act. - hermes_cli/_early_recovery.py: stdlib-only bootstrap repair invoked at the very top of main.py, before any third-party import. Probes the fragile core packages via real imports, force-reinstalls broken ones using the pyproject.toml pins, shares main.py's single-flight recovery lock, and never clears markers (the confirmed lifecycle stays with the full recovery path in main.py). - Probe/repair tables now have one canonical home in _early_recovery, reused by main.py so the two layers cannot drift. - Manual --force-reinstall fallback commands now print pinned specs via _lazy_refresh_repair_specs() instead of bare package names. - tests: entry-point lifecycle coverage proving a broken dotenv import crashes main.py without repair and imports cleanly with it, a stdlib-only import guard for _early_recovery, and unit coverage for marker gating, lock single-flight, pinned specs, and marker preservation. --- hermes_cli/_early_recovery.py | 226 +++++++++++++++++++ hermes_cli/main.py | 51 +++-- tests/hermes_cli/test_early_recovery.py | 277 ++++++++++++++++++++++++ 3 files changed, 536 insertions(+), 18 deletions(-) create mode 100644 hermes_cli/_early_recovery.py create mode 100644 tests/hermes_cli/test_early_recovery.py diff --git a/hermes_cli/_early_recovery.py b/hermes_cli/_early_recovery.py new file mode 100644 index 000000000000..428232d85735 --- /dev/null +++ b/hermes_cli/_early_recovery.py @@ -0,0 +1,226 @@ +"""Dependency-light venv recovery that runs BEFORE hermes_cli.main's imports. + +The ``hermes`` console entry point is ``hermes_cli.main:main``. Importing +``hermes_cli.main`` pulls in third-party packages at module level (``dotenv`` +via ``hermes_cli.env_loader``, ``yaml`` via ``hermes_cli.config``, ...). In +the exact failure state the update-recovery markers exist for — a failed lazy +backend refresh or interrupted core install that wiped a core package's +import files (#57828) — a normal launch crashes *while importing main.py*, +before ``_recover_from_interrupted_install()`` can run. The marker system is +unreachable precisely when it is needed most. + +This module is deliberately **stdlib-only** so importing it can never fail on +a corrupted venv. ``hermes_cli.main`` imports and calls +:func:`recover_if_needed` at the very top of its module body, before any +third-party import. + +Scope: this early pass only repairs enough for ``hermes_cli.main`` to become +importable again (force-reinstall of the known-fragile core packages, using +the pins from pyproject.toml). It NEVER clears the recovery markers — the +full, confirmed marker lifecycle stays with ``_recover_from_interrupted_install()`` +in main.py, which runs right after import succeeds. +""" + +from __future__ import annotations + +import importlib +import os +import subprocess +import sys +import time +from pathlib import Path + +# Core packages a failed lazy ``uv pip install`` is known to leave with intact +# distribution metadata but wiped import files (#57828). ``module`` is what we +# probe via a real import; ``attr`` guards against an empty/stub module. +# main.py's marker-recovery path reuses these tables — keep them here (the +# dependency-light module) so both layers probe and repair the same set. +LAZY_REFRESH_IMPORT_PROBES: tuple[tuple[str, str], ...] = ( + ("yaml", "SafeDumper"), + ("dotenv", "load_dotenv"), + ("click", "Command"), + ("certifi", "contents"), + ("rich", "print"), + ("cryptography", "__version__"), + ("jwt", "encode"), +) + +LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = { + "yaml": "PyYAML", + "dotenv": "python-dotenv", + "click": "click", + "certifi": "certifi", + "rich": "rich", + "cryptography": "cryptography", + "jwt": "PyJWT", +} + + +def _project_root() -> Path: + return Path(__file__).resolve().parent.parent + + +def _pinned_specs(packages: list[str], project_root: Path) -> list[str]: + """Map bare package names to their pinned specs from pyproject.toml. + + Stdlib-only (tomllib + naive requirement-head parsing — ``packaging`` may + itself be broken in the failure state this module exists for). Unknown + packages fall back to their bare name. + """ + pyproject = project_root / "pyproject.toml" + if not pyproject.is_file(): + return packages + try: + import tomllib + + with open(pyproject, "rb") as f: + raw_deps = tomllib.load(f).get("project", {}).get("dependencies", []) or [] + except Exception: + return packages + + name_to_spec: dict[str, str] = {} + for spec in raw_deps: + head = spec.split(";", 1)[0].strip() + bare = head + for op in ("==", ">=", "<=", "~=", ">", "<", "!="): + if op in bare: + bare = bare.split(op, 1)[0] + break + key = bare.strip().split("[", 1)[0].strip().lower() + if key: + name_to_spec[key] = head + return [name_to_spec.get(pkg.lower(), pkg) for pkg in packages] + + +def _probe_broken_packages() -> list[str]: + """Import-probe the fragile core packages in THIS process. + + Returns repair package names (deduped, probe order) for modules that fail + to import or lack their sentinel attribute. Failed imports leave nothing + in ``sys.modules``, so a post-repair retry in the same process works. + """ + broken: list[str] = [] + for mod_name, attr in LAZY_REFRESH_IMPORT_PROBES: + try: + mod = importlib.import_module(mod_name) + if not hasattr(mod, attr): + raise ImportError(f"{mod_name} missing {attr}") + except Exception: + pkg = LAZY_REFRESH_REPAIR_PACKAGES.get(mod_name) + if pkg and pkg not in broken: + broken.append(pkg) + return broken + + +def _run_repair_install(specs: list[str], project_root: Path) -> bool: + """ensurepip + ``pip install --force-reinstall`` the given specs. + + Streams nothing to stdout (``hermes acp`` speaks JSON-RPC on stdout); + output is captured and replayed to stderr only on failure. Never raises. + """ + try: + subprocess.run( + [sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], + cwd=project_root, + capture_output=True, + ) + except Exception: + pass + try: + result = subprocess.run( + [sys.executable, "-m", "pip", "install", "--force-reinstall", *specs], + cwd=project_root, + capture_output=True, + text=True, + ) + except Exception as exc: + print(f" ✗ Early venv repair could not run pip: {exc}", file=sys.stderr) + return False + if result.returncode != 0: + tail = (result.stderr or result.stdout or "")[-2000:] + if tail: + print(tail, file=sys.stderr) + return False + return True + + +def recover_if_needed( + project_root: Path | None = None, + argv: list[str] | None = None, +) -> None: + """Repair wiped core packages so ``hermes_cli.main`` can import at all. + + Fast path (no marker present) is two ``lstat`` calls. Only acts when a + recovery marker from a prior ``hermes update`` exists AND an import probe + confirms a core package is actually broken. Markers are intentionally + NOT cleared here — ``_recover_from_interrupted_install()`` in main.py owns + the confirmed marker lifecycle and runs immediately after import succeeds. + + Never raises: on any failure the import of main.py proceeds and surfaces + the real error. + """ + try: + args = sys.argv[1:] if argv is None else argv + # Same deliberately-loose match as main(): the real update flow writes + # and clears its own markers — a recovery install must not race it. + if "update" in args: + return + root = _project_root() if project_root is None else project_root + core_marker = root / ".update-incomplete" + lazy_marker = root / ".lazy-refresh-incomplete" + if not core_marker.exists() and not lazy_marker.exists(): + return + # Managed/Docker/PyPI installs have no source tree here — the marker + # is not ours to act on; main.py's recovery clears it. + if not (root / "pyproject.toml").is_file(): + return + + broken = _probe_broken_packages() + if not broken: + # Imports are fine — main.py will load and run full recovery. + return + + # Single-flight: share main.py's recovery lock so an early repair + # never races a concurrent full recovery into the same shared venv. + lock_path = root / ".update-incomplete.lock" + try: + fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.write(fd, f"{os.getpid()}\n".encode()) + os.close(fd) + except FileExistsError: + try: + if time.time() - lock_path.stat().st_mtime > 3600: + lock_path.unlink() + except OSError: + pass + return + except OSError: + pass # read-only fs / perms — proceed unlocked, install surfaces it + + try: + specs = _pinned_specs(broken, root) + print( + "⚠ Core package(s) broken by an interrupted update — " + f"repairing before launch: {', '.join(broken)}", + file=sys.stderr, + ) + if _run_repair_install(specs, root) and not _probe_broken_packages(): + print(" ✓ Core packages repaired.", file=sys.stderr) + else: + print( + " ✗ Automatic repair incomplete. Recover manually with:", + file=sys.stderr, + ) + print( + f" {sys.executable} -m pip install --force-reinstall " + + " ".join(specs), + file=sys.stderr, + ) + finally: + try: + lock_path.unlink() + except OSError: + pass + except Exception: + # Never block launch — the import of main.py will surface the truth. + pass diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 3313263c549a..b9bb4737e199 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -64,6 +64,25 @@ except ModuleNotFoundError: import os import sys +# Early venv self-heal — MUST run before any third-party import below. When +# a prior ``hermes update`` left a recovery marker and a core package's import +# files were wiped (#57828 — failed lazy backend refresh), the module-level +# ``from hermes_cli.env_loader import ...`` / ``from hermes_cli.config import +# ...`` imports further down would crash before ``main()`` ever reaches +# ``_recover_from_interrupted_install()``. ``_early_recovery`` is stdlib-only +# (safe to import on a corrupted venv), repairs just enough for this module to +# finish importing, and leaves the marker lifecycle to the full recovery path. +# The module import itself is unguarded on purpose: it lives in this same +# package directory, so if IT can't import, nothing else in hermes_cli can +# either. It is also the canonical home of the probe/repair tables reused by +# the full recovery path below. +from hermes_cli import _early_recovery as _early_recovery_mod + +try: + _early_recovery_mod.recover_if_needed() +except Exception: + pass + def _exit_after_oneshot(rc: object) -> None: """Exit one-shot mode without letting late native finalizers change rc. @@ -7724,9 +7743,12 @@ def _recover_lazy_refresh_marker_locked() -> None: "Leaving `.lazy-refresh-incomplete` for the next launch." ) print(" Recover manually with:") + all_specs = _lazy_refresh_repair_specs( + sorted(set(_LAZY_REFRESH_REPAIR_PACKAGES.values())) + ) print( f" {' '.join(install_prefix)} install --force-reinstall " - "PyYAML python-dotenv click certifi rich cryptography PyJWT" + + " ".join(shlex.quote(s) for s in all_specs) ) @@ -8283,25 +8305,16 @@ def _cleanup_quarantined_exes(scripts_dir: Path | None = None) -> None: # Import probes for venv corruption after a failed lazy ``uv pip install``. # Metadata can look fine while ``.py`` files were removed mid-install (#57828). +# Canonical tables live in the stdlib-only ``_early_recovery`` module (which +# also probes/repairs BEFORE this module's third-party imports can run) so the +# early and full recovery layers can never drift apart. _LAZY_REFRESH_IMPORT_PROBES: tuple[tuple[str, str], ...] = ( - ("yaml", "SafeDumper"), - ("dotenv", "load_dotenv"), - ("click", "Command"), - ("certifi", "contents"), - ("rich", "print"), - ("cryptography", "__version__"), - ("jwt", "encode"), + _early_recovery_mod.LAZY_REFRESH_IMPORT_PROBES ) -_LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = { - "yaml": "PyYAML", - "dotenv": "python-dotenv", - "click": "click", - "certifi": "certifi", - "rich": "rich", - "cryptography": "cryptography", - "jwt": "PyJWT", -} +_LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = ( + _early_recovery_mod.LAZY_REFRESH_REPAIR_PACKAGES +) def _run_package_only_install( @@ -8508,7 +8521,9 @@ def _repair_venv_via_import_probes( ): print(" ✓ Venv repair succeeded") return "repaired" - manual = " ".join(repr(p) for p in broken) + manual = " ".join( + shlex.quote(s) for s in _lazy_refresh_repair_specs(broken) + ) print(" ⚠ Venv repair incomplete. Run manually, then `hermes update`:") print( f" {' '.join(install_cmd_prefix)} install --force-reinstall {manual}" diff --git a/tests/hermes_cli/test_early_recovery.py b/tests/hermes_cli/test_early_recovery.py new file mode 100644 index 000000000000..984978e28d81 --- /dev/null +++ b/tests/hermes_cli/test_early_recovery.py @@ -0,0 +1,277 @@ +"""Tests for hermes_cli._early_recovery — the dependency-light bootstrap +repair that runs BEFORE hermes_cli.main's third-party imports (#57828 / #58004). + +Covers: +- entry-point lifecycle: a broken core import (dotenv) crashes the import of + hermes_cli.main WITHOUT early recovery, and imports fine when recovery runs + first (proving main.py invokes recovery before its third-party imports) +- recover_if_needed unit behavior: fast path, marker gating, update-argv skip, + lock single-flight, no marker clearing, pinned repair specs +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from hermes_cli import _early_recovery as er + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +# --------------------------------------------------------------------------- +# Entry-point lifecycle (subprocess, real imports) +# --------------------------------------------------------------------------- + +def _make_broken_dotenv_shadow(tmp_path: Path) -> Path: + """A sys.path dir shadowing ``dotenv`` with the #57828 failure state: + distribution metadata intact, import files wiped/broken.""" + shadow = tmp_path / "shadow" + shadow.mkdir() + (shadow / "dotenv.py").write_text( + "raise ImportError('import files wiped mid-install (#57828)')\n", + encoding="utf-8", + ) + return shadow + + +def _run_lifecycle_subprocess(tmp_path: Path, *, repair: bool) -> subprocess.CompletedProcess: + shadow = _make_broken_dotenv_shadow(tmp_path) + hermes_home = tmp_path / "hermes_home" + hermes_home.mkdir() + script = tmp_path / "lifecycle.py" + script.write_text( + textwrap.dedent( + f""" + import sys + + shadow = {str(shadow)!r} + sys.path.insert(0, shadow) + + # _early_recovery must be importable on the corrupted venv + # (stdlib-only) — this import itself is part of the contract. + import hermes_cli._early_recovery as er + + REPAIR = {repair!r} + + def recorder(*args, **kwargs): + print("EARLY_RECOVERY_CALLED", flush=True) + if REPAIR: + sys.path.remove(shadow) + sys.modules.pop("dotenv", None) + + er.recover_if_needed = recorder + + import hermes_cli.main # noqa: F401 + print("MAIN_IMPORTED_OK", flush=True) + """ + ), + encoding="utf-8", + ) + env = { + **os.environ, + "PYTHONPATH": str(REPO_ROOT), + "HERMES_HOME": str(hermes_home), + } + return subprocess.run( + [sys.executable, str(script)], + capture_output=True, + text=True, + cwd=REPO_ROOT, + env=env, + timeout=120, + ) + + +def test_broken_dotenv_crashes_main_import_without_repair(tmp_path): + """Negative control: the shadow really breaks importing hermes_cli.main, + and recovery was invoked BEFORE the crash (i.e. before third-party + imports) — so a real repair at that point can save the launch.""" + result = _run_lifecycle_subprocess(tmp_path, repair=False) + assert result.returncode != 0 + assert "EARLY_RECOVERY_CALLED" in result.stdout + assert "MAIN_IMPORTED_OK" not in result.stdout + assert "wiped mid-install" in result.stderr + + +def test_early_recovery_runs_before_main_imports_and_saves_launch(tmp_path): + """When recovery repairs the broken package, hermes_cli.main imports + cleanly — proving the recovery hook fires before env_loader/dotenv.""" + result = _run_lifecycle_subprocess(tmp_path, repair=True) + assert "EARLY_RECOVERY_CALLED" in result.stdout + assert "MAIN_IMPORTED_OK" in result.stdout, result.stderr + assert result.returncode == 0 + + +def test_early_recovery_module_is_stdlib_only(tmp_path): + """The module must import in a process where every non-stdlib import + fails — that is the whole point of its existence.""" + script = tmp_path / "stdlib_only.py" + script.write_text( + textwrap.dedent( + """ + import builtins + import sys + + STDLIB = set(sys.stdlib_module_names) | {"hermes_cli"} + real_import = builtins.__import__ + + def guard(name, *args, **kwargs): + top = name.split(".")[0] + if top not in STDLIB: + raise ImportError(f"non-stdlib import blocked: {name}") + return real_import(name, *args, **kwargs) + + builtins.__import__ = guard + import hermes_cli._early_recovery # noqa: F401 + print("STDLIB_ONLY_OK") + """ + ), + encoding="utf-8", + ) + result = subprocess.run( + [sys.executable, str(script)], + capture_output=True, + text=True, + cwd=REPO_ROOT, + env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}, + timeout=60, + ) + assert "STDLIB_ONLY_OK" in result.stdout, result.stderr + + +# --------------------------------------------------------------------------- +# recover_if_needed unit behavior +# --------------------------------------------------------------------------- + +def _project(tmp_path: Path, *, pyproject: bool = True) -> Path: + root = tmp_path / "proj" + root.mkdir(exist_ok=True) + if pyproject: + (root / "pyproject.toml").write_text( + '[project]\nname = "x"\ndependencies = [\n' + ' "PyYAML==6.0.2",\n' + ' "python-dotenv==1.2.2",\n' + ' "PyJWT[crypto]==2.13.0",\n' + "]\n", + encoding="utf-8", + ) + return root + + +def test_fast_path_no_marker_never_probes(tmp_path, monkeypatch): + root = _project(tmp_path) + probed = [] + monkeypatch.setattr(er, "_probe_broken_packages", lambda: probed.append(1) or []) + er.recover_if_needed(project_root=root, argv=[]) + assert probed == [] + + +def test_update_argv_skips_recovery(tmp_path, monkeypatch): + root = _project(tmp_path) + (root / ".lazy-refresh-incomplete").write_text("x", encoding="utf-8") + probed = [] + monkeypatch.setattr(er, "_probe_broken_packages", lambda: probed.append(1) or []) + er.recover_if_needed(project_root=root, argv=["update"]) + assert probed == [] + + +def test_no_pyproject_skips_and_preserves_marker(tmp_path, monkeypatch): + root = _project(tmp_path, pyproject=False) + marker = root / ".update-incomplete" + marker.write_text("x", encoding="utf-8") + monkeypatch.setattr(er, "_probe_broken_packages", lambda: ["PyYAML"]) + installs = [] + monkeypatch.setattr(er, "_run_repair_install", lambda specs, r: installs.append(specs) or True) + er.recover_if_needed(project_root=root, argv=[]) + assert installs == [] + assert marker.exists() + + +def test_marker_plus_broken_probe_repairs_with_pinned_specs(tmp_path, monkeypatch): + root = _project(tmp_path) + marker = root / ".lazy-refresh-incomplete" + marker.write_text("x", encoding="utf-8") + + probe_results = iter([["PyYAML", "python-dotenv"], []]) + monkeypatch.setattr(er, "_probe_broken_packages", lambda: next(probe_results)) + installs = [] + monkeypatch.setattr( + er, "_run_repair_install", lambda specs, r: installs.append(specs) or True + ) + + er.recover_if_needed(project_root=root, argv=[]) + + assert installs == [["PyYAML==6.0.2", "python-dotenv==1.2.2"]] + # Marker lifecycle belongs to main.py's full recovery — never cleared here. + assert marker.exists() + # Lock released for the full recovery pass. + assert not (root / ".update-incomplete.lock").exists() + + +def test_healthy_probe_skips_install(tmp_path, monkeypatch): + root = _project(tmp_path) + (root / ".update-incomplete").write_text("x", encoding="utf-8") + monkeypatch.setattr(er, "_probe_broken_packages", lambda: []) + installs = [] + monkeypatch.setattr(er, "_run_repair_install", lambda specs, r: installs.append(specs) or True) + er.recover_if_needed(project_root=root, argv=[]) + assert installs == [] + + +def test_lock_held_skips_repair(tmp_path, monkeypatch): + root = _project(tmp_path) + (root / ".lazy-refresh-incomplete").write_text("x", encoding="utf-8") + (root / ".update-incomplete.lock").write_text("123\n", encoding="utf-8") + monkeypatch.setattr(er, "_probe_broken_packages", lambda: ["PyYAML"]) + installs = [] + monkeypatch.setattr(er, "_run_repair_install", lambda specs, r: installs.append(specs) or True) + er.recover_if_needed(project_root=root, argv=[]) + assert installs == [] + # Fresh (non-stale) lock is left for its owner. + assert (root / ".update-incomplete.lock").exists() + + +def test_failed_repair_prints_manual_command_with_pins(tmp_path, monkeypatch, capsys): + root = _project(tmp_path) + (root / ".lazy-refresh-incomplete").write_text("x", encoding="utf-8") + monkeypatch.setattr(er, "_probe_broken_packages", lambda: ["PyJWT"]) + monkeypatch.setattr(er, "_run_repair_install", lambda specs, r: False) + er.recover_if_needed(project_root=root, argv=[]) + err = capsys.readouterr().err + assert "--force-reinstall" in err + assert "PyJWT[crypto]==2.13.0" in err + + +def test_pinned_specs_falls_back_to_bare_names_without_pyproject(tmp_path): + root = _project(tmp_path, pyproject=False) + assert er._pinned_specs(["PyYAML", "unknown-pkg"], root) == ["PyYAML", "unknown-pkg"] + + +def test_pinned_specs_strips_env_markers_and_matches_extras(tmp_path): + root = _project(tmp_path) + (root / "pyproject.toml").write_text( + '[project]\nname = "x"\ndependencies = [\n' + ' "cryptography==46.0.7; python_version >= \'3.11\'",\n' + ' "PyJWT[crypto]==2.13.0",\n' + "]\n", + encoding="utf-8", + ) + assert er._pinned_specs(["cryptography", "PyJWT"], root) == [ + "cryptography==46.0.7", + "PyJWT[crypto]==2.13.0", + ] + + +def test_probe_tables_shared_with_main(): + """The full recovery layer in main.py must probe/repair the same set as + the early layer — the tables have one canonical home.""" + import hermes_cli.main as m + + assert m._LAZY_REFRESH_IMPORT_PROBES == er.LAZY_REFRESH_IMPORT_PROBES + assert m._LAZY_REFRESH_REPAIR_PACKAGES == er.LAZY_REFRESH_REPAIR_PACKAGES