fix(update): prevent and self-heal half-updated venvs on Windows (#57659)

Root-causes the July 2026 Windows incident chain (locked _brotlicffi.pyd /
_sodium.pyd during install, then 'No module named annotated_doc' with
'hermes update' insisting 'Already up to date!'):

- hermes update: probe venv core imports even when the checkout is current;
  a half-updated venv (dep sync killed mid-flight by a locked .pyd) is now
  detected and repaired instead of being reported as up to date
- hermes update (Windows): after pausing gateways, refuse to mutate the venv
  while other processes run from the venv interpreter (the Desktop backend
  runs as python.exe so the hermes.exe shim guard never saw it); --force
  keeps the old behavior
- install.ps1 venv stage: disarm gateway autostart Scheduled Tasks before
  the kill sweep (they respawn the gateway inside the kill->delete window),
  make the sweep a bounded loop requiring 3 clean passes, and rename-then-
  delete the old venv (a rename succeeds even with mapped DLLs) with stale-
  dir cleanup on the next run
- desktop updater: 'venv shim still locked after 15s' now ABORTS the update
  hand-off (restarting our backend, surfacing the holder to the user)
  instead of 'proceeding anyway (force)' into guaranteed venv corruption;
  the unlock wait also re-kills respawned backends each poll tick
This commit is contained in:
Teknium 2026-07-03 03:24:08 -07:00 committed by GitHub
parent 741bd9ba42
commit b14d75f8af
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 507 additions and 20 deletions

View file

@ -8776,6 +8776,159 @@ def _wait_for_windows_update_gateway_exit(
return survivors
def _venv_core_imports_healthy() -> tuple[bool, str]:
"""Probe the project venv for the core imports the backend needs to boot.
Runs a tiny import check inside the venv interpreter (NOT this process
``hermes update`` may be driven by a different Python). Catches the
half-updated-venv state: git checkout current but a dependency sync that
failed or was killed partway (e.g. Windows access-denied on a loaded
.pyd), leaving imports like ``fastapi``'s new transitive deps missing.
Without this probe, ``hermes update`` on a current checkout prints
"Already up to date!" and returns without ever re-syncing dependencies
the user's install stays broken no matter how many times they update
(ryanc's incident, July 2026).
Returns ``(healthy, detail)``. Never raises; unknown states report
healthy so a probe failure can't force needless reinstalls.
"""
venv_dir = PROJECT_ROOT / "venv"
python_name = "python.exe" if _is_windows() else "python"
bin_dir = "Scripts" if _is_windows() else "bin"
venv_python = venv_dir / bin_dir / python_name
if not venv_python.exists():
return True, ""
# Core web/serve imports plus their newest transitive deps. Import (not
# just metadata) — a package can have intact dist-info but a missing
# module after an interrupted uninstall/install cycle.
check = (
"import importlib\n"
"mods = ['fastapi', 'uvicorn', 'pydantic', 'openai', 'yaml']\n"
"missing = []\n"
"for m in mods:\n"
" try: importlib.import_module(m)\n"
" except Exception as e: missing.append(f'{m}: {e}')\n"
"print('\\n'.join(missing))\n"
)
try:
result = subprocess.run(
[str(venv_python), "-c", check],
capture_output=True,
text=True,
timeout=60,
cwd=PROJECT_ROOT,
)
except Exception as exc:
logger.debug("venv health probe failed to run: %s", exc)
return True, ""
missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()]
if result.returncode != 0 and not missing:
# Interpreter itself is broken (e.g. deleted stdlib) — that IS unhealthy.
detail = (result.stderr or "").strip().splitlines()
return False, detail[0] if detail else "venv python failed to run"
if missing:
return False, "; ".join(missing[:4])
return True, ""
def _detect_venv_python_processes(
*, exclude_pids: set[int] | None = None
) -> list[tuple[int, str, str]]:
"""Find live processes running from the project venv's interpreter.
The hermes.exe shim guard misses the biggest lock-holder class on
Windows: the Desktop app's backend (``python.exe -m hermes_cli.main
serve``) and anything else running straight off ``venv\\Scripts\\python
(w).exe``. Those processes keep native ``.pyd`` extensions mapped, so a
dependency sync mid-update dies with access-denied and strands the venv
half-updated (ryanc's brotlicffi/_sodium.pyd incidents, July 2026).
Killing them from here is pointless the Desktop app supervises its
backend and respawns it within seconds so the caller should refuse and
tell the user to close the app instead. Returns ``(pid, name, cmdline)``
tuples; empty off-Windows / without psutil / when nothing matches. The
calling process and its ancestors are always excluded (a CLI ``hermes
update`` itself runs from the venv python). Never raises.
"""
if not _is_windows():
return []
try:
import psutil
except Exception:
return []
venv_dir = PROJECT_ROOT / "venv"
try:
venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep
except OSError:
venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep
skip: set[int] = set(exclude_pids or set())
skip.add(os.getpid())
try:
for anc in psutil.Process().parents():
skip.add(int(anc.pid))
except Exception:
pass
matches: list[tuple[int, str, str]] = []
try:
proc_iter = psutil.process_iter(["pid", "exe", "name", "cmdline"])
except Exception:
return []
for proc in proc_iter:
try:
info = proc.info
except Exception:
continue
pid = info.get("pid")
exe = info.get("exe")
if not exe or pid is None or int(pid) in skip:
continue
try:
exe_norm = str(Path(exe).resolve()).lower()
except (OSError, ValueError):
exe_norm = str(exe).lower()
if not exe_norm.startswith(venv_prefix):
continue
name = info.get("name") or Path(exe).name
cmdline = " ".join(info.get("cmdline") or [])[:120]
matches.append((int(pid), str(name), cmdline))
return matches
def _format_venv_python_holders_message(matches: list[tuple[int, str, str]]) -> str:
"""Explain which venv processes block the update and how to clear them."""
lines = [
"✗ Other Hermes processes are running from this install's venv:",
]
for pid, name, cmdline in matches[:6]:
hint = ""
low = cmdline.lower()
if "serve" in low or "dashboard" in low:
hint = " ← Hermes Desktop backend (close the desktop app)"
elif "gateway" in low:
hint = " ← gateway"
lines.append(f" PID {pid} {name} {cmdline}{hint}")
if len(matches) > 6:
lines.append(f" ... and {len(matches) - 6} more")
lines.append("")
lines.append(
" On Windows these keep native extension files (.pyd) locked, so the"
)
lines.append(
" dependency update would fail partway and leave a broken install."
)
lines.append(
" Close the Hermes desktop app / other Hermes terminals, then re-run:"
)
lines.append(" hermes update")
lines.append(" (or use `hermes update --force` to proceed anyway at your own risk)")
return "\n".join(lines)
def _pause_windows_gateways_for_update() -> dict | None:
"""Stop running Windows gateways before mutating the checkout or venv.
@ -9235,6 +9388,19 @@ def _cmd_update_impl(args, gateway_mode: bool):
_windows_gateway_resume,
)
# With gateways paused, anything still running from the venv interpreter
# (most commonly the Desktop app's `hermes serve` backend) will keep .pyd
# files locked and corrupt the dependency sync below. Refuse rather than
# race: killing the desktop backend is futile (the app supervises and
# respawns it), so the user must close the app. --force preserves the old
# behavior for users who know what they're doing.
if _is_windows() and not getattr(args, "force", False):
_venv_holders = _detect_venv_python_processes()
if _venv_holders:
print(_format_venv_python_holders_message(_venv_holders))
_resume_windows_gateways_after_update(_windows_gateway_resume)
sys.exit(2)
# Try git-based update first, fall back to ZIP download on Windows
# when git file I/O is broken (antivirus, NTFS filter drivers, etc.)
use_zip_update = False
@ -9436,7 +9602,41 @@ def _cmd_update_impl(args, gateway_mode: bool):
text=True,
check=False,
)
print("✓ Already up to date!")
# A current checkout does NOT imply a healthy install: a previous
# dependency sync may have failed partway (classic on Windows,
# where a running gateway/desktop backend keeps .pyd files locked
# and uv/pip dies with access-denied, stranding the venv between
# versions). Probe the venv's core imports and repair if broken —
# otherwise "Already up to date!" gaslights the user while their
# install stays bricked.
healthy, detail = _venv_core_imports_healthy()
if not healthy:
print("⚠ Checkout is current, but the venv is unhealthy:")
print(f" {detail}")
print("→ Repairing Python dependencies...")
_write_update_incomplete_marker()
from hermes_cli.managed_uv import ensure_uv
repair_uv = ensure_uv()
if repair_uv:
repair_env = {**os.environ, "VIRTUAL_ENV": str(PROJECT_ROOT / "venv")}
_install_python_dependencies_with_optional_fallback(
[repair_uv, "pip"], env=repair_env, group="all"
)
else:
_install_python_dependencies_with_optional_fallback(
[sys.executable, "-m", "pip"], group="all"
)
_clear_update_incomplete_marker()
healthy_after, detail_after = _venv_core_imports_healthy()
if healthy_after:
print("✓ Dependencies repaired!")
else:
print(f"⚠ Venv still unhealthy after repair: {detail_after}")
print(" Close all Hermes windows/gateways and re-run: hermes update")
else:
print("✓ Already up to date!")
_resume_windows_gateways_after_update(_windows_gateway_resume)
return