mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-15 14:22:43 +00:00
fix(update): prevent and self-heal half-updated venvs on Windows (#57659)
Root-causes the July 2026 Windows incident chain (locked _brotlicffi.pyd / _sodium.pyd during install, then 'No module named annotated_doc' with 'hermes update' insisting 'Already up to date!'): - hermes update: probe venv core imports even when the checkout is current; a half-updated venv (dep sync killed mid-flight by a locked .pyd) is now detected and repaired instead of being reported as up to date - hermes update (Windows): after pausing gateways, refuse to mutate the venv while other processes run from the venv interpreter (the Desktop backend runs as python.exe so the hermes.exe shim guard never saw it); --force keeps the old behavior - install.ps1 venv stage: disarm gateway autostart Scheduled Tasks before the kill sweep (they respawn the gateway inside the kill->delete window), make the sweep a bounded loop requiring 3 clean passes, and rename-then- delete the old venv (a rename succeeds even with mapped DLLs) with stale- dir cleanup on the next run - desktop updater: 'venv shim still locked after 15s' now ABORTS the update hand-off (restarting our backend, surfacing the holder to the user) instead of 'proceeding anyway (force)' into guaranteed venv corruption; the unlock wait also re-kills respawned backends each poll tick
This commit is contained in:
parent
741bd9ba42
commit
b14d75f8af
4 changed files with 507 additions and 20 deletions
|
|
@ -8776,6 +8776,159 @@ def _wait_for_windows_update_gateway_exit(
|
|||
return survivors
|
||||
|
||||
|
||||
def _venv_core_imports_healthy() -> tuple[bool, str]:
|
||||
"""Probe the project venv for the core imports the backend needs to boot.
|
||||
|
||||
Runs a tiny import check inside the venv interpreter (NOT this process —
|
||||
``hermes update`` may be driven by a different Python). Catches the
|
||||
half-updated-venv state: git checkout current but a dependency sync that
|
||||
failed or was killed partway (e.g. Windows access-denied on a loaded
|
||||
.pyd), leaving imports like ``fastapi``'s new transitive deps missing.
|
||||
Without this probe, ``hermes update`` on a current checkout prints
|
||||
"Already up to date!" and returns without ever re-syncing dependencies —
|
||||
the user's install stays broken no matter how many times they update
|
||||
(ryanc's incident, July 2026).
|
||||
|
||||
Returns ``(healthy, detail)``. Never raises; unknown states report
|
||||
healthy so a probe failure can't force needless reinstalls.
|
||||
"""
|
||||
venv_dir = PROJECT_ROOT / "venv"
|
||||
python_name = "python.exe" if _is_windows() else "python"
|
||||
bin_dir = "Scripts" if _is_windows() else "bin"
|
||||
venv_python = venv_dir / bin_dir / python_name
|
||||
if not venv_python.exists():
|
||||
return True, ""
|
||||
|
||||
# Core web/serve imports plus their newest transitive deps. Import (not
|
||||
# just metadata) — a package can have intact dist-info but a missing
|
||||
# module after an interrupted uninstall/install cycle.
|
||||
check = (
|
||||
"import importlib\n"
|
||||
"mods = ['fastapi', 'uvicorn', 'pydantic', 'openai', 'yaml']\n"
|
||||
"missing = []\n"
|
||||
"for m in mods:\n"
|
||||
" try: importlib.import_module(m)\n"
|
||||
" except Exception as e: missing.append(f'{m}: {e}')\n"
|
||||
"print('\\n'.join(missing))\n"
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(venv_python), "-c", check],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
cwd=PROJECT_ROOT,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("venv health probe failed to run: %s", exc)
|
||||
return True, ""
|
||||
|
||||
missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()]
|
||||
if result.returncode != 0 and not missing:
|
||||
# Interpreter itself is broken (e.g. deleted stdlib) — that IS unhealthy.
|
||||
detail = (result.stderr or "").strip().splitlines()
|
||||
return False, detail[0] if detail else "venv python failed to run"
|
||||
if missing:
|
||||
return False, "; ".join(missing[:4])
|
||||
return True, ""
|
||||
|
||||
|
||||
def _detect_venv_python_processes(
|
||||
*, exclude_pids: set[int] | None = None
|
||||
) -> list[tuple[int, str, str]]:
|
||||
"""Find live processes running from the project venv's interpreter.
|
||||
|
||||
The hermes.exe shim guard misses the biggest lock-holder class on
|
||||
Windows: the Desktop app's backend (``python.exe -m hermes_cli.main
|
||||
serve``) and anything else running straight off ``venv\\Scripts\\python
|
||||
(w).exe``. Those processes keep native ``.pyd`` extensions mapped, so a
|
||||
dependency sync mid-update dies with access-denied and strands the venv
|
||||
half-updated (ryanc's brotlicffi/_sodium.pyd incidents, July 2026).
|
||||
|
||||
Killing them from here is pointless — the Desktop app supervises its
|
||||
backend and respawns it within seconds — so the caller should refuse and
|
||||
tell the user to close the app instead. Returns ``(pid, name, cmdline)``
|
||||
tuples; empty off-Windows / without psutil / when nothing matches. The
|
||||
calling process and its ancestors are always excluded (a CLI ``hermes
|
||||
update`` itself runs from the venv python). Never raises.
|
||||
"""
|
||||
if not _is_windows():
|
||||
return []
|
||||
try:
|
||||
import psutil
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
venv_dir = PROJECT_ROOT / "venv"
|
||||
try:
|
||||
venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep
|
||||
except OSError:
|
||||
venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep
|
||||
|
||||
skip: set[int] = set(exclude_pids or set())
|
||||
skip.add(os.getpid())
|
||||
try:
|
||||
for anc in psutil.Process().parents():
|
||||
skip.add(int(anc.pid))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
matches: list[tuple[int, str, str]] = []
|
||||
try:
|
||||
proc_iter = psutil.process_iter(["pid", "exe", "name", "cmdline"])
|
||||
except Exception:
|
||||
return []
|
||||
for proc in proc_iter:
|
||||
try:
|
||||
info = proc.info
|
||||
except Exception:
|
||||
continue
|
||||
pid = info.get("pid")
|
||||
exe = info.get("exe")
|
||||
if not exe or pid is None or int(pid) in skip:
|
||||
continue
|
||||
try:
|
||||
exe_norm = str(Path(exe).resolve()).lower()
|
||||
except (OSError, ValueError):
|
||||
exe_norm = str(exe).lower()
|
||||
if not exe_norm.startswith(venv_prefix):
|
||||
continue
|
||||
name = info.get("name") or Path(exe).name
|
||||
cmdline = " ".join(info.get("cmdline") or [])[:120]
|
||||
matches.append((int(pid), str(name), cmdline))
|
||||
return matches
|
||||
|
||||
|
||||
def _format_venv_python_holders_message(matches: list[tuple[int, str, str]]) -> str:
|
||||
"""Explain which venv processes block the update and how to clear them."""
|
||||
lines = [
|
||||
"✗ Other Hermes processes are running from this install's venv:",
|
||||
]
|
||||
for pid, name, cmdline in matches[:6]:
|
||||
hint = ""
|
||||
low = cmdline.lower()
|
||||
if "serve" in low or "dashboard" in low:
|
||||
hint = " ← Hermes Desktop backend (close the desktop app)"
|
||||
elif "gateway" in low:
|
||||
hint = " ← gateway"
|
||||
lines.append(f" PID {pid} {name} {cmdline}{hint}")
|
||||
if len(matches) > 6:
|
||||
lines.append(f" ... and {len(matches) - 6} more")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
" On Windows these keep native extension files (.pyd) locked, so the"
|
||||
)
|
||||
lines.append(
|
||||
" dependency update would fail partway and leave a broken install."
|
||||
)
|
||||
lines.append(
|
||||
" Close the Hermes desktop app / other Hermes terminals, then re-run:"
|
||||
)
|
||||
lines.append(" hermes update")
|
||||
lines.append(" (or use `hermes update --force` to proceed anyway at your own risk)")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _pause_windows_gateways_for_update() -> dict | None:
|
||||
"""Stop running Windows gateways before mutating the checkout or venv.
|
||||
|
||||
|
|
@ -9235,6 +9388,19 @@ def _cmd_update_impl(args, gateway_mode: bool):
|
|||
_windows_gateway_resume,
|
||||
)
|
||||
|
||||
# With gateways paused, anything still running from the venv interpreter
|
||||
# (most commonly the Desktop app's `hermes serve` backend) will keep .pyd
|
||||
# files locked and corrupt the dependency sync below. Refuse rather than
|
||||
# race: killing the desktop backend is futile (the app supervises and
|
||||
# respawns it), so the user must close the app. --force preserves the old
|
||||
# behavior for users who know what they're doing.
|
||||
if _is_windows() and not getattr(args, "force", False):
|
||||
_venv_holders = _detect_venv_python_processes()
|
||||
if _venv_holders:
|
||||
print(_format_venv_python_holders_message(_venv_holders))
|
||||
_resume_windows_gateways_after_update(_windows_gateway_resume)
|
||||
sys.exit(2)
|
||||
|
||||
# Try git-based update first, fall back to ZIP download on Windows
|
||||
# when git file I/O is broken (antivirus, NTFS filter drivers, etc.)
|
||||
use_zip_update = False
|
||||
|
|
@ -9436,7 +9602,41 @@ def _cmd_update_impl(args, gateway_mode: bool):
|
|||
text=True,
|
||||
check=False,
|
||||
)
|
||||
print("✓ Already up to date!")
|
||||
|
||||
# A current checkout does NOT imply a healthy install: a previous
|
||||
# dependency sync may have failed partway (classic on Windows,
|
||||
# where a running gateway/desktop backend keeps .pyd files locked
|
||||
# and uv/pip dies with access-denied, stranding the venv between
|
||||
# versions). Probe the venv's core imports and repair if broken —
|
||||
# otherwise "Already up to date!" gaslights the user while their
|
||||
# install stays bricked.
|
||||
healthy, detail = _venv_core_imports_healthy()
|
||||
if not healthy:
|
||||
print("⚠ Checkout is current, but the venv is unhealthy:")
|
||||
print(f" {detail}")
|
||||
print("→ Repairing Python dependencies...")
|
||||
_write_update_incomplete_marker()
|
||||
from hermes_cli.managed_uv import ensure_uv
|
||||
|
||||
repair_uv = ensure_uv()
|
||||
if repair_uv:
|
||||
repair_env = {**os.environ, "VIRTUAL_ENV": str(PROJECT_ROOT / "venv")}
|
||||
_install_python_dependencies_with_optional_fallback(
|
||||
[repair_uv, "pip"], env=repair_env, group="all"
|
||||
)
|
||||
else:
|
||||
_install_python_dependencies_with_optional_fallback(
|
||||
[sys.executable, "-m", "pip"], group="all"
|
||||
)
|
||||
_clear_update_incomplete_marker()
|
||||
healthy_after, detail_after = _venv_core_imports_healthy()
|
||||
if healthy_after:
|
||||
print("✓ Dependencies repaired!")
|
||||
else:
|
||||
print(f"⚠ Venv still unhealthy after repair: {detail_after}")
|
||||
print(" Close all Hermes windows/gateways and re-run: hermes update")
|
||||
else:
|
||||
print("✓ Already up to date!")
|
||||
_resume_windows_gateways_after_update(_windows_gateway_resume)
|
||||
return
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue