hermes-agent/hermes_cli/update_cmd.py
brooklyn! 5e807390fd
Merge pull request #74487 from NousResearch/bb/update-eol-churn
fix(update): repair managed checkouts still running core.autocrlf=true
2026-07-30 04:53:24 -05:00

5086 lines
217 KiB
Python
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Hermes update pipeline — extracted from ``hermes_cli/main.py``.
Mechanical move (main.py decomposition): ``_cmd_update_impl``, ``_cmd_update_check``
and every module-level helper used only by the update path, plus the update-only
constants they read. Function bodies are lifted verbatim; the only mechanical
change is that references to helpers/constants that STAY in ``hermes_cli.main``
(and to moved-but-test-patched siblings) are routed through ``_m()`` — a lazy
``hermes_cli.main`` reference — so existing call sites and test monkeypatches
that target ``hermes_cli.main.<name>`` (``PROJECT_ROOT``, ``_is_windows``,
``_run_pre_update_backup``, ...) keep working unchanged. ``main.py`` re-imports
every public-ish name from here (``# noqa: F401``) so the argparse wiring and
the test-patch surface still resolve on ``hermes_cli.main``.
Three self-contained closures nested inside ``_cmd_update_impl``
(``_print_items``, ``_wait_for_service_active``, ``_service_restart_sec``) were
hoisted to module level; they capture no enclosing state (verified via
``symtable``). ``_restart_one_systemd_gateway_unit``, ``_resolve_manage_cmd``
and ``_on_unit_timeout`` DO capture enclosing locals and stay nested,
byte-identical.
Imports are one-way: ``hermes_cli.main`` imports this module, never the reverse
at import time (``_m()`` resolves lazily at call time, when main.py is fully
loaded, so there is no import cycle).
"""
import hashlib
import json
import logging
import os
import shlex
import shutil
import subprocess
import sys
import time as _time
from datetime import datetime
from pathlib import Path
from typing import Optional
from hermes_cli.config import get_hermes_home
logger = logging.getLogger(__name__)
def _m():
"""Lazy ``hermes_cli.main`` reference.
Lets callers keep patching ``hermes_cli.main.<helper>`` (the historical
test surface) and have those patches reach this code path, and defers the
import so ``hermes_cli.main`` -> ``hermes_cli.update_cmd`` stays one-way
at import time.
"""
from hermes_cli import main
return main
_UPDATE_RUNTIME_RELOAD_MODULES = (
"hermes_constants",
"tools.environments.local",
"tools.lazy_deps",
)
def _reload_updated_runtime_modules() -> None:
"""Reload update-sensitive modules after the checkout changes in-place.
``hermes update`` keeps running in the pre-pull Python process. After a
large update, modules already present in ``sys.modules`` can still expose
old symbols even though their source files on disk are new. Refresh the
small module set used by lazy-backend refresh before that step imports
newly-updated code paths.
"""
try:
import importlib
importlib.invalidate_caches()
for module_name in _UPDATE_RUNTIME_RELOAD_MODULES:
module = _m().sys.modules.get(module_name)
if module is None:
continue
try:
importlib.reload(module)
except Exception as exc:
logger.debug("Could not reload updated module %s: %s", module_name, exc)
except Exception as exc:
logger.debug("Could not refresh update runtime modules: %s", exc)
# Critical files that Hermes must be able to import immediately after an
# update/install. Most are imported on every CLI startup; ``web_server.py``
# is the desktop/dashboard backend path that a fresh Windows install launches
# right away. If any of these fail to parse after a pull, the user can be
# left with a bricked CLI or desktop backend. The post-pull syntax guard
# validates these and auto-rolls-back on failure.
_UPDATE_CRITICAL_FILES = (
"hermes_cli/main.py",
"hermes_cli/config.py",
"hermes_cli/__init__.py",
"hermes_cli/web_server.py",
"cli.py",
"run_agent.py",
"model_tools.py",
"toolsets.py",
"hermes_constants.py",
)
def _capture_head_sha(git_cmd, cwd) -> str | None:
"""Return the current HEAD SHA, or None if it can't be resolved."""
try:
result = subprocess.run(
git_cmd + ["rev-parse", "HEAD"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=True,
)
return result.stdout.strip() or None
except (subprocess.CalledProcessError, OSError):
return None
def _validate_critical_files_syntax(root) -> tuple[bool, str | None, str | None]:
"""Compile each file in ``_UPDATE_CRITICAL_FILES`` to catch SyntaxErrors.
These are the files imported on every ``hermes`` startup; if any of them
has a syntax error (orphan merge-conflict markers, bad ref to a name
that no longer exists, etc.) the CLI can't bootstrap at all. We validate
them after a successful ``git pull`` so we can auto-roll-back instead of
leaving the user with a bricked install.
The compiled ``.pyc`` is written to a temp directory rather than the
source tree's ``__pycache__/`` so we don't race with concurrent test
workers that walk the same dir, and so we don't leave a stale pyc
behind in production if the next interpreter run picks a different
Python version. The pyc is discarded on function return either way —
we only care about the compile-or-not signal.
Returns ``(ok, failing_path, error_message)``. ``ok=True`` means every
file parsed cleanly.
"""
import py_compile
import tempfile
root = Path(root)
with tempfile.TemporaryDirectory(prefix="hermes-syntax-check-") as tmpdir:
for relpath in _UPDATE_CRITICAL_FILES:
path = root / relpath
if not path.exists():
# Missing file is suspicious but not necessarily fatal — a future
# refactor may legitimately remove one of these. Skip and move on.
continue
# Mirror the relative path under the tmpdir so two different
# files with the same basename don't collide on the cfile name.
cfile = Path(tmpdir) / (relpath.replace("/", "__") + "c")
try:
py_compile.compile(str(path), cfile=str(cfile), doraise=True)
except py_compile.PyCompileError as exc:
return False, str(path), str(exc)
except OSError as exc:
return False, str(path), f"could not read: {exc}"
return True, None, None
def _gateway_prompt(prompt_text: str, default: str = "", timeout: float = 300.0) -> str:
"""File-based IPC prompt for gateway mode.
Writes a prompt marker file so the gateway can forward the question to the
user, then polls for a response file. Falls back to *default* on timeout.
Used by ``hermes update --gateway`` so interactive prompts (stash restore,
config migration) are forwarded to the messenger instead of being silently
skipped.
"""
import json as _json
import uuid as _uuid
from hermes_constants import get_hermes_home
home = get_hermes_home()
prompt_path = home / ".update_prompt.json"
response_path = home / ".update_response"
# Clean any stale response file
response_path.unlink(missing_ok=True)
payload = {
"prompt": prompt_text,
"default": default,
"id": str(_uuid.uuid4()),
}
tmp = prompt_path.with_suffix(".tmp")
tmp.write_text(_json.dumps(payload), encoding="utf-8")
tmp.replace(prompt_path)
# Poll for response
deadline = _time.monotonic() + timeout
while _time.monotonic() < deadline:
if response_path.exists():
try:
answer = response_path.read_text(encoding="utf-8").strip()
response_path.unlink(missing_ok=True)
prompt_path.unlink(missing_ok=True)
return answer if answer else default
except (OSError, ValueError):
pass
_time.sleep(0.5)
# Timeout — clean up and use default
prompt_path.unlink(missing_ok=True)
response_path.unlink(missing_ok=True)
print(f" (no response after {int(timeout)}s, using default: {default!r})")
return default
def _npm_bin_exists(bin_dir: Path, name: str) -> bool:
"""True when an npm bin shim for *name* exists (POSIX or Windows)."""
return any(
(bin_dir / candidate).exists()
for candidate in (name, f"{name}.cmd", f"{name}.ps1", f"{name}.exe")
)
def _web_build_toolchain_ready(*roots: Path) -> bool:
"""True when ``tsc`` and ``vite`` shims are reachable from any of *roots*.
Callers must pass every root the build would search; checking only one
reports a healthy tree as broken.
"""
bin_dirs = [
bin_dir
for bin_dir in (root / "node_modules" / ".bin" for root in roots)
if bin_dir.is_dir()
]
return bool(bin_dirs) and all(
any(_npm_bin_exists(bin_dir, tool) for bin_dir in bin_dirs)
for tool in ("tsc", "vite")
)
def _web_toolchain_roots(web_dir: Path) -> tuple[Path, ...]:
"""Roots whose ``node_modules/.bin`` can satisfy the web build.
``npm run build`` prepends ``node_modules/.bin`` for the package and each
of its ancestors, so shims hoisted to the workspace root and shims nested
under a package that owns its lockfile (#42973) are equally valid.
"""
return (web_dir, web_dir.parent)
def _print_curator_first_run_notice() -> None:
"""Print a short heads-up about the skill curator after `hermes update`.
Only fires when the curator is enabled AND has no recorded run yet, which
is exactly the window where the gateway ticker used to fire Curator
against a fresh skill library immediately after an update. We defer the
first real pass by one ``interval_hours``; this notice tells the user how
to preview or disable before then. Silent on steady state.
"""
try:
from agent import curator
except Exception:
return
try:
if not curator.is_enabled():
return
state = curator.load_state()
except Exception:
return
if state.get("last_run_at"):
# Curator has run before (real or already seeded) — no notice needed.
return
try:
hours = curator.get_interval_hours()
except Exception:
hours = 24 * 7
days = max(1, hours // 24)
print()
print(" Skill curator")
print(
f" Background skill maintenance is enabled. First pass is deferred "
f"~{days}d after installation; only agent-created skills are in "
f"scope and nothing is ever auto-deleted (archive is recoverable)."
)
print(" Preview now: hermes curator run --dry-run")
print(" Pause it: hermes curator pause")
print(
" Docs: https://hermes-agent.nousresearch.com/docs/user-guide/features/curator"
)
def _print_fts_optimize_available_notice() -> None:
"""Advertise the opt-in v23 search-index optimization after `hermes update`.
Only fires when the current profile's state.db is still on the legacy
(pre-v23) inline FTS layout. Leads with the reclaimable-space figure and
points at the exact command. Honors ``sessions.fts_optimize_notice``:
``advise`` (default) prints an advisory notice, ``require`` prints a
firmer required-upgrade notice, ``off`` suppresses it. Silent for
fresh/already-optimized installs.
"""
mode = "advise"
try:
from hermes_cli.config import load_config
mode = str(
((load_config() or {}).get("sessions") or {}).get(
"fts_optimize_notice", "advise"
)
).strip().lower()
except Exception:
mode = "advise"
if mode == "off":
return
try:
from hermes_constants import get_hermes_home
from hermes_state import SessionDB
except Exception:
return
db_path = get_hermes_home() / "state.db"
if not db_path.exists():
return
try:
size_gb = db_path.stat().st_size / (1024 ** 3)
except OSError:
return
# Skip the notice for trivially small DBs — the win isn't worth the nag.
if size_gb < 0.5:
return
db = None
interrupted = False
try:
db = SessionDB(db_path=db_path, read_only=True)
# read_only opens skip schema init, so probe the layout directly.
row = db._conn.execute(
"SELECT sql FROM sqlite_master "
"WHERE type = 'table' AND name = 'messages_fts'"
).fetchone()
# An interrupted `optimize-storage` run: the table is already the
# v23 shape, but backfill markers / demoted trash tables remain.
# Offer the command again — re-running resumes and finishes it.
interrupted = bool(
db._conn.execute(
"SELECT 1 FROM state_meta "
"WHERE key = 'fts_rebuild_high_water' LIMIT 1"
).fetchone()
or db._conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
"AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1"
).fetchone()
or db._conn.execute(
"SELECT 1 FROM state_meta WHERE key IN "
"('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1"
).fetchone()
)
except Exception:
return
finally:
if db is not None:
try:
db.close()
except Exception:
pass
sql = (row[0] if row else "") or ""
if not sql or ("tool_name" in sql and not interrupted):
# v23 layout already present (fresh/optimized) — nothing to offer.
return
if interrupted:
print()
print("◆ Session database optimization incomplete")
print(
" A previous `hermes sessions optimize-storage` run was "
"interrupted. Search still works; re-run the command to resume "
"and finish reclaiming disk:"
)
print(" hermes sessions optimize-storage")
return
# Concrete size framing — lead with the savings the user cares about.
est_reclaim = size_gb * 0.6
print()
if mode == "require":
print("◆ Session database upgrade required")
print(
f" Your search index uses the OLD storage layout and should be "
f"upgraded. The new layout typically frees ~60% of state.db "
f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is "
f"required for continued optimal operation."
)
else:
print("◆ Reclaim ~60% of your session database disk")
print(
f" Your search index uses the old storage layout. Upgrading it "
f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB "
f"of your current {size_gb:.1f} GB."
)
print(" Run when convenient: hermes sessions optimize-storage")
print(
" It runs in the foreground with a progress bar, is safe to "
"interrupt/re-run, and never changes your conversations."
)
def _print_curator_recent_run_notice() -> None:
"""Print the most recent curator run summary, exactly once.
The curator runs in the background (gateway tick + CLI session start),
so users learn about skill consolidations only by stumbling into a
rename. ``hermes update`` is a high-attention surface — surface the
most recent run's rename map here, once.
Show-once: state stamps ``last_run_summary_shown_at`` after printing.
Subsequent ``hermes update`` invocations skip the block until a newer
curator run lands. Silent when the curator has never run, when the
most recent summary has already been shown, or when the summary has
no rename information to display (no archives).
"""
try:
from agent import curator
except Exception:
return
try:
state = curator.load_state()
except Exception:
return
last_run_at = state.get("last_run_at")
if not last_run_at:
return # no curator run yet — first-run notice handles this case
if state.get("last_run_summary_shown_at") == last_run_at:
return # already shown for this run
summary = state.get("last_run_summary") or ""
if not summary:
return
# Only print when there's something interesting to show — i.e. the
# rename map block was appended (multi-line summary). A bare "auto:
# no changes; llm: no change" doesn't warrant interrupting the
# update flow.
if "\n" not in summary:
# Still stamp it shown so we don't reconsider it on every update.
try:
state["last_run_summary_shown_at"] = last_run_at
curator.save_state(state)
except Exception:
pass
return
# Format the timestamp as "Xh ago" for readability.
when = _format_time_ago(last_run_at)
print()
print(f" Skill curator — last run {when}")
for line in summary.splitlines():
print(f" {line}")
print(
" (This message shows once per curator run. "
"View anytime: hermes curator status)"
)
# Stamp shown so we don't repeat on the next update.
try:
state["last_run_summary_shown_at"] = last_run_at
curator.save_state(state)
except Exception:
pass
def _format_time_ago(iso_ts: str) -> str:
"""Render an ISO timestamp as `Xh ago` / `Xd ago` / `Xm ago`. Best effort."""
try:
from datetime import datetime, timezone
ts = datetime.fromisoformat(iso_ts.replace("Z", "+00:00"))
if ts.tzinfo is None:
ts = ts.replace(tzinfo=timezone.utc)
delta = datetime.now(timezone.utc) - ts
secs = int(delta.total_seconds())
if secs < 60:
return "just now"
if secs < 3600:
return f"{secs // 60}m ago"
if secs < 86400:
return f"{secs // 3600}h ago"
return f"{secs // 86400}d ago"
except Exception:
return "recently"
def _finish_dashboard_update_cleanup(node_failures: list[str]) -> None:
"""Refresh managed dashboards or stop stale manual ones after an update."""
if node_failures:
print()
print(" Leaving running dashboard process(es) untouched because the")
print(" Node.js dependency refresh did not complete.")
return
stop_result = _m()._kill_stale_dashboard_processes(restart_managed=True)
if not stop_result.get("unrecovered"):
return
print()
print(
"⚠ A web dashboard/serve process was stopped during update and could "
"not be auto-restarted."
)
print(" Re-launch it when you want the web UI back:")
print(" hermes dashboard --port <port>")
def _atomic_replace_dir(src: str, dst: str) -> None:
"""Replace directory *dst* with *src* without leaving *dst* half-deleted.
The naive ``rmtree(dst); copytree(src, dst)`` has a destructive window: if
the copy fails partway (common on the Windows ZIP-update path, which only
runs because file I/O is already flaky on that machine), the old directory
is already gone and nothing replaced it — the install is left with a
deleted tree (issue #49145, where ``ui-tui/`` vanished and broke the TUI).
Instead, stage the new copy into a sibling temp dir first; only once that
fully succeeds do we swap it in. A failure during staging raises with the
original *dst* still intact.
"""
staging = f"{dst}.hermes-update-staging"
backup = f"{dst}.hermes-update-old"
# Clear any leftovers from a previously-interrupted update.
for leftover in (staging, backup):
if os.path.exists(leftover):
shutil.rmtree(leftover, ignore_errors=True)
# 1. Stage the new copy. If this fails, dst is untouched.
shutil.copytree(src, staging)
# 2. Swap: move the live dir aside, move staging into place. Both moves are
# same-filesystem renames; if the second fails we restore the backup.
if os.path.exists(dst):
os.rename(dst, backup)
try:
os.rename(staging, dst)
except OSError:
if os.path.exists(backup) and not os.path.exists(dst):
os.rename(backup, dst) # roll back to the original
raise
# 3. New dir is in place; drop the old one (best-effort — never fatal).
if os.path.exists(backup):
shutil.rmtree(backup, ignore_errors=True)
def _update_via_zip(args):
"""Update Hermes Agent by downloading a ZIP archive.
Used on Windows when git file I/O is broken (antivirus, NTFS filter
drivers causing 'Invalid argument' errors on file creation).
"""
import tempfile
import zipfile
from urllib.request import urlretrieve
# The ZIP fallback exists for Windows git-file-I/O breakage. It pulls a
# static archive from GitHub, which is fine for the default "main"
# channel but would silently ignore --branch and update from main even
# if the user asked for something else — exactly the silent-divergence
# bug --branch was added to prevent. Refuse to proceed in that case
# rather than lie.
branch = _m()._resolve_update_branch(args)
if branch != "main":
print(
f"✗ --branch={branch} is not supported on the Windows ZIP-fallback "
"update path."
)
print(
" This path runs when git file I/O is broken on the system. "
"Either resolve the git-side breakage (typically an antivirus "
"or NTFS filter holding files open) and rerun `hermes update "
f"--branch {branch}`, or update against main with `hermes update`."
)
_m().sys.exit(1)
zip_url = (
f"https://github.com/NousResearch/hermes-agent/archive/refs/heads/{branch}.zip"
)
print("→ Downloading latest version...")
tmp_dir = tempfile.mkdtemp(prefix="hermes-update-")
try:
zip_path = os.path.join(tmp_dir, f"hermes-agent-{branch}.zip")
urlretrieve(zip_url, zip_path)
print("→ Extracting...")
import stat as _stat
with zipfile.ZipFile(zip_path, "r") as zf:
# Validate paths to prevent zip-slip (path traversal) AND reject
# symlink members. A GitHub source ZIP for hermes-agent itself
# should never contain symlinks — they'd point outside the
# extracted tree and let an attacker who can compromise the
# update mirror plant arbitrary files via the update path.
tmp_dir_real = os.path.realpath(tmp_dir)
for member in zf.infolist():
member_path = os.path.realpath(os.path.join(tmp_dir, member.filename))
if (
not member_path.startswith(tmp_dir_real + os.sep)
and member_path != tmp_dir_real
):
raise ValueError(
f"Zip-slip detected: {member.filename} escapes extraction directory"
)
# Unix mode lives in the upper 16 bits of external_attr;
# mask to the file-type bits.
mode = (member.external_attr >> 16) & 0o170000
if _stat.S_ISLNK(mode):
raise ValueError(
f"ZIP contains unsupported symlink member: {member.filename}"
)
zf.extractall(tmp_dir)
# GitHub ZIPs extract to hermes-agent-<branch>/
extracted = os.path.join(tmp_dir, f"hermes-agent-{branch}")
if not os.path.isdir(extracted):
# Try to find it
for d in os.listdir(tmp_dir):
candidate = os.path.join(tmp_dir, d)
if os.path.isdir(candidate) and d != "__MACOSX":
extracted = candidate
break
# Copy updated files over existing installation, preserving venv/node_modules/.git
preserve = {"venv", "node_modules", ".git", ".env"}
update_count = 0
for item in os.listdir(extracted):
if item in preserve:
continue
src = os.path.join(extracted, item)
dst = os.path.join(str(_m().PROJECT_ROOT), item)
if os.path.isdir(src):
# Atomic-ish replace: never leave dst half-deleted if the copy
# fails partway (the failure mode behind #49145 on Windows).
_atomic_replace_dir(src, dst)
else:
shutil.copy2(src, dst)
update_count += 1
print(f"✓ Updated {update_count} items from ZIP")
except Exception as e:
print(f"✗ ZIP update failed: {e}")
_m().sys.exit(1)
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
# Clear stale bytecode after ZIP extraction
removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT)
if removed:
print(
f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}"
)
_m()._record_bytecode_fingerprint()
# Reinstall Python dependencies. Prefer .[all], but if one optional extra
# breaks on this machine, keep base deps and reinstall the remaining extras
# individually so update does not silently strip working capabilities.
print("→ Updating Python dependencies...")
from hermes_cli.managed_uv import ensure_uv, update_managed_uv
# Keep managed uv current — runs `uv self update` if we already have one.
update_managed_uv()
uv_bin = ensure_uv()
pip_cmd = [_m().sys.executable, "-m", "pip"]
if not uv_bin:
uv_bin = _ensure_uv_for_termux(pip_cmd)
if uv_bin:
uv_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")}
if _m()._is_termux_env(uv_env):
uv_env.pop("PYTHONPATH", None)
uv_env.pop("PYTHONHOME", None)
_m()._install_python_dependencies_with_optional_fallback([uv_bin, "pip"], env=uv_env)
else:
# Use sys.executable to explicitly call the venv's pip module,
# avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu.
# Some environments lose pip inside the venv; bootstrap it back with
# ensurepip before trying the editable install.
try:
subprocess.run(
pip_cmd + ["--version"],
cwd=_m().PROJECT_ROOT,
check=True,
capture_output=True,
)
except subprocess.CalledProcessError:
subprocess.run(
[_m().sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"],
cwd=_m().PROJECT_ROOT,
check=True,
)
_m()._install_python_dependencies_with_optional_fallback(pip_cmd)
# ZIP path parity: heal the active memory provider's bridge packages
# after the dependency reinstall, same as the git-pull path (#53272,
# #70636).
_m()._refresh_active_memory_provider_dependencies()
node_failures = _update_node_dependencies()
_m()._build_web_ui(_m().PROJECT_ROOT / "web")
# Sync skills
try:
from tools.skills_sync import sync_skills
print("→ Syncing bundled skills...")
result = sync_skills(quiet=True)
if result["copied"]:
print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}")
if result.get("updated"):
print(
f"{len(result['updated'])} updated: {', '.join(result['updated'])}"
)
if result.get("user_modified"):
print(f" ~ {len(result['user_modified'])} user-modified (kept)")
print(
" → see them: hermes skills list-modified "
"(diff/reset to resume updates)"
)
if result.get("cleaned"):
print(f" {len(result['cleaned'])} removed from manifest")
if result.get("relocated"):
print(
f"{len(result['relocated'])} moved to new upstream paths: "
f"{', '.join(result['relocated'])}"
)
if not result["copied"] and not result.get("updated"):
print(" ✓ Skills are up to date")
except Exception:
pass
# Seed the model-catalog disk cache from the freshly-unpacked checkout
# (same rationale as the git-pull path in _cmd_update_impl). Non-fatal.
try:
from hermes_cli.model_catalog import seed_cache_from_checkout
if seed_cache_from_checkout(_m().PROJECT_ROOT):
print(" ✓ Model catalog cache refreshed from checkout")
except Exception as e:
logger.debug("Model catalog seed during zip update failed: %s", e)
# ── Post-update state.db integrity guard (#68474) ─────────────────
# Same as the git-pull path: verify state.db survived the ZIP update
# and auto-restore from the most recent pre-update snapshot if needed.
try:
from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity
_state_path = get_hermes_home() / "state.db"
if _state_path.exists():
_state_ok = verify_sqlite_integrity(
_state_path, check_header=True, run_pragma=True
)
if not _state_ok.get("valid"):
print()
print(
"⚠ state.db is corrupted after update: "
+ _state_ok.get("message", "unknown error")
)
_snap_root = _quick_snapshot_root(get_hermes_home())
if _snap_root.exists():
_snap_dirs = sorted(
(d for d in _snap_root.iterdir() if d.is_dir()),
reverse=True,
)
for _snap_dir in _snap_dirs:
_snap_state = _snap_dir / "state.db"
if _snap_state.exists():
_snap_ok = verify_sqlite_integrity(
_snap_state, check_header=True, run_pragma=True
)
if _snap_ok.get("valid"):
try:
import shutil as _shutil
_shutil.copy2(_snap_state, _state_path)
_restored_ok = verify_sqlite_integrity(
_state_path,
check_header=True,
run_pragma=True,
)
if _restored_ok.get("valid"):
print(
" ✓ Auto-restored from snapshot "
f"{_snap_dir.name}"
)
else:
print(
" ✗ Auto-restore FAILED — restored "
"copy also failed integrity"
)
break
except OSError as _exc:
print(
f" ✗ Auto-restore file copy failed: {_exc}"
)
break
except Exception as exc:
logger.debug(
"Post-update state.db integrity check (zip path) failed: %s", exc
)
print()
if node_failures:
print(
"⚠ Update partially complete — Node.js dependencies for "
f"{', '.join(node_failures)} did not refresh."
)
print(" Code and Python deps are updated, but the dashboard/TUI may")
print(" be in a mixed state until the Node deps are rebuilt.")
else:
print("✓ Update complete!")
try:
_print_curator_first_run_notice()
except Exception as e:
logger.debug("Curator first-run notice failed: %s", e)
try:
_print_curator_recent_run_notice()
except Exception as e:
logger.debug("Curator recent-run notice failed: %s", e)
# Don't stop a working dashboard when the Node refresh failed — see the
# git-update path for rationale (#30271).
_finish_dashboard_update_cleanup(node_failures)
def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[str]:
status = subprocess.run(
git_cmd + ["status", "--porcelain"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=True,
)
if not status.stdout.strip():
return None
# If the index has unmerged entries (e.g. from an interrupted merge/rebase),
# git stash will fail with "needs merge / could not write index". Clear the
# conflict state with `git reset` so the stash can proceed. Working-tree
# changes are preserved; only the index conflict markers are dropped.
unmerged = subprocess.run(
git_cmd + ["ls-files", "--unmerged"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if unmerged.stdout.strip():
print("→ Clearing unmerged index entries from a previous conflict...")
subprocess.run(git_cmd + ["reset"], cwd=cwd, capture_output=True)
from datetime import datetime, timezone
stash_name = datetime.now(timezone.utc).strftime(
"hermes-update-autostash-%Y%m%d-%H%M%S"
)
print("→ Local changes detected — stashing before update...")
prev_stash = subprocess.run(
git_cmd + ["rev-parse", "--verify", "refs/stash"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
).stdout.strip()
push = subprocess.run(
git_cmd + ["stash", "push", "--include-untracked", "-m", stash_name],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if push.stdout.strip():
print(push.stdout.strip())
stash_probe = subprocess.run(
git_cmd + ["rev-parse", "--verify", "refs/stash"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
stash_ref = stash_probe.stdout.strip()
stash_created = (
stash_probe.returncode == 0 and bool(stash_ref) and stash_ref != prev_stash
)
if push.returncode != 0:
if stash_created:
# git stash push exits non-zero when it saved everything but could
# not delete some swept untracked files from the working tree
# (e.g. a root-owned directory: "warning: failed to remove ...:
# Permission denied"). The stash entry is complete — the changes
# are safe — so this is not a failure. Leave the undeletable
# files in place and continue the update.
if push.stderr.strip():
print(push.stderr.strip())
print(
" ⚠ Some untracked files could not be removed from the "
"working tree (permission denied)."
)
print(
" They were still saved to the stash and were left in "
"place — the update will continue."
)
# A partially-failed stash push also aborts its working-tree
# cleanup for TRACKED modifications — they are saved in the stash
# but still dirty the tree, which would break the checkout/pull
# that follows. Safe to reset: everything is in the stash entry.
subprocess.run(
git_cmd + ["reset", "--hard", "HEAD"],
cwd=cwd,
capture_output=True,
)
else:
# No stash entry was created: the changes were NOT saved. This
# is a real failure — bail out before the update touches HEAD.
print("✗ Could not stash local changes — update aborted.")
if push.stderr.strip():
print(f" {push.stderr.strip().splitlines()[0]}")
print(
" Commit, stash, or clean up your local changes manually, "
"then re-run `hermes update`."
)
raise subprocess.CalledProcessError(
push.returncode, push.args, output=push.stdout, stderr=push.stderr
)
return stash_ref
def _resolve_stash_selector(
git_cmd: list[str], cwd: Path, stash_ref: str
) -> Optional[str]:
stash_list = subprocess.run(
git_cmd + ["stash", "list", "--format=%gd %H"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=True,
)
for line in stash_list.stdout.splitlines():
selector, _, commit = line.partition(" ")
if commit.strip() == stash_ref:
return selector.strip()
return None
def _print_stash_cleanup_guidance(
stash_ref: str, stash_selector: Optional[str] = None
) -> None:
print(
" Check `git status` first so you don't accidentally reapply the same change twice."
)
print(" Find the saved entry with: git stash list --format='%gd %H %s'")
if stash_selector:
print(f" Remove it with: git stash drop {stash_selector}")
else:
print(
f" Look for commit {stash_ref}, then drop its selector with: git stash drop stash@{{N}}"
)
def _stash_apply_failed_only_on_existing_untracked(stderr: str) -> bool:
"""True when a ``git stash apply`` failure is ONLY about untracked files
that already exist in the working tree.
This is the tail end of the permission-denied autostash class: ``git stash
push --include-untracked`` swept undeletable files (e.g. a root-owned
``packaging/`` directory) into the stash but could not remove them from
disk. On restore, git applies all tracked changes, then refuses to
overwrite those still-present files (``already exists, no checkout`` /
``could not restore untracked files from stash``) and exits non-zero even
though nothing was lost. Any other error line (e.g. ``would be
overwritten by merge`` / ``Aborting``) means the tracked apply itself
failed and this returns False.
"""
lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()]
if not lines:
return False
saw_untracked_error = False
for ln in lines:
if "already exists, no checkout" in ln:
saw_untracked_error = True
elif "could not restore untracked files from stash" in ln:
saw_untracked_error = True
elif ln.startswith(("warning:", "hint:")):
continue
else:
return False
return saw_untracked_error
def _restore_stashed_changes(
git_cmd: list[str],
cwd: Path,
stash_ref: str,
prompt_user: bool = False,
input_fn=None,
) -> bool:
if prompt_user:
print()
print("⚠ Local changes were stashed before updating.")
print(
" Restoring them may reapply local customizations onto the updated codebase."
)
print(" Review the result afterward if Hermes behaves unexpectedly.")
print("Restore local changes now? [Y/n]")
if input_fn is not None:
response = input_fn("Restore local changes now? [Y/n]", "y")
else:
response = input().strip().lower()
if response not in {"", "y", "yes"}:
print("Skipped restoring local changes.")
print("Your changes are still preserved in git stash.")
print(f"Restore manually with: git stash apply {stash_ref}")
return False
print("→ Restoring local changes...")
restore = subprocess.run(
git_cmd + ["stash", "apply", stash_ref],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
# Check for unmerged (conflicted) files — can happen even when returncode is 0
unmerged = subprocess.run(
git_cmd + ["diff", "--name-only", "--diff-filter=U"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
has_conflicts = bool(unmerged.stdout.strip())
if restore.returncode != 0 and not has_conflicts and (
_stash_apply_failed_only_on_existing_untracked(restore.stderr)
):
# Permission-denied autostash tail end: the tracked changes applied
# cleanly; the only "failure" is untracked files that never left the
# working tree (git could not delete them at stash time, so it now
# refuses to overwrite them). Their content was never touched —
# nothing is lost. Treat as restored.
print(
" ⚠ Some stashed untracked files already exist in the working "
"tree and were kept as-is."
)
elif restore.returncode != 0 or has_conflicts:
print("✗ Update pulled new code, but restoring local changes hit conflicts.")
if restore.stdout.strip():
print(restore.stdout.strip())
if restore.stderr.strip():
print(restore.stderr.strip())
# Show which files conflicted
conflicted_files = unmerged.stdout.strip()
if conflicted_files:
print("\nConflicted files:")
for f in conflicted_files.splitlines():
print(f"{f}")
print("\nYour stashed changes are preserved — nothing is lost.")
print(f" Stash ref: {stash_ref}")
# Always reset to clean state — leaving conflict markers in source
# files makes hermes completely unrunnable (SyntaxError on import).
# The user's changes are safe in the stash for manual recovery.
subprocess.run(
git_cmd + ["reset", "--hard", "HEAD"],
cwd=cwd,
capture_output=True,
)
print("Working tree reset to clean state.")
print(f"Restore your changes later with: git stash apply {stash_ref}")
# Don't sys.exit — the code update itself succeeded, only the stash
# restore had conflicts. Let cmd_update continue with pip install,
# skill sync, and gateway restart.
return False
stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref)
if stash_selector is None:
print(
"⚠ Local changes were restored, but Hermes couldn't find the stash entry to drop."
)
print(
" The stash was left in place. You can remove it manually after checking the result."
)
_print_stash_cleanup_guidance(stash_ref)
else:
drop = subprocess.run(
git_cmd + ["stash", "drop", stash_selector],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if drop.returncode != 0:
print(
"⚠ Local changes were restored, but Hermes couldn't drop the saved stash entry."
)
if drop.stdout.strip():
print(drop.stdout.strip())
if drop.stderr.strip():
print(drop.stderr.strip())
print(
" The stash was left in place. You can remove it manually after checking the result."
)
_print_stash_cleanup_guidance(stash_ref, stash_selector)
print("⚠ Local changes were restored on top of the updated codebase.")
print(" Review `git diff` / `git status` if Hermes behaves unexpectedly.")
return True
def _discard_stashed_changes(
git_cmd: list[str],
cwd: Path,
stash_ref: str,
) -> bool:
"""Throw away a stash created before an update, without applying it.
Used only on a NON-interactive update when the user has set
``updates.non_interactive_local_changes: discard`` — i.e. they've opted out
of keeping local source edits on this machine. Drops the stash entry
instead of re-applying it, so the working tree stays clean at the freshly
pulled HEAD. Unlike ``git reset --hard`` + ``git clean -fd``, this only
affects what was stashed (tracked changes + the untracked files we
explicitly captured) — ignored paths like node_modules/venv/build outputs
are never touched, since they were never stashed.
Returns True if the stash was dropped, False on a git failure (in which
case the stash is left in place for safety).
"""
stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref)
if stash_selector is None:
print(
"⚠ Configured to discard local changes on non-interactive update, "
"but Hermes couldn't find the stash entry to drop."
)
_print_stash_cleanup_guidance(stash_ref)
return False
drop = subprocess.run(
git_cmd + ["stash", "drop", stash_selector],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if drop.returncode != 0:
print(
"⚠ Configured to discard local changes, but Hermes couldn't drop "
"the saved stash entry."
)
if drop.stderr.strip():
print(f" {drop.stderr.strip().splitlines()[0]}")
_print_stash_cleanup_guidance(stash_ref, stash_selector)
return False
print("→ Discarded local source changes (updates.non_interactive_local_changes=discard).")
return True
OFFICIAL_REPO_URLS = {
"https://github.com/NousResearch/hermes-agent.git",
"git@github.com:NousResearch/hermes-agent.git",
"https://github.com/NousResearch/hermes-agent",
"git@github.com:NousResearch/hermes-agent",
}
OFFICIAL_REPO_URL = "https://github.com/NousResearch/hermes-agent.git"
SKIP_UPSTREAM_PROMPT_FILE = ".skip_upstream_prompt"
def _get_origin_url(git_cmd: list[str], cwd: Path) -> Optional[str]:
"""Get the URL of the origin remote, or None if not set."""
try:
result = subprocess.run(
git_cmd + ["remote", "get-url", "origin"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if result.returncode == 0:
return result.stdout.strip()
except Exception:
pass
return None
def _is_fork(origin_url: Optional[str]) -> bool:
"""Check if the origin remote points to a fork (not the official repo)."""
if not origin_url:
return False
# Normalize URL for comparison (strip trailing .git if present)
normalized = origin_url.rstrip("/")
if normalized.endswith(".git"):
normalized = normalized[:-4]
for official in OFFICIAL_REPO_URLS:
official_normalized = official.rstrip("/")
if official_normalized.endswith(".git"):
official_normalized = official_normalized[:-4]
if normalized == official_normalized:
return False
return True
def _has_upstream_remote(git_cmd: list[str], cwd: Path) -> bool:
"""Check if an 'upstream' remote already exists."""
try:
result = subprocess.run(
git_cmd + ["remote", "get-url", "upstream"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
return result.returncode == 0
except Exception:
return False
def _add_upstream_remote(git_cmd: list[str], cwd: Path) -> bool:
"""Add the official repo as the 'upstream' remote. Returns True on success."""
try:
result = subprocess.run(
git_cmd + ["remote", "add", "upstream", OFFICIAL_REPO_URL],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
return result.returncode == 0
except Exception:
return False
def _count_commits_between(git_cmd: list[str], cwd: Path, base: str, head: str) -> int:
"""Count commits on `head` that are not on `base`. Returns -1 on error."""
try:
result = subprocess.run(
git_cmd + ["rev-list", "--count", f"{base}..{head}"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if result.returncode == 0:
return int(result.stdout.strip())
except Exception:
pass
return -1
def _should_skip_upstream_prompt() -> bool:
"""Check if user previously declined to add upstream."""
from hermes_constants import get_hermes_home
return (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).exists()
def _mark_skip_upstream_prompt():
"""Create marker file to skip future upstream prompts."""
try:
from hermes_constants import get_hermes_home
(get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).touch()
except Exception:
pass
def _sync_fork_with_upstream(git_cmd: list[str], cwd: Path) -> bool:
"""Attempt to push updated main to origin (sync fork).
Returns True if push succeeded, False otherwise.
"""
try:
result = subprocess.run(
git_cmd + ["push", "origin", "main", "--force-with-lease"],
cwd=cwd,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
return result.returncode == 0
except Exception:
return False
def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path) -> None:
"""Check if fork is behind upstream and sync if safe.
This implements the fork upstream sync logic:
- If upstream remote doesn't exist, ask user if they want to add it
- Compare origin/main with upstream/main
- If origin/main is strictly behind upstream/main, pull from upstream
- Try to sync fork back to origin if possible
"""
has_upstream = _has_upstream_remote(git_cmd, cwd)
if not has_upstream:
# Check if user previously declined
if _should_skip_upstream_prompt():
return
# Ask user if they want to add upstream
print()
print(" Your fork is not tracking the official Hermes repository.")
print(" This means you may miss updates from NousResearch/hermes-agent.")
print()
try:
response = (
input("Add official repo as 'upstream' remote? [Y/n]: ").strip().lower()
)
except (EOFError, KeyboardInterrupt):
print()
response = "n"
if response in {"", "y", "yes"}:
print("→ Adding upstream remote...")
if _add_upstream_remote(git_cmd, cwd):
print(
" ✓ Added upstream: https://github.com/NousResearch/hermes-agent.git"
)
has_upstream = True
else:
print(" ✗ Failed to add upstream remote. Skipping upstream sync.")
return
else:
print(
" Skipped. Run 'git remote add upstream https://github.com/NousResearch/hermes-agent.git' to add later."
)
_mark_skip_upstream_prompt()
return
# Fetch upstream main only. This sync compares upstream/main with
# origin/main, so there's no reason to pull every upstream ref — and a bare
# fetch drags in thousands of auto-generated branches.
print()
print("→ Fetching upstream...")
try:
subprocess.run(
git_cmd + ["fetch", "upstream", "main", "--quiet"],
cwd=cwd,
capture_output=True,
check=True,
)
except subprocess.CalledProcessError:
print(" ✗ Failed to fetch upstream. Skipping upstream sync.")
return
# Compare origin/main with upstream/main
origin_ahead = _count_commits_between(git_cmd, cwd, "upstream/main", "origin/main")
upstream_ahead = _count_commits_between(
git_cmd, cwd, "origin/main", "upstream/main"
)
if origin_ahead < 0 or upstream_ahead < 0:
print(" ✗ Could not compare branches. Skipping upstream sync.")
return
# If origin/main has commits not on upstream, don't trample
if origin_ahead > 0:
print()
print(f" Your fork has {origin_ahead} commit(s) not on upstream.")
print(" Skipping upstream sync to preserve your changes.")
print(" If you want to merge upstream changes, run:")
print(" git pull upstream main")
return
# If upstream is not ahead, fork is up to date
if upstream_ahead == 0:
print(" ✓ Fork is up to date with upstream")
return
# origin/main is strictly behind upstream/main (can fast-forward)
print()
print(f"→ Fork is {upstream_ahead} commit(s) behind upstream")
print("→ Pulling from upstream...")
try:
subprocess.run(
git_cmd + ["pull", "--ff-only", "upstream", "main"],
cwd=cwd,
check=True,
)
except subprocess.CalledProcessError:
print(
" ✗ Failed to pull from upstream. You may need to resolve conflicts manually."
)
return
print(" ✓ Updated from upstream")
# Try to sync fork back to origin
print("→ Syncing fork...")
if _sync_fork_with_upstream(git_cmd, cwd):
print(" ✓ Fork synced with upstream")
else:
print(
" Got updates from upstream but couldn't push to fork (no write access?)"
)
print(" Your local repo is updated, but your fork on GitHub may be behind.")
def _invalidate_update_cache():
"""Delete the update-check cache for ALL profiles so no banner
reports a stale "commits behind" count after a successful update.
The git repo is shared across profiles — when one profile runs
``hermes update``, every profile is now current.
"""
homes = []
# Default profile home (Docker-aware — uses /opt/data in Docker)
from hermes_constants import get_default_hermes_root
default_home = get_default_hermes_root()
homes.append(default_home)
# Named profiles under <root>/profiles/
profiles_root = default_home / "profiles"
if profiles_root.is_dir():
for entry in profiles_root.iterdir():
if entry.is_dir():
homes.append(entry)
for home in homes:
try:
cache_file = home / ".update_check"
if cache_file.exists():
cache_file.unlink()
except Exception:
pass
def _write_marker_file(path: Path, *, label: str) -> None:
"""Drop an update-recovery breadcrumb. Never raises."""
if _m()._pytest_owns_live_checkout(path.parent):
logger.debug("Skipping %s marker under pytest (live checkout)", label)
return
try:
path.write_text(
f"started={_time.time()}\npid={os.getpid()}\n", encoding="utf-8"
)
except OSError as exc:
logger.debug("Could not write %s marker: %s", label, exc)
def _write_update_incomplete_marker() -> None:
"""Drop the interrupted core-install breadcrumb. Never raises."""
_write_marker_file(_m()._update_marker_path(), label="update-incomplete")
def _write_lazy_refresh_incomplete_marker() -> None:
"""Drop the interrupted lazy-refresh breadcrumb. Never raises."""
_write_marker_file(_m()._lazy_refresh_marker_path(), label="lazy-refresh-incomplete")
def _format_concurrent_instances_message(
matches: list[tuple[int, str]], scripts_dir: Path
) -> str:
"""Build a human-readable explanation + remediation hint for the user."""
shim = scripts_dir / "hermes.exe"
lines = ["✗ Another hermes.exe is running:"]
for pid, name in matches:
lines.append(f" PID {pid} {name}")
lines.append("")
lines.append(f" Updating now would fail to overwrite {shim} because")
lines.append(" Windows blocks REPLACE on a running executable.")
lines.append("")
lines.append(" Close Hermes Desktop, exit any open `hermes` REPLs, and")
lines.append(" stop the gateway (`hermes gateway stop`) before retrying.")
lines.append("")
if matches:
pid_args = " ".join(f"/PID {pid}" for pid, _ in matches)
lines.append(" If you've already closed everything and these PIDs are")
lines.append(" stale, terminate them directly, then retry the update:")
lines.append(f" taskkill {pid_args} /F")
lines.append("")
lines.append(" Override with `hermes update --force` if you've already")
lines.append(" confirmed those processes will not write to the venv.")
return "\n".join(lines)
def _upgrade_pip_before_lazy_refresh(
install_cmd_prefix: list[str],
*,
env: dict[str, str] | None = None,
) -> None:
"""Upgrade pip before lazy-backend refreshes.
Older pip (e.g. 24.0 on Python 3.11) can fail setuptools-backed source
builds during lazy installs and leave a partially-written venv (#57828).
Never raises.
"""
try:
_m()._run_package_only_install(
install_cmd_prefix + ["install", "--upgrade", "pip"],
env=env,
)
except subprocess.CalledProcessError as exc:
logger.debug("pip upgrade before lazy refresh failed: %s", exc)
def _refresh_active_lazy_features(
install_cmd_prefix: list[str] | None = None,
*,
env: dict[str, str] | None = None,
) -> bool:
"""Refresh lazy-installed backends after a code update.
When pyproject.toml's ``[all]`` extra was slimmed down (May 2026), most
optional backends moved to ``tools/lazy_deps.py`` and only install on
first use. ``hermes update`` runs ``uv pip install -e .[all]`` which
leaves those packages untouched — so if we bump a pin in
:data:`LAZY_DEPS` (CVE response, transitive bug fix), users who already
activated the backend keep the stale version forever.
This function asks lazy_deps which features the user has previously
activated and reinstalls them under the current pins. Features the
user never enabled stay quiet — no churn for cold backends.
Returns True when the venv is safe to use (refresh succeeded, or no
active lazy backends, or post-failure import repair succeeded). Returns
False when a failed lazy install left broken core imports that automatic
repair could not fix (#57828).
Never raises. A failure here must not block the rest of the update.
"""
try:
from tools import lazy_deps
except Exception as exc:
logger.debug("Lazy refresh skipped (import failed): %s", exc)
return True
try:
active = lazy_deps.active_features()
except Exception as exc:
logger.debug("Lazy refresh skipped (active_features failed): %s", exc)
return True
if not active:
return True
print()
print(f"→ Refreshing {len(active)} active lazy backend(s)...")
unexpected_failure = False
try:
results = lazy_deps.refresh_active_features(prompt=False)
except Exception as exc:
# refresh_active_features is documented as never-raise, but defend
# the update flow against future regressions.
print(f" ⚠ Lazy refresh failed unexpectedly: {exc}")
results = {}
unexpected_failure = True
refreshed = [f for f, s in results.items() if s == "refreshed"]
current = [f for f, s in results.items() if s == "current"]
failed = [(f, s) for f, s in results.items() if s.startswith("failed:")]
skipped = [(f, s) for f, s in results.items() if s.startswith("skipped:")]
if refreshed:
print(f"{len(refreshed)} refreshed: {', '.join(refreshed)}")
if current:
print(f"{len(current)} already current")
if skipped:
# Most common reason: security.allow_lazy_installs=false. Show one
# line so the user knows why; not an error.
names = ", ".join(f for f, _ in skipped)
reason = skipped[0][1].split(": ", 1)[-1]
print(f" · {len(skipped)} skipped ({reason}): {names}")
if not failed and not unexpected_failure:
return True
for feature, status in failed:
reason = status.split(": ", 1)[-1]
# Clip noisy pip stderr to keep update output legible.
if len(reason) > 200:
reason = reason[:200] + "..."
print(f"{feature} failed to refresh: {reason}")
if install_cmd_prefix is None:
print(" ⚠ Lazy refresh failed; rerun `hermes update` once resolved.")
return False
# Immediate import-based recovery — metadata-only verifiers miss the case
# where DISTRIBUTION-INFO remains but import files were wiped (#57828).
# Unavailable probes are indeterminate, not healthy — keep the lazy marker.
status = _m()._repair_venv_via_import_probes(install_cmd_prefix, env=env)
if status == "repaired":
print(
" Lazy backend(s) keep their previous version until refresh succeeds."
)
return True
if status == "healthy":
print(
" Lazy backend(s) keep their previous version; probed packages look intact."
)
print(" Rerun `hermes update` once the upstream issue is resolved.")
return True
if status == "indeterminate":
print(
" ⚠ Leaving `.lazy-refresh-incomplete` until import probes can confirm health."
)
return False
def _refresh_active_memory_provider_dependencies() -> None:
"""Refresh pip dependencies for the configured external memory provider.
Memory-provider bridge packages are declared in each provider's
``plugin.yaml`` (plus mode-dependent extras like Hindsight's
``hindsight-all``), NOT in Hermes' editable-install extras or
``LAZY_DEPS`` alone — so the core dependency reinstall above can strip
or downgrade them (#53272 mem0ai, #70636 hindsight-embed). Re-run the
provider's declared install for the ACTIVE provider only, after the
core install and lazy refresh, so the last write to any shared package
is the one the active provider needs.
Never raises. A failure here must not block the rest of the update.
"""
try:
from hermes_cli.config import load_config
cfg = load_config()
except Exception as exc:
logger.debug("Memory provider refresh skipped (config load failed): %s", exc)
return
provider = ""
if isinstance(cfg, dict):
memory_cfg = cfg.get("memory")
if isinstance(memory_cfg, dict):
if memory_cfg.get("enabled") is False:
return
provider = str(memory_cfg.get("provider") or "").strip()
# "default" / empty is the built-in file-backed store — no pip deps.
if not provider or provider in {"default", "builtin", "none"}:
return
try:
from hermes_cli.memory_setup import _install_dependencies
except Exception as exc:
logger.debug("Memory provider refresh skipped (import failed): %s", exc)
return
print()
print(f"→ Refreshing active memory provider dependencies ({provider})...")
try:
_install_dependencies(provider, force=True)
except Exception as exc:
print(f"{provider} dependencies failed to refresh: {exc}")
def _is_android_python() -> bool:
return _m().sys.platform == "android"
def _install_psutil_android_compat(
install_cmd_prefix: list[str],
*,
env: dict[str, str] | None = None,
) -> None:
"""Install psutil on Android by patching upstream platform detection.
psutil's setup currently gates Linux sources behind
``sys.platform.startswith('linux')``. On Termux Python reports
``sys.platform == 'android'``, so setup aborts with
"platform android is not supported" despite compiling fine when using the
Linux source path.
We patch only the extracted build tree used for this install attempt;
nothing is persisted in the repository.
Stopgap: remove this once https://github.com/giampaolo/psutil/pull/2762
merges and ships in a release. The standalone installer script uses the
same shared helper and should be removed together.
"""
import tempfile
import urllib.request
from hermes_cli.psutil_android import PSUTIL_URL, prepare_patched_psutil_sdist
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
archive = tmp_path / "psutil.tar.gz"
urllib.request.urlretrieve(PSUTIL_URL, archive)
src_root = prepare_patched_psutil_sdist(archive, tmp_path)
_m()._run_install_with_heartbeat(
install_cmd_prefix + ["install", "--no-build-isolation", str(src_root)],
env=env,
)
def _ensure_uv_for_termux(pip_cmd: list[str]) -> str | None:
"""Best-effort uv bootstrap on Termux for faster update installs.
The normal path (``ensure_uv()`` in managed_uv) installs the managed
standalone uv into ``$HERMES_HOME/bin/uv``, but on Termux the official
installer may not work (glibc vs bionic). Prefer a uv already on PATH
(e.g. ``pkg install uv``); only if there is none do we fall back to a
wheel-only ``pip install uv`` so we never source-build the Rust crate.
"""
from hermes_cli.managed_uv import resolve_uv
existing = resolve_uv()
if existing:
return existing
if not _m()._is_termux_env():
return None
# A Termux-packaged uv lands on PATH but not in the managed bin dir, so
# resolve_uv() misses it. Use it before pip, which has no Android wheel and
# would otherwise build uv from source on a low-memory device.
system_uv = shutil.which("uv")
if system_uv:
return system_uv
try:
print(" → Termux detected: trying to install uv for faster dependency updates...")
result = subprocess.run(
pip_cmd + ["install", "uv", "--only-binary", ":all:"],
cwd=_m().PROJECT_ROOT,
check=False,
)
if result.returncode != 0:
return None
except Exception:
pass
# After pip install, check managed path first, then PATH
return resolve_uv() or shutil.which("uv")
def _npm_manifest_paths() -> tuple[Path, ...]:
"""Manifests whose changes must defeat the update-skip.
The lockfile alone is NOT a sufficient key: on a local checkout a dev
can edit package.json (root or a workspace) without running npm — the
lockfile is then unchanged but `hermes update` is exactly the step
expected to sync node_modules (via the `npm install` fallback in
_run_npm_install_deterministic).
The workspace list is pulled from the root package.json's `workspaces`
globs (npm's own source of truth) rather than hardcoded, so adding a
workspace can never silently escape the skip key. The root install
(step 1, --workspaces=false) still hoists shared deps for EVERY
workspace — desktop included — so all of them belong in the key, not
just the ones step 2 installs. Falls back to hashing just root
manifests if package.json is unreadable (never skips more than main
would have installed).
"""
root_pkg = _m().PROJECT_ROOT / "package.json"
paths = [_m().PROJECT_ROOT / "package-lock.json", root_pkg]
try:
workspaces = json.loads(root_pkg.read_text(encoding="utf-8")).get(
"workspaces", []
)
if isinstance(workspaces, dict): # legacy {"packages": [...]} form
workspaces = workspaces.get("packages", [])
for pattern in workspaces:
for match in sorted(_m().PROJECT_ROOT.glob(str(pattern))):
manifest = match / "package.json"
if manifest.is_file():
paths.append(manifest)
except (OSError, json.JSONDecodeError, TypeError):
pass
return tuple(paths)
def _npm_manifests_digest() -> str | None:
"""Combined sha256 over the lockfile + all workspace package.json files.
Returns None when the lockfile is missing (never skip then).
"""
if not (_m().PROJECT_ROOT / "package-lock.json").exists():
return None
h = hashlib.sha256()
for p in _npm_manifest_paths():
h.update(str(p.relative_to(_m().PROJECT_ROOT)).encode())
try:
h.update(p.read_bytes())
except OSError:
h.update(b"<missing>")
return h.hexdigest()
def _npm_lockfile_changed(hermes_root: Path) -> bool:
current = _npm_manifests_digest()
if current is None:
return True
# Also check that node_modules exists; a matching hash with missing
# node_modules means the cache was recorded by another checkout.
if not (_m().PROJECT_ROOT / "node_modules").is_dir():
return True
# A matching lockfile hash over a tree whose web build toolchain never
# landed must NOT skip the reinstall — otherwise every later `hermes
# update` keeps rebuilding against a half-installed tree and serving a
# stale dist.
web_dir = _m().PROJECT_ROOT / "web"
if (web_dir / "package.json").is_file() and not _web_build_toolchain_ready(
*_web_toolchain_roots(web_dir)
):
return True
try:
# Key the cache by PROJECT_ROOT so parallel worktrees don't collide.
cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12]
cache_file = hermes_root / f".npm_lock_hash_{cache_key}"
if not cache_file.exists():
return True
return cache_file.read_text(encoding="utf-8").strip() != current
except OSError:
return True
def _record_npm_lockfile_hash(hermes_root: Path) -> None:
digest = _npm_manifests_digest()
if digest is None:
return
try:
cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12]
cache_file = hermes_root / f".npm_lock_hash_{cache_key}"
cache_file.write_text(digest, encoding="utf-8")
except OSError:
logger.debug("Could not write npm lockfile hash cache")
def _update_node_dependencies() -> list[str]:
"""Refresh Node deps in the repo root and update workspaces.
Returns the list of labels whose npm install failed (empty on success),
so the caller can treat a Node refresh failure as a partial update rather
than silently reporting ``Update complete!`` (#30271).
"""
if not (_m().PROJECT_ROOT / "package.json").exists():
return []
npm = _m()._resolve_node_runtime_npm()
if not npm:
# If the only npm reachable inside this WSL shell is the Windows one,
# flag it loudly: silently skipping leaves ui-tui deps stale while the
# rest of the update proceeds, and running it would corrupt the tree.
from hermes_constants import is_wsl
path_npm = shutil.which("npm")
if is_wsl() and path_npm and _m()._is_windows_npm_path(path_npm):
print("→ Updating Node.js dependencies...")
print(" ⚠ Skipped: only a Windows npm is reachable from this WSL shell.")
print(" Install Node.js inside the WSL distro (nvm, or your distro's")
print(" package manager), then re-run `hermes update`.")
failed = ["repo root"]
if any(
(_m().PROJECT_ROOT / workspace / "package.json").exists()
for workspace in ("ui-tui", "web")
):
failed.append("ui-tui, web workspaces")
return failed
return []
from hermes_constants import get_default_hermes_root
# This cache describes PROJECT_ROOT/node_modules, which is shared by every
# Hermes profile using this checkout. Keep one per-checkout cache under the
# shared Hermes root rather than rerunning npm once per named profile.
shared_hermes_root = get_default_hermes_root()
if not _m()._npm_lockfile_changed(shared_hermes_root):
logger.info("npm lockfile unchanged, skipping npm install")
return []
# With a single workspace lockfile the root install would cover ALL
# workspaces — but apps/desktop pulls in Electron as a devDependency,
# and its postinstall downloads a ~200MB binary. Most users don't
# need desktop during `hermes update`, so we install root-only first
# then add just the workspaces the CLI/TUI/web build actually requires.
# Desktop deps are installed on demand by the desktop launcher
# (see _desktop_build_needed).
print("→ Updating Node.js dependencies...")
def _partial_update_failure(*labels: str) -> list[str]:
print()
print(" ⚠ Node.js dependency refresh did not complete cleanly; the")
print(" installation may be in a mixed state (updated code, stale Node")
print(" deps). Fix npm and re-run `hermes update`.")
return list(labels)
extra_args = ["--no-fund", "--no-audit", "--progress=false"]
from hermes_constants import with_hermes_node_path
nixos_env = with_hermes_node_path(_m()._nixos_build_env())
# Step 1: root install (no workspace recursion).
# NOTE: capture_output=False here is deliberate (#18840) — optional
# postinstall scripts (e.g. @askjo/camofox-browser's browser-binary fetch)
# print download progress, and capturing it makes a long download look
# hung. The chatty npm-deprecation noise during `hermes update` comes from
# the *desktop* build, not this step; that one is captured to update.log.
root_args = [*extra_args, "--workspaces=false"]
root_result = _m()._run_npm_install_deterministic(
npm,
_m().PROJECT_ROOT,
extra_args=tuple(root_args),
capture_output=False,
env=nixos_env,
)
if root_result.returncode != 0:
print(" ⚠ npm install failed in repo root")
stderr = (root_result.stderr or "").strip() if root_result.stderr else ""
if stderr:
print(f" {stderr.splitlines()[-1]}")
return _partial_update_failure("repo root")
# Step 2: install only the workspaces update needs (ui-tui, web).
# --workspace selects specific workspaces; the rest (desktop) are skipped.
ws_args = [*extra_args, "--workspace", "ui-tui", "--workspace", "web"]
ws_result = _m()._run_npm_install_deterministic(
npm,
_m().PROJECT_ROOT,
extra_args=tuple(ws_args),
capture_output=False,
env=nixos_env,
)
if ws_result.returncode == 0:
_record_npm_lockfile_hash(shared_hermes_root)
print(" ✓ repo root + ui-tui, web workspaces (desktop skipped)")
return []
print(" ⚠ npm workspace install failed")
stderr = (ws_result.stderr or "").strip() if ws_result.stderr else ""
if stderr:
print(f" {stderr.splitlines()[-1]}")
return _partial_update_failure("ui-tui, web workspaces")
def _log_only_write(text: str) -> None:
"""Write ``text`` to ``~/.hermes/logs/update.log`` only, never the terminal.
During ``hermes update`` ``sys.stdout`` is an ``_UpdateOutputStream`` that
mirrors to both the terminal and ``update.log``. Loud, low-signal
subprocess output (npm installs, the Electron/vite build, the cua-driver
installer's "Next steps" wall) should be captured and tucked into the log
so failures stay debuggable, without flooding the user's terminal. This
reaches past the mirroring stream straight to the underlying log handle.
"""
if not text:
return
stream = _m().sys.stdout
log_file = getattr(stream, "_log", None)
if log_file is None:
return
try:
log_file.write(text if text.endswith("\n") else text + "\n")
log_file.flush()
except Exception:
pass
def _run_logged_subprocess(cmd, *, cwd=None, env=None):
"""Run ``cmd`` capturing combined output into update.log (not the terminal).
Returns the ``CompletedProcess`` (with ``stdout`` populated) so the caller
can decide whether to surface the captured output on failure.
"""
result = subprocess.run(
cmd,
cwd=cwd,
env=env,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
encoding="utf-8",
errors="replace",
)
_log_only_write(result.stdout or "")
return result
def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False):
"""Implement ``hermes update --check``: fetch and report without installing.
``branch`` selects which branch the check compares against. Default is
"main"; callers can pass another branch to ask "are there new commits
on origin/<branch>?" without performing the update.
``branch_explicit`` is True iff the caller passed --branch on the CLI.
Installs that can't honor non-default branches (e.g. Docker) surface a
one-line notice instead of silently dropping the flag.
"""
from hermes_cli.config import detect_install_method, recommended_update_command_for_method
method = detect_install_method(_m().PROJECT_ROOT)
if method == "docker":
# Docker can't ``git fetch`` from within the container. Surface the
# same long-form ``docker pull`` guidance ``hermes update`` (apply
# path) uses — telling the user to "reinstall via curl" or that
# ".git is missing" would point them at the wrong remediation.
from hermes_cli.config import format_docker_update_message
print(format_docker_update_message())
sys.exit(1)
if method in {"nix", "nixos"}:
print(recommended_update_command_for_method(method))
sys.exit(1)
git_dir = _m().PROJECT_ROOT / ".git"
if not git_dir.exists():
print("✗ Not a git repository — cannot check for updates.")
sys.exit(1)
git_cmd = ["git"]
if sys.platform == "win32":
git_cmd = ["git", "-c", "windows.appendAtomically=false"]
# Fetch only the branch we compare against; prefer upstream as the canonical
# reference. A bare `git fetch <remote>` pulls every ref, and this repo has
# thousands of auto-generated branches, so scope the fetch to <branch>.
# Note: upstream/<branch> may not exist for non-main branches (a fork's
# bb/gui has no upstream counterpart), so when the caller picks a
# non-default branch we skip the upstream probe and use origin directly.
# Installer checkouts are shallow (`git clone --depth 1`). A plain
# `git fetch` would unshallow the repo (dragging in the whole history —
# the exact cost the shallow clone avoided) and the rev-list count below
# would then report a huge bogus "behind" number. Detect shallow up front:
# fetch with --depth 1 to preserve the boundary and report presence-only.
is_shallow = (
subprocess.run(
git_cmd + ["rev-parse", "--is-shallow-repository"],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
).stdout.strip()
== "true"
)
depth_args = ["--depth", "1"] if is_shallow else []
if branch == "main":
# Probe locally (~6 ms) whether an 'upstream' remote exists at all
# before spending a network fetch on it. Non-fork installs have no
# 'upstream' remote, and the old flow burned a failed network attempt
# (~0.3-1 s) on every --check before falling back to origin.
has_upstream_remote = (
subprocess.run(
git_cmd + ["remote", "get-url", "upstream"],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
).returncode
== 0
)
fetch_result = None
if has_upstream_remote:
print("→ Fetching from upstream...")
fetch_result = subprocess.run(
git_cmd + ["fetch"] + depth_args + ["upstream", branch],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if fetch_result is not None and fetch_result.returncode == 0:
upstream_exists = True
compare_branch = f"upstream/{branch}"
else:
# No upstream remote, or the upstream fetch failed — use origin.
print("→ Fetching from origin...")
fetch_result = subprocess.run(
git_cmd + ["fetch"] + depth_args + ["origin", branch],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
upstream_exists = False
compare_branch = f"origin/{branch}"
else:
# Non-default branch: compare against origin/<branch> directly.
print("→ Fetching from origin...")
fetch_result = subprocess.run(
git_cmd + ["fetch"] + depth_args + ["origin", branch],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
upstream_exists = False
compare_branch = f"origin/{branch}"
if fetch_result.returncode != 0:
stderr = fetch_result.stderr.strip()
if "Could not resolve host" in stderr or "unable to access" in stderr:
print("✗ Network error — cannot reach the remote repository.")
elif "Authentication failed" in stderr or "could not read Username" in stderr:
print("✗ Authentication failed — check your git credentials or SSH key.")
else:
print("✗ Failed to fetch.")
if stderr:
print(f" {stderr.splitlines()[0]}")
sys.exit(1)
# Verify the compare ref actually exists before asking rev-list about it.
# Without this, `git rev-list HEAD..origin/<bogus> --count` exits 128 and
# (with check=True) raises CalledProcessError, surfacing a Python
# traceback. Friendlier to detect-and-report.
verify_result = subprocess.run(
git_cmd + ["rev-parse", "--verify", "--quiet", compare_branch],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if verify_result.returncode != 0:
print(f"✗ Branch '{branch}' not found on {compare_branch.split('/', 1)[0]}.")
sys.exit(1)
if is_shallow:
# No history to count across the shallow boundary. Compare tip SHAs and
# report presence-only (mirrors the banner's _check_via_local_git).
head_sha = subprocess.run(
git_cmd + ["rev-parse", "HEAD"],
cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
).stdout.strip()
target_sha = subprocess.run(
git_cmd + ["rev-parse", compare_branch],
cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
).stdout.strip()
if head_sha and target_sha and head_sha == target_sha:
print("✓ Already up to date.")
else:
print(f"⚕ Update available (behind {compare_branch}).")
from hermes_cli.config import recommended_update_command
print(f" Run '{recommended_update_command()}' to install.")
return
rev_result = subprocess.run(
git_cmd + ["rev-list", f"HEAD..{compare_branch}", "--count"],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=True,
)
behind = int(rev_result.stdout.strip())
if behind == 0:
print("✓ Already up to date.")
else:
commits_word = "commit" if behind == 1 else "commits"
print(f"⚕ Update available: {behind} {commits_word} behind {compare_branch}.")
from hermes_cli.config import recommended_update_command
print(f" Run '{recommended_update_command()}' to install.")
def _ensure_fhs_path_guard() -> None:
"""Ensure /usr/local/bin is on PATH for RHEL-family root non-login shells.
Mirrors the post-symlink probe added to ``scripts/install.sh`` so that
existing FHS-layout root installs on RHEL/CentOS/Rocky/Alma 8+ get
repaired on ``hermes update`` without requiring a reinstall. The
installer's assumption that ``/usr/local/bin`` is on PATH for every
standard shell breaks on those distros in non-login interactive shells
(su, sudo -s, tmux panes, some web terminals): /etc/bashrc doesn't
add /usr/local/bin and /root/.bash_profile doesn't either. Symptom:
``hermes`` prints ``command not found`` even though the symlink lives
at /usr/local/bin/hermes.
Silent no-op on: non-Linux, non-root, non-FHS installs, and any system
where ``bash -i -c 'command -v hermes'`` already resolves. Idempotent.
"""
if _m().sys.platform != "linux":
return
try:
if os.geteuid() != 0: # windows-footgun: ok — Linux FHS helper, guarded by sys.platform == "linux" above + AttributeError catch
return
except AttributeError:
return
# Only act when this is actually an FHS-layout install (command link at
# /usr/local/bin/hermes, code at /usr/local/lib/hermes-agent).
fhs_link = Path("/usr/local/bin/hermes")
if not fhs_link.is_symlink() and not fhs_link.exists():
return
# Probe a fresh non-login interactive bash the way the user will use it.
# ``bash -i -c`` sources ~/.bashrc but NOT ~/.bash_profile or /etc/profile,
# which is the exact scenario where RHEL root loses /usr/local/bin.
home = os.environ.get("HOME") or "/root"
try:
probe = subprocess.run(
[
"env",
"-i",
f"HOME={home}",
f"TERM={os.environ.get('TERM', 'dumb')}",
"bash",
"-i",
"-c",
"command -v hermes",
],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=10,
)
except (FileNotFoundError, subprocess.TimeoutExpired):
return # no bash or probe hung — don't block update on this
if probe.returncode == 0:
return # already on PATH, nothing to do
path_line = 'export PATH="/usr/local/bin:$PATH"'
path_comment = (
"# Hermes Agent — ensure /usr/local/bin is on PATH " "(RHEL non-login shells)"
)
wrote_any = False
for candidate in (".bashrc", ".bash_profile"):
cfg = Path(home) / candidate
if not cfg.is_file():
continue
try:
existing = cfg.read_text(errors="replace", encoding="utf-8")
except OSError:
continue
# Idempotency: skip if any uncommented PATH= line already references
# /usr/local/bin. Mirrors the grep pattern used by install.sh.
already_guarded = any(
"/usr/local/bin" in line
and "PATH" in line
and not line.lstrip().startswith("#")
for line in existing.splitlines()
)
if already_guarded:
continue
try:
with cfg.open("a", encoding="utf-8") as f:
f.write("\n" + path_comment + "\n" + path_line + "\n")
except OSError as e:
print(f" ⚠ Could not update {cfg}: {e}")
continue
print(f" ✓ Added /usr/local/bin to PATH in {cfg}")
wrote_any = True
if wrote_any:
print(" (reload your shell or run 'source ~/.bashrc' to pick it up)")
def _ensure_acp_launcher() -> None:
"""Self-heal: install a ``hermes-acp`` launcher next to the ``hermes`` one.
Mirrors the launcher block in ``scripts/install.sh`` so existing installs
gain the ACP command on ``hermes update`` without a reinstall. ACP hosts
(Zed, JetBrains, Buzz Desktop) spawn the agent by resolving the
``hermes-acp`` command name against the login-shell PATH; the console
script of that name lives inside the install's venv, which is not on that
PATH, so those hosts report Hermes as not installed even when it is.
The shim simply delegates to the sibling ``hermes`` launcher with the
``acp`` subcommand, which makes it correct for every install layout
(venv wrapper, FHS symlink, pipx/pip console script) without having to
reconstruct interpreter/entrypoint paths.
No-op on Windows (install.ps1 puts ``venv\\Scripts`` on the user PATH, so
``hermes-acp.exe`` already resolves) and wherever a ``hermes-acp`` is
already present next to the ``hermes`` command. Unwritable directories
(e.g. ``/usr/local/bin`` as non-root) are skipped silently. Idempotent.
"""
if _m().sys.platform == "win32":
return
for bin_dir in (Path.home() / ".local" / "bin", Path("/usr/local/bin")):
hermes_cmd = bin_dir / "hermes"
acp_cmd = bin_dir / "hermes-acp"
try:
if not (hermes_cmd.is_file() or hermes_cmd.is_symlink()):
continue
# Already present — a console script (pip/pipx install), an
# earlier shim, or a symlink. is_symlink() catches broken
# symlinks that exists() would miss; never follow-and-overwrite
# (the #21454 failure mode).
if acp_cmd.exists() or acp_cmd.is_symlink():
continue
shim = (
"#!/usr/bin/env bash\n"
"# Hermes Agent — ACP launcher (written by `hermes update`).\n"
"# ACP hosts (Zed, JetBrains, Buzz) resolve the agent by this\n"
"# command name on the login-shell PATH.\n"
f'exec "{hermes_cmd}" acp "$@"\n'
)
acp_cmd.write_text(shim, encoding="utf-8")
acp_cmd.chmod(acp_cmd.stat().st_mode | 0o755)
except OSError:
continue
print(f" ✓ Installed hermes-acp launcher → {acp_cmd}")
_PRE_UPDATE_SNAPSHOT_KEEP = 1
# Per-file size cap for the pre-update quick snapshot. Anything larger is
# skipped with a warning: the snapshot exists to protect small, hard-to-
# regenerate state (pairing JSONs, cron jobs, config, auth) — not to copy a
# multi-GB state.db on every update (observed: a 24 GB state.db added ~60s
# of wall time and silently ate 24 GB of disk per update).
_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE = 1 << 30 # 1 GiB
def _resolve_pre_update_backup_mode(args) -> str:
"""Resolve the pre-update backup mode: ``"off"``, ``"quick"``, or ``"full"``.
CLI flags win over config; ``--no-backup`` beats ``--backup`` when both
are set. Config accepts the mode strings plus legacy booleans:
``true`` → ``full`` (the old zip behavior), ``false`` → ``off``
(an explicit opt-out now disables the quick snapshot too — previously
it ran unconditionally, ignoring the user's setting). A missing key
defaults to ``quick``.
"""
if getattr(args, "no_backup", False):
return "off"
if getattr(args, "backup", False):
return "full"
try:
from hermes_cli.config import load_config
cfg = load_config()
except Exception as exc:
logging.getLogger(__name__).debug(
"Could not load config for pre-update backup: %s", exc
)
cfg = {}
updates_cfg = cfg.get("updates", {}) if isinstance(cfg, dict) else {}
raw = updates_cfg.get("pre_update_backup", "quick")
if raw is True:
return "full"
if raw is False:
return "off"
mode = str(raw).strip().lower()
if mode in ("off", "false", "none", "disabled"):
return "off"
if mode in ("full", "zip", "true"):
return "full"
if mode == "quick":
return "quick"
logging.getLogger(__name__).warning(
"Unknown updates.pre_update_backup value %r — using 'quick'", raw
)
return "quick"
def _run_pre_update_backup(args) -> Optional[str]:
"""Run the pre-update safety backup and return the quick-snapshot id.
Single consolidated mechanism gated on ``updates.pre_update_backup``:
- ``off`` — nothing runs. Explicit user opt-out is honored fully.
- ``quick`` (default) — a state snapshot of critical small files
(pairing JSONs, cron jobs, config, auth; see ``_QUICK_STATE_FILES``)
under ``state-snapshots/``. Files over 1 GiB are skipped with a
warning so a bloated state.db can never stall the update
(issues #15733, #34600 are the reason this safety net exists).
- ``full`` — the quick snapshot PLUS a full zip of HERMES_HOME under
``backups/`` (restorable via ``hermes import``; the #48200 wrong-path
wipe is the reason this level exists).
``--backup`` forces ``full`` for one run; ``--no-backup`` forces ``off``.
Never raises — a backup failure should not block the update itself.
Returns the quick-snapshot id (used by the post-update cron-jobs
restore safety net), or ``None`` when mode is ``off`` or the snapshot
failed.
"""
mode = _resolve_pre_update_backup_mode(args)
if mode == "off":
if getattr(args, "no_backup", False):
print("◆ Pre-update backup: skipped (--no-backup)")
print()
# Config-level off is silent — the user opted out; don't spam them
# on every update.
return None
snapshot_id = None
try:
from hermes_cli.backup import (
_quick_snapshot_root,
create_quick_snapshot,
verify_sqlite_integrity,
)
# NOTE: this function later does `from hermes_constants import
# get_hermes_home`, which makes the name function-local — the
# module-level import is shadowed and unbound here. Alias explicitly.
from hermes_cli.config import get_hermes_home as _get_home
snapshot_id = create_quick_snapshot(
label="pre-update",
keep=_PRE_UPDATE_SNAPSHOT_KEEP,
max_file_size=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE,
)
# After the snapshot, verify the source state.db is still intact.
# The snapshot was taken via _safe_copy_db (read-only SQLite backup
# API), but a concurrent process (antivirus, force-killed gateway
# releasing file handles, Windows filter driver) can corrupt the live
# file at any point. A silent zeroing at this point would proceed with
# the update and exit code 0 — exactly the #68474 symptom.
if snapshot_id:
_src_path = _get_home() / "state.db"
if _src_path.exists():
_integrity = verify_sqlite_integrity(
_src_path,
check_header=True,
run_pragma=True,
max_bytes=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE,
)
if not _integrity.get("valid"):
_msg = _integrity.get("message", "unknown error")
print(
f" ⚠ state.db integrity check FAILED after snapshot: {_msg}"
)
# Check if the snapshot itself is valid.
_snap_root = _quick_snapshot_root(_get_home())
_snap_state = _snap_root / snapshot_id / "state.db"
if _snap_state.exists():
_snap_ok = verify_sqlite_integrity(
_snap_state, check_header=True, run_pragma=True
)
if _snap_ok.get("valid"):
print(
" ✓ Snapshot copy is valid — continuing update."
)
print(
" If state.db is lost after update it will be auto-restored."
)
else:
print(
" ✗ Snapshot copy ALSO failed integrity — "
"the source was already corrupted before the backup."
)
else:
print(
" ⚠ Snapshot does not contain state.db (was skipped or too large)."
)
print()
if snapshot_id:
print(f"◆ Pre-update snapshot: {snapshot_id}")
except Exception as exc:
# Never let a snapshot failure block an update.
logging.getLogger(__name__).debug("Pre-update snapshot failed: %s", exc)
if mode != "full":
if snapshot_id:
print()
return snapshot_id
try:
from hermes_cli.backup import create_pre_update_backup
except Exception as exc:
print(
f"⚠ Pre-update backup: could not load backup module ({exc}); continuing update."
)
print()
return snapshot_id
try:
from hermes_cli.config import load_config
_keep = (load_config() or {}).get("updates", {}).get("backup_keep", 5)
except Exception:
_keep = 5
print("◆ Creating pre-update backup...")
t0 = _time.monotonic()
try:
out_path = create_pre_update_backup(keep=int(_keep))
except Exception as exc: # defensive — helper already swallows, but just in case
print(f" ⚠ Backup failed: {exc}")
print(" Continuing with update.")
print()
return snapshot_id
elapsed = _time.monotonic() - t0
if out_path is None:
print(" ⚠ Backup skipped (no files found or write failed); continuing update.")
print()
return snapshot_id
try:
size_bytes = out_path.stat().st_size
except OSError:
size_bytes = 0
# Human-readable size
size_str = f"{size_bytes} B"
for unit in ("KB", "MB", "GB"):
if size_bytes < 1024:
break
size_bytes /= 1024
size_str = f"{size_bytes:.1f} {unit}"
# Render path using display_hermes_home so the user sees ~/.hermes/...
try:
from hermes_constants import get_hermes_home, display_hermes_home
home = get_hermes_home()
try:
display_path = f"{display_hermes_home()}/{out_path.relative_to(home)}"
except ValueError:
display_path = str(out_path)
except Exception:
display_path = str(out_path)
print(f" Saved: {display_path} ({size_str}, {elapsed:.1f}s)")
print(f" Restore: hermes import {out_path}")
print(" Disable: set updates.pre_update_backup: quick (or off) in config.yaml")
print()
return snapshot_id
def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool:
"""Write a planned-stop marker into a specific profile home."""
try:
from datetime import timezone
from gateway.status import _get_process_start_time
from utils import atomic_json_write
record = {
"target_pid": pid,
"target_start_time": _get_process_start_time(pid),
"stopper_pid": os.getpid(),
"written_at": datetime.now(timezone.utc).isoformat(),
}
atomic_json_write(
Path(profile_path) / ".gateway-planned-stop.json",
record,
indent=None,
separators=(",", ":"),
)
return True
except (OSError, PermissionError):
return False
def _wait_for_windows_update_gateway_exit(
pids: list[int], *, timeout: float
) -> set[int]:
"""Wait for the given gateway PIDs to exit, returning survivors."""
if not pids:
return set()
from gateway.status import _pid_exists
remaining = set(pids)
deadline = _time.monotonic() + max(timeout, 0.0)
while remaining and _time.monotonic() < deadline:
for pid in list(remaining):
try:
if not _pid_exists(pid):
remaining.discard(pid)
except Exception:
remaining.discard(pid)
if remaining:
_time.sleep(0.25)
survivors: set[int] = set()
for pid in remaining:
try:
if _pid_exists(pid):
survivors.add(pid)
except Exception:
pass
return survivors
def _venv_core_imports_healthy() -> tuple[bool, str]:
"""Probe the project venv for the core imports the backend needs to boot.
Runs a tiny import check inside the venv interpreter (NOT this process —
``hermes update`` may be driven by a different Python). Catches the
half-updated-venv state: git checkout current but a dependency sync that
failed or was killed partway (e.g. Windows access-denied on a loaded
.pyd), leaving imports like ``fastapi``'s new transitive deps missing.
Without this probe, ``hermes update`` on a current checkout prints
"Already up to date!" and returns without ever re-syncing dependencies —
the user's install stays broken no matter how many times they update
(ryanc's incident, July 2026).
Returns ``(healthy, detail)``. Never raises; unknown states report
healthy so a probe failure can't force needless reinstalls.
"""
venv_dir = _m().PROJECT_ROOT / "venv"
python_name = "python.exe" if _m()._is_windows() else "python"
bin_dir = "Scripts" if _m()._is_windows() else "bin"
venv_python = venv_dir / bin_dir / python_name
if not venv_python.exists():
# No venv interpreter at all. In a dev checkout that's normal (the
# dev may run hermes from any interpreter), so report healthy to
# avoid forcing reinstalls. But on a MANAGED install (the Windows
# installer / desktop bootstrap stamps `.hermes-bootstrap-complete`,
# and an interrupted update leaves `.update-incomplete`), the venv
# IS the install — its absence means a repair got interrupted after
# the old venv was moved aside, and "Already up to date!" would
# gaslight the user while nothing can run.
managed_markers = (
_m().PROJECT_ROOT / ".hermes-bootstrap-complete",
_m()._update_marker_path(),
)
if any(m.exists() for m in managed_markers):
return False, f"venv python missing ({venv_python})"
return True, ""
# Core web/serve imports plus their newest transitive deps. Import (not
# just metadata) — a package can have intact dist-info but a missing
# module after an interrupted uninstall/install cycle.
check = (
"import importlib\n"
"mods = ['fastapi', 'uvicorn', 'pydantic', 'openai', 'yaml']\n"
"missing = []\n"
"for m in mods:\n"
" try: importlib.import_module(m)\n"
" except Exception as e: missing.append(f'{m}: {e}')\n"
"print('\\n'.join(missing))\n"
)
try:
result = subprocess.run(
[str(venv_python), "-c", check],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=60,
cwd=_m().PROJECT_ROOT,
)
except Exception as exc:
logger.debug("venv health probe failed to run: %s", exc)
return True, ""
missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()]
if result.returncode != 0 and not missing:
# Interpreter itself is broken (e.g. deleted stdlib) — that IS unhealthy.
detail = (result.stderr or "").strip().splitlines()
return False, detail[0] if detail else "venv python failed to run"
if missing:
return False, "; ".join(missing[:4])
return True, ""
def _detect_venv_python_processes(
*, exclude_pids: set[int] | None = None
) -> list[tuple[int, str, str]]:
"""Find live processes running from the project venv's interpreter.
The hermes.exe shim guard misses the biggest lock-holder class on
Windows: the Desktop app's backend (``python.exe -m hermes_cli.main
serve``) and anything else running straight off ``venv\\Scripts\\python
(w).exe``. Those processes keep native ``.pyd`` extensions mapped, so a
dependency sync mid-update dies with access-denied and strands the venv
half-updated (ryanc's brotlicffi/_sodium.pyd incidents, July 2026).
Killing them from here is pointless — the Desktop app supervises its
backend and respawns it within seconds — so the caller should refuse and
tell the user to close the app instead. Returns ``(pid, name, cmdline)``
tuples; empty off-Windows / without psutil / when nothing matches. The
calling process and its ancestors are always excluded (a CLI ``hermes
update`` itself runs from the venv python). Never raises.
"""
if not _m()._is_windows():
return []
try:
import psutil
except Exception:
return []
venv_dir = _m().PROJECT_ROOT / "venv"
try:
venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep
except OSError:
venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep
try:
root_prefix = str(_m().PROJECT_ROOT.resolve()).lower().rstrip(os.sep) + os.sep
except OSError:
root_prefix = str(_m().PROJECT_ROOT).lower().rstrip(os.sep) + os.sep
skip: set[int] = set(exclude_pids or set())
skip.add(os.getpid())
try:
for anc in psutil.Process().parents():
skip.add(int(anc.pid))
except Exception:
pass
matches: list[tuple[int, str, str]] = []
try:
proc_iter = psutil.process_iter(["pid", "exe", "name", "cmdline", "cwd"])
except Exception:
return []
for proc in proc_iter:
try:
info = proc.info
except Exception:
continue
pid = info.get("pid")
exe = info.get("exe")
if not exe or pid is None or int(pid) in skip:
continue
try:
exe_norm = str(Path(exe).resolve()).lower()
except (OSError, ValueError):
exe_norm = str(exe).lower()
cmdline_raw = " ".join(info.get("cmdline") or [])
cmdline_low = cmdline_raw.lower()
cwd_low = str(info.get("cwd") or "").lower().rstrip(os.sep) + os.sep
# Primary match: the executable itself lives under this venv
# (venv\Scripts\python(w).exe — the desktop backend / gateway case).
is_holder = exe_norm.startswith(venv_prefix)
# Fallback: uv/base-interpreter trampolines run a python whose exe is
# OUTSIDE the venv but which still imports from it and holds its .pyd
# files. Catch those by what they're running: a cmdline that references
# this venv's path, or a `-m hermes_cli.main ...` invocation tied to
# this install (install root in the cmdline or as the working dir).
if not is_holder and venv_prefix in cmdline_low:
is_holder = True
if not is_holder and "hermes_cli.main" in cmdline_low:
if root_prefix in cmdline_low or cwd_low.startswith(root_prefix):
is_holder = True
if not is_holder:
continue
name = info.get("name") or Path(exe).name
matches.append((int(pid), str(name), cmdline_raw[:120]))
return matches
def _format_venv_python_holders_message(matches: list[tuple[int, str, str]]) -> str:
"""Explain which venv processes block the update and how to clear them."""
lines = [
"✗ Other Hermes processes are running from this install's venv:",
]
for pid, name, cmdline in matches[:6]:
hint = ""
low = cmdline.lower()
if "serve" in low or "dashboard" in low:
hint = " ← Hermes Desktop backend (close the desktop app)"
elif "gateway" in low:
hint = " ← gateway"
lines.append(f" PID {pid} {name} {cmdline}{hint}")
if len(matches) > 6:
lines.append(f" ... and {len(matches) - 6} more")
lines.append("")
lines.append(
" On Windows these keep native extension files (.pyd) locked, so the"
)
lines.append(
" dependency update would fail partway and leave a broken install."
)
lines.append(
" Close the Hermes desktop app / other Hermes terminals, then re-run:"
)
lines.append(" hermes update")
lines.append(" (or use `hermes update --force-venv` to proceed anyway at your own risk)")
return "\n".join(lines)
def _pause_windows_gateways_for_update() -> dict | None:
"""Stop running Windows gateways before mutating the checkout or venv.
Windows scheduled/startup gateways run through pythonw.exe, so the generic
hermes.exe concurrent-instance guard does not see them. They still import
from the checkout and can keep files locked while ``git`` or ``uv`` updates
the install. Stop only PIDs that the gateway discovery code identifies.
"""
if not _m()._is_windows():
return None
try:
from gateway.status import terminate_pid
from hermes_cli.gateway import (
_capture_gateway_argv,
_get_restart_drain_timeout,
find_gateway_pids,
find_profile_gateway_processes,
)
except Exception as exc:
logger.debug("Could not prepare Windows gateway pause for update: %s", exc)
return None
try:
running_pids = list(dict.fromkeys(find_gateway_pids(all_profiles=True)))
except Exception as exc:
logger.debug("Could not discover Windows gateway PIDs before update: %s", exc)
return None
if not running_pids:
# No gateway is running right now, but the user may have installed an
# autostart entry (Scheduled Task or Startup-folder login item) — that
# is an explicit "I want a gateway" signal. A gateway that died between
# updates (e.g. the spawning terminal/TUI closed, taking its child with
# it) would otherwise never come back: the autostart entry only fires on
# the next login, and the update flow's resume path only relaunched
# gateways that were running when the update began. Cold-start one after
# the update so an installed gateway is actually up post-update. Users
# who run gateway-less (no autostart entry) get nothing forced on them.
try:
from hermes_cli import gateway_windows
if gateway_windows.is_installed():
return {
"resume_needed": True,
"profiles": {},
"unmapped_pids": [],
"unmapped": [],
"cold_start_if_installed": True,
}
except Exception as exc:
logger.debug(
"Could not check Windows gateway autostart state before update: %s",
exc,
)
return None
profile_processes = {}
try:
profile_processes = {
proc.pid: proc for proc in find_profile_gateway_processes()
}
except Exception as exc:
logger.debug("Could not map Windows gateway PIDs to profiles: %s", exc)
profiles: dict[str, int] = {}
mapped_pids = []
for pid in running_pids:
proc = profile_processes.get(pid)
if proc is None:
continue
profiles[str(proc.profile)] = int(pid)
mapped_pids.append(int(pid))
_write_update_planned_stop_marker(Path(proc.path), int(pid))
print("→ Stopping Windows gateway process(es) before updating Hermes...")
try:
drain_timeout = max(float(_get_restart_drain_timeout()), 1.0)
except Exception:
drain_timeout = 10.0
survivors = _m()._wait_for_windows_update_gateway_exit(
mapped_pids,
timeout=drain_timeout,
)
unmapped_pids = [pid for pid in running_pids if pid not in profile_processes]
# Snapshot each unmapped gateway's command line *before* we force-kill it,
# so ``_resume_windows_gateways_after_update`` can respawn it by replaying
# its own argv. Unmapped gateways are ones with no profile→PID-file mapping
# — e.g. a Windows Scheduled Task running ``pythonw.exe -m hermes_cli.main
# gateway run``. Without this snapshot they were force-killed and never
# restarted (the "Restart manually after update" dead-end from #50090).
unmapped: list[dict] = []
for pid in unmapped_pids:
argv = None
try:
argv = _capture_gateway_argv(int(pid))
except Exception as exc:
logger.debug("Could not capture argv for unmapped gateway %s: %s", pid, exc)
unmapped.append({"pid": int(pid), "argv": argv})
force_killed = []
for pid in sorted(set(survivors).union(unmapped_pids)):
try:
terminate_pid(int(pid), force=True)
force_killed.append(int(pid))
except (ProcessLookupError, PermissionError, OSError):
pass
if profiles:
print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}")
if force_killed:
print(f" → Force-stopped {len(force_killed)} gateway process(es)")
if unmapped_pids:
respawnable = sum(1 for u in unmapped if u.get("argv"))
print(
f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping"
)
if respawnable < len(unmapped_pids):
# Some had no recoverable command line (psutil missing, access
# denied, already gone): those still need a manual restart.
print(" Restart manually after update: hermes gateway run")
return {
"resume_needed": True,
"profiles": profiles,
"unmapped_pids": unmapped_pids,
"unmapped": unmapped,
}
def _cold_start_windows_gateway_after_update() -> None:
"""Start a fresh detached gateway after update when one is installed but down.
Invoked from ``_resume_windows_gateways_after_update`` for the
``cold_start_if_installed`` case: no gateway was running when the update
began, but an autostart entry (Scheduled Task / Startup-folder login item)
is installed, signalling the user wants a gateway. Unlike the relaunch
paths — which watch an old PID and respawn once it exits — this is a direct
fresh spawn via the same hidden-console + breakaway path that
``hermes gateway start`` uses (``gateway_windows._spawn_detached``).
Best-effort and idempotent: re-checks that nothing is running first so a
concurrent start (e.g. the autostart entry firing) can't produce a
duplicate gateway.
"""
if not _m()._is_windows():
return
try:
from hermes_cli import gateway_windows
from hermes_cli.gateway import find_gateway_pids
except Exception as exc:
logger.debug("Could not load Windows gateway cold-start helpers: %s", exc)
return
# Re-check liveness right before spawning — between pause and resume the
# autostart entry may have already brought a gateway up, or a leftover
# process may have re-registered. Don't double-start.
try:
if list(find_gateway_pids(all_profiles=True)):
return
except Exception as exc:
logger.debug("Could not re-check gateway liveness before cold-start: %s", exc)
return
try:
pid = gateway_windows._spawn_detached()
except Exception as exc:
logger.debug("Could not cold-start Windows gateway after update: %s", exc)
return
if pid:
print()
print(f" ✓ Starting Windows gateway after update (PID {pid})")
def _for_each_systemd_gateway_unit(
list_units_stdout: str,
*,
process_unit,
on_unit_timeout,
) -> None:
"""Process each ``hermes-gateway*.service`` from ``systemctl list-units``.
``subprocess.TimeoutExpired`` raised by ``process_unit`` is isolated to
that unit via ``on_unit_timeout`` so one wedged systemctl call cannot
abort the rest of the fleet (#68523).
"""
for line in (list_units_stdout or "").strip().splitlines():
parts = line.split()
if not parts:
continue
unit = parts[0]
if not unit.endswith(".service"):
continue
# list-units is already pattern-filtered, but keep the name gate so a
# stray non-gateway line cannot enter the restart path.
if not unit.startswith("hermes-gateway"):
continue
svc_name = unit.removesuffix(".service")
try:
process_unit(svc_name)
except subprocess.TimeoutExpired as exc:
on_unit_timeout(svc_name, exc)
def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None:
"""Print an explicit incomplete-update warning for unrestarted units."""
if not failed_units:
return
# Preserve discovery order while de-duplicating.
seen = set()
ordered = []
for name in failed_units:
if name in seen:
continue
seen.add(name)
ordered.append(name)
print()
print("⚠ Update incomplete — some gateway units were not restarted:")
for name in ordered:
print(f" - {name}")
print(" Skipped units may still be running pre-update code (mixed")
print(" sys.modules). Restart them manually, then verify:")
print(" hermes gateway status")
print(" systemctl --user restart <unit> # user-scope")
print(" sudo systemctl restart <unit> # system-scope")
def _refresh_windows_gateway_launchers() -> None:
"""Regenerate installed Windows gateway launcher scripts after update.
The Scheduled Task / Startup-folder launchers (``gateway.cmd`` +
``gateway.vbs``) are persistence artifacts written once at install time —
``hermes update`` never touched them, so installs created before the
hidden-console rework (aa2ae36c3f) kept launching the gateway through
``pythonw.exe`` forever: every descendant spawn flashed a conhost
(#54220/#56747) and, since #70344, the console-less gateway died at
startup with ``RuntimeError: sys.stderr is None`` (#71671).
The task's /TR points at a stable script path, so rewriting the files in
place retargets the task without any schtasks call (no UAC needed).
``_write_task_script`` is idempotent and renders from current code, so
this is a no-op for modern installs. Best-effort: a failed refresh must
never fail the update.
"""
if not _m()._is_windows():
return
try:
from hermes_cli import gateway_windows
if not gateway_windows.is_installed():
return
gateway_windows._write_task_script()
print(" ✓ Refreshed Windows gateway launcher scripts")
except Exception as exc:
logger.debug("Could not refresh Windows gateway launchers after update: %s", exc)
def _resume_windows_gateways_after_update(token: dict | None) -> None:
"""Restart Windows profile gateways previously paused for update."""
if not token or not token.get("resume_needed"):
return
token["resume_needed"] = False
if not _m()._is_windows():
return
# Regenerate the persisted launcher scripts before respawning anything,
# so a legacy pythonw-era Scheduled Task / Startup entry comes back on
# the current hidden-console design at the next login too.
_m()._refresh_windows_gateway_launchers()
profiles = token.get("profiles") or {}
unmapped = token.get("unmapped") or []
cold_start = bool(token.get("cold_start_if_installed"))
if not profiles and not any(u.get("argv") for u in unmapped):
if cold_start:
_m()._cold_start_windows_gateway_after_update()
return
try:
from hermes_cli.gateway import (
launch_detached_gateway_restart_by_cmdline,
launch_detached_profile_gateway_restart,
)
except Exception as exc:
logger.debug("Could not load Windows gateway restart helper: %s", exc)
return
relaunched = []
for profile, old_pid in sorted(profiles.items()):
try:
if launch_detached_profile_gateway_restart(str(profile), int(old_pid)):
relaunched.append(str(profile))
except Exception as exc:
logger.debug(
"Could not restart Windows gateway profile %s after update: %s",
profile,
exc,
)
# Respawn unmapped gateways (no profile→PID-file mapping, e.g. a Scheduled
# Task) by replaying the argv we snapshotted before force-killing them.
unmapped_relaunched = 0
for entry in unmapped:
argv = entry.get("argv")
old_pid = entry.get("pid")
if not argv or not old_pid:
continue
try:
if launch_detached_gateway_restart_by_cmdline(int(old_pid), list(argv)):
unmapped_relaunched += 1
except Exception as exc:
logger.debug(
"Could not restart unmapped Windows gateway (pid %s) after update: %s",
old_pid,
exc,
)
if relaunched:
print()
print(f" ✓ Restarting Windows gateway profile(s): {', '.join(relaunched)}")
if unmapped_relaunched:
if not relaunched:
print()
print(
f" ✓ Restarting {unmapped_relaunched} unmapped Windows gateway process(es)"
)
def _discard_lockfile_churn(git_cmd, repo_root):
"""Restore tracked ``package-lock.json`` files that npm dirtied locally.
npm rewrites lockfiles non-deterministically at install/build time. On a
managed install those diffs are never intentional, so we discard them so
``hermes update`` sees a clean tree instead of autostashing every run.
Best-effort; only ever touches files named ``package-lock.json``.
"""
try:
diff = subprocess.run(
git_cmd + ["diff", "--name-only"],
cwd=repo_root,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if diff.returncode != 0:
return
dirty_package_dirs = {
Path(line.strip()).parent
for line in diff.stdout.splitlines()
if line.strip().endswith("package.json")
}
dirty = [
line.strip()
for line in diff.stdout.splitlines()
if line.strip().endswith("package-lock.json")
and Path(line.strip()).parent not in dirty_package_dirs
]
if not dirty:
return
subprocess.run(
git_cmd + ["checkout", "--", *dirty],
cwd=repo_root,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=False,
)
print(f"→ Discarded npm lockfile churn ({len(dirty)} file(s))")
except Exception:
# Never let lockfile cleanup block an update.
pass
def _normalize_managed_eol(git_cmd, repo_root):
"""Take a managed checkout off ``core.autocrlf=true`` without leaving it dirty.
Git for Windows ships ``core.autocrlf=true`` in its system config, which
renormalizes this repo's LF text files to CRLF in the working tree. That
breaks ``git checkout`` on update with "Your local changes would be
overwritten", so ``install.ps1`` pins ``core.autocrlf=false`` on the managed
clone (#67730). Checkouts created before that landed never got the pin and
cannot receive it — the bootstrap installer reuses its build-pinned
``install.ps1`` forever — so ``hermes update``, which ships with the checkout
itself, is the only path left that can fix them.
The pin and the cleanup are one operation. Under ``autocrlf=true`` git
compares normalized content, so a CRLF working tree reads clean; pinning
alone would expose every text file as modified and hand the update an
autostash of the whole tree. So the pin is written only after the tree is
verified clean under it, and a checkout we cannot fully normalize is left
exactly as it was. Best-effort: never blocks an update.
"""
# -c, not config: evaluate the tree as it WOULD look pinned, without
# persisting anything we might not be able to follow through on.
probe = git_cmd + ["-c", "core.autocrlf=false"]
def _dirty(*extra):
out = subprocess.run(
probe + ["diff", "-z", "--name-only", *extra],
cwd=repo_root,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if out.returncode != 0:
return None
return {p for p in out.stdout.split("\0") if p}
def _eol_only():
all_dirty, real_dirty = _dirty(), _dirty("--ignore-cr-at-eol")
if all_dirty is None or real_dirty is None:
return None
return all_dirty - real_dirty
try:
effective = subprocess.run(
git_cmd + ["config", "--get", "core.autocrlf"],
cwd=repo_root,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
# Only "true" rewrites LF to CRLF on checkout. Unset, false, and input
# all leave the working tree alone, so there is nothing to repair.
if effective.stdout.strip().lower() != "true":
return
eol_only = _eol_only()
if eol_only is None:
return
if eol_only:
# Pathspec over stdin, not argv: a fully renormalized checkout is
# thousands of paths, well past the Windows command-line limit.
subprocess.run(
probe
+ ["checkout", "--pathspec-from-file=-", "--pathspec-file-nul", "--"],
cwd=repo_root,
input="\0".join(sorted(eol_only)),
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=False,
)
if _eol_only():
# Still dirty — persisting the pin here would only surface churn
# we failed to clear. Leave the checkout as we found it.
return
print(f"→ Normalized line-ending churn ({len(eol_only)} file(s))")
subprocess.run(
git_cmd + ["config", "core.autocrlf", "false"],
cwd=repo_root,
capture_output=True,
check=False,
)
except Exception:
# Never let line-ending cleanup block an update.
pass
def _cmd_update_impl(args, gateway_mode: bool):
"""Body of ``cmd_update`` — kept separate so the wrapper can always
restore stdio even on ``sys.exit``."""
# In gateway mode, use file-based IPC for prompts instead of stdin
gw_input_fn = (
(lambda prompt, default="": _gateway_prompt(prompt, default))
if gateway_mode
else None
)
assume_yes = bool(getattr(args, "yes", False))
# Whether this update is running without a human at the keyboard.
# Interactive terminal updates always stash-and-ask (unchanged behavior);
# only non-interactive updates (desktop/chat app, gateway, `--yes`) consult
# the `updates.non_interactive_local_changes` config setting to decide
# whether to auto-restore stashed local source changes or throw them away.
_non_interactive_update = (
gateway_mode
or assume_yes
or not (sys.stdin.isatty() and sys.stdout.isatty())
)
discard_local_changes = False
if _non_interactive_update:
try:
from hermes_cli.config import load_config
_update_cfg = (load_config() or {}).get("updates", {})
if isinstance(_update_cfg, dict):
_mode = str(_update_cfg.get("non_interactive_local_changes", "stash")).lower()
discard_local_changes = _mode == "discard"
except Exception as exc:
# Never let a config read failure change the safe default.
logger.debug("Could not read updates.non_interactive_local_changes: %s", exc)
discard_local_changes = False
print("⚕ Updating Hermes Agent...")
print()
# On Windows, abort early if another hermes.exe is holding the venv shim
# open. Continuing would result in a string of WinError 32 warnings and
# then either a deferred-rename leftover or a failed git-pull fast path
# that silently falls back to the slower ZIP route. See issue #26670.
if _m()._is_windows() and not getattr(args, "force", False):
scripts_dir = _m()._venv_scripts_dir()
if scripts_dir is not None:
concurrent = _m()._detect_concurrent_hermes_instances(scripts_dir)
if concurrent:
print(_format_concurrent_instances_message(concurrent, scripts_dir))
sys.exit(2)
# Pre-update backup — runs before any git/file mutation so users can
# always roll back to the exact state they had before this update.
# Returns the quick-snapshot id (or None when disabled/failed); the
# post-update cron-jobs safety net uses it to detect job loss.
pre_update_snapshot_id = _m()._run_pre_update_backup(args)
_windows_gateway_resume = _m()._pause_windows_gateways_for_update()
if _windows_gateway_resume:
import atexit as _atexit
_atexit.register(
_m()._resume_windows_gateways_after_update,
_windows_gateway_resume,
)
# With gateways paused, anything still running from the venv interpreter
# (most commonly the Desktop app's `hermes serve` backend) will keep .pyd
# files locked and corrupt the dependency sync below. Refuse rather than
# race: killing the desktop backend is futile (the app supervises and
# respawns it), so the user must close the app. Deliberately NOT bypassed
# by plain --force: the desktop bootstrap updater passes --force to skip
# the hermes.exe shim guard above, but its lock probe only checks the shim
# and app.asar — a non-desktop venv python holding a .pyd would sail
# through and corrupt the sync (the exact failure this guard exists for).
# --force-venv is the explicit escape hatch.
if _m()._is_windows() and not getattr(args, "force_venv", False):
_venv_holders = _m()._detect_venv_python_processes()
if _venv_holders:
print(_format_venv_python_holders_message(_venv_holders))
_m()._resume_windows_gateways_after_update(_windows_gateway_resume)
sys.exit(2)
# Try git-based update first, fall back to ZIP download on Windows
# when git file I/O is broken (antivirus, NTFS filter drivers, etc.)
use_zip_update = False
git_dir = _m().PROJECT_ROOT / ".git"
if not git_dir.exists():
if sys.platform == "win32":
use_zip_update = True
else:
print("✗ Not a git repository. Please reinstall:")
print(
" curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash"
)
sys.exit(1)
# On Windows, git can fail with "unable to write loose object file: Invalid argument"
# due to filesystem atomicity issues. Set the recommended workaround.
if sys.platform == "win32" and git_dir.exists():
subprocess.run(
[
"git",
"-c",
"windows.appendAtomically=false",
"config",
"windows.appendAtomically",
"false",
],
cwd=_m().PROJECT_ROOT,
check=False,
capture_output=True,
)
# Build git command once — reused for fork detection and the update itself.
git_cmd = ["git"]
if sys.platform == "win32":
git_cmd = ["git", "-c", "windows.appendAtomically=false"]
# Discard npm lockfile churn before any stash/branch logic. npm rewrites
# tracked package-lock.json files non-deterministically at install/build
# time (platform-specific optional deps, ideallyInert annotations, etc.),
# which is never an intentional edit on a managed install but leaves the
# tree dirty — forcing an autostash on every update and making branch
# switches fragile. Restoring them first lets the common case (only
# lockfile churn) update with a clean tree.
_discard_lockfile_churn(git_cmd, _m().PROJECT_ROOT)
# Same rationale, different generator: line-ending churn is machine-made
# dirt on a managed checkout, so clear it (and stop generating it) before
# the stash/branch logic rather than autostashing the entire tree.
_normalize_managed_eol(git_cmd, _m().PROJECT_ROOT)
# Detect if we're updating from a fork (before any branch logic)
origin_url = _m()._get_origin_url(git_cmd, _m().PROJECT_ROOT)
is_fork = _is_fork(origin_url)
if is_fork:
print("⚠ Updating from fork:")
print(f" {origin_url}")
print()
if use_zip_update:
# ZIP-based update for Windows when git is broken
try:
_update_via_zip(args)
finally:
_m()._resume_windows_gateways_after_update(_windows_gateway_resume)
return
# Fetch and pull
try:
# Resolve the target branch up front so the fetch can be scoped to it.
# A bare `git fetch origin` pulls every ref, and this repo carries
# thousands of auto-generated branches — an unscoped fetch can stall for
# minutes on a non-single-branch checkout. Fetch only what we update
# against.
branch = _m()._resolve_update_branch(args)
print("→ Fetching updates...")
fetch_result = subprocess.run(
git_cmd + ["fetch", "origin", branch],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if fetch_result.returncode != 0:
stderr = fetch_result.stderr.strip()
if "Could not resolve host" in stderr or "unable to access" in stderr:
print("✗ Network error — cannot reach the remote repository.")
print(f" {stderr.splitlines()[0]}" if stderr else "")
elif (
"Authentication failed" in stderr or "could not read Username" in stderr
):
print(
"✗ Authentication failed — check your git credentials or SSH key."
)
else:
print("✗ Failed to fetch updates from origin.")
if stderr:
print(f" {stderr.splitlines()[0]}")
sys.exit(1)
# Get current branch (returns literal "HEAD" when detached)
result = subprocess.run(
git_cmd + ["rev-parse", "--abbrev-ref", "HEAD"],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=True,
)
current_branch = result.stdout.strip()
# If user is on a different branch than the update target, switch
# to the target. When the target is "main" this is the historical
# "always update against main" behavior; for any other target it's
# the same thing — get HEAD onto the requested branch first, then
# fast-forward.
if current_branch != branch:
label = (
"detached HEAD"
if current_branch == "HEAD"
else f"branch '{current_branch}'"
)
print(f" ⚠ Currently on {label} — switching to {branch} for update...")
# Stash before checkout so uncommitted work isn't lost
auto_stash_ref = _m()._stash_local_changes_if_needed(git_cmd, _m().PROJECT_ROOT)
checkout_result = subprocess.run(
git_cmd + ["checkout", branch],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if checkout_result.returncode != 0:
# Local checkout doesn't have this branch yet. Try to set
# it up as a tracking branch of origin/<branch>. This is
# the common case when the requested branch exists upstream
# but was never checked out locally.
track_result = subprocess.run(
git_cmd + ["checkout", "-B", branch, f"origin/{branch}"],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if track_result.returncode != 0:
# Restore the user's prior branch + stash before bailing
# so we don't leave them stranded in a weird state.
if auto_stash_ref is not None:
_m()._restore_stashed_changes(
git_cmd,
_m().PROJECT_ROOT,
auto_stash_ref,
prompt_user=False,
input_fn=gw_input_fn,
)
print(f"✗ Branch '{branch}' does not exist locally or on origin.")
if track_result.stderr.strip():
print(f" {track_result.stderr.strip().splitlines()[0]}")
sys.exit(1)
else:
auto_stash_ref = _m()._stash_local_changes_if_needed(git_cmd, _m().PROJECT_ROOT)
prompt_for_restore = (
auto_stash_ref is not None
and not assume_yes
and (gateway_mode or (sys.stdin.isatty() and sys.stdout.isatty()))
)
# Check if there are updates
result = subprocess.run(
git_cmd + ["rev-list", f"HEAD..origin/{branch}", "--count"],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=True,
)
commit_count = int(result.stdout.strip())
if commit_count == 0:
_invalidate_update_cache()
# Even if origin is up to date, the fork may be behind upstream
if is_fork and branch == "main":
_m()._sync_with_upstream_if_needed(git_cmd, _m().PROJECT_ROOT)
# Restore stash and switch back to original branch if we moved
if auto_stash_ref is not None:
_m()._restore_stashed_changes(
git_cmd,
_m().PROJECT_ROOT,
auto_stash_ref,
prompt_user=prompt_for_restore,
input_fn=gw_input_fn,
)
if current_branch not in {branch, "HEAD"}:
subprocess.run(
git_cmd + ["checkout", current_branch],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
check=False,
)
# "No new commits" does not mean the managed interpreter is safe.
# uv can retain the same CPython patch while python-build-standalone
# refreshes the embedded SQLite underneath it. Keep the existing
# update-boundary hook active on this retry path too.
from hermes_cli.managed_uv import ensure_uv, update_managed_uv
runtime_repairs = []
update_managed_uv(repair_observer=runtime_repairs.append)
ensure_uv(repair_observer=runtime_repairs.append)
runtime_repaired = next(
(result for result in runtime_repairs if result.repaired),
None,
)
# A current checkout does NOT imply a healthy install: a previous
# dependency sync may have failed partway (classic on Windows,
# where a running gateway/desktop backend keeps .pyd files locked
# and uv/pip dies with access-denied, stranding the venv between
# versions). Probe the venv's core imports and repair if broken —
# otherwise "Already up to date!" gaslights the user while their
# install stays bricked.
healthy, detail = _venv_core_imports_healthy()
if not healthy:
print("⚠ Checkout is current, but the venv is unhealthy:")
print(f" {detail}")
print("→ Repairing Python dependencies...")
_write_update_incomplete_marker()
from hermes_cli.managed_uv import ensure_uv
repair_uv = ensure_uv()
# A managed install whose venv is gone entirely (interrupted
# repair after the old venv was moved aside) needs the venv
# recreated before dependencies can be installed into it.
venv_python_missing = not (
_m().PROJECT_ROOT
/ "venv"
/ ("Scripts" if _m()._is_windows() else "bin")
/ ("python.exe" if _m()._is_windows() else "python")
).exists()
if venv_python_missing and repair_uv:
print("→ Recreating virtual environment...")
subprocess.run(
[repair_uv, "venv", "venv"],
cwd=_m().PROJECT_ROOT,
check=False,
)
if repair_uv:
repair_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")}
_m()._install_python_dependencies_with_optional_fallback(
[repair_uv, "pip"], env=repair_env, group="all"
)
else:
_m()._install_python_dependencies_with_optional_fallback(
[sys.executable, "-m", "pip"], group="all"
)
_m()._clear_update_incomplete_marker()
healthy_after, detail_after = _venv_core_imports_healthy()
if healthy_after:
print("✓ Dependencies repaired!")
else:
print(f"⚠ Venv still unhealthy after repair: {detail_after}")
print(" Close all Hermes windows/gateways and re-run: hermes update")
else:
print("✓ Already up to date!")
if runtime_repaired is not None and not _m()._is_windows():
print()
print(
"⚠ Restart required to finish the managed Python runtime repair."
)
print(
" Any running Hermes gateways, Desktop backends, or other "
"long-lived processes still use the previous runtime."
)
print(" Restart each of them to pick up the repaired runtime.")
_m()._resume_windows_gateways_after_update(_windows_gateway_resume)
return
print(f"→ Found {commit_count} new commit(s)")
print("→ Pulling updates...")
update_succeeded = False
# Capture the pre-pull SHA so we can auto-roll-back if the new code
# has a syntax error in a critical-path file (PR #28452 incident:
# orphan merge-conflict markers in hermes_cli/config.py bricked
# every user who ran ``hermes update`` for the 7 minutes between
# the bad commit and the fix landing).
pre_pull_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT)
try:
# Merge the ref we already fetched above (→ Fetching updates...)
# instead of `git pull`, which performs a SECOND network fetch of
# the same branch (~0.5-1.5 s of redundant round-trip per update).
# `merge --ff-only origin/<branch>` is byte-identical in effect to
# `pull --ff-only origin <branch>` given the fresh tracking ref;
# the divergence fallback below is unchanged.
pull_result = subprocess.run(
git_cmd + ["merge", "--ff-only", f"origin/{branch}"],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if pull_result.returncode != 0:
# ff-only failed — local and remote have diverged (e.g. upstream
# force-pushed or rebase). Since local changes are already
# stashed, reset to match the remote exactly.
print(
" ⚠ Fast-forward not possible (history diverged), resetting to match remote..."
)
reset_result = subprocess.run(
git_cmd + ["reset", "--hard", f"origin/{branch}"],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if reset_result.returncode != 0:
print(f"✗ Failed to reset to origin/{branch}.")
if reset_result.stderr.strip():
print(f" {reset_result.stderr.strip()}")
print(
f" Try manually: git fetch origin && git reset --hard origin/{branch}"
)
sys.exit(1)
# Post-pull syntax guard: validate critical-path files actually
# parse before declaring the update successful. If a bad commit
# made it through CI (e.g. admin-merge bypass of a failing
# ruff check), this catches it on the user side and rolls back
# so the CLI stays bootable. The user can then retry ``hermes
# update`` later once a fix lands upstream.
syntax_ok, failing_path, syntax_error = _validate_critical_files_syntax(
_m().PROJECT_ROOT
)
if not syntax_ok:
print()
print("✗ Pulled code has a syntax error in a critical file:")
print(f" {failing_path}")
if syntax_error:
# py_compile errors can be multi-line; show the first
# ~6 lines so the user sees the actual SyntaxError text.
for line in str(syntax_error).splitlines()[:6]:
print(f" {line}")
if pre_pull_sha:
print()
print(f"→ Rolling back to {pre_pull_sha[:10]}...")
rollback_result = subprocess.run(
git_cmd + ["reset", "--hard", pre_pull_sha],
cwd=_m().PROJECT_ROOT,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
)
if rollback_result.returncode == 0:
print(" ✓ Rollback complete — your install is unchanged.")
print(" Try ``hermes update`` again later once a fix lands.")
else:
print(" ✗ Rollback failed. Recover manually with:")
print(f" cd {_m().PROJECT_ROOT} && git reset --hard {pre_pull_sha}")
if rollback_result.stderr.strip():
print(f" ({rollback_result.stderr.strip().splitlines()[0]})")
else:
print()
print(" Could not capture pre-pull SHA — recover manually with:")
print(f" cd {_m().PROJECT_ROOT} && git reflog && git reset --hard <prev-sha>")
sys.exit(1)
update_succeeded = True
finally:
if auto_stash_ref is not None:
# Don't attempt stash restore if the code update itself failed —
# working tree is in an unknown state.
if not update_succeeded:
print(
f" Local changes preserved in stash (ref: {auto_stash_ref})"
)
print(" Restore manually with: git stash apply")
elif discard_local_changes:
# Non-interactive update + user opted into discarding local
# source edits (updates.non_interactive_local_changes:
# discard). Throw the stash away instead of re-applying it.
_m()._discard_stashed_changes(
git_cmd,
_m().PROJECT_ROOT,
auto_stash_ref,
)
else:
_m()._restore_stashed_changes(
git_cmd,
_m().PROJECT_ROOT,
auto_stash_ref,
prompt_user=prompt_for_restore,
input_fn=gw_input_fn,
)
_invalidate_update_cache()
# Clear stale .pyc bytecode cache — prevents ImportError on gateway
# restart when updated source references names that didn't exist in
# the old bytecode (e.g. get_hermes_home added to hermes_constants).
removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT)
if removed:
print(
f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}"
)
_m()._record_bytecode_fingerprint()
# Fork upstream sync logic (only for main branch on forks)
if is_fork and branch == "main":
_m()._sync_with_upstream_if_needed(git_cmd, _m().PROJECT_ROOT)
# Reinstall Python dependencies. Prefer .[all], but if one optional extra
# breaks on this machine, keep base deps and reinstall the remaining extras
# individually so update does not silently strip working capabilities.
#
# Drop the core-install breadcrumb BEFORE touching the venv. If the
# install is killed mid-flight (Ctrl-C, terminal close, WSL OOM), the
# marker survives and the next ``hermes`` launch finishes the install
# via ``_recover_from_interrupted_install``. Cleared after the core
# ``.[all]`` install completes — lazy refresh uses a separate marker.
_write_update_incomplete_marker()
print("→ Updating Python dependencies...")
from hermes_cli.managed_uv import ensure_uv, update_managed_uv
# Keep managed uv current — runs `uv self update` if we already have one.
update_managed_uv()
uv_bin = ensure_uv()
pip_cmd = [sys.executable, "-m", "pip"]
if not uv_bin:
uv_bin = _ensure_uv_for_termux(pip_cmd)
install_group = "all"
if uv_bin:
uv_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")}
if _m()._is_termux_env(uv_env):
uv_env.pop("PYTHONPATH", None)
uv_env.pop("PYTHONHOME", None)
install_group = "termux-all"
print(" → Termux detected: using uv + curated termux-all optional profile...")
if _m()._is_termux_env(uv_env) and _is_android_python():
print(" → Termux/Android detected: prebuilding psutil with Linux source path compatibility...")
_install_psutil_android_compat([uv_bin, "pip"], env=uv_env)
_m()._install_python_dependencies_with_optional_fallback(
[uv_bin, "pip"], env=uv_env, group=install_group
)
else:
# Use sys.executable to explicitly call the venv's pip module,
# avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu.
# Some environments lose pip inside the venv; bootstrap it back with
# ensurepip before trying the editable install.
pip_cmd = [sys.executable, "-m", "pip"]
try:
subprocess.run(
pip_cmd + ["--version"],
cwd=_m().PROJECT_ROOT,
check=True,
capture_output=True,
)
except subprocess.CalledProcessError:
subprocess.run(
[sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"],
cwd=_m().PROJECT_ROOT,
check=True,
)
if _m()._is_termux_env():
install_group = "termux-all"
print(" → Termux detected: using curated termux-all optional profile...")
if _m()._is_termux_env() and _is_android_python():
print(" → Termux/Android detected: prebuilding psutil with Linux source path compatibility...")
_install_psutil_android_compat(pip_cmd)
_m()._install_python_dependencies_with_optional_fallback(pip_cmd, group=install_group)
install_prefix = [uv_bin, "pip"] if uv_bin else pip_cmd
lazy_env = uv_env if uv_bin else None
# Core ``.[all]`` install finished. Clear the generic core breadcrumb
# before the lazy-refresh phase — that phase uses its own marker so a
# later lazy failure cannot be "healed" by clearing the core marker
# based on a narrow 7-package import probe (#58004 review).
_m()._clear_update_incomplete_marker()
# The update process is still the old Python interpreter process. Run
# one final cache/module refresh immediately before lazy backend
# refresh, which imports newly-pulled modules that may depend on fresh
# symbols in hermes_constants or lazy_deps. The dependency install
# above may also have regenerated bytecode from build-cache copies —
# this second sweep catches those stragglers (#60242, #65240).
removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT)
if removed:
print(
f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}"
)
_m()._record_bytecode_fingerprint()
_m()._reload_updated_runtime_modules()
# Upgrade pip before lazy refreshes — stale pip can fail source builds
# and leave partially-written packages (#57828).
_write_lazy_refresh_incomplete_marker()
_m()._upgrade_pip_before_lazy_refresh(install_prefix, env=lazy_env)
# Lazy refresh can corrupt the venv when a backend install fails.
# Clear the lazy marker only when refresh/repair is confirmed healthy.
lazy_ok = _m()._refresh_active_lazy_features(install_prefix, env=lazy_env)
if lazy_ok:
_m()._clear_lazy_refresh_incomplete_marker()
else:
print(
" ⚠ Lazy-refresh recovery incomplete — run `hermes` again "
"to finish import-based venv repair."
)
# Heal the active memory provider's bridge packages last — the core
# reinstall + lazy refresh above may have stripped or downgraded
# plugin.yaml-declared deps that aren't in extras (#53272, #70636).
_m()._refresh_active_memory_provider_dependencies()
node_failures = _update_node_dependencies()
_m()._build_web_ui(_m().PROJECT_ROOT / "web")
# Rebuild the desktop app if the source tree changed since the last
# build. ``hermes desktop --build-only`` uses the content-hash stamp
# internally, so this is effectively a no-op when nothing changed.
# Only bother if the user has a desktop app installed (indicated by
# an existing packaged executable or desktop dist); people who have
# never run ``hermes desktop`` shouldn't be forced into a full
# Electron build by ``hermes update``.
desktop_dir = _m().PROJECT_ROOT / "apps" / "desktop"
has_desktop_app = _m()._desktop_packaged_executable(desktop_dir) is not None or _m()._desktop_dist_exists(desktop_dir)
if (desktop_dir / "package.json").exists() and _m()._resolve_node_runtime_npm() and has_desktop_app:
print("→ Checking if desktop app needs rebuilding...")
# Consult the content-hash stamp IN-PROCESS first. The spawned
# `hermes desktop --build-only` subprocess re-imports the whole
# CLI stack (~1-3 s) just to reach the same _m()._desktop_build_needed
# check; when the stamp already says "up to date" we can skip the
# spawn entirely. The update path never passes --source, so the
# subprocess would run with source_mode=False — mirror that here.
# Any error in the pre-check falls through to the subprocess.
_skip_desktop_build = False
try:
_skip_desktop_build = not _m()._desktop_build_needed(
desktop_dir, _m().PROJECT_ROOT, source_mode=False
)
except Exception:
_skip_desktop_build = False
if _skip_desktop_build:
print(" ✓ Desktop app up to date")
else:
_desktop_build_cmd = [sys.executable, "-m", "hermes_cli.main", "desktop", "--build-only"]
# Capture the (very loud) Electron/vite build output into
# update.log instead of streaming it to the terminal. On the rare
# nonzero exit, retry once after waiting again for the venv — this
# covers a still-settling rebuild window the first wait didn't fully
# catch — then surface the captured tail so the failure is
# debuggable.
#
# Start the build subprocess with the Hermes-managed Node on PATH:
# when `hermes update` runs inside the desktop updater chain
# (Desktop → hermes-setup → hermes update), the shell PATH
# customizations are lost, so a bare-PATH child would fail with
# `node: not found` before cmd_gui can self-heal.
from hermes_constants import with_hermes_node_path
_build_env = with_hermes_node_path()
build_result = _m()._run_logged_subprocess(_desktop_build_cmd, cwd=_m().PROJECT_ROOT, env=_build_env)
if build_result.returncode != 0:
build_result = _m()._run_logged_subprocess(_desktop_build_cmd, cwd=_m().PROJECT_ROOT, env=_build_env)
if build_result.returncode != 0:
print(" ⚠ Desktop build failed (non-fatal; run `hermes desktop` to retry)")
tail = "\n".join((build_result.stdout or "").strip().splitlines()[-15:])
if tail:
print(tail)
from hermes_constants import display_hermes_home as _dhh
print(f" Full build log: {_dhh()}/logs/update.log")
else:
print(" ✓ Desktop app up to date")
print()
print("✓ Code updated!")
# ── Post-update state.db integrity guard (#68474) ─────────────────
# Verify that state.db survived the update intact. If the live file
# is now corrupted (zeroed, missing header, integrity failure),
# automatically restore from the pre-update snapshot rather than
# letting the user discover silently that their sessions are gone.
try:
from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity
_state_path = get_hermes_home() / "state.db"
if _state_path.exists():
_state_ok = verify_sqlite_integrity(
_state_path,
check_header=True,
run_pragma=True,
)
if _state_ok.get("valid"):
logger.debug(
"Post-update state.db integrity check: %s",
_state_ok.get("message"),
)
else:
print()
print(
"⚠ state.db is corrupted after update: "
+ _state_ok.get("message", "unknown error")
)
_pre_snap_id = pre_update_snapshot_id
if _pre_snap_id:
_snap_state = (
_quick_snapshot_root(get_hermes_home())
/ _pre_snap_id
/ "state.db"
)
if _snap_state.exists():
_snap_ok = verify_sqlite_integrity(
_snap_state, check_header=True, run_pragma=True
)
if _snap_ok.get("valid"):
try:
import shutil as _shutil
_shutil.copy2(_snap_state, _state_path)
_restored_ok = verify_sqlite_integrity(
_state_path,
check_header=True,
run_pragma=True,
)
if _restored_ok.get("valid"):
print(
" ✓ Auto-restored from pre-update "
f"snapshot ({_pre_snap_id})"
)
else:
print(
" ✗ Auto-restore FAILED — restored "
"copy also failed integrity"
)
except OSError as _exc:
print(
f" ✗ Auto-restore file copy failed: {_exc}"
)
else:
print(
" ✗ Pre-update snapshot also failed integrity"
)
else:
print(
" ⚠ Pre-update snapshot does not contain state.db"
)
else:
print(" ⚠ No pre-update snapshot was taken")
print()
except Exception as exc:
logger.debug("Post-update state.db integrity check failed: %s", exc)
# Seed the model-catalog disk cache from the freshly-pulled checkout.
# The repo ships the canonical catalog at
# website/static/api/model-catalog.json, and `git pull` just made it
# current — so copy it straight over ~/.hermes/cache/model_catalog.json
# instead of waiting on a network fetch (which can be bot-gated or hit a
# Portal hiccup). Keeps the model picker's curated/free lists in sync
# with the version the user just installed. Non-fatal on failure: the
# normal network refresh still applies on the next picker open.
try:
from hermes_cli.model_catalog import seed_cache_from_checkout
if seed_cache_from_checkout(_m().PROJECT_ROOT):
print(" ✓ Model catalog cache refreshed from checkout")
except Exception as e:
logger.debug("Model catalog seed during update failed: %s", e)
# Sync bundled skills (copies new, updates changed, respects user deletions)
try:
from tools.skills_sync import sync_skills
print()
print("→ Syncing bundled skills...")
result = sync_skills(quiet=True)
if result["copied"]:
print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}")
if result.get("updated"):
print(
f"{len(result['updated'])} updated: {', '.join(result['updated'])}"
)
if result.get("user_modified"):
print(f" ~ {len(result['user_modified'])} user-modified (kept)")
print(
" → see them: hermes skills list-modified "
"(diff/reset to resume updates)"
)
if result.get("cleaned"):
print(f" {len(result['cleaned'])} removed from manifest")
if result.get("relocated"):
print(
f"{len(result['relocated'])} moved to new upstream paths: "
f"{', '.join(result['relocated'])}"
)
if not result["copied"] and not result.get("updated"):
print(" ✓ Skills are up to date")
except Exception as e:
logger.debug("Skills sync during update failed: %s", e)
# Sync bundled skills to all profiles (including the active one).
# seed_profile_skills() uses subprocess with an explicit HERMES_HOME so
# it is not affected by sync_skills()'s module-level HERMES_HOME cache,
# which means the active profile is reliably synced regardless of whether
# the caller's HERMES_HOME env var points at the default or a named profile.
try:
from hermes_cli.profiles import (
list_profiles,
seed_profile_skills,
)
all_profiles = list_profiles()
if all_profiles:
print()
print("→ Syncing bundled skills to all profiles...")
for p in all_profiles:
try:
r = seed_profile_skills(p.path, quiet=True)
if r and r.get("skipped_opt_out"):
status = "opted out (--no-skills)"
elif r:
copied = len(r.get("copied", []))
updated = len(r.get("updated", []))
modified = len(r.get("user_modified", []))
parts = []
if copied:
parts.append(f"+{copied} new")
if updated:
parts.append(f"{updated} updated")
if modified:
parts.append(f"~{modified} user-modified")
status = ", ".join(parts) if parts else "up to date"
else:
status = "sync failed"
print(f" {p.name}: {status}")
except Exception as pe:
print(f" {p.name}: error ({pe})")
except Exception:
pass # profiles module not available or no profiles
# Backfill per-profile .env files for profiles created before the
# .env-seeding fix (#44792). Copies the default install's .env so
# those profiles keep the credentials they were effectively using.
try:
from hermes_cli.profiles import backfill_profile_envs
backfilled = backfill_profile_envs(quiet=True)
if backfilled:
print()
print(
f"→ Seeded .env for {len(backfilled)} profile(s) "
f"(copied from default): {', '.join(backfilled)}"
)
except Exception:
pass # profiles module not available or no profiles
# Sync Honcho host blocks to all profiles
try:
from plugins.memory.honcho.cli import sync_honcho_profiles_quiet
synced = sync_honcho_profiles_quiet()
if synced:
print(f"\n-> Honcho: synced {synced} profile(s)")
except Exception:
pass # honcho plugin not installed or not configured
# Check for config migrations
print()
print("→ Checking configuration for new options...")
from hermes_cli.config import (
get_missing_env_vars,
get_missing_config_fields,
check_config_version,
migrate_config,
)
missing_env = get_missing_env_vars(required_only=True)
missing_config = get_missing_config_fields()
current_ver, latest_ver = check_config_version()
has_new_options = bool(missing_env or missing_config)
version_bump_only = (
not has_new_options and current_ver < latest_ver
)
needs_migration = has_new_options or current_ver < latest_ver
if version_bump_only:
# Nothing for the user to fill in — only the config format version
# changed (new defaults already merge in transparently). Asking
# "configure new options now?" here is misleading: saying yes just
# bumps the version and looks like a no-op (issue: ScottFive /
# Tt2021). Apply it silently and say what actually happened.
print()
print(
f" Updating config format (v{current_ver} → v{latest_ver})…"
)
try:
migrate_config(interactive=False, quiet=True)
print(" ✓ Config format updated (no new settings to configure)")
except Exception as _mig_err:
print(f" ⚠️ Config format update failed: {_mig_err}")
print(" Run 'hermes config migrate' to retry.")
elif needs_migration:
print()
# Show WHAT changed, not just a count, so the user can make an
# informed yes/no decision (previously the prompt named nothing).
def _print_items(items, label, key, fallback_key=None):
if not items:
return
print(f" {label}:")
shown = items[:8]
for it in shown:
if isinstance(it, dict):
name = it.get(key) or (fallback_key and it.get(fallback_key)) or "?"
desc = (it.get("description") or "").strip()
else:
# Defensive: some callers/mocks pass bare name strings.
name = str(it)
desc = ""
if desc:
print(f"{name}{desc}")
else:
print(f"{name}")
extra = len(items) - len(shown)
if extra > 0:
print(f" … and {extra} more")
if missing_env:
print(
f" ⚠️ {len(missing_env)} new required setting(s) need configuration"
)
_print_items(missing_env, "New settings", "name")
if missing_config:
print(f" {len(missing_config)} new config option(s) available")
_print_items(missing_config, "New options", "key")
print()
if assume_yes:
print(
" --yes: auto-applying config migration (skipping API-key prompts)."
)
response = "y"
elif gateway_mode:
response = (
_gateway_prompt(
"Would you like to configure new options now? [Y/n]", "n"
)
.strip()
.lower()
)
elif not (sys.stdin.isatty() and sys.stdout.isatty()):
print(" Non-interactive session — applying safe config migrations.")
response = "auto"
else:
try:
response = (
input("Would you like to configure them now? [Y/n]: ")
.strip()
.lower()
)
except EOFError:
response = "n"
if response in {"", "y", "yes", "auto"}:
print()
# Gateway mode, --yes, and non-interactive update contexts
# (dashboard / web server actions) cannot prompt for API keys.
# Still run the non-interactive migration pass before restarting
# so new default config fields and version bumps are written
# before the freshly updated gateway validates config at startup.
interactive_migration = not (
gateway_mode or assume_yes or response == "auto"
)
results = migrate_config(interactive=interactive_migration, quiet=False)
if results["env_added"] or results["config_added"]:
print()
print("✓ Configuration updated!")
if (gateway_mode or assume_yes or response == "auto") and missing_env:
print(" API keys require manual entry: hermes config migrate")
else:
print()
print("Skipped. Run 'hermes config migrate' later to configure.")
else:
print(" ✓ Configuration is up to date")
# Safety net: config-version migrations have been observed to leave
# cron/jobs.json valid-but-empty, silently dropping every scheduled
# job (issue #34600). The desktop scheduler can also overwrite with
# its own small set, causing partial loss (issue #52144). If the
# live file now has fewer jobs than the pre-update snapshot, restore
# it and warn loudly.
try:
from hermes_cli.backup import restore_cron_jobs_if_emptied
cron_restore = restore_cron_jobs_if_emptied(pre_update_snapshot_id)
if cron_restore:
print()
print(
" ⚠️ cron/jobs.json lost jobs during this update — "
f"restored {cron_restore['job_count']} job(s) from "
f"pre-update snapshot {cron_restore['snapshot_id']}."
)
except Exception as exc:
# Never let the cron safety net break an otherwise-good update.
logger.debug("Cron jobs auto-restore check failed: %s", exc)
print()
if node_failures:
print(
"⚠ Update partially complete — Node.js dependencies for "
f"{', '.join(node_failures)} did not refresh."
)
print(" Code and Python deps are updated, but the dashboard/TUI may")
print(" be in a mixed state until the Node deps are rebuilt.")
else:
print("✓ Update complete!")
# Search-index optimization notice (v23). Existing installs keep their
# working search index untouched on update; the compact v23 layout —
# which reclaims a large fraction of state.db on heavy users — is
# opt-in. Surface it here (the moment the user is already thinking
# about their install) with the exact command and the concrete size
# win. Show-once-ish: only when a legacy index is actually present.
try:
_print_fts_optimize_available_notice()
except Exception as e:
logger.debug("FTS optimize notice failed: %s", e)
# Curator first-run heads-up. Only prints when curator is enabled AND
# has never run — i.e. the window where the ticker would otherwise
# have fired against a fresh skill library. Kept silent on steady
# state so we don't nag.
try:
_print_curator_first_run_notice()
except Exception as e:
logger.debug("Curator first-run notice failed: %s", e)
# Most-recent curator run notice — show-once per run. Surfaces the
# rename map (`old-name → umbrella`) on the high-attention update
# surface so users learn about consolidations without having to
# check `hermes curator status`. Self-stamps after printing so it
# never repeats for the same run.
try:
_print_curator_recent_run_notice()
except Exception as e:
logger.debug("Curator recent-run notice failed: %s", e)
# Repair RHEL-family root installs where /usr/local/bin isn't on PATH
# for non-login interactive shells. No-op on every other platform.
try:
_ensure_fhs_path_guard()
except Exception as e:
logger.debug("FHS PATH guard check failed: %s", e)
# Self-heal the hermes-acp launcher for installs that predate it, so
# ACP hosts (Zed, JetBrains, Buzz) can resolve Hermes on PATH without
# a reinstall. No-op on Windows and when already present.
try:
_ensure_acp_launcher()
except Exception as e:
logger.debug("hermes-acp launcher self-heal failed: %s", e)
# Refresh the cua-driver binary used by the Computer Use toolset.
# The upstream installer is gated on supported platforms and on the
# binary already being on PATH, so this is a no-op for users who
# don't have it. Tying the refresh to ``hermes update`` gives users a
# predictable cadence (matches when they pull new agent code) without
# adding startup latency or a per-launch GitHub API call.
try:
refresh_cua_driver = True
try:
from hermes_cli.config import load_config
_update_cfg = (load_config() or {}).get("updates", {})
if isinstance(_update_cfg, dict):
refresh_cua_driver = bool(
_update_cfg.get("refresh_cua_driver", True)
)
except Exception as cfg_exc:
logger.debug("Could not read updates.refresh_cua_driver: %s", cfg_exc)
if (
refresh_cua_driver
and sys.platform in ("darwin", "win32", "linux")
and shutil.which("cua-driver")
):
from hermes_cli.tools_config import install_cua_driver
print()
print("→ Refreshing cua-driver (Computer Use)...")
# require_confirmed_update: only run the (multi-minute,
# silent) upstream installer when the driver's native
# check-update verb positively reports a newer release.
# An indeterminate check (offline, rate-limited, old
# driver) keeps the installed version — `hermes update`
# must stay fast; `hermes computer-use install --upgrade`
# remains the force path.
install_cua_driver(
upgrade=True,
require_confirmed_update=True,
show_installer_progress=False,
)
except Exception as e:
logger.debug("cua-driver refresh failed: %s", e)
# Write exit code *before* the gateway restart attempt.
# When running as ``hermes update --gateway`` (spawned by the gateway's
# /update command), this process lives inside the gateway's systemd
# cgroup. A graceful SIGUSR1 restart keeps the drain loop alive long
# enough for the exit-code marker to be written below, but the
# fallback ``systemctl restart`` path (see below) kills everything in
# the cgroup (KillMode=mixed → SIGKILL to remaining processes),
# including us and the wrapping bash shell. The shell never reaches
# its ``printf $status > .update_exit_code`` epilogue, so the
# exit-code marker file would never be created. The new gateway's
# update watcher would then poll for 30 minutes and send a spurious
# timeout message.
#
# Writing the marker here — after git pull + pip install succeed but
# before we attempt the restart — ensures the new gateway sees it
# regardless of how we die.
if gateway_mode:
_exit_code_path = get_hermes_home() / ".update_exit_code"
try:
_exit_code_path.write_text("0", encoding="utf-8")
except OSError:
pass
gateway_fleet_restart_incomplete = False
# Auto-restart ALL gateways after update.
# The code update (git pull) is shared across all profiles, so every
# running gateway needs restarting to pick up the new code.
try:
from hermes_cli.gateway import (
is_macos,
supports_systemd_services,
_ensure_user_systemd_env,
find_gateway_pids,
find_profile_gateway_processes,
_prepare_profile_gateway_update_restart,
_get_service_pids,
_graceful_restart_via_sigusr1,
_wait_for_gateway_exit,
)
import signal as _signal
def _wait_for_service_active(
scope_cmd_: list,
svc_name_: str,
timeout: float = 10.0,
) -> bool:
"""Poll ``systemctl is-active`` until the unit reports active.
systemd's Stopped -> Started transition after a graceful exit
(or a hard restart) is not instantaneous; a one-shot check
races that window and falsely reports the unit as down.
Poll every 0.5s up to ``timeout`` seconds before giving up.
"""
deadline = _time.monotonic() + max(timeout, 0.5)
while True:
try:
_verify = subprocess.run(
scope_cmd_ + ["is-active", svc_name_],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=5,
)
if _verify.stdout.strip() == "active":
return True
except (FileNotFoundError, subprocess.TimeoutExpired):
pass
if _time.monotonic() >= deadline:
return False
_time.sleep(0.5)
def _service_restart_sec(
scope_cmd_: list,
svc_name_: str,
default: float = 0.0,
) -> float:
"""Read the unit's ``RestartUSec`` (RestartSec) in seconds.
After a graceful exit-75, systemd waits ``RestartSec`` before
respawning the unit. Callers that poll for ``is-active``
must use a timeout >= ``RestartSec`` + transition slack, or
they'll give up *during* the cooldown window and wrongly
conclude the unit didn't relaunch.
"""
try:
_show = subprocess.run(
scope_cmd_
+ [
"show",
svc_name_,
"--property=RestartUSec",
"--value",
],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=5,
)
except (FileNotFoundError, subprocess.TimeoutExpired):
return default
raw = (_show.stdout or "").strip()
# systemd emits values like "30s", "100ms", "1min 30s", or
# "infinity". Parse conservatively; on any miss return default.
if not raw or raw == "infinity":
return default
total = 0.0
matched = False
for part in raw.split():
for _suf, _mult in (
("ms", 0.001),
("us", 0.000001),
("min", 60.0),
("s", 1.0),
):
if part.endswith(_suf):
try:
total += float(part[: -len(_suf)]) * _mult
matched = True
except ValueError:
pass
break
return total if matched else default
_manage_cmd_cache: dict = {}
def _resolve_manage_cmd(scope_: str, scope_cmd_: list, svc_name_: str):
"""Resolve the command prefix for manage-units operations.
Read-only systemctl calls (``is-active``, ``show``,
``list-units``) work unprivileged, but manage-units verbs
(``reset-failed``, ``start``, ``restart``) on a *system*
service trigger a polkit ``org.freedesktop.systemd1.manage-units``
authentication prompt when run as a non-root user. That
interactive prompt runs inside our captured subprocess with a
10-15s timeout — the user sees the prompt flash and "exit
directly" before they can answer, and the resulting
TimeoutExpired used to be swallowed silently.
Strategy: if root, plain systemctl. If not root, try
non-interactive sudo (``sudo -n``) — first a blanket probe,
then a targeted ``systemctl reset-failed`` probe so a
least-privilege sudoers entry scoped to
``systemctl ... hermes-gateway*`` also qualifies
(``reset-failed`` is an idempotent no-op we run before every
privileged restart anyway). If neither works, return None —
the caller must SKIP the restart (without draining the
gateway first!) and tell the user how to restart manually.
``--no-ask-password`` guarantees polkit can never hang a
captured subprocess on this path.
"""
if scope_ in _manage_cmd_cache:
return _manage_cmd_cache[scope_]
cmd = scope_cmd_ + ["--no-ask-password"]
if (
scope_ == "system"
and hasattr(os, "geteuid")
and os.geteuid() != 0 # windows-footgun: ok — systemd path, Linux-only
):
sudo_cmd = ["sudo", "-n"] + scope_cmd_ + ["--no-ask-password"]
sudo_ok = False
try:
_probe = subprocess.run(
["sudo", "-n", "true"],
capture_output=True,
timeout=5,
)
sudo_ok = _probe.returncode == 0
if not sudo_ok:
# Blanket sudo refused — a targeted sudoers entry
# (NOPASSWD for systemctl ... hermes-gateway*)
# may still allow the exact commands we need.
_probe = subprocess.run(
sudo_cmd + ["reset-failed", svc_name_],
capture_output=True,
timeout=5,
)
sudo_ok = _probe.returncode == 0
except (FileNotFoundError, subprocess.TimeoutExpired):
sudo_ok = False
cmd = sudo_cmd if sudo_ok else None
_manage_cmd_cache[scope_] = cmd
return cmd
# Drain budget for graceful SIGUSR1 restarts. The gateway drains
# for up to ``agent.restart_drain_timeout`` (default 60s) before
# exiting with code 75; we wait slightly longer so the drain
# completes before we fall back to a hard restart. On older
# systemd units without SIGUSR1 wiring this wait just times out
# and we fall back to ``systemctl restart`` (the old behaviour).
try:
from hermes_constants import (
DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT as _DEFAULT_DRAIN,
)
except Exception:
_DEFAULT_DRAIN = 60.0
_cfg_drain = None
try:
from hermes_cli.config import load_config
_cfg_agent = load_config().get("agent") or {}
_cfg_drain = _cfg_agent.get("restart_drain_timeout")
except Exception:
pass
try:
_drain_budget = (
float(_cfg_drain)
if _cfg_drain is not None
else float(_DEFAULT_DRAIN)
)
except (TypeError, ValueError):
_drain_budget = float(_DEFAULT_DRAIN)
# Add a 15s margin so the drain loop + final exit finish before
# we escalate to ``systemctl restart`` / SIGTERM.
_drain_budget = max(_drain_budget, 30.0) + 15.0
restarted_services = []
failed_or_stale_units = []
killed_pids = set()
relaunched_profiles = []
externally_supervised_profiles = []
# --- Systemd services (Linux) ---
# Discover all hermes-gateway* units (default + profiles)
if supports_systemd_services():
try:
_ensure_user_systemd_env()
except Exception:
pass
for scope, scope_cmd in [
("user", ["systemctl", "--user"]),
("system", ["systemctl"]),
]:
try:
result = subprocess.run(
scope_cmd
+ [
"list-units",
"hermes-gateway*",
"--plain",
"--no-legend",
"--no-pager",
],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=10,
)
except FileNotFoundError:
continue
except subprocess.TimeoutExpired as exc:
# Discovery timeout — skip this scope, keep the other.
print(
f" ⚠ systemctl timed out listing {scope}-scope "
f"gateway units ({exc.cmd if exc.cmd else 'unknown command'}). "
f"Check the gateway with: hermes gateway status"
)
continue
def _restart_one_systemd_gateway_unit(svc_name: str) -> None:
# Check if active
check = subprocess.run(
scope_cmd + ["is-active", svc_name],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=5,
)
if check.stdout.strip() != "active":
return
# Resolve how we may run manage-units verbs
# (reset-failed/start/restart) for this scope.
# None ⇒ no non-interactive privilege path; we
# must avoid those verbs entirely or polkit will
# throw an interactive auth prompt inside our
# captured 10-15s subprocess (the user sees it
# flash and "exit directly" — reported June 2026).
_manage_cmd = _resolve_manage_cmd(
scope, scope_cmd, svc_name
)
# Prefer a graceful SIGUSR1 restart so in-flight
# agent runs drain instead of being SIGKILLed.
# The gateway's SIGUSR1 handler calls
# request_restart(via_service=True) → drain →
# exit; systemd's Restart=always respawns the unit.
_main_pid = 0
try:
_show = subprocess.run(
scope_cmd
+ [
"show",
svc_name,
"--property=MainPID",
"--value",
],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=5,
)
_main_pid = int((_show.stdout or "").strip() or 0)
except (
ValueError,
subprocess.TimeoutExpired,
FileNotFoundError,
):
_main_pid = 0
_graceful_ok = False
if _main_pid > 0:
print(
f"{svc_name}: draining (up to {int(_drain_budget)}s)..."
)
_graceful_ok = _graceful_restart_via_sigusr1(
_main_pid,
drain_timeout=_drain_budget,
)
if _graceful_ok:
# Gateway exited after a planned restart.
# ``Restart=always`` means systemd WILL respawn
# the unit — but only after
# ``RestartSec`` (default 60s on our unit
# file). That 60s wait is a crash-loop guard,
# and is the right default when the gateway
# dies unexpectedly. For a voluntary restart
# on update, it's dead time the user watches.
#
# Shortcut it: ``reset-failed`` + ``start``
# skips RestartSec entirely (we're manually
# initiating the unit, not waiting for
# systemd's auto-restart logic). Takes about
# as long as the process takes to come up
# (~1-3s on a warm box).
#
# If the unit is already active because
# RestartSec elapsed while we were draining,
# ``start`` is a no-op and we fall through to
# the poll below. Either way we collapse the
# 60s+ delay to a ~5s one.
#
# The shortcut needs manage-units privileges.
# Without them (system service, non-root, no
# passwordless sudo) skip it — systemd's own
# auto-restart still relaunches the unit after
# RestartSec, no privileges required.
if _manage_cmd is not None:
subprocess.run(
_manage_cmd + ["reset-failed", svc_name],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=10,
)
subprocess.run(
_manage_cmd + ["start", svc_name],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=15,
)
# Short poll: the gateway should be up
# within a few seconds now that we
# bypassed RestartSec.
if _wait_for_service_active(
scope_cmd,
svc_name,
timeout=10.0,
):
restarted_services.append(svc_name)
return
# Passive poll: systemd's auto-restart fires
# after RestartSec regardless of privileges.
# This is the primary path when _manage_cmd is
# None, and the fallback when the explicit
# start didn't take.
_restart_sec = _service_restart_sec(
scope_cmd,
svc_name,
default=0.0,
)
_post_drain_timeout = max(
10.0,
_restart_sec + 10.0,
)
if _manage_cmd is None and _restart_sec > 5.0:
print(
f"{svc_name}: waiting for systemd "
f"auto-restart (~{int(_restart_sec)}s; "
"no root for an immediate restart)..."
)
if _wait_for_service_active(
scope_cmd,
svc_name,
timeout=_post_drain_timeout,
):
restarted_services.append(svc_name)
return
# Process exited but wasn't respawned (older
# unit without Restart=on-failure or
# RestartForceExitStatus=75). Fall through
# to systemctl start/restart.
print(
f"{svc_name} drained but didn't relaunch — forcing restart"
)
# Forcing a restart requires manage-units
# privileges. Without a non-interactive path,
# running systemctl here would spawn a polkit
# auth prompt inside a captured 10-15s subprocess
# — it flashes and dies before the user can
# answer. Skip with clear instructions instead.
if _manage_cmd is None:
failed_or_stale_units.append(svc_name)
print(
f"{svc_name} is a system service and restarting it needs root.\n"
f" Restart it manually to load the new version:\n"
f" sudo systemctl restart {svc_name}\n"
f" To let `hermes update` restart it automatically, allow\n"
f" passwordless sudo for systemctl, or run updates with sudo."
)
return
# Fallback: blunt systemctl restart. This is
# what the old code always did; we get here only
# when the graceful path failed (unit missing
# SIGUSR1 wiring, drain exceeded the budget,
# restart-policy mismatch).
#
# Always `reset-failed` first. If systemd's own
# auto-restart attempts already parked the unit
# in a failed state (transient CHDIR / OOM /
# filesystem race after our drain + exit-75),
# a plain `systemctl restart` can wedge against
# the RestartSec backoff and leave the unit
# dead. Clearing the failed state first makes
# the restart idempotent. Mirrors the recovery
# path in `hermes gateway restart`
# (`systemd_restart()`) as of PR #20949.
subprocess.run(
_manage_cmd + ["reset-failed", svc_name],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=10,
)
restart = subprocess.run(
_manage_cmd + ["restart", svc_name],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=15,
)
if restart.returncode == 0:
# Verify the service actually survived the
# restart. systemctl restart returns 0 even
# if the new process crashes immediately.
if _wait_for_service_active(
scope_cmd,
svc_name,
timeout=10.0,
):
restarted_services.append(svc_name)
else:
# Retry once — transient startup failures
# (stale module cache, import race) often
# resolve on the second attempt. Again
# clear any failed state first so the
# retry isn't blocked by the previous
# crash.
print(
f"{svc_name} died after restart, retrying..."
)
subprocess.run(
_manage_cmd + ["reset-failed", svc_name],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=10,
)
subprocess.run(
_manage_cmd + ["restart", svc_name],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=15,
)
if _wait_for_service_active(
scope_cmd,
svc_name,
timeout=10.0,
):
restarted_services.append(svc_name)
print(f"{svc_name} recovered on retry")
else:
failed_or_stale_units.append(svc_name)
_scope_flag = "--user " if scope == "user" else ""
_sudo_hint = "sudo " if scope == "system" else ""
print(
f"{svc_name} failed to stay running after restart.\n"
f" Check logs: {_sudo_hint}journalctl {_scope_flag}-u {svc_name} --since '2 min ago'\n"
f" Recover manually:\n"
f" {_sudo_hint}systemctl {_scope_flag}reset-failed {svc_name}\n"
f" {_sudo_hint}systemctl {_scope_flag}restart {svc_name}"
)
else:
failed_or_stale_units.append(svc_name)
print(
f" ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}"
)
def _on_unit_timeout(svc_name: str, exc: subprocess.TimeoutExpired) -> None:
# Isolate the timeout to this unit and keep going
# (#68523). A scope-wide handler used to abort every
# later gateway and leave the fleet on mixed code.
failed_or_stale_units.append(svc_name)
print(
f" ⚠ systemctl timed out restarting {svc_name} "
f"({exc.cmd if exc.cmd else 'unknown command'}); "
f"continuing with remaining gateways"
)
_for_each_systemd_gateway_unit(
result.stdout,
process_unit=_restart_one_systemd_gateway_unit,
on_unit_timeout=_on_unit_timeout,
)
# --- Launchd services (macOS) ---
if is_macos():
try:
from hermes_cli.gateway import (
launchd_restart,
get_launchd_label,
get_launchd_plist_path,
)
plist_path = get_launchd_plist_path()
if plist_path.exists():
check = subprocess.run(
["launchctl", "list", get_launchd_label()],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=5,
)
if check.returncode == 0:
try:
launchd_restart()
restarted_services.append(get_launchd_label())
except subprocess.CalledProcessError as e:
stderr = (getattr(e, "stderr", "") or "").strip()
print(f" ⚠ Gateway restart failed: {stderr}")
except (FileNotFoundError, subprocess.TimeoutExpired, ImportError):
pass
# --- Manual (non-service) gateways ---
# Kill any remaining gateway processes not managed by a service.
# Exclude PIDs that belong to just-restarted services so we don't
# immediately kill the process that systemd/launchd just spawned.
service_pids = _get_service_pids()
manual_pids = find_gateway_pids(
exclude_pids=service_pids, all_profiles=True
)
profile_processes = {
proc.pid: proc
for proc in find_profile_gateway_processes(exclude_pids=service_pids)
if proc.pid in manual_pids
}
for pid, proc in profile_processes.items():
restart_mode = _prepare_profile_gateway_update_restart(
proc.profile, pid
)
if restart_mode is None:
continue
# Prefer a graceful SIGUSR1 drain so in-flight agent runs
# finish before the watcher respawns the gateway. If the
# gateway doesn't support SIGUSR1 or doesn't exit within
# the drain budget, fall back to SIGTERM — the watcher
# still sees the exit and relaunches either way.
# Announce the drain first: this wait can hold for the full
# budget per gateway with no other output, and on surfaces
# that stream update progress (the desktop updater most of
# all) the silence reads as a hung update (#44515).
print(
f"{proc.profile}: draining gateway PID {pid} "
f"(up to {int(_drain_budget)}s)..."
)
drained = _graceful_restart_via_sigusr1(
pid,
drain_timeout=_drain_budget,
)
if not drained:
try:
os.kill(pid, _signal.SIGTERM)
except (ProcessLookupError, PermissionError):
pass
# Wait for the old process to fully exit before the watcher
# spawns the new gateway. Telegram holds the previous
# getUpdates long-poll session open on its servers for up to
# ~30s after the client disconnects. If the new gateway
# connects before that window expires it receives a 409
# Conflict, which _handle_polling_conflict() recovers from
# via back-off retries — but a brief wait here reduces the
# chance of hitting that path at all, especially on fast
# machines where the watcher loop restarts in < 1s.
# We wait up to 5s for the process to exit (the OS-level
# close, not the Telegram server-side expiry), then let the
# watcher take over. The Telegram adapter's retry logic
# handles any remaining 409s if the server session is still
# live when the new gateway polls.
_wait_for_gateway_exit(timeout=5.0, force_after=None)
killed_pids.add(pid)
if restart_mode == "external-supervisor":
externally_supervised_profiles.append(proc.profile)
else:
relaunched_profiles.append(proc.profile)
for pid in manual_pids:
if pid in profile_processes:
continue
try:
os.kill(pid, _signal.SIGTERM)
killed_pids.add(pid)
except (ProcessLookupError, PermissionError):
pass
if restarted_services or killed_pids:
print()
for svc in restarted_services:
print(f" ✓ Restarted {svc}")
if relaunched_profiles:
names = ", ".join(relaunched_profiles)
print(f" ✓ Restarting manual gateway profile(s): {names}")
if externally_supervised_profiles:
names = ", ".join(externally_supervised_profiles)
print(
" ✓ Handed gateway profile(s) back to their external "
f"supervisor: {names}"
)
unmapped_count = (
len(killed_pids)
- len(relaunched_profiles)
- len(externally_supervised_profiles)
)
if unmapped_count:
print(f" → Stopped {unmapped_count} manual gateway process(es)")
print(" Restart manually: hermes gateway run")
if unmapped_count > 1:
print(
" (or: hermes -p <profile> gateway run for each profile)"
)
if failed_or_stale_units:
gateway_fleet_restart_incomplete = True
if gateway_mode:
_exit_code_path = get_hermes_home() / ".update_exit_code"
try:
_exit_code_path.write_text("1", encoding="utf-8")
except OSError:
pass
_warn_incomplete_gateway_fleet_restart(failed_or_stale_units)
if not restarted_services and not killed_pids:
# No gateways were running — nothing to do
pass
# --- Post-restart survivor sweep -----------------------------
# Issue #17648: some gateways ignore SIGTERM (stuck drain,
# blocked I/O, PID dead but zombie). The detached profile
# watchers wait 120s for the old PID to exit — if it never
# does, no respawn happens and the user keeps hitting
# ImportError against a stale sys.modules. Give the
# graceful paths a brief window to complete, then SIGKILL
# any remaining pre-update PIDs so the watcher / service
# manager can relaunch with fresh code.
try:
_time.sleep(3.0)
_service_pids_after = _get_service_pids()
_surviving = find_gateway_pids(
exclude_pids=_service_pids_after,
all_profiles=True,
)
# Scope to PIDs we already tried to kill during this
# update (killed_pids). Anything new is a gateway that
# started AFTER our restart attempt — respecting user
# intent, we don't kill those.
_stuck = [pid for pid in _surviving if pid in killed_pids]
if _stuck:
print()
print(
f"{len(_stuck)} gateway process(es) ignored SIGTERM — force-killing"
)
from gateway.status import terminate_pid as _terminate_pid
for pid in _stuck:
try:
# Routes through taskkill /T /F on Windows,
# SIGKILL on POSIX — _signal.SIGKILL doesn't
# exist on Windows so the old raw os.kill call
# used to crash the entire update path.
_terminate_pid(pid, force=True)
except (ProcessLookupError, PermissionError, OSError):
pass
# Give the OS a beat to reap the processes so the
# watchers see them exit and respawn.
_time.sleep(1.5)
except Exception as _sweep_exc:
logger.debug("Post-restart survivor sweep failed: %s", _sweep_exc)
except Exception as e:
logger.debug("Gateway restart during update failed: %s", e)
_m()._resume_windows_gateways_after_update(_windows_gateway_resume)
# Warn if legacy Hermes gateway unit files are still installed.
# When both hermes.service (from a pre-rename install) and the
# current hermes-gateway.service are enabled, they SIGTERM-fight
# for the same bot token (see PR #11909). Flagging here means
# every `hermes update` surfaces the issue until the user migrates.
try:
from hermes_cli.gateway import (
has_legacy_hermes_units,
_find_legacy_hermes_units,
supports_systemd_services,
)
if supports_systemd_services() and has_legacy_hermes_units():
print()
print("⚠ Legacy Hermes gateway unit(s) detected:")
for name, path, is_sys in _find_legacy_hermes_units():
scope = "system" if is_sys else "user"
print(f" {path} ({scope} scope)")
print()
print(" These pre-rename units (hermes.service) fight the current")
print(" hermes-gateway.service for the bot token and cause SIGTERM")
print(" flap loops. Remove them with:")
print()
print(" hermes gateway migrate-legacy")
print()
print(" (add `sudo` if any are in system scope)")
except Exception as e:
logger.debug("Legacy unit check during update failed: %s", e)
# Restart a managed dashboard through systemd, or stop stale manual
# dashboard processes. Raw-killing a systemd-owned dashboard PID makes
# systemd treat it as a clean stop, leaving the Cloudflare origin dead.
# Preserve the safety rule above: a failed Node refresh leaves the
# currently running dashboard untouched.
_finish_dashboard_update_cleanup(node_failures)
print()
print("Tip: You can now select a provider and model:")
print(" hermes model # Select provider and model")
if gateway_fleet_restart_incomplete:
# Code update itself succeeded, but at least one gateway still
# runs pre-update modules — surface that as a failed update so
# automation / operators do not treat the fleet as healthy.
sys.exit(1)
except subprocess.CalledProcessError as e:
if sys.platform == "win32":
print(f"⚠ Git update failed: {e}")
print("→ Falling back to ZIP download...")
print()
_update_via_zip(args)
else:
print(f"✗ Update failed: {e}")
sys.exit(1)
# --- Hoisted from the body of _cmd_update_impl (self-contained, no closure state) ---
def _print_items(items, label, key, fallback_key=None):
if not items:
return
print(f" {label}:")
shown = items[:8]
for it in shown:
if isinstance(it, dict):
name = it.get(key) or (fallback_key and it.get(fallback_key)) or "?"
desc = (it.get("description") or "").strip()
else:
# Defensive: some callers/mocks pass bare name strings.
name = str(it)
desc = ""
if desc:
print(f"{name}{desc}")
else:
print(f"{name}")
extra = len(items) - len(shown)
if extra > 0:
print(f" … and {extra} more")
def _wait_for_service_active(
scope_cmd_: list,
svc_name_: str,
timeout: float = 10.0,
) -> bool:
"""Poll ``systemctl is-active`` until the unit reports active.
systemd's Stopped -> Started transition after a graceful exit
(or a hard restart) is not instantaneous; a one-shot check
races that window and falsely reports the unit as down.
Poll every 0.5s up to ``timeout`` seconds before giving up.
"""
deadline = _time.monotonic() + max(timeout, 0.5)
while True:
try:
_verify = subprocess.run(
scope_cmd_ + ["is-active", svc_name_],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=5,
)
if _verify.stdout.strip() == "active":
return True
except (FileNotFoundError, subprocess.TimeoutExpired):
pass
if _time.monotonic() >= deadline:
return False
_time.sleep(0.5)
def _service_restart_sec(
scope_cmd_: list,
svc_name_: str,
default: float = 0.0,
) -> float:
"""Read the unit's ``RestartUSec`` (RestartSec) in seconds.
After a graceful exit-75, systemd waits ``RestartSec`` before
respawning the unit. Callers that poll for ``is-active``
must use a timeout >= ``RestartSec`` + transition slack, or
they'll give up *during* the cooldown window and wrongly
conclude the unit didn't relaunch.
"""
try:
_show = subprocess.run(
scope_cmd_
+ [
"show",
svc_name_,
"--property=RestartUSec",
"--value",
],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
timeout=5,
)
except (FileNotFoundError, subprocess.TimeoutExpired):
return default
raw = (_show.stdout or "").strip()
# systemd emits values like "30s", "100ms", "1min 30s", or
# "infinity". Parse conservatively; on any miss return default.
if not raw or raw == "infinity":
return default
total = 0.0
matched = False
for part in raw.split():
for _suf, _mult in (
("ms", 0.001),
("us", 0.000001),
("min", 60.0),
("s", 1.0),
):
if part.endswith(_suf):
try:
total += float(part[: -len(_suf)]) * _mult
matched = True
except ValueError:
pass
break
return total if matched else default