mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
Salvaged from PR #67807 by @f-trycua onto current main. - Foreground gate: discover delivery_mode support from the live tools/list inputSchema.properties (fail closed), not the never-shipped input.delivery_mode capability token - bring_to_front: standalone strict-schema MCP tool (inject_session=False), separate approval scope, requires foreground - Verdict precedence: confirmed > unverifiable (verify before retry) > suspected_noop/refusal (escalate); surfaced as explicit verdict field - Typed cua_browser_* route inside computer_use (browser_route.py) with exact-binding, adapter-injected session, snapshot-scoped refs - Per-Hermes-session backend isolation + release_computer_use_session seam wired into AIAgent.close() - Recorded 0.9 tools/list fixture replaces fabricated capability tokens
5003 lines
213 KiB
Python
5003 lines
213 KiB
Python
"""Hermes update pipeline — extracted from ``hermes_cli/main.py``.
|
||
|
||
Mechanical move (main.py decomposition): ``_cmd_update_impl``, ``_cmd_update_check``
|
||
and every module-level helper used only by the update path, plus the update-only
|
||
constants they read. Function bodies are lifted verbatim; the only mechanical
|
||
change is that references to helpers/constants that STAY in ``hermes_cli.main``
|
||
(and to moved-but-test-patched siblings) are routed through ``_m()`` — a lazy
|
||
``hermes_cli.main`` reference — so existing call sites and test monkeypatches
|
||
that target ``hermes_cli.main.<name>`` (``PROJECT_ROOT``, ``_is_windows``,
|
||
``_run_pre_update_backup``, ...) keep working unchanged. ``main.py`` re-imports
|
||
every public-ish name from here (``# noqa: F401``) so the argparse wiring and
|
||
the test-patch surface still resolve on ``hermes_cli.main``.
|
||
|
||
Three self-contained closures nested inside ``_cmd_update_impl``
|
||
(``_print_items``, ``_wait_for_service_active``, ``_service_restart_sec``) were
|
||
hoisted to module level; they capture no enclosing state (verified via
|
||
``symtable``). ``_restart_one_systemd_gateway_unit``, ``_resolve_manage_cmd``
|
||
and ``_on_unit_timeout`` DO capture enclosing locals and stay nested,
|
||
byte-identical.
|
||
|
||
Imports are one-way: ``hermes_cli.main`` imports this module, never the reverse
|
||
at import time (``_m()`` resolves lazily at call time, when main.py is fully
|
||
loaded, so there is no import cycle).
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import shlex
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import time as _time
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Optional
|
||
|
||
from hermes_cli.config import get_hermes_home
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def _m():
|
||
"""Lazy ``hermes_cli.main`` reference.
|
||
|
||
Lets callers keep patching ``hermes_cli.main.<helper>`` (the historical
|
||
test surface) and have those patches reach this code path, and defers the
|
||
import so ``hermes_cli.main`` -> ``hermes_cli.update_cmd`` stays one-way
|
||
at import time.
|
||
"""
|
||
from hermes_cli import main
|
||
|
||
return main
|
||
|
||
|
||
_UPDATE_RUNTIME_RELOAD_MODULES = (
|
||
"hermes_constants",
|
||
"tools.environments.local",
|
||
"tools.lazy_deps",
|
||
)
|
||
|
||
def _reload_updated_runtime_modules() -> None:
|
||
"""Reload update-sensitive modules after the checkout changes in-place.
|
||
|
||
``hermes update`` keeps running in the pre-pull Python process. After a
|
||
large update, modules already present in ``sys.modules`` can still expose
|
||
old symbols even though their source files on disk are new. Refresh the
|
||
small module set used by lazy-backend refresh before that step imports
|
||
newly-updated code paths.
|
||
"""
|
||
try:
|
||
import importlib
|
||
|
||
importlib.invalidate_caches()
|
||
for module_name in _UPDATE_RUNTIME_RELOAD_MODULES:
|
||
module = _m().sys.modules.get(module_name)
|
||
if module is None:
|
||
continue
|
||
try:
|
||
importlib.reload(module)
|
||
except Exception as exc:
|
||
logger.debug("Could not reload updated module %s: %s", module_name, exc)
|
||
except Exception as exc:
|
||
logger.debug("Could not refresh update runtime modules: %s", exc)
|
||
|
||
# Critical files that Hermes must be able to import immediately after an
|
||
# update/install. Most are imported on every CLI startup; ``web_server.py``
|
||
# is the desktop/dashboard backend path that a fresh Windows install launches
|
||
# right away. If any of these fail to parse after a pull, the user can be
|
||
# left with a bricked CLI or desktop backend. The post-pull syntax guard
|
||
# validates these and auto-rolls-back on failure.
|
||
_UPDATE_CRITICAL_FILES = (
|
||
"hermes_cli/main.py",
|
||
"hermes_cli/config.py",
|
||
"hermes_cli/__init__.py",
|
||
"hermes_cli/web_server.py",
|
||
"cli.py",
|
||
"run_agent.py",
|
||
"model_tools.py",
|
||
"toolsets.py",
|
||
"hermes_constants.py",
|
||
)
|
||
|
||
def _capture_head_sha(git_cmd, cwd) -> str | None:
|
||
"""Return the current HEAD SHA, or None if it can't be resolved."""
|
||
try:
|
||
result = subprocess.run(
|
||
git_cmd + ["rev-parse", "HEAD"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
check=True,
|
||
)
|
||
return result.stdout.strip() or None
|
||
except (subprocess.CalledProcessError, OSError):
|
||
return None
|
||
|
||
def _validate_critical_files_syntax(root) -> tuple[bool, str | None, str | None]:
|
||
"""Compile each file in ``_UPDATE_CRITICAL_FILES`` to catch SyntaxErrors.
|
||
|
||
These are the files imported on every ``hermes`` startup; if any of them
|
||
has a syntax error (orphan merge-conflict markers, bad ref to a name
|
||
that no longer exists, etc.) the CLI can't bootstrap at all. We validate
|
||
them after a successful ``git pull`` so we can auto-roll-back instead of
|
||
leaving the user with a bricked install.
|
||
|
||
The compiled ``.pyc`` is written to a temp directory rather than the
|
||
source tree's ``__pycache__/`` so we don't race with concurrent test
|
||
workers that walk the same dir, and so we don't leave a stale pyc
|
||
behind in production if the next interpreter run picks a different
|
||
Python version. The pyc is discarded on function return either way —
|
||
we only care about the compile-or-not signal.
|
||
|
||
Returns ``(ok, failing_path, error_message)``. ``ok=True`` means every
|
||
file parsed cleanly.
|
||
"""
|
||
import py_compile
|
||
import tempfile
|
||
|
||
root = Path(root)
|
||
with tempfile.TemporaryDirectory(prefix="hermes-syntax-check-") as tmpdir:
|
||
for relpath in _UPDATE_CRITICAL_FILES:
|
||
path = root / relpath
|
||
if not path.exists():
|
||
# Missing file is suspicious but not necessarily fatal — a future
|
||
# refactor may legitimately remove one of these. Skip and move on.
|
||
continue
|
||
# Mirror the relative path under the tmpdir so two different
|
||
# files with the same basename don't collide on the cfile name.
|
||
cfile = Path(tmpdir) / (relpath.replace("/", "__") + "c")
|
||
try:
|
||
py_compile.compile(str(path), cfile=str(cfile), doraise=True)
|
||
except py_compile.PyCompileError as exc:
|
||
return False, str(path), str(exc)
|
||
except OSError as exc:
|
||
return False, str(path), f"could not read: {exc}"
|
||
return True, None, None
|
||
|
||
def _gateway_prompt(prompt_text: str, default: str = "", timeout: float = 300.0) -> str:
|
||
"""File-based IPC prompt for gateway mode.
|
||
|
||
Writes a prompt marker file so the gateway can forward the question to the
|
||
user, then polls for a response file. Falls back to *default* on timeout.
|
||
|
||
Used by ``hermes update --gateway`` so interactive prompts (stash restore,
|
||
config migration) are forwarded to the messenger instead of being silently
|
||
skipped.
|
||
"""
|
||
import json as _json
|
||
import uuid as _uuid
|
||
from hermes_constants import get_hermes_home
|
||
|
||
home = get_hermes_home()
|
||
prompt_path = home / ".update_prompt.json"
|
||
response_path = home / ".update_response"
|
||
|
||
# Clean any stale response file
|
||
response_path.unlink(missing_ok=True)
|
||
|
||
payload = {
|
||
"prompt": prompt_text,
|
||
"default": default,
|
||
"id": str(_uuid.uuid4()),
|
||
}
|
||
tmp = prompt_path.with_suffix(".tmp")
|
||
tmp.write_text(_json.dumps(payload), encoding="utf-8")
|
||
tmp.replace(prompt_path)
|
||
|
||
# Poll for response
|
||
deadline = _time.monotonic() + timeout
|
||
while _time.monotonic() < deadline:
|
||
if response_path.exists():
|
||
try:
|
||
answer = response_path.read_text(encoding="utf-8").strip()
|
||
response_path.unlink(missing_ok=True)
|
||
prompt_path.unlink(missing_ok=True)
|
||
return answer if answer else default
|
||
except (OSError, ValueError):
|
||
pass
|
||
_time.sleep(0.5)
|
||
|
||
# Timeout — clean up and use default
|
||
prompt_path.unlink(missing_ok=True)
|
||
response_path.unlink(missing_ok=True)
|
||
print(f" (no response after {int(timeout)}s, using default: {default!r})")
|
||
return default
|
||
|
||
def _npm_bin_exists(bin_dir: Path, name: str) -> bool:
|
||
"""True when an npm bin shim for *name* exists (POSIX or Windows)."""
|
||
return any(
|
||
(bin_dir / candidate).exists()
|
||
for candidate in (name, f"{name}.cmd", f"{name}.ps1", f"{name}.exe")
|
||
)
|
||
|
||
def _web_build_toolchain_ready(*roots: Path) -> bool:
|
||
"""True when ``tsc`` and ``vite`` shims are reachable from any of *roots*.
|
||
|
||
Callers must pass every root the build would search; checking only one
|
||
reports a healthy tree as broken.
|
||
"""
|
||
bin_dirs = [
|
||
bin_dir
|
||
for bin_dir in (root / "node_modules" / ".bin" for root in roots)
|
||
if bin_dir.is_dir()
|
||
]
|
||
return bool(bin_dirs) and all(
|
||
any(_npm_bin_exists(bin_dir, tool) for bin_dir in bin_dirs)
|
||
for tool in ("tsc", "vite")
|
||
)
|
||
|
||
def _web_toolchain_roots(web_dir: Path) -> tuple[Path, ...]:
|
||
"""Roots whose ``node_modules/.bin`` can satisfy the web build.
|
||
|
||
``npm run build`` prepends ``node_modules/.bin`` for the package and each
|
||
of its ancestors, so shims hoisted to the workspace root and shims nested
|
||
under a package that owns its lockfile (#42973) are equally valid.
|
||
"""
|
||
return (web_dir, web_dir.parent)
|
||
|
||
def _print_curator_first_run_notice() -> None:
|
||
"""Print a short heads-up about the skill curator after `hermes update`.
|
||
|
||
Only fires when the curator is enabled AND has no recorded run yet, which
|
||
is exactly the window where the gateway ticker used to fire Curator
|
||
against a fresh skill library immediately after an update. We defer the
|
||
first real pass by one ``interval_hours``; this notice tells the user how
|
||
to preview or disable before then. Silent on steady state.
|
||
"""
|
||
try:
|
||
from agent import curator
|
||
except Exception:
|
||
return
|
||
try:
|
||
if not curator.is_enabled():
|
||
return
|
||
state = curator.load_state()
|
||
except Exception:
|
||
return
|
||
if state.get("last_run_at"):
|
||
# Curator has run before (real or already seeded) — no notice needed.
|
||
return
|
||
try:
|
||
hours = curator.get_interval_hours()
|
||
except Exception:
|
||
hours = 24 * 7
|
||
days = max(1, hours // 24)
|
||
print()
|
||
print("ℹ Skill curator")
|
||
print(
|
||
f" Background skill maintenance is enabled. First pass is deferred "
|
||
f"~{days}d after installation; only agent-created skills are in "
|
||
f"scope and nothing is ever auto-deleted (archive is recoverable)."
|
||
)
|
||
print(" Preview now: hermes curator run --dry-run")
|
||
print(" Pause it: hermes curator pause")
|
||
print(
|
||
" Docs: https://hermes-agent.nousresearch.com/docs/user-guide/features/curator"
|
||
)
|
||
|
||
def _print_fts_optimize_available_notice() -> None:
|
||
"""Advertise the opt-in v23 search-index optimization after `hermes update`.
|
||
|
||
Only fires when the current profile's state.db is still on the legacy
|
||
(pre-v23) inline FTS layout. Leads with the reclaimable-space figure and
|
||
points at the exact command. Honors ``sessions.fts_optimize_notice``:
|
||
``advise`` (default) prints an advisory notice, ``require`` prints a
|
||
firmer required-upgrade notice, ``off`` suppresses it. Silent for
|
||
fresh/already-optimized installs.
|
||
"""
|
||
mode = "advise"
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
|
||
mode = str(
|
||
((load_config() or {}).get("sessions") or {}).get(
|
||
"fts_optimize_notice", "advise"
|
||
)
|
||
).strip().lower()
|
||
except Exception:
|
||
mode = "advise"
|
||
if mode == "off":
|
||
return
|
||
|
||
try:
|
||
from hermes_constants import get_hermes_home
|
||
from hermes_state import SessionDB
|
||
except Exception:
|
||
return
|
||
db_path = get_hermes_home() / "state.db"
|
||
if not db_path.exists():
|
||
return
|
||
try:
|
||
size_gb = db_path.stat().st_size / (1024 ** 3)
|
||
except OSError:
|
||
return
|
||
# Skip the notice for trivially small DBs — the win isn't worth the nag.
|
||
if size_gb < 0.5:
|
||
return
|
||
db = None
|
||
interrupted = False
|
||
try:
|
||
db = SessionDB(db_path=db_path, read_only=True)
|
||
# read_only opens skip schema init, so probe the layout directly.
|
||
row = db._conn.execute(
|
||
"SELECT sql FROM sqlite_master "
|
||
"WHERE type = 'table' AND name = 'messages_fts'"
|
||
).fetchone()
|
||
# An interrupted `optimize-storage` run: the table is already the
|
||
# v23 shape, but backfill markers / demoted trash tables remain.
|
||
# Offer the command again — re-running resumes and finishes it.
|
||
interrupted = bool(
|
||
db._conn.execute(
|
||
"SELECT 1 FROM state_meta "
|
||
"WHERE key = 'fts_rebuild_high_water' LIMIT 1"
|
||
).fetchone()
|
||
or db._conn.execute(
|
||
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
|
||
"AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1"
|
||
).fetchone()
|
||
or db._conn.execute(
|
||
"SELECT 1 FROM state_meta WHERE key IN "
|
||
"('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1"
|
||
).fetchone()
|
||
)
|
||
except Exception:
|
||
return
|
||
finally:
|
||
if db is not None:
|
||
try:
|
||
db.close()
|
||
except Exception:
|
||
pass
|
||
sql = (row[0] if row else "") or ""
|
||
if not sql or ("tool_name" in sql and not interrupted):
|
||
# v23 layout already present (fresh/optimized) — nothing to offer.
|
||
return
|
||
|
||
if interrupted:
|
||
print()
|
||
print("◆ Session database optimization incomplete")
|
||
print(
|
||
" A previous `hermes sessions optimize-storage` run was "
|
||
"interrupted. Search still works; re-run the command to resume "
|
||
"and finish reclaiming disk:"
|
||
)
|
||
print(" hermes sessions optimize-storage")
|
||
return
|
||
|
||
# Concrete size framing — lead with the savings the user cares about.
|
||
est_reclaim = size_gb * 0.6
|
||
print()
|
||
if mode == "require":
|
||
print("◆ Session database upgrade required")
|
||
print(
|
||
f" Your search index uses the OLD storage layout and should be "
|
||
f"upgraded. The new layout typically frees ~60% of state.db "
|
||
f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is "
|
||
f"required for continued optimal operation."
|
||
)
|
||
else:
|
||
print("◆ Reclaim ~60% of your session database disk")
|
||
print(
|
||
f" Your search index uses the old storage layout. Upgrading it "
|
||
f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB "
|
||
f"of your current {size_gb:.1f} GB."
|
||
)
|
||
print(" Run when convenient: hermes sessions optimize-storage")
|
||
print(
|
||
" It runs in the foreground with a progress bar, is safe to "
|
||
"interrupt/re-run, and never changes your conversations."
|
||
)
|
||
|
||
def _print_curator_recent_run_notice() -> None:
|
||
"""Print the most recent curator run summary, exactly once.
|
||
|
||
The curator runs in the background (gateway tick + CLI session start),
|
||
so users learn about skill consolidations only by stumbling into a
|
||
rename. ``hermes update`` is a high-attention surface — surface the
|
||
most recent run's rename map here, once.
|
||
|
||
Show-once: state stamps ``last_run_summary_shown_at`` after printing.
|
||
Subsequent ``hermes update`` invocations skip the block until a newer
|
||
curator run lands. Silent when the curator has never run, when the
|
||
most recent summary has already been shown, or when the summary has
|
||
no rename information to display (no archives).
|
||
"""
|
||
try:
|
||
from agent import curator
|
||
except Exception:
|
||
return
|
||
try:
|
||
state = curator.load_state()
|
||
except Exception:
|
||
return
|
||
|
||
last_run_at = state.get("last_run_at")
|
||
if not last_run_at:
|
||
return # no curator run yet — first-run notice handles this case
|
||
|
||
if state.get("last_run_summary_shown_at") == last_run_at:
|
||
return # already shown for this run
|
||
|
||
summary = state.get("last_run_summary") or ""
|
||
if not summary:
|
||
return
|
||
|
||
# Only print when there's something interesting to show — i.e. the
|
||
# rename map block was appended (multi-line summary). A bare "auto:
|
||
# no changes; llm: no change" doesn't warrant interrupting the
|
||
# update flow.
|
||
if "\n" not in summary:
|
||
# Still stamp it shown so we don't reconsider it on every update.
|
||
try:
|
||
state["last_run_summary_shown_at"] = last_run_at
|
||
curator.save_state(state)
|
||
except Exception:
|
||
pass
|
||
return
|
||
|
||
# Format the timestamp as "Xh ago" for readability.
|
||
when = _format_time_ago(last_run_at)
|
||
print()
|
||
print(f"ℹ Skill curator — last run {when}")
|
||
for line in summary.splitlines():
|
||
print(f" {line}")
|
||
print(
|
||
" (This message shows once per curator run. "
|
||
"View anytime: hermes curator status)"
|
||
)
|
||
|
||
# Stamp shown so we don't repeat on the next update.
|
||
try:
|
||
state["last_run_summary_shown_at"] = last_run_at
|
||
curator.save_state(state)
|
||
except Exception:
|
||
pass
|
||
|
||
def _format_time_ago(iso_ts: str) -> str:
|
||
"""Render an ISO timestamp as `Xh ago` / `Xd ago` / `Xm ago`. Best effort."""
|
||
try:
|
||
from datetime import datetime, timezone
|
||
ts = datetime.fromisoformat(iso_ts.replace("Z", "+00:00"))
|
||
if ts.tzinfo is None:
|
||
ts = ts.replace(tzinfo=timezone.utc)
|
||
delta = datetime.now(timezone.utc) - ts
|
||
secs = int(delta.total_seconds())
|
||
if secs < 60:
|
||
return "just now"
|
||
if secs < 3600:
|
||
return f"{secs // 60}m ago"
|
||
if secs < 86400:
|
||
return f"{secs // 3600}h ago"
|
||
return f"{secs // 86400}d ago"
|
||
except Exception:
|
||
return "recently"
|
||
|
||
def _finish_dashboard_update_cleanup(node_failures: list[str]) -> None:
|
||
"""Refresh managed dashboards or stop stale manual ones after an update."""
|
||
if node_failures:
|
||
print()
|
||
print(" ℹ Leaving running dashboard process(es) untouched because the")
|
||
print(" Node.js dependency refresh did not complete.")
|
||
return
|
||
|
||
stop_result = _m()._kill_stale_dashboard_processes(restart_managed=True)
|
||
if not stop_result.get("unrecovered"):
|
||
return
|
||
|
||
print()
|
||
print(
|
||
"⚠ A web dashboard/serve process was stopped during update and could "
|
||
"not be auto-restarted."
|
||
)
|
||
print(" Re-launch it when you want the web UI back:")
|
||
print(" hermes dashboard --port <port>")
|
||
|
||
def _atomic_replace_dir(src: str, dst: str) -> None:
|
||
"""Replace directory *dst* with *src* without leaving *dst* half-deleted.
|
||
|
||
The naive ``rmtree(dst); copytree(src, dst)`` has a destructive window: if
|
||
the copy fails partway (common on the Windows ZIP-update path, which only
|
||
runs because file I/O is already flaky on that machine), the old directory
|
||
is already gone and nothing replaced it — the install is left with a
|
||
deleted tree (issue #49145, where ``ui-tui/`` vanished and broke the TUI).
|
||
|
||
Instead, stage the new copy into a sibling temp dir first; only once that
|
||
fully succeeds do we swap it in. A failure during staging raises with the
|
||
original *dst* still intact.
|
||
"""
|
||
staging = f"{dst}.hermes-update-staging"
|
||
backup = f"{dst}.hermes-update-old"
|
||
# Clear any leftovers from a previously-interrupted update.
|
||
for leftover in (staging, backup):
|
||
if os.path.exists(leftover):
|
||
shutil.rmtree(leftover, ignore_errors=True)
|
||
|
||
# 1. Stage the new copy. If this fails, dst is untouched.
|
||
shutil.copytree(src, staging)
|
||
# 2. Swap: move the live dir aside, move staging into place. Both moves are
|
||
# same-filesystem renames; if the second fails we restore the backup.
|
||
if os.path.exists(dst):
|
||
os.rename(dst, backup)
|
||
try:
|
||
os.rename(staging, dst)
|
||
except OSError:
|
||
if os.path.exists(backup) and not os.path.exists(dst):
|
||
os.rename(backup, dst) # roll back to the original
|
||
raise
|
||
# 3. New dir is in place; drop the old one (best-effort — never fatal).
|
||
if os.path.exists(backup):
|
||
shutil.rmtree(backup, ignore_errors=True)
|
||
|
||
def _update_via_zip(args):
|
||
"""Update Hermes Agent by downloading a ZIP archive.
|
||
|
||
Used on Windows when git file I/O is broken (antivirus, NTFS filter
|
||
drivers causing 'Invalid argument' errors on file creation).
|
||
"""
|
||
import tempfile
|
||
import zipfile
|
||
from urllib.request import urlretrieve
|
||
|
||
# The ZIP fallback exists for Windows git-file-I/O breakage. It pulls a
|
||
# static archive from GitHub, which is fine for the default "main"
|
||
# channel but would silently ignore --branch and update from main even
|
||
# if the user asked for something else — exactly the silent-divergence
|
||
# bug --branch was added to prevent. Refuse to proceed in that case
|
||
# rather than lie.
|
||
branch = _m()._resolve_update_branch(args)
|
||
if branch != "main":
|
||
print(
|
||
f"✗ --branch={branch} is not supported on the Windows ZIP-fallback "
|
||
"update path."
|
||
)
|
||
print(
|
||
" This path runs when git file I/O is broken on the system. "
|
||
"Either resolve the git-side breakage (typically an antivirus "
|
||
"or NTFS filter holding files open) and rerun `hermes update "
|
||
f"--branch {branch}`, or update against main with `hermes update`."
|
||
)
|
||
_m().sys.exit(1)
|
||
zip_url = (
|
||
f"https://github.com/NousResearch/hermes-agent/archive/refs/heads/{branch}.zip"
|
||
)
|
||
|
||
print("→ Downloading latest version...")
|
||
tmp_dir = tempfile.mkdtemp(prefix="hermes-update-")
|
||
try:
|
||
zip_path = os.path.join(tmp_dir, f"hermes-agent-{branch}.zip")
|
||
urlretrieve(zip_url, zip_path)
|
||
|
||
print("→ Extracting...")
|
||
import stat as _stat
|
||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||
# Validate paths to prevent zip-slip (path traversal) AND reject
|
||
# symlink members. A GitHub source ZIP for hermes-agent itself
|
||
# should never contain symlinks — they'd point outside the
|
||
# extracted tree and let an attacker who can compromise the
|
||
# update mirror plant arbitrary files via the update path.
|
||
tmp_dir_real = os.path.realpath(tmp_dir)
|
||
for member in zf.infolist():
|
||
member_path = os.path.realpath(os.path.join(tmp_dir, member.filename))
|
||
if (
|
||
not member_path.startswith(tmp_dir_real + os.sep)
|
||
and member_path != tmp_dir_real
|
||
):
|
||
raise ValueError(
|
||
f"Zip-slip detected: {member.filename} escapes extraction directory"
|
||
)
|
||
# Unix mode lives in the upper 16 bits of external_attr;
|
||
# mask to the file-type bits.
|
||
mode = (member.external_attr >> 16) & 0o170000
|
||
if _stat.S_ISLNK(mode):
|
||
raise ValueError(
|
||
f"ZIP contains unsupported symlink member: {member.filename}"
|
||
)
|
||
zf.extractall(tmp_dir)
|
||
|
||
# GitHub ZIPs extract to hermes-agent-<branch>/
|
||
extracted = os.path.join(tmp_dir, f"hermes-agent-{branch}")
|
||
if not os.path.isdir(extracted):
|
||
# Try to find it
|
||
for d in os.listdir(tmp_dir):
|
||
candidate = os.path.join(tmp_dir, d)
|
||
if os.path.isdir(candidate) and d != "__MACOSX":
|
||
extracted = candidate
|
||
break
|
||
|
||
# Copy updated files over existing installation, preserving venv/node_modules/.git
|
||
preserve = {"venv", "node_modules", ".git", ".env"}
|
||
update_count = 0
|
||
for item in os.listdir(extracted):
|
||
if item in preserve:
|
||
continue
|
||
src = os.path.join(extracted, item)
|
||
dst = os.path.join(str(_m().PROJECT_ROOT), item)
|
||
if os.path.isdir(src):
|
||
# Atomic-ish replace: never leave dst half-deleted if the copy
|
||
# fails partway (the failure mode behind #49145 on Windows).
|
||
_atomic_replace_dir(src, dst)
|
||
else:
|
||
shutil.copy2(src, dst)
|
||
update_count += 1
|
||
|
||
print(f"✓ Updated {update_count} items from ZIP")
|
||
|
||
except Exception as e:
|
||
print(f"✗ ZIP update failed: {e}")
|
||
_m().sys.exit(1)
|
||
finally:
|
||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||
|
||
# Clear stale bytecode after ZIP extraction
|
||
removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT)
|
||
if removed:
|
||
print(
|
||
f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}"
|
||
)
|
||
_m()._record_bytecode_fingerprint()
|
||
|
||
# Reinstall Python dependencies. Prefer .[all], but if one optional extra
|
||
# breaks on this machine, keep base deps and reinstall the remaining extras
|
||
# individually so update does not silently strip working capabilities.
|
||
print("→ Updating Python dependencies...")
|
||
|
||
from hermes_cli.managed_uv import ensure_uv, update_managed_uv
|
||
|
||
# Keep managed uv current — runs `uv self update` if we already have one.
|
||
update_managed_uv()
|
||
|
||
uv_bin = ensure_uv()
|
||
|
||
pip_cmd = [_m().sys.executable, "-m", "pip"]
|
||
if not uv_bin:
|
||
uv_bin = _ensure_uv_for_termux(pip_cmd)
|
||
if uv_bin:
|
||
uv_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")}
|
||
if _m()._is_termux_env(uv_env):
|
||
uv_env.pop("PYTHONPATH", None)
|
||
uv_env.pop("PYTHONHOME", None)
|
||
_m()._install_python_dependencies_with_optional_fallback([uv_bin, "pip"], env=uv_env)
|
||
else:
|
||
# Use sys.executable to explicitly call the venv's pip module,
|
||
# avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu.
|
||
# Some environments lose pip inside the venv; bootstrap it back with
|
||
# ensurepip before trying the editable install.
|
||
try:
|
||
subprocess.run(
|
||
pip_cmd + ["--version"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
check=True,
|
||
capture_output=True,
|
||
)
|
||
except subprocess.CalledProcessError:
|
||
subprocess.run(
|
||
[_m().sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
check=True,
|
||
)
|
||
_m()._install_python_dependencies_with_optional_fallback(pip_cmd)
|
||
|
||
# ZIP path parity: heal the active memory provider's bridge packages
|
||
# after the dependency reinstall, same as the git-pull path (#53272,
|
||
# #70636).
|
||
_m()._refresh_active_memory_provider_dependencies()
|
||
|
||
node_failures = _update_node_dependencies()
|
||
_m()._build_web_ui(_m().PROJECT_ROOT / "web")
|
||
|
||
# Sync skills
|
||
try:
|
||
from tools.skills_sync import sync_skills
|
||
|
||
print("→ Syncing bundled skills...")
|
||
result = sync_skills(quiet=True)
|
||
if result["copied"]:
|
||
print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}")
|
||
if result.get("updated"):
|
||
print(
|
||
f" ↑ {len(result['updated'])} updated: {', '.join(result['updated'])}"
|
||
)
|
||
if result.get("user_modified"):
|
||
print(f" ~ {len(result['user_modified'])} user-modified (kept)")
|
||
print(
|
||
" → see them: hermes skills list-modified "
|
||
"(diff/reset to resume updates)"
|
||
)
|
||
if result.get("cleaned"):
|
||
print(f" − {len(result['cleaned'])} removed from manifest")
|
||
if result.get("relocated"):
|
||
print(
|
||
f" → {len(result['relocated'])} moved to new upstream paths: "
|
||
f"{', '.join(result['relocated'])}"
|
||
)
|
||
if not result["copied"] and not result.get("updated"):
|
||
print(" ✓ Skills are up to date")
|
||
except Exception:
|
||
pass
|
||
|
||
# Seed the model-catalog disk cache from the freshly-unpacked checkout
|
||
# (same rationale as the git-pull path in _cmd_update_impl). Non-fatal.
|
||
try:
|
||
from hermes_cli.model_catalog import seed_cache_from_checkout
|
||
|
||
if seed_cache_from_checkout(_m().PROJECT_ROOT):
|
||
print(" ✓ Model catalog cache refreshed from checkout")
|
||
except Exception as e:
|
||
logger.debug("Model catalog seed during zip update failed: %s", e)
|
||
|
||
# ── Post-update state.db integrity guard (#68474) ─────────────────
|
||
# Same as the git-pull path: verify state.db survived the ZIP update
|
||
# and auto-restore from the most recent pre-update snapshot if needed.
|
||
try:
|
||
from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity
|
||
|
||
_state_path = get_hermes_home() / "state.db"
|
||
if _state_path.exists():
|
||
_state_ok = verify_sqlite_integrity(
|
||
_state_path, check_header=True, run_pragma=True
|
||
)
|
||
if not _state_ok.get("valid"):
|
||
print()
|
||
print(
|
||
"⚠ state.db is corrupted after update: "
|
||
+ _state_ok.get("message", "unknown error")
|
||
)
|
||
_snap_root = _quick_snapshot_root(get_hermes_home())
|
||
if _snap_root.exists():
|
||
_snap_dirs = sorted(
|
||
(d for d in _snap_root.iterdir() if d.is_dir()),
|
||
reverse=True,
|
||
)
|
||
for _snap_dir in _snap_dirs:
|
||
_snap_state = _snap_dir / "state.db"
|
||
if _snap_state.exists():
|
||
_snap_ok = verify_sqlite_integrity(
|
||
_snap_state, check_header=True, run_pragma=True
|
||
)
|
||
if _snap_ok.get("valid"):
|
||
try:
|
||
import shutil as _shutil
|
||
|
||
_shutil.copy2(_snap_state, _state_path)
|
||
_restored_ok = verify_sqlite_integrity(
|
||
_state_path,
|
||
check_header=True,
|
||
run_pragma=True,
|
||
)
|
||
if _restored_ok.get("valid"):
|
||
print(
|
||
" ✓ Auto-restored from snapshot "
|
||
f"{_snap_dir.name}"
|
||
)
|
||
else:
|
||
print(
|
||
" ✗ Auto-restore FAILED — restored "
|
||
"copy also failed integrity"
|
||
)
|
||
break
|
||
except OSError as _exc:
|
||
print(
|
||
f" ✗ Auto-restore file copy failed: {_exc}"
|
||
)
|
||
break
|
||
except Exception as exc:
|
||
logger.debug(
|
||
"Post-update state.db integrity check (zip path) failed: %s", exc
|
||
)
|
||
|
||
print()
|
||
if node_failures:
|
||
print(
|
||
"⚠ Update partially complete — Node.js dependencies for "
|
||
f"{', '.join(node_failures)} did not refresh."
|
||
)
|
||
print(" Code and Python deps are updated, but the dashboard/TUI may")
|
||
print(" be in a mixed state until the Node deps are rebuilt.")
|
||
else:
|
||
print("✓ Update complete!")
|
||
try:
|
||
_print_curator_first_run_notice()
|
||
except Exception as e:
|
||
logger.debug("Curator first-run notice failed: %s", e)
|
||
try:
|
||
_print_curator_recent_run_notice()
|
||
except Exception as e:
|
||
logger.debug("Curator recent-run notice failed: %s", e)
|
||
# Don't stop a working dashboard when the Node refresh failed — see the
|
||
# git-update path for rationale (#30271).
|
||
_finish_dashboard_update_cleanup(node_failures)
|
||
|
||
def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[str]:
|
||
status = subprocess.run(
|
||
git_cmd + ["status", "--porcelain"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
check=True,
|
||
)
|
||
if not status.stdout.strip():
|
||
return None
|
||
|
||
# If the index has unmerged entries (e.g. from an interrupted merge/rebase),
|
||
# git stash will fail with "needs merge / could not write index". Clear the
|
||
# conflict state with `git reset` so the stash can proceed. Working-tree
|
||
# changes are preserved; only the index conflict markers are dropped.
|
||
unmerged = subprocess.run(
|
||
git_cmd + ["ls-files", "--unmerged"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if unmerged.stdout.strip():
|
||
print("→ Clearing unmerged index entries from a previous conflict...")
|
||
subprocess.run(git_cmd + ["reset"], cwd=cwd, capture_output=True)
|
||
|
||
from datetime import datetime, timezone
|
||
|
||
stash_name = datetime.now(timezone.utc).strftime(
|
||
"hermes-update-autostash-%Y%m%d-%H%M%S"
|
||
)
|
||
print("→ Local changes detected — stashing before update...")
|
||
prev_stash = subprocess.run(
|
||
git_cmd + ["rev-parse", "--verify", "refs/stash"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
).stdout.strip()
|
||
push = subprocess.run(
|
||
git_cmd + ["stash", "push", "--include-untracked", "-m", stash_name],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if push.stdout.strip():
|
||
print(push.stdout.strip())
|
||
stash_probe = subprocess.run(
|
||
git_cmd + ["rev-parse", "--verify", "refs/stash"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
stash_ref = stash_probe.stdout.strip()
|
||
stash_created = (
|
||
stash_probe.returncode == 0 and bool(stash_ref) and stash_ref != prev_stash
|
||
)
|
||
|
||
if push.returncode != 0:
|
||
if stash_created:
|
||
# git stash push exits non-zero when it saved everything but could
|
||
# not delete some swept untracked files from the working tree
|
||
# (e.g. a root-owned directory: "warning: failed to remove ...:
|
||
# Permission denied"). The stash entry is complete — the changes
|
||
# are safe — so this is not a failure. Leave the undeletable
|
||
# files in place and continue the update.
|
||
if push.stderr.strip():
|
||
print(push.stderr.strip())
|
||
print(
|
||
" ⚠ Some untracked files could not be removed from the "
|
||
"working tree (permission denied)."
|
||
)
|
||
print(
|
||
" They were still saved to the stash and were left in "
|
||
"place — the update will continue."
|
||
)
|
||
# A partially-failed stash push also aborts its working-tree
|
||
# cleanup for TRACKED modifications — they are saved in the stash
|
||
# but still dirty the tree, which would break the checkout/pull
|
||
# that follows. Safe to reset: everything is in the stash entry.
|
||
subprocess.run(
|
||
git_cmd + ["reset", "--hard", "HEAD"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
)
|
||
else:
|
||
# No stash entry was created: the changes were NOT saved. This
|
||
# is a real failure — bail out before the update touches HEAD.
|
||
print("✗ Could not stash local changes — update aborted.")
|
||
if push.stderr.strip():
|
||
print(f" {push.stderr.strip().splitlines()[0]}")
|
||
print(
|
||
" Commit, stash, or clean up your local changes manually, "
|
||
"then re-run `hermes update`."
|
||
)
|
||
raise subprocess.CalledProcessError(
|
||
push.returncode, push.args, output=push.stdout, stderr=push.stderr
|
||
)
|
||
|
||
return stash_ref
|
||
|
||
def _resolve_stash_selector(
|
||
git_cmd: list[str], cwd: Path, stash_ref: str
|
||
) -> Optional[str]:
|
||
stash_list = subprocess.run(
|
||
git_cmd + ["stash", "list", "--format=%gd %H"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
check=True,
|
||
)
|
||
for line in stash_list.stdout.splitlines():
|
||
selector, _, commit = line.partition(" ")
|
||
if commit.strip() == stash_ref:
|
||
return selector.strip()
|
||
return None
|
||
|
||
def _print_stash_cleanup_guidance(
|
||
stash_ref: str, stash_selector: Optional[str] = None
|
||
) -> None:
|
||
print(
|
||
" Check `git status` first so you don't accidentally reapply the same change twice."
|
||
)
|
||
print(" Find the saved entry with: git stash list --format='%gd %H %s'")
|
||
if stash_selector:
|
||
print(f" Remove it with: git stash drop {stash_selector}")
|
||
else:
|
||
print(
|
||
f" Look for commit {stash_ref}, then drop its selector with: git stash drop stash@{{N}}"
|
||
)
|
||
|
||
def _stash_apply_failed_only_on_existing_untracked(stderr: str) -> bool:
|
||
"""True when a ``git stash apply`` failure is ONLY about untracked files
|
||
that already exist in the working tree.
|
||
|
||
This is the tail end of the permission-denied autostash class: ``git stash
|
||
push --include-untracked`` swept undeletable files (e.g. a root-owned
|
||
``packaging/`` directory) into the stash but could not remove them from
|
||
disk. On restore, git applies all tracked changes, then refuses to
|
||
overwrite those still-present files (``already exists, no checkout`` /
|
||
``could not restore untracked files from stash``) and exits non-zero even
|
||
though nothing was lost. Any other error line (e.g. ``would be
|
||
overwritten by merge`` / ``Aborting``) means the tracked apply itself
|
||
failed and this returns False.
|
||
"""
|
||
lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()]
|
||
if not lines:
|
||
return False
|
||
saw_untracked_error = False
|
||
for ln in lines:
|
||
if "already exists, no checkout" in ln:
|
||
saw_untracked_error = True
|
||
elif "could not restore untracked files from stash" in ln:
|
||
saw_untracked_error = True
|
||
elif ln.startswith(("warning:", "hint:")):
|
||
continue
|
||
else:
|
||
return False
|
||
return saw_untracked_error
|
||
|
||
def _restore_stashed_changes(
|
||
git_cmd: list[str],
|
||
cwd: Path,
|
||
stash_ref: str,
|
||
prompt_user: bool = False,
|
||
input_fn=None,
|
||
) -> bool:
|
||
if prompt_user:
|
||
print()
|
||
print("⚠ Local changes were stashed before updating.")
|
||
print(
|
||
" Restoring them may reapply local customizations onto the updated codebase."
|
||
)
|
||
print(" Review the result afterward if Hermes behaves unexpectedly.")
|
||
print("Restore local changes now? [Y/n]")
|
||
if input_fn is not None:
|
||
response = input_fn("Restore local changes now? [Y/n]", "y")
|
||
else:
|
||
response = input().strip().lower()
|
||
if response not in {"", "y", "yes"}:
|
||
print("Skipped restoring local changes.")
|
||
print("Your changes are still preserved in git stash.")
|
||
print(f"Restore manually with: git stash apply {stash_ref}")
|
||
return False
|
||
|
||
print("→ Restoring local changes...")
|
||
restore = subprocess.run(
|
||
git_cmd + ["stash", "apply", stash_ref],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
|
||
# Check for unmerged (conflicted) files — can happen even when returncode is 0
|
||
unmerged = subprocess.run(
|
||
git_cmd + ["diff", "--name-only", "--diff-filter=U"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
has_conflicts = bool(unmerged.stdout.strip())
|
||
|
||
if restore.returncode != 0 and not has_conflicts and (
|
||
_stash_apply_failed_only_on_existing_untracked(restore.stderr)
|
||
):
|
||
# Permission-denied autostash tail end: the tracked changes applied
|
||
# cleanly; the only "failure" is untracked files that never left the
|
||
# working tree (git could not delete them at stash time, so it now
|
||
# refuses to overwrite them). Their content was never touched —
|
||
# nothing is lost. Treat as restored.
|
||
print(
|
||
" ⚠ Some stashed untracked files already exist in the working "
|
||
"tree and were kept as-is."
|
||
)
|
||
elif restore.returncode != 0 or has_conflicts:
|
||
print("✗ Update pulled new code, but restoring local changes hit conflicts.")
|
||
if restore.stdout.strip():
|
||
print(restore.stdout.strip())
|
||
if restore.stderr.strip():
|
||
print(restore.stderr.strip())
|
||
|
||
# Show which files conflicted
|
||
conflicted_files = unmerged.stdout.strip()
|
||
if conflicted_files:
|
||
print("\nConflicted files:")
|
||
for f in conflicted_files.splitlines():
|
||
print(f" • {f}")
|
||
|
||
print("\nYour stashed changes are preserved — nothing is lost.")
|
||
print(f" Stash ref: {stash_ref}")
|
||
|
||
# Always reset to clean state — leaving conflict markers in source
|
||
# files makes hermes completely unrunnable (SyntaxError on import).
|
||
# The user's changes are safe in the stash for manual recovery.
|
||
subprocess.run(
|
||
git_cmd + ["reset", "--hard", "HEAD"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
)
|
||
print("Working tree reset to clean state.")
|
||
print(f"Restore your changes later with: git stash apply {stash_ref}")
|
||
# Don't sys.exit — the code update itself succeeded, only the stash
|
||
# restore had conflicts. Let cmd_update continue with pip install,
|
||
# skill sync, and gateway restart.
|
||
return False
|
||
|
||
stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref)
|
||
if stash_selector is None:
|
||
print(
|
||
"⚠ Local changes were restored, but Hermes couldn't find the stash entry to drop."
|
||
)
|
||
print(
|
||
" The stash was left in place. You can remove it manually after checking the result."
|
||
)
|
||
_print_stash_cleanup_guidance(stash_ref)
|
||
else:
|
||
drop = subprocess.run(
|
||
git_cmd + ["stash", "drop", stash_selector],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if drop.returncode != 0:
|
||
print(
|
||
"⚠ Local changes were restored, but Hermes couldn't drop the saved stash entry."
|
||
)
|
||
if drop.stdout.strip():
|
||
print(drop.stdout.strip())
|
||
if drop.stderr.strip():
|
||
print(drop.stderr.strip())
|
||
print(
|
||
" The stash was left in place. You can remove it manually after checking the result."
|
||
)
|
||
_print_stash_cleanup_guidance(stash_ref, stash_selector)
|
||
|
||
print("⚠ Local changes were restored on top of the updated codebase.")
|
||
print(" Review `git diff` / `git status` if Hermes behaves unexpectedly.")
|
||
return True
|
||
|
||
def _discard_stashed_changes(
|
||
git_cmd: list[str],
|
||
cwd: Path,
|
||
stash_ref: str,
|
||
) -> bool:
|
||
"""Throw away a stash created before an update, without applying it.
|
||
|
||
Used only on a NON-interactive update when the user has set
|
||
``updates.non_interactive_local_changes: discard`` — i.e. they've opted out
|
||
of keeping local source edits on this machine. Drops the stash entry
|
||
instead of re-applying it, so the working tree stays clean at the freshly
|
||
pulled HEAD. Unlike ``git reset --hard`` + ``git clean -fd``, this only
|
||
affects what was stashed (tracked changes + the untracked files we
|
||
explicitly captured) — ignored paths like node_modules/venv/build outputs
|
||
are never touched, since they were never stashed.
|
||
|
||
Returns True if the stash was dropped, False on a git failure (in which
|
||
case the stash is left in place for safety).
|
||
"""
|
||
stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref)
|
||
if stash_selector is None:
|
||
print(
|
||
"⚠ Configured to discard local changes on non-interactive update, "
|
||
"but Hermes couldn't find the stash entry to drop."
|
||
)
|
||
_print_stash_cleanup_guidance(stash_ref)
|
||
return False
|
||
|
||
drop = subprocess.run(
|
||
git_cmd + ["stash", "drop", stash_selector],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if drop.returncode != 0:
|
||
print(
|
||
"⚠ Configured to discard local changes, but Hermes couldn't drop "
|
||
"the saved stash entry."
|
||
)
|
||
if drop.stderr.strip():
|
||
print(f" {drop.stderr.strip().splitlines()[0]}")
|
||
_print_stash_cleanup_guidance(stash_ref, stash_selector)
|
||
return False
|
||
|
||
print("→ Discarded local source changes (updates.non_interactive_local_changes=discard).")
|
||
return True
|
||
|
||
OFFICIAL_REPO_URLS = {
|
||
"https://github.com/NousResearch/hermes-agent.git",
|
||
"git@github.com:NousResearch/hermes-agent.git",
|
||
"https://github.com/NousResearch/hermes-agent",
|
||
"git@github.com:NousResearch/hermes-agent",
|
||
}
|
||
|
||
OFFICIAL_REPO_URL = "https://github.com/NousResearch/hermes-agent.git"
|
||
|
||
SKIP_UPSTREAM_PROMPT_FILE = ".skip_upstream_prompt"
|
||
|
||
def _get_origin_url(git_cmd: list[str], cwd: Path) -> Optional[str]:
|
||
"""Get the URL of the origin remote, or None if not set."""
|
||
try:
|
||
result = subprocess.run(
|
||
git_cmd + ["remote", "get-url", "origin"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if result.returncode == 0:
|
||
return result.stdout.strip()
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
def _is_fork(origin_url: Optional[str]) -> bool:
|
||
"""Check if the origin remote points to a fork (not the official repo)."""
|
||
if not origin_url:
|
||
return False
|
||
# Normalize URL for comparison (strip trailing .git if present)
|
||
normalized = origin_url.rstrip("/")
|
||
if normalized.endswith(".git"):
|
||
normalized = normalized[:-4]
|
||
for official in OFFICIAL_REPO_URLS:
|
||
official_normalized = official.rstrip("/")
|
||
if official_normalized.endswith(".git"):
|
||
official_normalized = official_normalized[:-4]
|
||
if normalized == official_normalized:
|
||
return False
|
||
return True
|
||
|
||
def _has_upstream_remote(git_cmd: list[str], cwd: Path) -> bool:
|
||
"""Check if an 'upstream' remote already exists."""
|
||
try:
|
||
result = subprocess.run(
|
||
git_cmd + ["remote", "get-url", "upstream"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
return result.returncode == 0
|
||
except Exception:
|
||
return False
|
||
|
||
def _add_upstream_remote(git_cmd: list[str], cwd: Path) -> bool:
|
||
"""Add the official repo as the 'upstream' remote. Returns True on success."""
|
||
try:
|
||
result = subprocess.run(
|
||
git_cmd + ["remote", "add", "upstream", OFFICIAL_REPO_URL],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
return result.returncode == 0
|
||
except Exception:
|
||
return False
|
||
|
||
def _count_commits_between(git_cmd: list[str], cwd: Path, base: str, head: str) -> int:
|
||
"""Count commits on `head` that are not on `base`. Returns -1 on error."""
|
||
try:
|
||
result = subprocess.run(
|
||
git_cmd + ["rev-list", "--count", f"{base}..{head}"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if result.returncode == 0:
|
||
return int(result.stdout.strip())
|
||
except Exception:
|
||
pass
|
||
return -1
|
||
|
||
def _should_skip_upstream_prompt() -> bool:
|
||
"""Check if user previously declined to add upstream."""
|
||
from hermes_constants import get_hermes_home
|
||
|
||
return (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).exists()
|
||
|
||
def _mark_skip_upstream_prompt():
|
||
"""Create marker file to skip future upstream prompts."""
|
||
try:
|
||
from hermes_constants import get_hermes_home
|
||
|
||
(get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).touch()
|
||
except Exception:
|
||
pass
|
||
|
||
def _sync_fork_with_upstream(git_cmd: list[str], cwd: Path) -> bool:
|
||
"""Attempt to push updated main to origin (sync fork).
|
||
|
||
Returns True if push succeeded, False otherwise.
|
||
"""
|
||
try:
|
||
result = subprocess.run(
|
||
git_cmd + ["push", "origin", "main", "--force-with-lease"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
return result.returncode == 0
|
||
except Exception:
|
||
return False
|
||
|
||
def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path) -> None:
|
||
"""Check if fork is behind upstream and sync if safe.
|
||
|
||
This implements the fork upstream sync logic:
|
||
- If upstream remote doesn't exist, ask user if they want to add it
|
||
- Compare origin/main with upstream/main
|
||
- If origin/main is strictly behind upstream/main, pull from upstream
|
||
- Try to sync fork back to origin if possible
|
||
"""
|
||
has_upstream = _has_upstream_remote(git_cmd, cwd)
|
||
|
||
if not has_upstream:
|
||
# Check if user previously declined
|
||
if _should_skip_upstream_prompt():
|
||
return
|
||
|
||
# Ask user if they want to add upstream
|
||
print()
|
||
print("ℹ Your fork is not tracking the official Hermes repository.")
|
||
print(" This means you may miss updates from NousResearch/hermes-agent.")
|
||
print()
|
||
try:
|
||
response = (
|
||
input("Add official repo as 'upstream' remote? [Y/n]: ").strip().lower()
|
||
)
|
||
except (EOFError, KeyboardInterrupt):
|
||
print()
|
||
response = "n"
|
||
|
||
if response in {"", "y", "yes"}:
|
||
print("→ Adding upstream remote...")
|
||
if _add_upstream_remote(git_cmd, cwd):
|
||
print(
|
||
" ✓ Added upstream: https://github.com/NousResearch/hermes-agent.git"
|
||
)
|
||
has_upstream = True
|
||
else:
|
||
print(" ✗ Failed to add upstream remote. Skipping upstream sync.")
|
||
return
|
||
else:
|
||
print(
|
||
" Skipped. Run 'git remote add upstream https://github.com/NousResearch/hermes-agent.git' to add later."
|
||
)
|
||
_mark_skip_upstream_prompt()
|
||
return
|
||
|
||
# Fetch upstream main only. This sync compares upstream/main with
|
||
# origin/main, so there's no reason to pull every upstream ref — and a bare
|
||
# fetch drags in thousands of auto-generated branches.
|
||
print()
|
||
print("→ Fetching upstream...")
|
||
try:
|
||
subprocess.run(
|
||
git_cmd + ["fetch", "upstream", "main", "--quiet"],
|
||
cwd=cwd,
|
||
capture_output=True,
|
||
check=True,
|
||
)
|
||
except subprocess.CalledProcessError:
|
||
print(" ✗ Failed to fetch upstream. Skipping upstream sync.")
|
||
return
|
||
|
||
# Compare origin/main with upstream/main
|
||
origin_ahead = _count_commits_between(git_cmd, cwd, "upstream/main", "origin/main")
|
||
upstream_ahead = _count_commits_between(
|
||
git_cmd, cwd, "origin/main", "upstream/main"
|
||
)
|
||
|
||
if origin_ahead < 0 or upstream_ahead < 0:
|
||
print(" ✗ Could not compare branches. Skipping upstream sync.")
|
||
return
|
||
|
||
# If origin/main has commits not on upstream, don't trample
|
||
if origin_ahead > 0:
|
||
print()
|
||
print(f"ℹ Your fork has {origin_ahead} commit(s) not on upstream.")
|
||
print(" Skipping upstream sync to preserve your changes.")
|
||
print(" If you want to merge upstream changes, run:")
|
||
print(" git pull upstream main")
|
||
return
|
||
|
||
# If upstream is not ahead, fork is up to date
|
||
if upstream_ahead == 0:
|
||
print(" ✓ Fork is up to date with upstream")
|
||
return
|
||
|
||
# origin/main is strictly behind upstream/main (can fast-forward)
|
||
print()
|
||
print(f"→ Fork is {upstream_ahead} commit(s) behind upstream")
|
||
print("→ Pulling from upstream...")
|
||
|
||
try:
|
||
subprocess.run(
|
||
git_cmd + ["pull", "--ff-only", "upstream", "main"],
|
||
cwd=cwd,
|
||
check=True,
|
||
)
|
||
except subprocess.CalledProcessError:
|
||
print(
|
||
" ✗ Failed to pull from upstream. You may need to resolve conflicts manually."
|
||
)
|
||
return
|
||
|
||
print(" ✓ Updated from upstream")
|
||
|
||
# Try to sync fork back to origin
|
||
print("→ Syncing fork...")
|
||
if _sync_fork_with_upstream(git_cmd, cwd):
|
||
print(" ✓ Fork synced with upstream")
|
||
else:
|
||
print(
|
||
" ℹ Got updates from upstream but couldn't push to fork (no write access?)"
|
||
)
|
||
print(" Your local repo is updated, but your fork on GitHub may be behind.")
|
||
|
||
def _invalidate_update_cache():
|
||
"""Delete the update-check cache for ALL profiles so no banner
|
||
reports a stale "commits behind" count after a successful update.
|
||
|
||
The git repo is shared across profiles — when one profile runs
|
||
``hermes update``, every profile is now current.
|
||
"""
|
||
homes = []
|
||
# Default profile home (Docker-aware — uses /opt/data in Docker)
|
||
from hermes_constants import get_default_hermes_root
|
||
|
||
default_home = get_default_hermes_root()
|
||
homes.append(default_home)
|
||
# Named profiles under <root>/profiles/
|
||
profiles_root = default_home / "profiles"
|
||
if profiles_root.is_dir():
|
||
for entry in profiles_root.iterdir():
|
||
if entry.is_dir():
|
||
homes.append(entry)
|
||
for home in homes:
|
||
try:
|
||
cache_file = home / ".update_check"
|
||
if cache_file.exists():
|
||
cache_file.unlink()
|
||
except Exception:
|
||
pass
|
||
|
||
def _write_marker_file(path: Path, *, label: str) -> None:
|
||
"""Drop an update-recovery breadcrumb. Never raises."""
|
||
try:
|
||
path.write_text(
|
||
f"started={_time.time()}\npid={os.getpid()}\n", encoding="utf-8"
|
||
)
|
||
except OSError as exc:
|
||
logger.debug("Could not write %s marker: %s", label, exc)
|
||
|
||
def _write_update_incomplete_marker() -> None:
|
||
"""Drop the interrupted core-install breadcrumb. Never raises."""
|
||
_write_marker_file(_m()._update_marker_path(), label="update-incomplete")
|
||
|
||
def _write_lazy_refresh_incomplete_marker() -> None:
|
||
"""Drop the interrupted lazy-refresh breadcrumb. Never raises."""
|
||
_write_marker_file(_m()._lazy_refresh_marker_path(), label="lazy-refresh-incomplete")
|
||
|
||
def _format_concurrent_instances_message(
|
||
matches: list[tuple[int, str]], scripts_dir: Path
|
||
) -> str:
|
||
"""Build a human-readable explanation + remediation hint for the user."""
|
||
shim = scripts_dir / "hermes.exe"
|
||
lines = ["✗ Another hermes.exe is running:"]
|
||
for pid, name in matches:
|
||
lines.append(f" PID {pid} {name}")
|
||
lines.append("")
|
||
lines.append(f" Updating now would fail to overwrite {shim} because")
|
||
lines.append(" Windows blocks REPLACE on a running executable.")
|
||
lines.append("")
|
||
lines.append(" Close Hermes Desktop, exit any open `hermes` REPLs, and")
|
||
lines.append(" stop the gateway (`hermes gateway stop`) before retrying.")
|
||
lines.append("")
|
||
if matches:
|
||
pid_args = " ".join(f"/PID {pid}" for pid, _ in matches)
|
||
lines.append(" If you've already closed everything and these PIDs are")
|
||
lines.append(" stale, terminate them directly, then retry the update:")
|
||
lines.append(f" taskkill {pid_args} /F")
|
||
lines.append("")
|
||
lines.append(" Override with `hermes update --force` if you've already")
|
||
lines.append(" confirmed those processes will not write to the venv.")
|
||
return "\n".join(lines)
|
||
|
||
def _upgrade_pip_before_lazy_refresh(
|
||
install_cmd_prefix: list[str],
|
||
*,
|
||
env: dict[str, str] | None = None,
|
||
) -> None:
|
||
"""Upgrade pip before lazy-backend refreshes.
|
||
|
||
Older pip (e.g. 24.0 on Python 3.11) can fail setuptools-backed source
|
||
builds during lazy installs and leave a partially-written venv (#57828).
|
||
Never raises.
|
||
"""
|
||
try:
|
||
_m()._run_package_only_install(
|
||
install_cmd_prefix + ["install", "--upgrade", "pip"],
|
||
env=env,
|
||
)
|
||
except subprocess.CalledProcessError as exc:
|
||
logger.debug("pip upgrade before lazy refresh failed: %s", exc)
|
||
|
||
def _refresh_active_lazy_features(
|
||
install_cmd_prefix: list[str] | None = None,
|
||
*,
|
||
env: dict[str, str] | None = None,
|
||
) -> bool:
|
||
"""Refresh lazy-installed backends after a code update.
|
||
|
||
When pyproject.toml's ``[all]`` extra was slimmed down (May 2026), most
|
||
optional backends moved to ``tools/lazy_deps.py`` and only install on
|
||
first use. ``hermes update`` runs ``uv pip install -e .[all]`` which
|
||
leaves those packages untouched — so if we bump a pin in
|
||
:data:`LAZY_DEPS` (CVE response, transitive bug fix), users who already
|
||
activated the backend keep the stale version forever.
|
||
|
||
This function asks lazy_deps which features the user has previously
|
||
activated and reinstalls them under the current pins. Features the
|
||
user never enabled stay quiet — no churn for cold backends.
|
||
|
||
Returns True when the venv is safe to use (refresh succeeded, or no
|
||
active lazy backends, or post-failure import repair succeeded). Returns
|
||
False when a failed lazy install left broken core imports that automatic
|
||
repair could not fix (#57828).
|
||
|
||
Never raises. A failure here must not block the rest of the update.
|
||
"""
|
||
try:
|
||
from tools import lazy_deps
|
||
except Exception as exc:
|
||
logger.debug("Lazy refresh skipped (import failed): %s", exc)
|
||
return True
|
||
|
||
try:
|
||
active = lazy_deps.active_features()
|
||
except Exception as exc:
|
||
logger.debug("Lazy refresh skipped (active_features failed): %s", exc)
|
||
return True
|
||
|
||
if not active:
|
||
return True
|
||
|
||
print()
|
||
print(f"→ Refreshing {len(active)} active lazy backend(s)...")
|
||
|
||
unexpected_failure = False
|
||
try:
|
||
results = lazy_deps.refresh_active_features(prompt=False)
|
||
except Exception as exc:
|
||
# refresh_active_features is documented as never-raise, but defend
|
||
# the update flow against future regressions.
|
||
print(f" ⚠ Lazy refresh failed unexpectedly: {exc}")
|
||
results = {}
|
||
unexpected_failure = True
|
||
|
||
refreshed = [f for f, s in results.items() if s == "refreshed"]
|
||
current = [f for f, s in results.items() if s == "current"]
|
||
failed = [(f, s) for f, s in results.items() if s.startswith("failed:")]
|
||
skipped = [(f, s) for f, s in results.items() if s.startswith("skipped:")]
|
||
|
||
if refreshed:
|
||
print(f" ↑ {len(refreshed)} refreshed: {', '.join(refreshed)}")
|
||
if current:
|
||
print(f" ✓ {len(current)} already current")
|
||
if skipped:
|
||
# Most common reason: security.allow_lazy_installs=false. Show one
|
||
# line so the user knows why; not an error.
|
||
names = ", ".join(f for f, _ in skipped)
|
||
reason = skipped[0][1].split(": ", 1)[-1]
|
||
print(f" · {len(skipped)} skipped ({reason}): {names}")
|
||
|
||
if not failed and not unexpected_failure:
|
||
return True
|
||
|
||
for feature, status in failed:
|
||
reason = status.split(": ", 1)[-1]
|
||
# Clip noisy pip stderr to keep update output legible.
|
||
if len(reason) > 200:
|
||
reason = reason[:200] + "..."
|
||
print(f" ⚠ {feature} failed to refresh: {reason}")
|
||
|
||
if install_cmd_prefix is None:
|
||
print(" ⚠ Lazy refresh failed; rerun `hermes update` once resolved.")
|
||
return False
|
||
|
||
# Immediate import-based recovery — metadata-only verifiers miss the case
|
||
# where DISTRIBUTION-INFO remains but import files were wiped (#57828).
|
||
# Unavailable probes are indeterminate, not healthy — keep the lazy marker.
|
||
status = _m()._repair_venv_via_import_probes(install_cmd_prefix, env=env)
|
||
if status == "repaired":
|
||
print(
|
||
" Lazy backend(s) keep their previous version until refresh succeeds."
|
||
)
|
||
return True
|
||
if status == "healthy":
|
||
print(
|
||
" Lazy backend(s) keep their previous version; probed packages look intact."
|
||
)
|
||
print(" Rerun `hermes update` once the upstream issue is resolved.")
|
||
return True
|
||
if status == "indeterminate":
|
||
print(
|
||
" ⚠ Leaving `.lazy-refresh-incomplete` until import probes can confirm health."
|
||
)
|
||
return False
|
||
|
||
def _refresh_active_memory_provider_dependencies() -> None:
|
||
"""Refresh pip dependencies for the configured external memory provider.
|
||
|
||
Memory-provider bridge packages are declared in each provider's
|
||
``plugin.yaml`` (plus mode-dependent extras like Hindsight's
|
||
``hindsight-all``), NOT in Hermes' editable-install extras or
|
||
``LAZY_DEPS`` alone — so the core dependency reinstall above can strip
|
||
or downgrade them (#53272 mem0ai, #70636 hindsight-embed). Re-run the
|
||
provider's declared install for the ACTIVE provider only, after the
|
||
core install and lazy refresh, so the last write to any shared package
|
||
is the one the active provider needs.
|
||
|
||
Never raises. A failure here must not block the rest of the update.
|
||
"""
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
|
||
cfg = load_config()
|
||
except Exception as exc:
|
||
logger.debug("Memory provider refresh skipped (config load failed): %s", exc)
|
||
return
|
||
|
||
provider = ""
|
||
if isinstance(cfg, dict):
|
||
memory_cfg = cfg.get("memory")
|
||
if isinstance(memory_cfg, dict):
|
||
if memory_cfg.get("enabled") is False:
|
||
return
|
||
provider = str(memory_cfg.get("provider") or "").strip()
|
||
|
||
# "default" / empty is the built-in file-backed store — no pip deps.
|
||
if not provider or provider in {"default", "builtin", "none"}:
|
||
return
|
||
|
||
try:
|
||
from hermes_cli.memory_setup import _install_dependencies
|
||
except Exception as exc:
|
||
logger.debug("Memory provider refresh skipped (import failed): %s", exc)
|
||
return
|
||
|
||
print()
|
||
print(f"→ Refreshing active memory provider dependencies ({provider})...")
|
||
|
||
try:
|
||
_install_dependencies(provider, force=True)
|
||
except Exception as exc:
|
||
print(f" ⚠ {provider} dependencies failed to refresh: {exc}")
|
||
|
||
def _is_android_python() -> bool:
|
||
return _m().sys.platform == "android"
|
||
|
||
def _install_psutil_android_compat(
|
||
install_cmd_prefix: list[str],
|
||
*,
|
||
env: dict[str, str] | None = None,
|
||
) -> None:
|
||
"""Install psutil on Android by patching upstream platform detection.
|
||
|
||
psutil's setup currently gates Linux sources behind
|
||
``sys.platform.startswith('linux')``. On Termux Python reports
|
||
``sys.platform == 'android'``, so setup aborts with
|
||
"platform android is not supported" despite compiling fine when using the
|
||
Linux source path.
|
||
|
||
We patch only the extracted build tree used for this install attempt;
|
||
nothing is persisted in the repository.
|
||
|
||
Stopgap: remove this once https://github.com/giampaolo/psutil/pull/2762
|
||
merges and ships in a release. The standalone installer script uses the
|
||
same shared helper and should be removed together.
|
||
"""
|
||
import tempfile
|
||
import urllib.request
|
||
from hermes_cli.psutil_android import PSUTIL_URL, prepare_patched_psutil_sdist
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
tmp_path = Path(tmp)
|
||
archive = tmp_path / "psutil.tar.gz"
|
||
urllib.request.urlretrieve(PSUTIL_URL, archive)
|
||
src_root = prepare_patched_psutil_sdist(archive, tmp_path)
|
||
|
||
_m()._run_install_with_heartbeat(
|
||
install_cmd_prefix + ["install", "--no-build-isolation", str(src_root)],
|
||
env=env,
|
||
)
|
||
|
||
def _ensure_uv_for_termux(pip_cmd: list[str]) -> str | None:
|
||
"""Best-effort uv bootstrap on Termux for faster update installs.
|
||
|
||
The normal path (``ensure_uv()`` in managed_uv) installs the managed
|
||
standalone uv into ``$HERMES_HOME/bin/uv``, but on Termux the official
|
||
installer may not work (glibc vs bionic). Prefer a uv already on PATH
|
||
(e.g. ``pkg install uv``); only if there is none do we fall back to a
|
||
wheel-only ``pip install uv`` so we never source-build the Rust crate.
|
||
"""
|
||
from hermes_cli.managed_uv import resolve_uv
|
||
|
||
existing = resolve_uv()
|
||
if existing:
|
||
return existing
|
||
if not _m()._is_termux_env():
|
||
return None
|
||
# A Termux-packaged uv lands on PATH but not in the managed bin dir, so
|
||
# resolve_uv() misses it. Use it before pip, which has no Android wheel and
|
||
# would otherwise build uv from source on a low-memory device.
|
||
system_uv = shutil.which("uv")
|
||
if system_uv:
|
||
return system_uv
|
||
try:
|
||
print(" → Termux detected: trying to install uv for faster dependency updates...")
|
||
result = subprocess.run(
|
||
pip_cmd + ["install", "uv", "--only-binary", ":all:"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
check=False,
|
||
)
|
||
if result.returncode != 0:
|
||
return None
|
||
except Exception:
|
||
pass
|
||
# After pip install, check managed path first, then PATH
|
||
return resolve_uv() or shutil.which("uv")
|
||
|
||
def _npm_manifest_paths() -> tuple[Path, ...]:
|
||
"""Manifests whose changes must defeat the update-skip.
|
||
|
||
The lockfile alone is NOT a sufficient key: on a local checkout a dev
|
||
can edit package.json (root or a workspace) without running npm — the
|
||
lockfile is then unchanged but `hermes update` is exactly the step
|
||
expected to sync node_modules (via the `npm install` fallback in
|
||
_run_npm_install_deterministic).
|
||
|
||
The workspace list is pulled from the root package.json's `workspaces`
|
||
globs (npm's own source of truth) rather than hardcoded, so adding a
|
||
workspace can never silently escape the skip key. The root install
|
||
(step 1, --workspaces=false) still hoists shared deps for EVERY
|
||
workspace — desktop included — so all of them belong in the key, not
|
||
just the ones step 2 installs. Falls back to hashing just root
|
||
manifests if package.json is unreadable (never skips more than main
|
||
would have installed).
|
||
"""
|
||
root_pkg = _m().PROJECT_ROOT / "package.json"
|
||
paths = [_m().PROJECT_ROOT / "package-lock.json", root_pkg]
|
||
try:
|
||
workspaces = json.loads(root_pkg.read_text(encoding="utf-8")).get(
|
||
"workspaces", []
|
||
)
|
||
if isinstance(workspaces, dict): # legacy {"packages": [...]} form
|
||
workspaces = workspaces.get("packages", [])
|
||
for pattern in workspaces:
|
||
for match in sorted(_m().PROJECT_ROOT.glob(str(pattern))):
|
||
manifest = match / "package.json"
|
||
if manifest.is_file():
|
||
paths.append(manifest)
|
||
except (OSError, json.JSONDecodeError, TypeError):
|
||
pass
|
||
return tuple(paths)
|
||
|
||
def _npm_manifests_digest() -> str | None:
|
||
"""Combined sha256 over the lockfile + all workspace package.json files.
|
||
|
||
Returns None when the lockfile is missing (never skip then).
|
||
"""
|
||
if not (_m().PROJECT_ROOT / "package-lock.json").exists():
|
||
return None
|
||
h = hashlib.sha256()
|
||
for p in _npm_manifest_paths():
|
||
h.update(str(p.relative_to(_m().PROJECT_ROOT)).encode())
|
||
try:
|
||
h.update(p.read_bytes())
|
||
except OSError:
|
||
h.update(b"<missing>")
|
||
return h.hexdigest()
|
||
|
||
def _npm_lockfile_changed(hermes_root: Path) -> bool:
|
||
current = _npm_manifests_digest()
|
||
if current is None:
|
||
return True
|
||
# Also check that node_modules exists; a matching hash with missing
|
||
# node_modules means the cache was recorded by another checkout.
|
||
if not (_m().PROJECT_ROOT / "node_modules").is_dir():
|
||
return True
|
||
# A matching lockfile hash over a tree whose web build toolchain never
|
||
# landed must NOT skip the reinstall — otherwise every later `hermes
|
||
# update` keeps rebuilding against a half-installed tree and serving a
|
||
# stale dist.
|
||
web_dir = _m().PROJECT_ROOT / "web"
|
||
if (web_dir / "package.json").is_file() and not _web_build_toolchain_ready(
|
||
*_web_toolchain_roots(web_dir)
|
||
):
|
||
return True
|
||
try:
|
||
# Key the cache by PROJECT_ROOT so parallel worktrees don't collide.
|
||
cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12]
|
||
cache_file = hermes_root / f".npm_lock_hash_{cache_key}"
|
||
if not cache_file.exists():
|
||
return True
|
||
return cache_file.read_text(encoding="utf-8").strip() != current
|
||
except OSError:
|
||
return True
|
||
|
||
def _record_npm_lockfile_hash(hermes_root: Path) -> None:
|
||
digest = _npm_manifests_digest()
|
||
if digest is None:
|
||
return
|
||
try:
|
||
cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12]
|
||
cache_file = hermes_root / f".npm_lock_hash_{cache_key}"
|
||
cache_file.write_text(digest, encoding="utf-8")
|
||
except OSError:
|
||
logger.debug("Could not write npm lockfile hash cache")
|
||
|
||
def _update_node_dependencies() -> list[str]:
|
||
"""Refresh Node deps in the repo root and update workspaces.
|
||
|
||
Returns the list of labels whose npm install failed (empty on success),
|
||
so the caller can treat a Node refresh failure as a partial update rather
|
||
than silently reporting ``Update complete!`` (#30271).
|
||
"""
|
||
if not (_m().PROJECT_ROOT / "package.json").exists():
|
||
return []
|
||
|
||
npm = _m()._resolve_node_runtime_npm()
|
||
if not npm:
|
||
# If the only npm reachable inside this WSL shell is the Windows one,
|
||
# flag it loudly: silently skipping leaves ui-tui deps stale while the
|
||
# rest of the update proceeds, and running it would corrupt the tree.
|
||
from hermes_constants import is_wsl
|
||
|
||
path_npm = shutil.which("npm")
|
||
if is_wsl() and path_npm and _m()._is_windows_npm_path(path_npm):
|
||
print("→ Updating Node.js dependencies...")
|
||
print(" ⚠ Skipped: only a Windows npm is reachable from this WSL shell.")
|
||
print(" Install Node.js inside the WSL distro (nvm, or your distro's")
|
||
print(" package manager), then re-run `hermes update`.")
|
||
failed = ["repo root"]
|
||
if any(
|
||
(_m().PROJECT_ROOT / workspace / "package.json").exists()
|
||
for workspace in ("ui-tui", "web")
|
||
):
|
||
failed.append("ui-tui, web workspaces")
|
||
return failed
|
||
return []
|
||
|
||
from hermes_constants import get_default_hermes_root
|
||
|
||
# This cache describes PROJECT_ROOT/node_modules, which is shared by every
|
||
# Hermes profile using this checkout. Keep one per-checkout cache under the
|
||
# shared Hermes root rather than rerunning npm once per named profile.
|
||
shared_hermes_root = get_default_hermes_root()
|
||
if not _m()._npm_lockfile_changed(shared_hermes_root):
|
||
logger.info("npm lockfile unchanged, skipping npm install")
|
||
return []
|
||
|
||
# With a single workspace lockfile the root install would cover ALL
|
||
# workspaces — but apps/desktop pulls in Electron as a devDependency,
|
||
# and its postinstall downloads a ~200MB binary. Most users don't
|
||
# need desktop during `hermes update`, so we install root-only first
|
||
# then add just the workspaces the CLI/TUI/web build actually requires.
|
||
# Desktop deps are installed on demand by the desktop launcher
|
||
# (see _desktop_build_needed).
|
||
print("→ Updating Node.js dependencies...")
|
||
|
||
def _partial_update_failure(*labels: str) -> list[str]:
|
||
print()
|
||
print(" ⚠ Node.js dependency refresh did not complete cleanly; the")
|
||
print(" installation may be in a mixed state (updated code, stale Node")
|
||
print(" deps). Fix npm and re-run `hermes update`.")
|
||
return list(labels)
|
||
|
||
extra_args = ["--no-fund", "--no-audit", "--progress=false"]
|
||
|
||
from hermes_constants import with_hermes_node_path
|
||
|
||
nixos_env = with_hermes_node_path(_m()._nixos_build_env())
|
||
|
||
# Step 1: root install (no workspace recursion).
|
||
# NOTE: capture_output=False here is deliberate (#18840) — optional
|
||
# postinstall scripts (e.g. @askjo/camofox-browser's browser-binary fetch)
|
||
# print download progress, and capturing it makes a long download look
|
||
# hung. The chatty npm-deprecation noise during `hermes update` comes from
|
||
# the *desktop* build, not this step; that one is captured to update.log.
|
||
root_args = [*extra_args, "--workspaces=false"]
|
||
root_result = _m()._run_npm_install_deterministic(
|
||
npm,
|
||
_m().PROJECT_ROOT,
|
||
extra_args=tuple(root_args),
|
||
capture_output=False,
|
||
env=nixos_env,
|
||
)
|
||
if root_result.returncode != 0:
|
||
print(" ⚠ npm install failed in repo root")
|
||
stderr = (root_result.stderr or "").strip() if root_result.stderr else ""
|
||
if stderr:
|
||
print(f" {stderr.splitlines()[-1]}")
|
||
return _partial_update_failure("repo root")
|
||
|
||
# Step 2: install only the workspaces update needs (ui-tui, web).
|
||
# --workspace selects specific workspaces; the rest (desktop) are skipped.
|
||
ws_args = [*extra_args, "--workspace", "ui-tui", "--workspace", "web"]
|
||
ws_result = _m()._run_npm_install_deterministic(
|
||
npm,
|
||
_m().PROJECT_ROOT,
|
||
extra_args=tuple(ws_args),
|
||
capture_output=False,
|
||
env=nixos_env,
|
||
)
|
||
if ws_result.returncode == 0:
|
||
_record_npm_lockfile_hash(shared_hermes_root)
|
||
print(" ✓ repo root + ui-tui, web workspaces (desktop skipped)")
|
||
return []
|
||
|
||
print(" ⚠ npm workspace install failed")
|
||
stderr = (ws_result.stderr or "").strip() if ws_result.stderr else ""
|
||
if stderr:
|
||
print(f" {stderr.splitlines()[-1]}")
|
||
return _partial_update_failure("ui-tui, web workspaces")
|
||
|
||
def _log_only_write(text: str) -> None:
|
||
"""Write ``text`` to ``~/.hermes/logs/update.log`` only, never the terminal.
|
||
|
||
During ``hermes update`` ``sys.stdout`` is an ``_UpdateOutputStream`` that
|
||
mirrors to both the terminal and ``update.log``. Loud, low-signal
|
||
subprocess output (npm installs, the Electron/vite build, the cua-driver
|
||
installer's "Next steps" wall) should be captured and tucked into the log
|
||
so failures stay debuggable, without flooding the user's terminal. This
|
||
reaches past the mirroring stream straight to the underlying log handle.
|
||
"""
|
||
if not text:
|
||
return
|
||
stream = _m().sys.stdout
|
||
log_file = getattr(stream, "_log", None)
|
||
if log_file is None:
|
||
return
|
||
try:
|
||
log_file.write(text if text.endswith("\n") else text + "\n")
|
||
log_file.flush()
|
||
except Exception:
|
||
pass
|
||
|
||
def _run_logged_subprocess(cmd, *, cwd=None, env=None):
|
||
"""Run ``cmd`` capturing combined output into update.log (not the terminal).
|
||
|
||
Returns the ``CompletedProcess`` (with ``stdout`` populated) so the caller
|
||
can decide whether to surface the captured output on failure.
|
||
"""
|
||
result = subprocess.run(
|
||
cmd,
|
||
cwd=cwd,
|
||
env=env,
|
||
check=False,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.STDOUT,
|
||
text=True,
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)
|
||
_log_only_write(result.stdout or "")
|
||
return result
|
||
|
||
def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False):
|
||
"""Implement ``hermes update --check``: fetch and report without installing.
|
||
|
||
``branch`` selects which branch the check compares against. Default is
|
||
"main"; callers can pass another branch to ask "are there new commits
|
||
on origin/<branch>?" without performing the update.
|
||
|
||
``branch_explicit`` is True iff the caller passed --branch on the CLI.
|
||
Installs that can't honor non-default branches (e.g. Docker) surface a
|
||
one-line notice instead of silently dropping the flag.
|
||
"""
|
||
from hermes_cli.config import detect_install_method, recommended_update_command_for_method
|
||
method = detect_install_method(_m().PROJECT_ROOT)
|
||
if method == "docker":
|
||
# Docker can't ``git fetch`` from within the container. Surface the
|
||
# same long-form ``docker pull`` guidance ``hermes update`` (apply
|
||
# path) uses — telling the user to "reinstall via curl" or that
|
||
# ".git is missing" would point them at the wrong remediation.
|
||
from hermes_cli.config import format_docker_update_message
|
||
print(format_docker_update_message())
|
||
sys.exit(1)
|
||
|
||
if method in {"nix", "nixos"}:
|
||
print(recommended_update_command_for_method(method))
|
||
sys.exit(1)
|
||
|
||
git_dir = _m().PROJECT_ROOT / ".git"
|
||
if not git_dir.exists():
|
||
print("✗ Not a git repository — cannot check for updates.")
|
||
sys.exit(1)
|
||
|
||
git_cmd = ["git"]
|
||
if sys.platform == "win32":
|
||
git_cmd = ["git", "-c", "windows.appendAtomically=false"]
|
||
|
||
# Fetch only the branch we compare against; prefer upstream as the canonical
|
||
# reference. A bare `git fetch <remote>` pulls every ref, and this repo has
|
||
# thousands of auto-generated branches, so scope the fetch to <branch>.
|
||
# Note: upstream/<branch> may not exist for non-main branches (a fork's
|
||
# bb/gui has no upstream counterpart), so when the caller picks a
|
||
# non-default branch we skip the upstream probe and use origin directly.
|
||
# Installer checkouts are shallow (`git clone --depth 1`). A plain
|
||
# `git fetch` would unshallow the repo (dragging in the whole history —
|
||
# the exact cost the shallow clone avoided) and the rev-list count below
|
||
# would then report a huge bogus "behind" number. Detect shallow up front:
|
||
# fetch with --depth 1 to preserve the boundary and report presence-only.
|
||
is_shallow = (
|
||
subprocess.run(
|
||
git_cmd + ["rev-parse", "--is-shallow-repository"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
).stdout.strip()
|
||
== "true"
|
||
)
|
||
depth_args = ["--depth", "1"] if is_shallow else []
|
||
|
||
if branch == "main":
|
||
# Probe locally (~6 ms) whether an 'upstream' remote exists at all
|
||
# before spending a network fetch on it. Non-fork installs have no
|
||
# 'upstream' remote, and the old flow burned a failed network attempt
|
||
# (~0.3-1 s) on every --check before falling back to origin.
|
||
has_upstream_remote = (
|
||
subprocess.run(
|
||
git_cmd + ["remote", "get-url", "upstream"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
).returncode
|
||
== 0
|
||
)
|
||
fetch_result = None
|
||
if has_upstream_remote:
|
||
print("→ Fetching from upstream...")
|
||
fetch_result = subprocess.run(
|
||
git_cmd + ["fetch"] + depth_args + ["upstream", branch],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if fetch_result is not None and fetch_result.returncode == 0:
|
||
upstream_exists = True
|
||
compare_branch = f"upstream/{branch}"
|
||
else:
|
||
# No upstream remote, or the upstream fetch failed — use origin.
|
||
print("→ Fetching from origin...")
|
||
fetch_result = subprocess.run(
|
||
git_cmd + ["fetch"] + depth_args + ["origin", branch],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
upstream_exists = False
|
||
compare_branch = f"origin/{branch}"
|
||
else:
|
||
# Non-default branch: compare against origin/<branch> directly.
|
||
print("→ Fetching from origin...")
|
||
fetch_result = subprocess.run(
|
||
git_cmd + ["fetch"] + depth_args + ["origin", branch],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
upstream_exists = False
|
||
compare_branch = f"origin/{branch}"
|
||
|
||
if fetch_result.returncode != 0:
|
||
stderr = fetch_result.stderr.strip()
|
||
if "Could not resolve host" in stderr or "unable to access" in stderr:
|
||
print("✗ Network error — cannot reach the remote repository.")
|
||
elif "Authentication failed" in stderr or "could not read Username" in stderr:
|
||
print("✗ Authentication failed — check your git credentials or SSH key.")
|
||
else:
|
||
print("✗ Failed to fetch.")
|
||
if stderr:
|
||
print(f" {stderr.splitlines()[0]}")
|
||
sys.exit(1)
|
||
|
||
# Verify the compare ref actually exists before asking rev-list about it.
|
||
# Without this, `git rev-list HEAD..origin/<bogus> --count` exits 128 and
|
||
# (with check=True) raises CalledProcessError, surfacing a Python
|
||
# traceback. Friendlier to detect-and-report.
|
||
verify_result = subprocess.run(
|
||
git_cmd + ["rev-parse", "--verify", "--quiet", compare_branch],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if verify_result.returncode != 0:
|
||
print(f"✗ Branch '{branch}' not found on {compare_branch.split('/', 1)[0]}.")
|
||
sys.exit(1)
|
||
|
||
if is_shallow:
|
||
# No history to count across the shallow boundary. Compare tip SHAs and
|
||
# report presence-only (mirrors the banner's _check_via_local_git).
|
||
head_sha = subprocess.run(
|
||
git_cmd + ["rev-parse", "HEAD"],
|
||
cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
|
||
).stdout.strip()
|
||
target_sha = subprocess.run(
|
||
git_cmd + ["rev-parse", compare_branch],
|
||
cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
|
||
).stdout.strip()
|
||
if head_sha and target_sha and head_sha == target_sha:
|
||
print("✓ Already up to date.")
|
||
else:
|
||
print(f"⚕ Update available (behind {compare_branch}).")
|
||
from hermes_cli.config import recommended_update_command
|
||
|
||
print(f" Run '{recommended_update_command()}' to install.")
|
||
return
|
||
|
||
rev_result = subprocess.run(
|
||
git_cmd + ["rev-list", f"HEAD..{compare_branch}", "--count"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
check=True,
|
||
)
|
||
behind = int(rev_result.stdout.strip())
|
||
|
||
if behind == 0:
|
||
print("✓ Already up to date.")
|
||
else:
|
||
commits_word = "commit" if behind == 1 else "commits"
|
||
print(f"⚕ Update available: {behind} {commits_word} behind {compare_branch}.")
|
||
from hermes_cli.config import recommended_update_command
|
||
|
||
print(f" Run '{recommended_update_command()}' to install.")
|
||
|
||
def _ensure_fhs_path_guard() -> None:
|
||
"""Ensure /usr/local/bin is on PATH for RHEL-family root non-login shells.
|
||
|
||
Mirrors the post-symlink probe added to ``scripts/install.sh`` so that
|
||
existing FHS-layout root installs on RHEL/CentOS/Rocky/Alma 8+ get
|
||
repaired on ``hermes update`` without requiring a reinstall. The
|
||
installer's assumption that ``/usr/local/bin`` is on PATH for every
|
||
standard shell breaks on those distros in non-login interactive shells
|
||
(su, sudo -s, tmux panes, some web terminals): /etc/bashrc doesn't
|
||
add /usr/local/bin and /root/.bash_profile doesn't either. Symptom:
|
||
``hermes`` prints ``command not found`` even though the symlink lives
|
||
at /usr/local/bin/hermes.
|
||
|
||
Silent no-op on: non-Linux, non-root, non-FHS installs, and any system
|
||
where ``bash -i -c 'command -v hermes'`` already resolves. Idempotent.
|
||
"""
|
||
if _m().sys.platform != "linux":
|
||
return
|
||
try:
|
||
if os.geteuid() != 0: # windows-footgun: ok — Linux FHS helper, guarded by sys.platform == "linux" above + AttributeError catch
|
||
return
|
||
except AttributeError:
|
||
return
|
||
# Only act when this is actually an FHS-layout install (command link at
|
||
# /usr/local/bin/hermes, code at /usr/local/lib/hermes-agent).
|
||
fhs_link = Path("/usr/local/bin/hermes")
|
||
if not fhs_link.is_symlink() and not fhs_link.exists():
|
||
return
|
||
|
||
# Probe a fresh non-login interactive bash the way the user will use it.
|
||
# ``bash -i -c`` sources ~/.bashrc but NOT ~/.bash_profile or /etc/profile,
|
||
# which is the exact scenario where RHEL root loses /usr/local/bin.
|
||
home = os.environ.get("HOME") or "/root"
|
||
try:
|
||
probe = subprocess.run(
|
||
[
|
||
"env",
|
||
"-i",
|
||
f"HOME={home}",
|
||
f"TERM={os.environ.get('TERM', 'dumb')}",
|
||
"bash",
|
||
"-i",
|
||
"-c",
|
||
"command -v hermes",
|
||
],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=10,
|
||
)
|
||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||
return # no bash or probe hung — don't block update on this
|
||
if probe.returncode == 0:
|
||
return # already on PATH, nothing to do
|
||
|
||
path_line = 'export PATH="/usr/local/bin:$PATH"'
|
||
path_comment = (
|
||
"# Hermes Agent — ensure /usr/local/bin is on PATH " "(RHEL non-login shells)"
|
||
)
|
||
wrote_any = False
|
||
for candidate in (".bashrc", ".bash_profile"):
|
||
cfg = Path(home) / candidate
|
||
if not cfg.is_file():
|
||
continue
|
||
try:
|
||
existing = cfg.read_text(errors="replace", encoding="utf-8")
|
||
except OSError:
|
||
continue
|
||
# Idempotency: skip if any uncommented PATH= line already references
|
||
# /usr/local/bin. Mirrors the grep pattern used by install.sh.
|
||
already_guarded = any(
|
||
"/usr/local/bin" in line
|
||
and "PATH" in line
|
||
and not line.lstrip().startswith("#")
|
||
for line in existing.splitlines()
|
||
)
|
||
if already_guarded:
|
||
continue
|
||
try:
|
||
with cfg.open("a", encoding="utf-8") as f:
|
||
f.write("\n" + path_comment + "\n" + path_line + "\n")
|
||
except OSError as e:
|
||
print(f" ⚠ Could not update {cfg}: {e}")
|
||
continue
|
||
print(f" ✓ Added /usr/local/bin to PATH in {cfg}")
|
||
wrote_any = True
|
||
if wrote_any:
|
||
print(" (reload your shell or run 'source ~/.bashrc' to pick it up)")
|
||
|
||
def _ensure_acp_launcher() -> None:
|
||
"""Self-heal: install a ``hermes-acp`` launcher next to the ``hermes`` one.
|
||
|
||
Mirrors the launcher block in ``scripts/install.sh`` so existing installs
|
||
gain the ACP command on ``hermes update`` without a reinstall. ACP hosts
|
||
(Zed, JetBrains, Buzz Desktop) spawn the agent by resolving the
|
||
``hermes-acp`` command name against the login-shell PATH; the console
|
||
script of that name lives inside the install's venv, which is not on that
|
||
PATH, so those hosts report Hermes as not installed even when it is.
|
||
|
||
The shim simply delegates to the sibling ``hermes`` launcher with the
|
||
``acp`` subcommand, which makes it correct for every install layout
|
||
(venv wrapper, FHS symlink, pipx/pip console script) without having to
|
||
reconstruct interpreter/entrypoint paths.
|
||
|
||
No-op on Windows (install.ps1 puts ``venv\\Scripts`` on the user PATH, so
|
||
``hermes-acp.exe`` already resolves) and wherever a ``hermes-acp`` is
|
||
already present next to the ``hermes`` command. Unwritable directories
|
||
(e.g. ``/usr/local/bin`` as non-root) are skipped silently. Idempotent.
|
||
"""
|
||
if _m().sys.platform == "win32":
|
||
return
|
||
for bin_dir in (Path.home() / ".local" / "bin", Path("/usr/local/bin")):
|
||
hermes_cmd = bin_dir / "hermes"
|
||
acp_cmd = bin_dir / "hermes-acp"
|
||
try:
|
||
if not (hermes_cmd.is_file() or hermes_cmd.is_symlink()):
|
||
continue
|
||
# Already present — a console script (pip/pipx install), an
|
||
# earlier shim, or a symlink. is_symlink() catches broken
|
||
# symlinks that exists() would miss; never follow-and-overwrite
|
||
# (the #21454 failure mode).
|
||
if acp_cmd.exists() or acp_cmd.is_symlink():
|
||
continue
|
||
shim = (
|
||
"#!/usr/bin/env bash\n"
|
||
"# Hermes Agent — ACP launcher (written by `hermes update`).\n"
|
||
"# ACP hosts (Zed, JetBrains, Buzz) resolve the agent by this\n"
|
||
"# command name on the login-shell PATH.\n"
|
||
f'exec "{hermes_cmd}" acp "$@"\n'
|
||
)
|
||
acp_cmd.write_text(shim, encoding="utf-8")
|
||
acp_cmd.chmod(acp_cmd.stat().st_mode | 0o755)
|
||
except OSError:
|
||
continue
|
||
print(f" ✓ Installed hermes-acp launcher → {acp_cmd}")
|
||
|
||
_PRE_UPDATE_SNAPSHOT_KEEP = 1
|
||
|
||
# Per-file size cap for the pre-update quick snapshot. Anything larger is
|
||
# skipped with a warning: the snapshot exists to protect small, hard-to-
|
||
# regenerate state (pairing JSONs, cron jobs, config, auth) — not to copy a
|
||
# multi-GB state.db on every update (observed: a 24 GB state.db added ~60s
|
||
# of wall time and silently ate 24 GB of disk per update).
|
||
_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE = 1 << 30 # 1 GiB
|
||
|
||
def _resolve_pre_update_backup_mode(args) -> str:
|
||
"""Resolve the pre-update backup mode: ``"off"``, ``"quick"``, or ``"full"``.
|
||
|
||
CLI flags win over config; ``--no-backup`` beats ``--backup`` when both
|
||
are set. Config accepts the mode strings plus legacy booleans:
|
||
``true`` → ``full`` (the old zip behavior), ``false`` → ``off``
|
||
(an explicit opt-out now disables the quick snapshot too — previously
|
||
it ran unconditionally, ignoring the user's setting). A missing key
|
||
defaults to ``quick``.
|
||
"""
|
||
if getattr(args, "no_backup", False):
|
||
return "off"
|
||
if getattr(args, "backup", False):
|
||
return "full"
|
||
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
|
||
cfg = load_config()
|
||
except Exception as exc:
|
||
logging.getLogger(__name__).debug(
|
||
"Could not load config for pre-update backup: %s", exc
|
||
)
|
||
cfg = {}
|
||
|
||
updates_cfg = cfg.get("updates", {}) if isinstance(cfg, dict) else {}
|
||
raw = updates_cfg.get("pre_update_backup", "quick")
|
||
|
||
if raw is True:
|
||
return "full"
|
||
if raw is False:
|
||
return "off"
|
||
mode = str(raw).strip().lower()
|
||
if mode in ("off", "false", "none", "disabled"):
|
||
return "off"
|
||
if mode in ("full", "zip", "true"):
|
||
return "full"
|
||
if mode == "quick":
|
||
return "quick"
|
||
logging.getLogger(__name__).warning(
|
||
"Unknown updates.pre_update_backup value %r — using 'quick'", raw
|
||
)
|
||
return "quick"
|
||
|
||
def _run_pre_update_backup(args) -> Optional[str]:
|
||
"""Run the pre-update safety backup and return the quick-snapshot id.
|
||
|
||
Single consolidated mechanism gated on ``updates.pre_update_backup``:
|
||
|
||
- ``off`` — nothing runs. Explicit user opt-out is honored fully.
|
||
- ``quick`` (default) — a state snapshot of critical small files
|
||
(pairing JSONs, cron jobs, config, auth; see ``_QUICK_STATE_FILES``)
|
||
under ``state-snapshots/``. Files over 1 GiB are skipped with a
|
||
warning so a bloated state.db can never stall the update
|
||
(issues #15733, #34600 are the reason this safety net exists).
|
||
- ``full`` — the quick snapshot PLUS a full zip of HERMES_HOME under
|
||
``backups/`` (restorable via ``hermes import``; the #48200 wrong-path
|
||
wipe is the reason this level exists).
|
||
|
||
``--backup`` forces ``full`` for one run; ``--no-backup`` forces ``off``.
|
||
Never raises — a backup failure should not block the update itself.
|
||
|
||
Returns the quick-snapshot id (used by the post-update cron-jobs
|
||
restore safety net), or ``None`` when mode is ``off`` or the snapshot
|
||
failed.
|
||
"""
|
||
mode = _resolve_pre_update_backup_mode(args)
|
||
|
||
if mode == "off":
|
||
if getattr(args, "no_backup", False):
|
||
print("◆ Pre-update backup: skipped (--no-backup)")
|
||
print()
|
||
# Config-level off is silent — the user opted out; don't spam them
|
||
# on every update.
|
||
return None
|
||
|
||
snapshot_id = None
|
||
try:
|
||
from hermes_cli.backup import (
|
||
_quick_snapshot_root,
|
||
create_quick_snapshot,
|
||
verify_sqlite_integrity,
|
||
)
|
||
|
||
# NOTE: this function later does `from hermes_constants import
|
||
# get_hermes_home`, which makes the name function-local — the
|
||
# module-level import is shadowed and unbound here. Alias explicitly.
|
||
from hermes_cli.config import get_hermes_home as _get_home
|
||
|
||
snapshot_id = create_quick_snapshot(
|
||
label="pre-update",
|
||
keep=_PRE_UPDATE_SNAPSHOT_KEEP,
|
||
max_file_size=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE,
|
||
)
|
||
|
||
# After the snapshot, verify the source state.db is still intact.
|
||
# The snapshot was taken via _safe_copy_db (read-only SQLite backup
|
||
# API), but a concurrent process (antivirus, force-killed gateway
|
||
# releasing file handles, Windows filter driver) can corrupt the live
|
||
# file at any point. A silent zeroing at this point would proceed with
|
||
# the update and exit code 0 — exactly the #68474 symptom.
|
||
if snapshot_id:
|
||
_src_path = _get_home() / "state.db"
|
||
if _src_path.exists():
|
||
_integrity = verify_sqlite_integrity(
|
||
_src_path,
|
||
check_header=True,
|
||
run_pragma=True,
|
||
max_bytes=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE,
|
||
)
|
||
if not _integrity.get("valid"):
|
||
_msg = _integrity.get("message", "unknown error")
|
||
print(
|
||
f" ⚠ state.db integrity check FAILED after snapshot: {_msg}"
|
||
)
|
||
# Check if the snapshot itself is valid.
|
||
_snap_root = _quick_snapshot_root(_get_home())
|
||
_snap_state = _snap_root / snapshot_id / "state.db"
|
||
if _snap_state.exists():
|
||
_snap_ok = verify_sqlite_integrity(
|
||
_snap_state, check_header=True, run_pragma=True
|
||
)
|
||
if _snap_ok.get("valid"):
|
||
print(
|
||
" ✓ Snapshot copy is valid — continuing update."
|
||
)
|
||
print(
|
||
" If state.db is lost after update it will be auto-restored."
|
||
)
|
||
else:
|
||
print(
|
||
" ✗ Snapshot copy ALSO failed integrity — "
|
||
"the source was already corrupted before the backup."
|
||
)
|
||
else:
|
||
print(
|
||
" ⚠ Snapshot does not contain state.db (was skipped or too large)."
|
||
)
|
||
print()
|
||
if snapshot_id:
|
||
print(f"◆ Pre-update snapshot: {snapshot_id}")
|
||
except Exception as exc:
|
||
# Never let a snapshot failure block an update.
|
||
logging.getLogger(__name__).debug("Pre-update snapshot failed: %s", exc)
|
||
|
||
if mode != "full":
|
||
if snapshot_id:
|
||
print()
|
||
return snapshot_id
|
||
|
||
try:
|
||
from hermes_cli.backup import create_pre_update_backup
|
||
except Exception as exc:
|
||
print(
|
||
f"⚠ Pre-update backup: could not load backup module ({exc}); continuing update."
|
||
)
|
||
print()
|
||
return snapshot_id
|
||
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
|
||
_keep = (load_config() or {}).get("updates", {}).get("backup_keep", 5)
|
||
except Exception:
|
||
_keep = 5
|
||
|
||
print("◆ Creating pre-update backup...")
|
||
t0 = _time.monotonic()
|
||
try:
|
||
out_path = create_pre_update_backup(keep=int(_keep))
|
||
except Exception as exc: # defensive — helper already swallows, but just in case
|
||
print(f" ⚠ Backup failed: {exc}")
|
||
print(" Continuing with update.")
|
||
print()
|
||
return snapshot_id
|
||
|
||
elapsed = _time.monotonic() - t0
|
||
|
||
if out_path is None:
|
||
print(" ⚠ Backup skipped (no files found or write failed); continuing update.")
|
||
print()
|
||
return snapshot_id
|
||
|
||
try:
|
||
size_bytes = out_path.stat().st_size
|
||
except OSError:
|
||
size_bytes = 0
|
||
|
||
# Human-readable size
|
||
size_str = f"{size_bytes} B"
|
||
for unit in ("KB", "MB", "GB"):
|
||
if size_bytes < 1024:
|
||
break
|
||
size_bytes /= 1024
|
||
size_str = f"{size_bytes:.1f} {unit}"
|
||
|
||
# Render path using display_hermes_home so the user sees ~/.hermes/...
|
||
try:
|
||
from hermes_constants import get_hermes_home, display_hermes_home
|
||
|
||
home = get_hermes_home()
|
||
try:
|
||
display_path = f"{display_hermes_home()}/{out_path.relative_to(home)}"
|
||
except ValueError:
|
||
display_path = str(out_path)
|
||
except Exception:
|
||
display_path = str(out_path)
|
||
|
||
print(f" Saved: {display_path} ({size_str}, {elapsed:.1f}s)")
|
||
print(f" Restore: hermes import {out_path}")
|
||
print(" Disable: set updates.pre_update_backup: quick (or off) in config.yaml")
|
||
print()
|
||
return snapshot_id
|
||
|
||
def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool:
|
||
"""Write a planned-stop marker into a specific profile home."""
|
||
try:
|
||
from datetime import timezone
|
||
|
||
from gateway.status import _get_process_start_time
|
||
from utils import atomic_json_write
|
||
|
||
record = {
|
||
"target_pid": pid,
|
||
"target_start_time": _get_process_start_time(pid),
|
||
"stopper_pid": os.getpid(),
|
||
"written_at": datetime.now(timezone.utc).isoformat(),
|
||
}
|
||
atomic_json_write(
|
||
Path(profile_path) / ".gateway-planned-stop.json",
|
||
record,
|
||
indent=None,
|
||
separators=(",", ":"),
|
||
)
|
||
return True
|
||
except (OSError, PermissionError):
|
||
return False
|
||
|
||
def _wait_for_windows_update_gateway_exit(
|
||
pids: list[int], *, timeout: float
|
||
) -> set[int]:
|
||
"""Wait for the given gateway PIDs to exit, returning survivors."""
|
||
if not pids:
|
||
return set()
|
||
|
||
from gateway.status import _pid_exists
|
||
|
||
remaining = set(pids)
|
||
deadline = _time.monotonic() + max(timeout, 0.0)
|
||
while remaining and _time.monotonic() < deadline:
|
||
for pid in list(remaining):
|
||
try:
|
||
if not _pid_exists(pid):
|
||
remaining.discard(pid)
|
||
except Exception:
|
||
remaining.discard(pid)
|
||
if remaining:
|
||
_time.sleep(0.25)
|
||
|
||
survivors: set[int] = set()
|
||
for pid in remaining:
|
||
try:
|
||
if _pid_exists(pid):
|
||
survivors.add(pid)
|
||
except Exception:
|
||
pass
|
||
return survivors
|
||
|
||
def _venv_core_imports_healthy() -> tuple[bool, str]:
|
||
"""Probe the project venv for the core imports the backend needs to boot.
|
||
|
||
Runs a tiny import check inside the venv interpreter (NOT this process —
|
||
``hermes update`` may be driven by a different Python). Catches the
|
||
half-updated-venv state: git checkout current but a dependency sync that
|
||
failed or was killed partway (e.g. Windows access-denied on a loaded
|
||
.pyd), leaving imports like ``fastapi``'s new transitive deps missing.
|
||
Without this probe, ``hermes update`` on a current checkout prints
|
||
"Already up to date!" and returns without ever re-syncing dependencies —
|
||
the user's install stays broken no matter how many times they update
|
||
(ryanc's incident, July 2026).
|
||
|
||
Returns ``(healthy, detail)``. Never raises; unknown states report
|
||
healthy so a probe failure can't force needless reinstalls.
|
||
"""
|
||
venv_dir = _m().PROJECT_ROOT / "venv"
|
||
python_name = "python.exe" if _m()._is_windows() else "python"
|
||
bin_dir = "Scripts" if _m()._is_windows() else "bin"
|
||
venv_python = venv_dir / bin_dir / python_name
|
||
if not venv_python.exists():
|
||
# No venv interpreter at all. In a dev checkout that's normal (the
|
||
# dev may run hermes from any interpreter), so report healthy to
|
||
# avoid forcing reinstalls. But on a MANAGED install (the Windows
|
||
# installer / desktop bootstrap stamps `.hermes-bootstrap-complete`,
|
||
# and an interrupted update leaves `.update-incomplete`), the venv
|
||
# IS the install — its absence means a repair got interrupted after
|
||
# the old venv was moved aside, and "Already up to date!" would
|
||
# gaslight the user while nothing can run.
|
||
managed_markers = (
|
||
_m().PROJECT_ROOT / ".hermes-bootstrap-complete",
|
||
_m()._update_marker_path(),
|
||
)
|
||
if any(m.exists() for m in managed_markers):
|
||
return False, f"venv python missing ({venv_python})"
|
||
return True, ""
|
||
|
||
# Core web/serve imports plus their newest transitive deps. Import (not
|
||
# just metadata) — a package can have intact dist-info but a missing
|
||
# module after an interrupted uninstall/install cycle.
|
||
check = (
|
||
"import importlib\n"
|
||
"mods = ['fastapi', 'uvicorn', 'pydantic', 'openai', 'yaml']\n"
|
||
"missing = []\n"
|
||
"for m in mods:\n"
|
||
" try: importlib.import_module(m)\n"
|
||
" except Exception as e: missing.append(f'{m}: {e}')\n"
|
||
"print('\\n'.join(missing))\n"
|
||
)
|
||
try:
|
||
result = subprocess.run(
|
||
[str(venv_python), "-c", check],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=60,
|
||
cwd=_m().PROJECT_ROOT,
|
||
)
|
||
except Exception as exc:
|
||
logger.debug("venv health probe failed to run: %s", exc)
|
||
return True, ""
|
||
|
||
missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()]
|
||
if result.returncode != 0 and not missing:
|
||
# Interpreter itself is broken (e.g. deleted stdlib) — that IS unhealthy.
|
||
detail = (result.stderr or "").strip().splitlines()
|
||
return False, detail[0] if detail else "venv python failed to run"
|
||
if missing:
|
||
return False, "; ".join(missing[:4])
|
||
return True, ""
|
||
|
||
def _detect_venv_python_processes(
|
||
*, exclude_pids: set[int] | None = None
|
||
) -> list[tuple[int, str, str]]:
|
||
"""Find live processes running from the project venv's interpreter.
|
||
|
||
The hermes.exe shim guard misses the biggest lock-holder class on
|
||
Windows: the Desktop app's backend (``python.exe -m hermes_cli.main
|
||
serve``) and anything else running straight off ``venv\\Scripts\\python
|
||
(w).exe``. Those processes keep native ``.pyd`` extensions mapped, so a
|
||
dependency sync mid-update dies with access-denied and strands the venv
|
||
half-updated (ryanc's brotlicffi/_sodium.pyd incidents, July 2026).
|
||
|
||
Killing them from here is pointless — the Desktop app supervises its
|
||
backend and respawns it within seconds — so the caller should refuse and
|
||
tell the user to close the app instead. Returns ``(pid, name, cmdline)``
|
||
tuples; empty off-Windows / without psutil / when nothing matches. The
|
||
calling process and its ancestors are always excluded (a CLI ``hermes
|
||
update`` itself runs from the venv python). Never raises.
|
||
"""
|
||
if not _m()._is_windows():
|
||
return []
|
||
try:
|
||
import psutil
|
||
except Exception:
|
||
return []
|
||
|
||
venv_dir = _m().PROJECT_ROOT / "venv"
|
||
try:
|
||
venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep
|
||
except OSError:
|
||
venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep
|
||
try:
|
||
root_prefix = str(_m().PROJECT_ROOT.resolve()).lower().rstrip(os.sep) + os.sep
|
||
except OSError:
|
||
root_prefix = str(_m().PROJECT_ROOT).lower().rstrip(os.sep) + os.sep
|
||
|
||
skip: set[int] = set(exclude_pids or set())
|
||
skip.add(os.getpid())
|
||
try:
|
||
for anc in psutil.Process().parents():
|
||
skip.add(int(anc.pid))
|
||
except Exception:
|
||
pass
|
||
|
||
matches: list[tuple[int, str, str]] = []
|
||
try:
|
||
proc_iter = psutil.process_iter(["pid", "exe", "name", "cmdline", "cwd"])
|
||
except Exception:
|
||
return []
|
||
for proc in proc_iter:
|
||
try:
|
||
info = proc.info
|
||
except Exception:
|
||
continue
|
||
pid = info.get("pid")
|
||
exe = info.get("exe")
|
||
if not exe or pid is None or int(pid) in skip:
|
||
continue
|
||
try:
|
||
exe_norm = str(Path(exe).resolve()).lower()
|
||
except (OSError, ValueError):
|
||
exe_norm = str(exe).lower()
|
||
cmdline_raw = " ".join(info.get("cmdline") or [])
|
||
cmdline_low = cmdline_raw.lower()
|
||
cwd_low = str(info.get("cwd") or "").lower().rstrip(os.sep) + os.sep
|
||
|
||
# Primary match: the executable itself lives under this venv
|
||
# (venv\Scripts\python(w).exe — the desktop backend / gateway case).
|
||
is_holder = exe_norm.startswith(venv_prefix)
|
||
# Fallback: uv/base-interpreter trampolines run a python whose exe is
|
||
# OUTSIDE the venv but which still imports from it and holds its .pyd
|
||
# files. Catch those by what they're running: a cmdline that references
|
||
# this venv's path, or a `-m hermes_cli.main ...` invocation tied to
|
||
# this install (install root in the cmdline or as the working dir).
|
||
if not is_holder and venv_prefix in cmdline_low:
|
||
is_holder = True
|
||
if not is_holder and "hermes_cli.main" in cmdline_low:
|
||
if root_prefix in cmdline_low or cwd_low.startswith(root_prefix):
|
||
is_holder = True
|
||
if not is_holder:
|
||
continue
|
||
name = info.get("name") or Path(exe).name
|
||
matches.append((int(pid), str(name), cmdline_raw[:120]))
|
||
return matches
|
||
|
||
def _format_venv_python_holders_message(matches: list[tuple[int, str, str]]) -> str:
|
||
"""Explain which venv processes block the update and how to clear them."""
|
||
lines = [
|
||
"✗ Other Hermes processes are running from this install's venv:",
|
||
]
|
||
for pid, name, cmdline in matches[:6]:
|
||
hint = ""
|
||
low = cmdline.lower()
|
||
if "serve" in low or "dashboard" in low:
|
||
hint = " ← Hermes Desktop backend (close the desktop app)"
|
||
elif "gateway" in low:
|
||
hint = " ← gateway"
|
||
lines.append(f" PID {pid} {name} {cmdline}{hint}")
|
||
if len(matches) > 6:
|
||
lines.append(f" ... and {len(matches) - 6} more")
|
||
lines.append("")
|
||
lines.append(
|
||
" On Windows these keep native extension files (.pyd) locked, so the"
|
||
)
|
||
lines.append(
|
||
" dependency update would fail partway and leave a broken install."
|
||
)
|
||
lines.append(
|
||
" Close the Hermes desktop app / other Hermes terminals, then re-run:"
|
||
)
|
||
lines.append(" hermes update")
|
||
lines.append(" (or use `hermes update --force-venv` to proceed anyway at your own risk)")
|
||
return "\n".join(lines)
|
||
|
||
def _pause_windows_gateways_for_update() -> dict | None:
|
||
"""Stop running Windows gateways before mutating the checkout or venv.
|
||
|
||
Windows scheduled/startup gateways run through pythonw.exe, so the generic
|
||
hermes.exe concurrent-instance guard does not see them. They still import
|
||
from the checkout and can keep files locked while ``git`` or ``uv`` updates
|
||
the install. Stop only PIDs that the gateway discovery code identifies.
|
||
"""
|
||
if not _m()._is_windows():
|
||
return None
|
||
|
||
try:
|
||
from gateway.status import terminate_pid
|
||
from hermes_cli.gateway import (
|
||
_capture_gateway_argv,
|
||
_get_restart_drain_timeout,
|
||
find_gateway_pids,
|
||
find_profile_gateway_processes,
|
||
)
|
||
except Exception as exc:
|
||
logger.debug("Could not prepare Windows gateway pause for update: %s", exc)
|
||
return None
|
||
|
||
try:
|
||
running_pids = list(dict.fromkeys(find_gateway_pids(all_profiles=True)))
|
||
except Exception as exc:
|
||
logger.debug("Could not discover Windows gateway PIDs before update: %s", exc)
|
||
return None
|
||
if not running_pids:
|
||
# No gateway is running right now, but the user may have installed an
|
||
# autostart entry (Scheduled Task or Startup-folder login item) — that
|
||
# is an explicit "I want a gateway" signal. A gateway that died between
|
||
# updates (e.g. the spawning terminal/TUI closed, taking its child with
|
||
# it) would otherwise never come back: the autostart entry only fires on
|
||
# the next login, and the update flow's resume path only relaunched
|
||
# gateways that were running when the update began. Cold-start one after
|
||
# the update so an installed gateway is actually up post-update. Users
|
||
# who run gateway-less (no autostart entry) get nothing forced on them.
|
||
try:
|
||
from hermes_cli import gateway_windows
|
||
|
||
if gateway_windows.is_installed():
|
||
return {
|
||
"resume_needed": True,
|
||
"profiles": {},
|
||
"unmapped_pids": [],
|
||
"unmapped": [],
|
||
"cold_start_if_installed": True,
|
||
}
|
||
except Exception as exc:
|
||
logger.debug(
|
||
"Could not check Windows gateway autostart state before update: %s",
|
||
exc,
|
||
)
|
||
return None
|
||
|
||
profile_processes = {}
|
||
try:
|
||
profile_processes = {
|
||
proc.pid: proc for proc in find_profile_gateway_processes()
|
||
}
|
||
except Exception as exc:
|
||
logger.debug("Could not map Windows gateway PIDs to profiles: %s", exc)
|
||
|
||
profiles: dict[str, int] = {}
|
||
mapped_pids = []
|
||
for pid in running_pids:
|
||
proc = profile_processes.get(pid)
|
||
if proc is None:
|
||
continue
|
||
profiles[str(proc.profile)] = int(pid)
|
||
mapped_pids.append(int(pid))
|
||
_write_update_planned_stop_marker(Path(proc.path), int(pid))
|
||
|
||
print("→ Stopping Windows gateway process(es) before updating Hermes...")
|
||
try:
|
||
drain_timeout = max(float(_get_restart_drain_timeout()), 1.0)
|
||
except Exception:
|
||
drain_timeout = 10.0
|
||
survivors = _m()._wait_for_windows_update_gateway_exit(
|
||
mapped_pids,
|
||
timeout=drain_timeout,
|
||
)
|
||
unmapped_pids = [pid for pid in running_pids if pid not in profile_processes]
|
||
|
||
# Snapshot each unmapped gateway's command line *before* we force-kill it,
|
||
# so ``_resume_windows_gateways_after_update`` can respawn it by replaying
|
||
# its own argv. Unmapped gateways are ones with no profile→PID-file mapping
|
||
# — e.g. a Windows Scheduled Task running ``pythonw.exe -m hermes_cli.main
|
||
# gateway run``. Without this snapshot they were force-killed and never
|
||
# restarted (the "Restart manually after update" dead-end from #50090).
|
||
unmapped: list[dict] = []
|
||
for pid in unmapped_pids:
|
||
argv = None
|
||
try:
|
||
argv = _capture_gateway_argv(int(pid))
|
||
except Exception as exc:
|
||
logger.debug("Could not capture argv for unmapped gateway %s: %s", pid, exc)
|
||
unmapped.append({"pid": int(pid), "argv": argv})
|
||
|
||
force_killed = []
|
||
for pid in sorted(set(survivors).union(unmapped_pids)):
|
||
try:
|
||
terminate_pid(int(pid), force=True)
|
||
force_killed.append(int(pid))
|
||
except (ProcessLookupError, PermissionError, OSError):
|
||
pass
|
||
|
||
if profiles:
|
||
print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}")
|
||
if force_killed:
|
||
print(f" → Force-stopped {len(force_killed)} gateway process(es)")
|
||
|
||
if unmapped_pids:
|
||
respawnable = sum(1 for u in unmapped if u.get("argv"))
|
||
print(
|
||
f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping"
|
||
)
|
||
if respawnable < len(unmapped_pids):
|
||
# Some had no recoverable command line (psutil missing, access
|
||
# denied, already gone): those still need a manual restart.
|
||
print(" Restart manually after update: hermes gateway run")
|
||
|
||
return {
|
||
"resume_needed": True,
|
||
"profiles": profiles,
|
||
"unmapped_pids": unmapped_pids,
|
||
"unmapped": unmapped,
|
||
}
|
||
|
||
def _cold_start_windows_gateway_after_update() -> None:
|
||
"""Start a fresh detached gateway after update when one is installed but down.
|
||
|
||
Invoked from ``_resume_windows_gateways_after_update`` for the
|
||
``cold_start_if_installed`` case: no gateway was running when the update
|
||
began, but an autostart entry (Scheduled Task / Startup-folder login item)
|
||
is installed, signalling the user wants a gateway. Unlike the relaunch
|
||
paths — which watch an old PID and respawn once it exits — this is a direct
|
||
fresh spawn via the same hidden-console + breakaway path that
|
||
``hermes gateway start`` uses (``gateway_windows._spawn_detached``).
|
||
|
||
Best-effort and idempotent: re-checks that nothing is running first so a
|
||
concurrent start (e.g. the autostart entry firing) can't produce a
|
||
duplicate gateway.
|
||
"""
|
||
if not _m()._is_windows():
|
||
return
|
||
try:
|
||
from hermes_cli import gateway_windows
|
||
from hermes_cli.gateway import find_gateway_pids
|
||
except Exception as exc:
|
||
logger.debug("Could not load Windows gateway cold-start helpers: %s", exc)
|
||
return
|
||
|
||
# Re-check liveness right before spawning — between pause and resume the
|
||
# autostart entry may have already brought a gateway up, or a leftover
|
||
# process may have re-registered. Don't double-start.
|
||
try:
|
||
if list(find_gateway_pids(all_profiles=True)):
|
||
return
|
||
except Exception as exc:
|
||
logger.debug("Could not re-check gateway liveness before cold-start: %s", exc)
|
||
return
|
||
|
||
try:
|
||
pid = gateway_windows._spawn_detached()
|
||
except Exception as exc:
|
||
logger.debug("Could not cold-start Windows gateway after update: %s", exc)
|
||
return
|
||
|
||
if pid:
|
||
print()
|
||
print(f" ✓ Starting Windows gateway after update (PID {pid})")
|
||
|
||
def _for_each_systemd_gateway_unit(
|
||
list_units_stdout: str,
|
||
*,
|
||
process_unit,
|
||
on_unit_timeout,
|
||
) -> None:
|
||
"""Process each ``hermes-gateway*.service`` from ``systemctl list-units``.
|
||
|
||
``subprocess.TimeoutExpired`` raised by ``process_unit`` is isolated to
|
||
that unit via ``on_unit_timeout`` so one wedged systemctl call cannot
|
||
abort the rest of the fleet (#68523).
|
||
"""
|
||
for line in (list_units_stdout or "").strip().splitlines():
|
||
parts = line.split()
|
||
if not parts:
|
||
continue
|
||
unit = parts[0]
|
||
if not unit.endswith(".service"):
|
||
continue
|
||
# list-units is already pattern-filtered, but keep the name gate so a
|
||
# stray non-gateway line cannot enter the restart path.
|
||
if not unit.startswith("hermes-gateway"):
|
||
continue
|
||
svc_name = unit.removesuffix(".service")
|
||
try:
|
||
process_unit(svc_name)
|
||
except subprocess.TimeoutExpired as exc:
|
||
on_unit_timeout(svc_name, exc)
|
||
|
||
def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None:
|
||
"""Print an explicit incomplete-update warning for unrestarted units."""
|
||
if not failed_units:
|
||
return
|
||
# Preserve discovery order while de-duplicating.
|
||
seen = set()
|
||
ordered = []
|
||
for name in failed_units:
|
||
if name in seen:
|
||
continue
|
||
seen.add(name)
|
||
ordered.append(name)
|
||
print()
|
||
print("⚠ Update incomplete — some gateway units were not restarted:")
|
||
for name in ordered:
|
||
print(f" - {name}")
|
||
print(" Skipped units may still be running pre-update code (mixed")
|
||
print(" sys.modules). Restart them manually, then verify:")
|
||
print(" hermes gateway status")
|
||
print(" systemctl --user restart <unit> # user-scope")
|
||
print(" sudo systemctl restart <unit> # system-scope")
|
||
|
||
def _refresh_windows_gateway_launchers() -> None:
|
||
"""Regenerate installed Windows gateway launcher scripts after update.
|
||
|
||
The Scheduled Task / Startup-folder launchers (``gateway.cmd`` +
|
||
``gateway.vbs``) are persistence artifacts written once at install time —
|
||
``hermes update`` never touched them, so installs created before the
|
||
hidden-console rework (aa2ae36c3f) kept launching the gateway through
|
||
``pythonw.exe`` forever: every descendant spawn flashed a conhost
|
||
(#54220/#56747) and, since #70344, the console-less gateway died at
|
||
startup with ``RuntimeError: sys.stderr is None`` (#71671).
|
||
|
||
The task's /TR points at a stable script path, so rewriting the files in
|
||
place retargets the task without any schtasks call (no UAC needed).
|
||
``_write_task_script`` is idempotent and renders from current code, so
|
||
this is a no-op for modern installs. Best-effort: a failed refresh must
|
||
never fail the update.
|
||
"""
|
||
if not _m()._is_windows():
|
||
return
|
||
try:
|
||
from hermes_cli import gateway_windows
|
||
|
||
if not gateway_windows.is_installed():
|
||
return
|
||
gateway_windows._write_task_script()
|
||
print(" ✓ Refreshed Windows gateway launcher scripts")
|
||
except Exception as exc:
|
||
logger.debug("Could not refresh Windows gateway launchers after update: %s", exc)
|
||
|
||
def _resume_windows_gateways_after_update(token: dict | None) -> None:
|
||
"""Restart Windows profile gateways previously paused for update."""
|
||
if not token or not token.get("resume_needed"):
|
||
return
|
||
token["resume_needed"] = False
|
||
if not _m()._is_windows():
|
||
return
|
||
|
||
# Regenerate the persisted launcher scripts before respawning anything,
|
||
# so a legacy pythonw-era Scheduled Task / Startup entry comes back on
|
||
# the current hidden-console design at the next login too.
|
||
_m()._refresh_windows_gateway_launchers()
|
||
|
||
profiles = token.get("profiles") or {}
|
||
unmapped = token.get("unmapped") or []
|
||
cold_start = bool(token.get("cold_start_if_installed"))
|
||
if not profiles and not any(u.get("argv") for u in unmapped):
|
||
if cold_start:
|
||
_m()._cold_start_windows_gateway_after_update()
|
||
return
|
||
|
||
try:
|
||
from hermes_cli.gateway import (
|
||
launch_detached_gateway_restart_by_cmdline,
|
||
launch_detached_profile_gateway_restart,
|
||
)
|
||
except Exception as exc:
|
||
logger.debug("Could not load Windows gateway restart helper: %s", exc)
|
||
return
|
||
|
||
relaunched = []
|
||
for profile, old_pid in sorted(profiles.items()):
|
||
try:
|
||
if launch_detached_profile_gateway_restart(str(profile), int(old_pid)):
|
||
relaunched.append(str(profile))
|
||
except Exception as exc:
|
||
logger.debug(
|
||
"Could not restart Windows gateway profile %s after update: %s",
|
||
profile,
|
||
exc,
|
||
)
|
||
|
||
# Respawn unmapped gateways (no profile→PID-file mapping, e.g. a Scheduled
|
||
# Task) by replaying the argv we snapshotted before force-killing them.
|
||
unmapped_relaunched = 0
|
||
for entry in unmapped:
|
||
argv = entry.get("argv")
|
||
old_pid = entry.get("pid")
|
||
if not argv or not old_pid:
|
||
continue
|
||
try:
|
||
if launch_detached_gateway_restart_by_cmdline(int(old_pid), list(argv)):
|
||
unmapped_relaunched += 1
|
||
except Exception as exc:
|
||
logger.debug(
|
||
"Could not restart unmapped Windows gateway (pid %s) after update: %s",
|
||
old_pid,
|
||
exc,
|
||
)
|
||
|
||
if relaunched:
|
||
print()
|
||
print(f" ✓ Restarting Windows gateway profile(s): {', '.join(relaunched)}")
|
||
if unmapped_relaunched:
|
||
if not relaunched:
|
||
print()
|
||
print(
|
||
f" ✓ Restarting {unmapped_relaunched} unmapped Windows gateway process(es)"
|
||
)
|
||
|
||
def _discard_lockfile_churn(git_cmd, repo_root):
|
||
"""Restore tracked ``package-lock.json`` files that npm dirtied locally.
|
||
|
||
npm rewrites lockfiles non-deterministically at install/build time. On a
|
||
managed install those diffs are never intentional, so we discard them so
|
||
``hermes update`` sees a clean tree instead of autostashing every run.
|
||
Best-effort; only ever touches files named ``package-lock.json``.
|
||
"""
|
||
try:
|
||
diff = subprocess.run(
|
||
git_cmd + ["diff", "--name-only"],
|
||
cwd=repo_root,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if diff.returncode != 0:
|
||
return
|
||
dirty_package_dirs = {
|
||
Path(line.strip()).parent
|
||
for line in diff.stdout.splitlines()
|
||
if line.strip().endswith("package.json")
|
||
}
|
||
dirty = [
|
||
line.strip()
|
||
for line in diff.stdout.splitlines()
|
||
if line.strip().endswith("package-lock.json")
|
||
and Path(line.strip()).parent not in dirty_package_dirs
|
||
]
|
||
if not dirty:
|
||
return
|
||
subprocess.run(
|
||
git_cmd + ["checkout", "--", *dirty],
|
||
cwd=repo_root,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
check=False,
|
||
)
|
||
print(f"→ Discarded npm lockfile churn ({len(dirty)} file(s))")
|
||
except Exception:
|
||
# Never let lockfile cleanup block an update.
|
||
pass
|
||
|
||
def _cmd_update_impl(args, gateway_mode: bool):
|
||
"""Body of ``cmd_update`` — kept separate so the wrapper can always
|
||
restore stdio even on ``sys.exit``."""
|
||
# In gateway mode, use file-based IPC for prompts instead of stdin
|
||
gw_input_fn = (
|
||
(lambda prompt, default="": _gateway_prompt(prompt, default))
|
||
if gateway_mode
|
||
else None
|
||
)
|
||
assume_yes = bool(getattr(args, "yes", False))
|
||
|
||
# Whether this update is running without a human at the keyboard.
|
||
# Interactive terminal updates always stash-and-ask (unchanged behavior);
|
||
# only non-interactive updates (desktop/chat app, gateway, `--yes`) consult
|
||
# the `updates.non_interactive_local_changes` config setting to decide
|
||
# whether to auto-restore stashed local source changes or throw them away.
|
||
_non_interactive_update = (
|
||
gateway_mode
|
||
or assume_yes
|
||
or not (sys.stdin.isatty() and sys.stdout.isatty())
|
||
)
|
||
discard_local_changes = False
|
||
if _non_interactive_update:
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
|
||
_update_cfg = (load_config() or {}).get("updates", {})
|
||
if isinstance(_update_cfg, dict):
|
||
_mode = str(_update_cfg.get("non_interactive_local_changes", "stash")).lower()
|
||
discard_local_changes = _mode == "discard"
|
||
except Exception as exc:
|
||
# Never let a config read failure change the safe default.
|
||
logger.debug("Could not read updates.non_interactive_local_changes: %s", exc)
|
||
discard_local_changes = False
|
||
|
||
print("⚕ Updating Hermes Agent...")
|
||
print()
|
||
|
||
# On Windows, abort early if another hermes.exe is holding the venv shim
|
||
# open. Continuing would result in a string of WinError 32 warnings and
|
||
# then either a deferred-rename leftover or a failed git-pull fast path
|
||
# that silently falls back to the slower ZIP route. See issue #26670.
|
||
if _m()._is_windows() and not getattr(args, "force", False):
|
||
scripts_dir = _m()._venv_scripts_dir()
|
||
if scripts_dir is not None:
|
||
concurrent = _m()._detect_concurrent_hermes_instances(scripts_dir)
|
||
if concurrent:
|
||
print(_format_concurrent_instances_message(concurrent, scripts_dir))
|
||
sys.exit(2)
|
||
|
||
# Pre-update backup — runs before any git/file mutation so users can
|
||
# always roll back to the exact state they had before this update.
|
||
# Returns the quick-snapshot id (or None when disabled/failed); the
|
||
# post-update cron-jobs safety net uses it to detect job loss.
|
||
pre_update_snapshot_id = _m()._run_pre_update_backup(args)
|
||
|
||
_windows_gateway_resume = _m()._pause_windows_gateways_for_update()
|
||
if _windows_gateway_resume:
|
||
import atexit as _atexit
|
||
|
||
_atexit.register(
|
||
_m()._resume_windows_gateways_after_update,
|
||
_windows_gateway_resume,
|
||
)
|
||
|
||
# With gateways paused, anything still running from the venv interpreter
|
||
# (most commonly the Desktop app's `hermes serve` backend) will keep .pyd
|
||
# files locked and corrupt the dependency sync below. Refuse rather than
|
||
# race: killing the desktop backend is futile (the app supervises and
|
||
# respawns it), so the user must close the app. Deliberately NOT bypassed
|
||
# by plain --force: the desktop bootstrap updater passes --force to skip
|
||
# the hermes.exe shim guard above, but its lock probe only checks the shim
|
||
# and app.asar — a non-desktop venv python holding a .pyd would sail
|
||
# through and corrupt the sync (the exact failure this guard exists for).
|
||
# --force-venv is the explicit escape hatch.
|
||
if _m()._is_windows() and not getattr(args, "force_venv", False):
|
||
_venv_holders = _m()._detect_venv_python_processes()
|
||
if _venv_holders:
|
||
print(_format_venv_python_holders_message(_venv_holders))
|
||
_m()._resume_windows_gateways_after_update(_windows_gateway_resume)
|
||
sys.exit(2)
|
||
|
||
# Try git-based update first, fall back to ZIP download on Windows
|
||
# when git file I/O is broken (antivirus, NTFS filter drivers, etc.)
|
||
use_zip_update = False
|
||
git_dir = _m().PROJECT_ROOT / ".git"
|
||
|
||
if not git_dir.exists():
|
||
if sys.platform == "win32":
|
||
use_zip_update = True
|
||
else:
|
||
print("✗ Not a git repository. Please reinstall:")
|
||
print(
|
||
" curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash"
|
||
)
|
||
sys.exit(1)
|
||
|
||
# On Windows, git can fail with "unable to write loose object file: Invalid argument"
|
||
# due to filesystem atomicity issues. Set the recommended workaround.
|
||
if sys.platform == "win32" and git_dir.exists():
|
||
subprocess.run(
|
||
[
|
||
"git",
|
||
"-c",
|
||
"windows.appendAtomically=false",
|
||
"config",
|
||
"windows.appendAtomically",
|
||
"false",
|
||
],
|
||
cwd=_m().PROJECT_ROOT,
|
||
check=False,
|
||
capture_output=True,
|
||
)
|
||
|
||
# Build git command once — reused for fork detection and the update itself.
|
||
git_cmd = ["git"]
|
||
if sys.platform == "win32":
|
||
git_cmd = ["git", "-c", "windows.appendAtomically=false"]
|
||
|
||
# Discard npm lockfile churn before any stash/branch logic. npm rewrites
|
||
# tracked package-lock.json files non-deterministically at install/build
|
||
# time (platform-specific optional deps, ideallyInert annotations, etc.),
|
||
# which is never an intentional edit on a managed install but leaves the
|
||
# tree dirty — forcing an autostash on every update and making branch
|
||
# switches fragile. Restoring them first lets the common case (only
|
||
# lockfile churn) update with a clean tree.
|
||
_discard_lockfile_churn(git_cmd, _m().PROJECT_ROOT)
|
||
|
||
# Detect if we're updating from a fork (before any branch logic)
|
||
origin_url = _m()._get_origin_url(git_cmd, _m().PROJECT_ROOT)
|
||
is_fork = _is_fork(origin_url)
|
||
|
||
if is_fork:
|
||
print("⚠ Updating from fork:")
|
||
print(f" {origin_url}")
|
||
print()
|
||
|
||
if use_zip_update:
|
||
# ZIP-based update for Windows when git is broken
|
||
try:
|
||
_update_via_zip(args)
|
||
finally:
|
||
_m()._resume_windows_gateways_after_update(_windows_gateway_resume)
|
||
return
|
||
|
||
# Fetch and pull
|
||
try:
|
||
|
||
# Resolve the target branch up front so the fetch can be scoped to it.
|
||
# A bare `git fetch origin` pulls every ref, and this repo carries
|
||
# thousands of auto-generated branches — an unscoped fetch can stall for
|
||
# minutes on a non-single-branch checkout. Fetch only what we update
|
||
# against.
|
||
branch = _m()._resolve_update_branch(args)
|
||
|
||
print("→ Fetching updates...")
|
||
fetch_result = subprocess.run(
|
||
git_cmd + ["fetch", "origin", branch],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if fetch_result.returncode != 0:
|
||
stderr = fetch_result.stderr.strip()
|
||
if "Could not resolve host" in stderr or "unable to access" in stderr:
|
||
print("✗ Network error — cannot reach the remote repository.")
|
||
print(f" {stderr.splitlines()[0]}" if stderr else "")
|
||
elif (
|
||
"Authentication failed" in stderr or "could not read Username" in stderr
|
||
):
|
||
print(
|
||
"✗ Authentication failed — check your git credentials or SSH key."
|
||
)
|
||
else:
|
||
print("✗ Failed to fetch updates from origin.")
|
||
if stderr:
|
||
print(f" {stderr.splitlines()[0]}")
|
||
sys.exit(1)
|
||
|
||
# Get current branch (returns literal "HEAD" when detached)
|
||
result = subprocess.run(
|
||
git_cmd + ["rev-parse", "--abbrev-ref", "HEAD"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
check=True,
|
||
)
|
||
current_branch = result.stdout.strip()
|
||
|
||
# If user is on a different branch than the update target, switch
|
||
# to the target. When the target is "main" this is the historical
|
||
# "always update against main" behavior; for any other target it's
|
||
# the same thing — get HEAD onto the requested branch first, then
|
||
# fast-forward.
|
||
if current_branch != branch:
|
||
label = (
|
||
"detached HEAD"
|
||
if current_branch == "HEAD"
|
||
else f"branch '{current_branch}'"
|
||
)
|
||
print(f" ⚠ Currently on {label} — switching to {branch} for update...")
|
||
# Stash before checkout so uncommitted work isn't lost
|
||
auto_stash_ref = _m()._stash_local_changes_if_needed(git_cmd, _m().PROJECT_ROOT)
|
||
checkout_result = subprocess.run(
|
||
git_cmd + ["checkout", branch],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if checkout_result.returncode != 0:
|
||
# Local checkout doesn't have this branch yet. Try to set
|
||
# it up as a tracking branch of origin/<branch>. This is
|
||
# the common case when the requested branch exists upstream
|
||
# but was never checked out locally.
|
||
track_result = subprocess.run(
|
||
git_cmd + ["checkout", "-B", branch, f"origin/{branch}"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if track_result.returncode != 0:
|
||
# Restore the user's prior branch + stash before bailing
|
||
# so we don't leave them stranded in a weird state.
|
||
if auto_stash_ref is not None:
|
||
_m()._restore_stashed_changes(
|
||
git_cmd,
|
||
_m().PROJECT_ROOT,
|
||
auto_stash_ref,
|
||
prompt_user=False,
|
||
input_fn=gw_input_fn,
|
||
)
|
||
print(f"✗ Branch '{branch}' does not exist locally or on origin.")
|
||
if track_result.stderr.strip():
|
||
print(f" {track_result.stderr.strip().splitlines()[0]}")
|
||
sys.exit(1)
|
||
else:
|
||
auto_stash_ref = _m()._stash_local_changes_if_needed(git_cmd, _m().PROJECT_ROOT)
|
||
|
||
prompt_for_restore = (
|
||
auto_stash_ref is not None
|
||
and not assume_yes
|
||
and (gateway_mode or (sys.stdin.isatty() and sys.stdout.isatty()))
|
||
)
|
||
|
||
# Check if there are updates
|
||
result = subprocess.run(
|
||
git_cmd + ["rev-list", f"HEAD..origin/{branch}", "--count"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
check=True,
|
||
)
|
||
commit_count = int(result.stdout.strip())
|
||
|
||
if commit_count == 0:
|
||
_invalidate_update_cache()
|
||
|
||
# Even if origin is up to date, the fork may be behind upstream
|
||
if is_fork and branch == "main":
|
||
_m()._sync_with_upstream_if_needed(git_cmd, _m().PROJECT_ROOT)
|
||
|
||
# Restore stash and switch back to original branch if we moved
|
||
if auto_stash_ref is not None:
|
||
_m()._restore_stashed_changes(
|
||
git_cmd,
|
||
_m().PROJECT_ROOT,
|
||
auto_stash_ref,
|
||
prompt_user=prompt_for_restore,
|
||
input_fn=gw_input_fn,
|
||
)
|
||
if current_branch not in {branch, "HEAD"}:
|
||
subprocess.run(
|
||
git_cmd + ["checkout", current_branch],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
check=False,
|
||
)
|
||
|
||
# "No new commits" does not mean the managed interpreter is safe.
|
||
# uv can retain the same CPython patch while python-build-standalone
|
||
# refreshes the embedded SQLite underneath it. Keep the existing
|
||
# update-boundary hook active on this retry path too.
|
||
from hermes_cli.managed_uv import ensure_uv, update_managed_uv
|
||
|
||
runtime_repairs = []
|
||
update_managed_uv(repair_observer=runtime_repairs.append)
|
||
ensure_uv(repair_observer=runtime_repairs.append)
|
||
runtime_repaired = next(
|
||
(result for result in runtime_repairs if result.repaired),
|
||
None,
|
||
)
|
||
|
||
# A current checkout does NOT imply a healthy install: a previous
|
||
# dependency sync may have failed partway (classic on Windows,
|
||
# where a running gateway/desktop backend keeps .pyd files locked
|
||
# and uv/pip dies with access-denied, stranding the venv between
|
||
# versions). Probe the venv's core imports and repair if broken —
|
||
# otherwise "Already up to date!" gaslights the user while their
|
||
# install stays bricked.
|
||
healthy, detail = _venv_core_imports_healthy()
|
||
if not healthy:
|
||
print("⚠ Checkout is current, but the venv is unhealthy:")
|
||
print(f" {detail}")
|
||
print("→ Repairing Python dependencies...")
|
||
_write_update_incomplete_marker()
|
||
from hermes_cli.managed_uv import ensure_uv
|
||
|
||
repair_uv = ensure_uv()
|
||
# A managed install whose venv is gone entirely (interrupted
|
||
# repair after the old venv was moved aside) needs the venv
|
||
# recreated before dependencies can be installed into it.
|
||
venv_python_missing = not (
|
||
_m().PROJECT_ROOT
|
||
/ "venv"
|
||
/ ("Scripts" if _m()._is_windows() else "bin")
|
||
/ ("python.exe" if _m()._is_windows() else "python")
|
||
).exists()
|
||
if venv_python_missing and repair_uv:
|
||
print("→ Recreating virtual environment...")
|
||
subprocess.run(
|
||
[repair_uv, "venv", "venv"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
check=False,
|
||
)
|
||
if repair_uv:
|
||
repair_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")}
|
||
_m()._install_python_dependencies_with_optional_fallback(
|
||
[repair_uv, "pip"], env=repair_env, group="all"
|
||
)
|
||
else:
|
||
_m()._install_python_dependencies_with_optional_fallback(
|
||
[sys.executable, "-m", "pip"], group="all"
|
||
)
|
||
_m()._clear_update_incomplete_marker()
|
||
healthy_after, detail_after = _venv_core_imports_healthy()
|
||
if healthy_after:
|
||
print("✓ Dependencies repaired!")
|
||
else:
|
||
print(f"⚠ Venv still unhealthy after repair: {detail_after}")
|
||
print(" Close all Hermes windows/gateways and re-run: hermes update")
|
||
else:
|
||
print("✓ Already up to date!")
|
||
if runtime_repaired is not None and not _m()._is_windows():
|
||
print()
|
||
print(
|
||
"⚠ Restart required to finish the managed Python runtime repair."
|
||
)
|
||
print(
|
||
" Any running Hermes gateways, Desktop backends, or other "
|
||
"long-lived processes still use the previous runtime."
|
||
)
|
||
print(
|
||
" Restart each of them before removing the parked venv"
|
||
+ (
|
||
f": {runtime_repaired.backup_venv}"
|
||
if runtime_repaired.backup_venv is not None
|
||
else "."
|
||
)
|
||
)
|
||
_m()._resume_windows_gateways_after_update(_windows_gateway_resume)
|
||
return
|
||
|
||
print(f"→ Found {commit_count} new commit(s)")
|
||
|
||
print("→ Pulling updates...")
|
||
update_succeeded = False
|
||
# Capture the pre-pull SHA so we can auto-roll-back if the new code
|
||
# has a syntax error in a critical-path file (PR #28452 incident:
|
||
# orphan merge-conflict markers in hermes_cli/config.py bricked
|
||
# every user who ran ``hermes update`` for the 7 minutes between
|
||
# the bad commit and the fix landing).
|
||
pre_pull_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT)
|
||
try:
|
||
# Merge the ref we already fetched above (→ Fetching updates...)
|
||
# instead of `git pull`, which performs a SECOND network fetch of
|
||
# the same branch (~0.5-1.5 s of redundant round-trip per update).
|
||
# `merge --ff-only origin/<branch>` is byte-identical in effect to
|
||
# `pull --ff-only origin <branch>` given the fresh tracking ref;
|
||
# the divergence fallback below is unchanged.
|
||
pull_result = subprocess.run(
|
||
git_cmd + ["merge", "--ff-only", f"origin/{branch}"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if pull_result.returncode != 0:
|
||
# ff-only failed — local and remote have diverged (e.g. upstream
|
||
# force-pushed or rebase). Since local changes are already
|
||
# stashed, reset to match the remote exactly.
|
||
print(
|
||
" ⚠ Fast-forward not possible (history diverged), resetting to match remote..."
|
||
)
|
||
reset_result = subprocess.run(
|
||
git_cmd + ["reset", "--hard", f"origin/{branch}"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if reset_result.returncode != 0:
|
||
print(f"✗ Failed to reset to origin/{branch}.")
|
||
if reset_result.stderr.strip():
|
||
print(f" {reset_result.stderr.strip()}")
|
||
print(
|
||
f" Try manually: git fetch origin && git reset --hard origin/{branch}"
|
||
)
|
||
sys.exit(1)
|
||
|
||
# Post-pull syntax guard: validate critical-path files actually
|
||
# parse before declaring the update successful. If a bad commit
|
||
# made it through CI (e.g. admin-merge bypass of a failing
|
||
# ruff check), this catches it on the user side and rolls back
|
||
# so the CLI stays bootable. The user can then retry ``hermes
|
||
# update`` later once a fix lands upstream.
|
||
syntax_ok, failing_path, syntax_error = _validate_critical_files_syntax(
|
||
_m().PROJECT_ROOT
|
||
)
|
||
if not syntax_ok:
|
||
print()
|
||
print("✗ Pulled code has a syntax error in a critical file:")
|
||
print(f" {failing_path}")
|
||
if syntax_error:
|
||
# py_compile errors can be multi-line; show the first
|
||
# ~6 lines so the user sees the actual SyntaxError text.
|
||
for line in str(syntax_error).splitlines()[:6]:
|
||
print(f" {line}")
|
||
if pre_pull_sha:
|
||
print()
|
||
print(f"→ Rolling back to {pre_pull_sha[:10]}...")
|
||
rollback_result = subprocess.run(
|
||
git_cmd + ["reset", "--hard", pre_pull_sha],
|
||
cwd=_m().PROJECT_ROOT,
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
)
|
||
if rollback_result.returncode == 0:
|
||
print(" ✓ Rollback complete — your install is unchanged.")
|
||
print(" Try ``hermes update`` again later once a fix lands.")
|
||
else:
|
||
print(" ✗ Rollback failed. Recover manually with:")
|
||
print(f" cd {_m().PROJECT_ROOT} && git reset --hard {pre_pull_sha}")
|
||
if rollback_result.stderr.strip():
|
||
print(f" ({rollback_result.stderr.strip().splitlines()[0]})")
|
||
else:
|
||
print()
|
||
print(" Could not capture pre-pull SHA — recover manually with:")
|
||
print(f" cd {_m().PROJECT_ROOT} && git reflog && git reset --hard <prev-sha>")
|
||
sys.exit(1)
|
||
|
||
update_succeeded = True
|
||
finally:
|
||
if auto_stash_ref is not None:
|
||
# Don't attempt stash restore if the code update itself failed —
|
||
# working tree is in an unknown state.
|
||
if not update_succeeded:
|
||
print(
|
||
f" ℹ️ Local changes preserved in stash (ref: {auto_stash_ref})"
|
||
)
|
||
print(" Restore manually with: git stash apply")
|
||
elif discard_local_changes:
|
||
# Non-interactive update + user opted into discarding local
|
||
# source edits (updates.non_interactive_local_changes:
|
||
# discard). Throw the stash away instead of re-applying it.
|
||
_m()._discard_stashed_changes(
|
||
git_cmd,
|
||
_m().PROJECT_ROOT,
|
||
auto_stash_ref,
|
||
)
|
||
else:
|
||
_m()._restore_stashed_changes(
|
||
git_cmd,
|
||
_m().PROJECT_ROOT,
|
||
auto_stash_ref,
|
||
prompt_user=prompt_for_restore,
|
||
input_fn=gw_input_fn,
|
||
)
|
||
|
||
_invalidate_update_cache()
|
||
|
||
# Clear stale .pyc bytecode cache — prevents ImportError on gateway
|
||
# restart when updated source references names that didn't exist in
|
||
# the old bytecode (e.g. get_hermes_home added to hermes_constants).
|
||
removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT)
|
||
if removed:
|
||
print(
|
||
f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}"
|
||
)
|
||
_m()._record_bytecode_fingerprint()
|
||
|
||
# Fork upstream sync logic (only for main branch on forks)
|
||
if is_fork and branch == "main":
|
||
_m()._sync_with_upstream_if_needed(git_cmd, _m().PROJECT_ROOT)
|
||
|
||
# Reinstall Python dependencies. Prefer .[all], but if one optional extra
|
||
# breaks on this machine, keep base deps and reinstall the remaining extras
|
||
# individually so update does not silently strip working capabilities.
|
||
#
|
||
# Drop the core-install breadcrumb BEFORE touching the venv. If the
|
||
# install is killed mid-flight (Ctrl-C, terminal close, WSL OOM), the
|
||
# marker survives and the next ``hermes`` launch finishes the install
|
||
# via ``_recover_from_interrupted_install``. Cleared after the core
|
||
# ``.[all]`` install completes — lazy refresh uses a separate marker.
|
||
_write_update_incomplete_marker()
|
||
print("→ Updating Python dependencies...")
|
||
from hermes_cli.managed_uv import ensure_uv, update_managed_uv
|
||
|
||
# Keep managed uv current — runs `uv self update` if we already have one.
|
||
update_managed_uv()
|
||
|
||
uv_bin = ensure_uv()
|
||
|
||
pip_cmd = [sys.executable, "-m", "pip"]
|
||
if not uv_bin:
|
||
uv_bin = _ensure_uv_for_termux(pip_cmd)
|
||
install_group = "all"
|
||
|
||
if uv_bin:
|
||
uv_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")}
|
||
if _m()._is_termux_env(uv_env):
|
||
uv_env.pop("PYTHONPATH", None)
|
||
uv_env.pop("PYTHONHOME", None)
|
||
install_group = "termux-all"
|
||
print(" → Termux detected: using uv + curated termux-all optional profile...")
|
||
if _m()._is_termux_env(uv_env) and _is_android_python():
|
||
print(" → Termux/Android detected: prebuilding psutil with Linux source path compatibility...")
|
||
_install_psutil_android_compat([uv_bin, "pip"], env=uv_env)
|
||
_m()._install_python_dependencies_with_optional_fallback(
|
||
[uv_bin, "pip"], env=uv_env, group=install_group
|
||
)
|
||
else:
|
||
# Use sys.executable to explicitly call the venv's pip module,
|
||
# avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu.
|
||
# Some environments lose pip inside the venv; bootstrap it back with
|
||
# ensurepip before trying the editable install.
|
||
pip_cmd = [sys.executable, "-m", "pip"]
|
||
try:
|
||
subprocess.run(
|
||
pip_cmd + ["--version"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
check=True,
|
||
capture_output=True,
|
||
)
|
||
except subprocess.CalledProcessError:
|
||
subprocess.run(
|
||
[sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"],
|
||
cwd=_m().PROJECT_ROOT,
|
||
check=True,
|
||
)
|
||
if _m()._is_termux_env():
|
||
install_group = "termux-all"
|
||
print(" → Termux detected: using curated termux-all optional profile...")
|
||
if _m()._is_termux_env() and _is_android_python():
|
||
print(" → Termux/Android detected: prebuilding psutil with Linux source path compatibility...")
|
||
_install_psutil_android_compat(pip_cmd)
|
||
_m()._install_python_dependencies_with_optional_fallback(pip_cmd, group=install_group)
|
||
|
||
install_prefix = [uv_bin, "pip"] if uv_bin else pip_cmd
|
||
lazy_env = uv_env if uv_bin else None
|
||
|
||
# Core ``.[all]`` install finished. Clear the generic core breadcrumb
|
||
# before the lazy-refresh phase — that phase uses its own marker so a
|
||
# later lazy failure cannot be "healed" by clearing the core marker
|
||
# based on a narrow 7-package import probe (#58004 review).
|
||
_m()._clear_update_incomplete_marker()
|
||
|
||
# The update process is still the old Python interpreter process. Run
|
||
# one final cache/module refresh immediately before lazy backend
|
||
# refresh, which imports newly-pulled modules that may depend on fresh
|
||
# symbols in hermes_constants or lazy_deps. The dependency install
|
||
# above may also have regenerated bytecode from build-cache copies —
|
||
# this second sweep catches those stragglers (#60242, #65240).
|
||
removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT)
|
||
if removed:
|
||
print(
|
||
f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}"
|
||
)
|
||
_m()._record_bytecode_fingerprint()
|
||
_m()._reload_updated_runtime_modules()
|
||
|
||
# Upgrade pip before lazy refreshes — stale pip can fail source builds
|
||
# and leave partially-written packages (#57828).
|
||
_write_lazy_refresh_incomplete_marker()
|
||
_m()._upgrade_pip_before_lazy_refresh(install_prefix, env=lazy_env)
|
||
|
||
# Lazy refresh can corrupt the venv when a backend install fails.
|
||
# Clear the lazy marker only when refresh/repair is confirmed healthy.
|
||
lazy_ok = _m()._refresh_active_lazy_features(install_prefix, env=lazy_env)
|
||
if lazy_ok:
|
||
_m()._clear_lazy_refresh_incomplete_marker()
|
||
else:
|
||
print(
|
||
" ⚠ Lazy-refresh recovery incomplete — run `hermes` again "
|
||
"to finish import-based venv repair."
|
||
)
|
||
|
||
# Heal the active memory provider's bridge packages last — the core
|
||
# reinstall + lazy refresh above may have stripped or downgraded
|
||
# plugin.yaml-declared deps that aren't in extras (#53272, #70636).
|
||
_m()._refresh_active_memory_provider_dependencies()
|
||
|
||
node_failures = _update_node_dependencies()
|
||
_m()._build_web_ui(_m().PROJECT_ROOT / "web")
|
||
|
||
# Rebuild the desktop app if the source tree changed since the last
|
||
# build. ``hermes desktop --build-only`` uses the content-hash stamp
|
||
# internally, so this is effectively a no-op when nothing changed.
|
||
# Only bother if the user has a desktop app installed (indicated by
|
||
# an existing packaged executable or desktop dist); people who have
|
||
# never run ``hermes desktop`` shouldn't be forced into a full
|
||
# Electron build by ``hermes update``.
|
||
desktop_dir = _m().PROJECT_ROOT / "apps" / "desktop"
|
||
has_desktop_app = _m()._desktop_packaged_executable(desktop_dir) is not None or _m()._desktop_dist_exists(desktop_dir)
|
||
if (desktop_dir / "package.json").exists() and _m()._resolve_node_runtime_npm() and has_desktop_app:
|
||
print("→ Checking if desktop app needs rebuilding...")
|
||
# Consult the content-hash stamp IN-PROCESS first. The spawned
|
||
# `hermes desktop --build-only` subprocess re-imports the whole
|
||
# CLI stack (~1-3 s) just to reach the same _m()._desktop_build_needed
|
||
# check; when the stamp already says "up to date" we can skip the
|
||
# spawn entirely. The update path never passes --source, so the
|
||
# subprocess would run with source_mode=False — mirror that here.
|
||
# Any error in the pre-check falls through to the subprocess.
|
||
_skip_desktop_build = False
|
||
try:
|
||
_skip_desktop_build = not _m()._desktop_build_needed(
|
||
desktop_dir, _m().PROJECT_ROOT, source_mode=False
|
||
)
|
||
except Exception:
|
||
_skip_desktop_build = False
|
||
if _skip_desktop_build:
|
||
print(" ✓ Desktop app up to date")
|
||
else:
|
||
_desktop_build_cmd = [sys.executable, "-m", "hermes_cli.main", "desktop", "--build-only"]
|
||
# Capture the (very loud) Electron/vite build output into
|
||
# update.log instead of streaming it to the terminal. On the rare
|
||
# nonzero exit, retry once after waiting again for the venv — this
|
||
# covers a still-settling rebuild window the first wait didn't fully
|
||
# catch — then surface the captured tail so the failure is
|
||
# debuggable.
|
||
#
|
||
# Start the build subprocess with the Hermes-managed Node on PATH:
|
||
# when `hermes update` runs inside the desktop updater chain
|
||
# (Desktop → hermes-setup → hermes update), the shell PATH
|
||
# customizations are lost, so a bare-PATH child would fail with
|
||
# `node: not found` before cmd_gui can self-heal.
|
||
from hermes_constants import with_hermes_node_path
|
||
|
||
_build_env = with_hermes_node_path()
|
||
build_result = _m()._run_logged_subprocess(_desktop_build_cmd, cwd=_m().PROJECT_ROOT, env=_build_env)
|
||
if build_result.returncode != 0:
|
||
build_result = _m()._run_logged_subprocess(_desktop_build_cmd, cwd=_m().PROJECT_ROOT, env=_build_env)
|
||
if build_result.returncode != 0:
|
||
print(" ⚠ Desktop build failed (non-fatal; run `hermes desktop` to retry)")
|
||
tail = "\n".join((build_result.stdout or "").strip().splitlines()[-15:])
|
||
if tail:
|
||
print(tail)
|
||
from hermes_constants import display_hermes_home as _dhh
|
||
print(f" Full build log: {_dhh()}/logs/update.log")
|
||
else:
|
||
print(" ✓ Desktop app up to date")
|
||
|
||
print()
|
||
print("✓ Code updated!")
|
||
|
||
# ── Post-update state.db integrity guard (#68474) ─────────────────
|
||
# Verify that state.db survived the update intact. If the live file
|
||
# is now corrupted (zeroed, missing header, integrity failure),
|
||
# automatically restore from the pre-update snapshot rather than
|
||
# letting the user discover silently that their sessions are gone.
|
||
try:
|
||
from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity
|
||
|
||
_state_path = get_hermes_home() / "state.db"
|
||
if _state_path.exists():
|
||
_state_ok = verify_sqlite_integrity(
|
||
_state_path,
|
||
check_header=True,
|
||
run_pragma=True,
|
||
)
|
||
if _state_ok.get("valid"):
|
||
logger.debug(
|
||
"Post-update state.db integrity check: %s",
|
||
_state_ok.get("message"),
|
||
)
|
||
else:
|
||
print()
|
||
print(
|
||
"⚠ state.db is corrupted after update: "
|
||
+ _state_ok.get("message", "unknown error")
|
||
)
|
||
_pre_snap_id = pre_update_snapshot_id
|
||
if _pre_snap_id:
|
||
_snap_state = (
|
||
_quick_snapshot_root(get_hermes_home())
|
||
/ _pre_snap_id
|
||
/ "state.db"
|
||
)
|
||
if _snap_state.exists():
|
||
_snap_ok = verify_sqlite_integrity(
|
||
_snap_state, check_header=True, run_pragma=True
|
||
)
|
||
if _snap_ok.get("valid"):
|
||
try:
|
||
import shutil as _shutil
|
||
|
||
_shutil.copy2(_snap_state, _state_path)
|
||
_restored_ok = verify_sqlite_integrity(
|
||
_state_path,
|
||
check_header=True,
|
||
run_pragma=True,
|
||
)
|
||
if _restored_ok.get("valid"):
|
||
print(
|
||
" ✓ Auto-restored from pre-update "
|
||
f"snapshot ({_pre_snap_id})"
|
||
)
|
||
else:
|
||
print(
|
||
" ✗ Auto-restore FAILED — restored "
|
||
"copy also failed integrity"
|
||
)
|
||
except OSError as _exc:
|
||
print(
|
||
f" ✗ Auto-restore file copy failed: {_exc}"
|
||
)
|
||
else:
|
||
print(
|
||
" ✗ Pre-update snapshot also failed integrity"
|
||
)
|
||
else:
|
||
print(
|
||
" ⚠ Pre-update snapshot does not contain state.db"
|
||
)
|
||
else:
|
||
print(" ⚠ No pre-update snapshot was taken")
|
||
print()
|
||
except Exception as exc:
|
||
logger.debug("Post-update state.db integrity check failed: %s", exc)
|
||
|
||
# Seed the model-catalog disk cache from the freshly-pulled checkout.
|
||
# The repo ships the canonical catalog at
|
||
# website/static/api/model-catalog.json, and `git pull` just made it
|
||
# current — so copy it straight over ~/.hermes/cache/model_catalog.json
|
||
# instead of waiting on a network fetch (which can be bot-gated or hit a
|
||
# Portal hiccup). Keeps the model picker's curated/free lists in sync
|
||
# with the version the user just installed. Non-fatal on failure: the
|
||
# normal network refresh still applies on the next picker open.
|
||
try:
|
||
from hermes_cli.model_catalog import seed_cache_from_checkout
|
||
|
||
if seed_cache_from_checkout(_m().PROJECT_ROOT):
|
||
print(" ✓ Model catalog cache refreshed from checkout")
|
||
except Exception as e:
|
||
logger.debug("Model catalog seed during update failed: %s", e)
|
||
|
||
# Sync bundled skills (copies new, updates changed, respects user deletions)
|
||
try:
|
||
from tools.skills_sync import sync_skills
|
||
|
||
print()
|
||
print("→ Syncing bundled skills...")
|
||
result = sync_skills(quiet=True)
|
||
if result["copied"]:
|
||
print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}")
|
||
if result.get("updated"):
|
||
print(
|
||
f" ↑ {len(result['updated'])} updated: {', '.join(result['updated'])}"
|
||
)
|
||
if result.get("user_modified"):
|
||
print(f" ~ {len(result['user_modified'])} user-modified (kept)")
|
||
print(
|
||
" → see them: hermes skills list-modified "
|
||
"(diff/reset to resume updates)"
|
||
)
|
||
if result.get("cleaned"):
|
||
print(f" − {len(result['cleaned'])} removed from manifest")
|
||
if result.get("relocated"):
|
||
print(
|
||
f" → {len(result['relocated'])} moved to new upstream paths: "
|
||
f"{', '.join(result['relocated'])}"
|
||
)
|
||
if not result["copied"] and not result.get("updated"):
|
||
print(" ✓ Skills are up to date")
|
||
except Exception as e:
|
||
logger.debug("Skills sync during update failed: %s", e)
|
||
|
||
# Sync bundled skills to all profiles (including the active one).
|
||
# seed_profile_skills() uses subprocess with an explicit HERMES_HOME so
|
||
# it is not affected by sync_skills()'s module-level HERMES_HOME cache,
|
||
# which means the active profile is reliably synced regardless of whether
|
||
# the caller's HERMES_HOME env var points at the default or a named profile.
|
||
try:
|
||
from hermes_cli.profiles import (
|
||
list_profiles,
|
||
seed_profile_skills,
|
||
)
|
||
|
||
all_profiles = list_profiles()
|
||
if all_profiles:
|
||
print()
|
||
print("→ Syncing bundled skills to all profiles...")
|
||
for p in all_profiles:
|
||
try:
|
||
r = seed_profile_skills(p.path, quiet=True)
|
||
if r and r.get("skipped_opt_out"):
|
||
status = "opted out (--no-skills)"
|
||
elif r:
|
||
copied = len(r.get("copied", []))
|
||
updated = len(r.get("updated", []))
|
||
modified = len(r.get("user_modified", []))
|
||
parts = []
|
||
if copied:
|
||
parts.append(f"+{copied} new")
|
||
if updated:
|
||
parts.append(f"↑{updated} updated")
|
||
if modified:
|
||
parts.append(f"~{modified} user-modified")
|
||
status = ", ".join(parts) if parts else "up to date"
|
||
else:
|
||
status = "sync failed"
|
||
print(f" {p.name}: {status}")
|
||
except Exception as pe:
|
||
print(f" {p.name}: error ({pe})")
|
||
except Exception:
|
||
pass # profiles module not available or no profiles
|
||
|
||
# Backfill per-profile .env files for profiles created before the
|
||
# .env-seeding fix (#44792). Copies the default install's .env so
|
||
# those profiles keep the credentials they were effectively using.
|
||
try:
|
||
from hermes_cli.profiles import backfill_profile_envs
|
||
|
||
backfilled = backfill_profile_envs(quiet=True)
|
||
if backfilled:
|
||
print()
|
||
print(
|
||
f"→ Seeded .env for {len(backfilled)} profile(s) "
|
||
f"(copied from default): {', '.join(backfilled)}"
|
||
)
|
||
except Exception:
|
||
pass # profiles module not available or no profiles
|
||
|
||
# Sync Honcho host blocks to all profiles
|
||
try:
|
||
from plugins.memory.honcho.cli import sync_honcho_profiles_quiet
|
||
|
||
synced = sync_honcho_profiles_quiet()
|
||
if synced:
|
||
print(f"\n-> Honcho: synced {synced} profile(s)")
|
||
except Exception:
|
||
pass # honcho plugin not installed or not configured
|
||
|
||
# Check for config migrations
|
||
print()
|
||
print("→ Checking configuration for new options...")
|
||
|
||
from hermes_cli.config import (
|
||
get_missing_env_vars,
|
||
get_missing_config_fields,
|
||
check_config_version,
|
||
migrate_config,
|
||
)
|
||
|
||
missing_env = get_missing_env_vars(required_only=True)
|
||
missing_config = get_missing_config_fields()
|
||
current_ver, latest_ver = check_config_version()
|
||
|
||
has_new_options = bool(missing_env or missing_config)
|
||
version_bump_only = (
|
||
not has_new_options and current_ver < latest_ver
|
||
)
|
||
needs_migration = has_new_options or current_ver < latest_ver
|
||
|
||
if version_bump_only:
|
||
# Nothing for the user to fill in — only the config format version
|
||
# changed (new defaults already merge in transparently). Asking
|
||
# "configure new options now?" here is misleading: saying yes just
|
||
# bumps the version and looks like a no-op (issue: ScottFive /
|
||
# Tt2021). Apply it silently and say what actually happened.
|
||
print()
|
||
print(
|
||
f" ℹ Updating config format (v{current_ver} → v{latest_ver})…"
|
||
)
|
||
try:
|
||
migrate_config(interactive=False, quiet=True)
|
||
print(" ✓ Config format updated (no new settings to configure)")
|
||
except Exception as _mig_err:
|
||
print(f" ⚠️ Config format update failed: {_mig_err}")
|
||
print(" Run 'hermes config migrate' to retry.")
|
||
elif needs_migration:
|
||
print()
|
||
# Show WHAT changed, not just a count, so the user can make an
|
||
# informed yes/no decision (previously the prompt named nothing).
|
||
def _print_items(items, label, key, fallback_key=None):
|
||
if not items:
|
||
return
|
||
print(f" {label}:")
|
||
shown = items[:8]
|
||
for it in shown:
|
||
if isinstance(it, dict):
|
||
name = it.get(key) or (fallback_key and it.get(fallback_key)) or "?"
|
||
desc = (it.get("description") or "").strip()
|
||
else:
|
||
# Defensive: some callers/mocks pass bare name strings.
|
||
name = str(it)
|
||
desc = ""
|
||
if desc:
|
||
print(f" • {name} — {desc}")
|
||
else:
|
||
print(f" • {name}")
|
||
extra = len(items) - len(shown)
|
||
if extra > 0:
|
||
print(f" … and {extra} more")
|
||
|
||
if missing_env:
|
||
print(
|
||
f" ⚠️ {len(missing_env)} new required setting(s) need configuration"
|
||
)
|
||
_print_items(missing_env, "New settings", "name")
|
||
if missing_config:
|
||
print(f" ℹ️ {len(missing_config)} new config option(s) available")
|
||
_print_items(missing_config, "New options", "key")
|
||
|
||
print()
|
||
if assume_yes:
|
||
print(
|
||
" ℹ --yes: auto-applying config migration (skipping API-key prompts)."
|
||
)
|
||
response = "y"
|
||
elif gateway_mode:
|
||
response = (
|
||
_gateway_prompt(
|
||
"Would you like to configure new options now? [Y/n]", "n"
|
||
)
|
||
.strip()
|
||
.lower()
|
||
)
|
||
elif not (sys.stdin.isatty() and sys.stdout.isatty()):
|
||
print(" ℹ Non-interactive session — applying safe config migrations.")
|
||
response = "auto"
|
||
else:
|
||
try:
|
||
response = (
|
||
input("Would you like to configure them now? [Y/n]: ")
|
||
.strip()
|
||
.lower()
|
||
)
|
||
except EOFError:
|
||
response = "n"
|
||
|
||
if response in {"", "y", "yes", "auto"}:
|
||
print()
|
||
# Gateway mode, --yes, and non-interactive update contexts
|
||
# (dashboard / web server actions) cannot prompt for API keys.
|
||
# Still run the non-interactive migration pass before restarting
|
||
# so new default config fields and version bumps are written
|
||
# before the freshly updated gateway validates config at startup.
|
||
interactive_migration = not (
|
||
gateway_mode or assume_yes or response == "auto"
|
||
)
|
||
results = migrate_config(interactive=interactive_migration, quiet=False)
|
||
|
||
if results["env_added"] or results["config_added"]:
|
||
print()
|
||
print("✓ Configuration updated!")
|
||
if (gateway_mode or assume_yes or response == "auto") and missing_env:
|
||
print(" ℹ API keys require manual entry: hermes config migrate")
|
||
else:
|
||
print()
|
||
print("Skipped. Run 'hermes config migrate' later to configure.")
|
||
else:
|
||
print(" ✓ Configuration is up to date")
|
||
|
||
# Safety net: config-version migrations have been observed to leave
|
||
# cron/jobs.json valid-but-empty, silently dropping every scheduled
|
||
# job (issue #34600). The desktop scheduler can also overwrite with
|
||
# its own small set, causing partial loss (issue #52144). If the
|
||
# live file now has fewer jobs than the pre-update snapshot, restore
|
||
# it and warn loudly.
|
||
try:
|
||
from hermes_cli.backup import restore_cron_jobs_if_emptied
|
||
|
||
cron_restore = restore_cron_jobs_if_emptied(pre_update_snapshot_id)
|
||
if cron_restore:
|
||
print()
|
||
print(
|
||
" ⚠️ cron/jobs.json lost jobs during this update — "
|
||
f"restored {cron_restore['job_count']} job(s) from "
|
||
f"pre-update snapshot {cron_restore['snapshot_id']}."
|
||
)
|
||
except Exception as exc:
|
||
# Never let the cron safety net break an otherwise-good update.
|
||
logger.debug("Cron jobs auto-restore check failed: %s", exc)
|
||
|
||
print()
|
||
if node_failures:
|
||
print(
|
||
"⚠ Update partially complete — Node.js dependencies for "
|
||
f"{', '.join(node_failures)} did not refresh."
|
||
)
|
||
print(" Code and Python deps are updated, but the dashboard/TUI may")
|
||
print(" be in a mixed state until the Node deps are rebuilt.")
|
||
else:
|
||
print("✓ Update complete!")
|
||
|
||
# Search-index optimization notice (v23). Existing installs keep their
|
||
# working search index untouched on update; the compact v23 layout —
|
||
# which reclaims a large fraction of state.db on heavy users — is
|
||
# opt-in. Surface it here (the moment the user is already thinking
|
||
# about their install) with the exact command and the concrete size
|
||
# win. Show-once-ish: only when a legacy index is actually present.
|
||
try:
|
||
_print_fts_optimize_available_notice()
|
||
except Exception as e:
|
||
logger.debug("FTS optimize notice failed: %s", e)
|
||
|
||
# Curator first-run heads-up. Only prints when curator is enabled AND
|
||
# has never run — i.e. the window where the ticker would otherwise
|
||
# have fired against a fresh skill library. Kept silent on steady
|
||
# state so we don't nag.
|
||
try:
|
||
_print_curator_first_run_notice()
|
||
except Exception as e:
|
||
logger.debug("Curator first-run notice failed: %s", e)
|
||
|
||
# Most-recent curator run notice — show-once per run. Surfaces the
|
||
# rename map (`old-name → umbrella`) on the high-attention update
|
||
# surface so users learn about consolidations without having to
|
||
# check `hermes curator status`. Self-stamps after printing so it
|
||
# never repeats for the same run.
|
||
try:
|
||
_print_curator_recent_run_notice()
|
||
except Exception as e:
|
||
logger.debug("Curator recent-run notice failed: %s", e)
|
||
|
||
# Repair RHEL-family root installs where /usr/local/bin isn't on PATH
|
||
# for non-login interactive shells. No-op on every other platform.
|
||
try:
|
||
_ensure_fhs_path_guard()
|
||
except Exception as e:
|
||
logger.debug("FHS PATH guard check failed: %s", e)
|
||
|
||
# Self-heal the hermes-acp launcher for installs that predate it, so
|
||
# ACP hosts (Zed, JetBrains, Buzz) can resolve Hermes on PATH without
|
||
# a reinstall. No-op on Windows and when already present.
|
||
try:
|
||
_ensure_acp_launcher()
|
||
except Exception as e:
|
||
logger.debug("hermes-acp launcher self-heal failed: %s", e)
|
||
|
||
# Refresh the cua-driver binary used by the Computer Use toolset.
|
||
# The upstream installer is gated on supported platforms and on the
|
||
# binary already being on PATH, so this is a no-op for users who
|
||
# don't have it. Tying the refresh to ``hermes update`` gives users a
|
||
# predictable cadence (matches when they pull new agent code) without
|
||
# adding startup latency or a per-launch GitHub API call.
|
||
try:
|
||
refresh_cua_driver = True
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
|
||
_update_cfg = (load_config() or {}).get("updates", {})
|
||
if isinstance(_update_cfg, dict):
|
||
refresh_cua_driver = bool(
|
||
_update_cfg.get("refresh_cua_driver", True)
|
||
)
|
||
except Exception as cfg_exc:
|
||
logger.debug("Could not read updates.refresh_cua_driver: %s", cfg_exc)
|
||
|
||
if (
|
||
refresh_cua_driver
|
||
and sys.platform in ("darwin", "win32", "linux")
|
||
and shutil.which("cua-driver")
|
||
):
|
||
from hermes_cli.tools_config import install_cua_driver
|
||
|
||
print()
|
||
print("→ Refreshing cua-driver (Computer Use)...")
|
||
# require_confirmed_update: only run the (multi-minute,
|
||
# silent) upstream installer when the driver's native
|
||
# check-update verb positively reports a newer release.
|
||
# An indeterminate check (offline, rate-limited, old
|
||
# driver) keeps the installed version — `hermes update`
|
||
# must stay fast; `hermes computer-use install --upgrade`
|
||
# remains the force path.
|
||
install_cua_driver(
|
||
upgrade=True,
|
||
require_confirmed_update=True,
|
||
show_installer_progress=False,
|
||
)
|
||
except Exception as e:
|
||
logger.debug("cua-driver refresh failed: %s", e)
|
||
|
||
# Write exit code *before* the gateway restart attempt.
|
||
# When running as ``hermes update --gateway`` (spawned by the gateway's
|
||
# /update command), this process lives inside the gateway's systemd
|
||
# cgroup. A graceful SIGUSR1 restart keeps the drain loop alive long
|
||
# enough for the exit-code marker to be written below, but the
|
||
# fallback ``systemctl restart`` path (see below) kills everything in
|
||
# the cgroup (KillMode=mixed → SIGKILL to remaining processes),
|
||
# including us and the wrapping bash shell. The shell never reaches
|
||
# its ``printf $status > .update_exit_code`` epilogue, so the
|
||
# exit-code marker file would never be created. The new gateway's
|
||
# update watcher would then poll for 30 minutes and send a spurious
|
||
# timeout message.
|
||
#
|
||
# Writing the marker here — after git pull + pip install succeed but
|
||
# before we attempt the restart — ensures the new gateway sees it
|
||
# regardless of how we die.
|
||
if gateway_mode:
|
||
_exit_code_path = get_hermes_home() / ".update_exit_code"
|
||
try:
|
||
_exit_code_path.write_text("0", encoding="utf-8")
|
||
except OSError:
|
||
pass
|
||
|
||
gateway_fleet_restart_incomplete = False
|
||
|
||
# Auto-restart ALL gateways after update.
|
||
# The code update (git pull) is shared across all profiles, so every
|
||
# running gateway needs restarting to pick up the new code.
|
||
try:
|
||
from hermes_cli.gateway import (
|
||
is_macos,
|
||
supports_systemd_services,
|
||
_ensure_user_systemd_env,
|
||
find_gateway_pids,
|
||
find_profile_gateway_processes,
|
||
_prepare_profile_gateway_update_restart,
|
||
_get_service_pids,
|
||
_graceful_restart_via_sigusr1,
|
||
_wait_for_gateway_exit,
|
||
)
|
||
import signal as _signal
|
||
|
||
def _wait_for_service_active(
|
||
scope_cmd_: list,
|
||
svc_name_: str,
|
||
timeout: float = 10.0,
|
||
) -> bool:
|
||
"""Poll ``systemctl is-active`` until the unit reports active.
|
||
|
||
systemd's Stopped -> Started transition after a graceful exit
|
||
(or a hard restart) is not instantaneous; a one-shot check
|
||
races that window and falsely reports the unit as down.
|
||
Poll every 0.5s up to ``timeout`` seconds before giving up.
|
||
"""
|
||
deadline = _time.monotonic() + max(timeout, 0.5)
|
||
while True:
|
||
try:
|
||
_verify = subprocess.run(
|
||
scope_cmd_ + ["is-active", svc_name_],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=5,
|
||
)
|
||
if _verify.stdout.strip() == "active":
|
||
return True
|
||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||
pass
|
||
if _time.monotonic() >= deadline:
|
||
return False
|
||
_time.sleep(0.5)
|
||
|
||
def _service_restart_sec(
|
||
scope_cmd_: list,
|
||
svc_name_: str,
|
||
default: float = 0.0,
|
||
) -> float:
|
||
"""Read the unit's ``RestartUSec`` (RestartSec) in seconds.
|
||
|
||
After a graceful exit-75, systemd waits ``RestartSec`` before
|
||
respawning the unit. Callers that poll for ``is-active``
|
||
must use a timeout >= ``RestartSec`` + transition slack, or
|
||
they'll give up *during* the cooldown window and wrongly
|
||
conclude the unit didn't relaunch.
|
||
"""
|
||
try:
|
||
_show = subprocess.run(
|
||
scope_cmd_
|
||
+ [
|
||
"show",
|
||
svc_name_,
|
||
"--property=RestartUSec",
|
||
"--value",
|
||
],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=5,
|
||
)
|
||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||
return default
|
||
raw = (_show.stdout or "").strip()
|
||
# systemd emits values like "30s", "100ms", "1min 30s", or
|
||
# "infinity". Parse conservatively; on any miss return default.
|
||
if not raw or raw == "infinity":
|
||
return default
|
||
total = 0.0
|
||
matched = False
|
||
for part in raw.split():
|
||
for _suf, _mult in (
|
||
("ms", 0.001),
|
||
("us", 0.000001),
|
||
("min", 60.0),
|
||
("s", 1.0),
|
||
):
|
||
if part.endswith(_suf):
|
||
try:
|
||
total += float(part[: -len(_suf)]) * _mult
|
||
matched = True
|
||
except ValueError:
|
||
pass
|
||
break
|
||
return total if matched else default
|
||
|
||
_manage_cmd_cache: dict = {}
|
||
|
||
def _resolve_manage_cmd(scope_: str, scope_cmd_: list, svc_name_: str):
|
||
"""Resolve the command prefix for manage-units operations.
|
||
|
||
Read-only systemctl calls (``is-active``, ``show``,
|
||
``list-units``) work unprivileged, but manage-units verbs
|
||
(``reset-failed``, ``start``, ``restart``) on a *system*
|
||
service trigger a polkit ``org.freedesktop.systemd1.manage-units``
|
||
authentication prompt when run as a non-root user. That
|
||
interactive prompt runs inside our captured subprocess with a
|
||
10-15s timeout — the user sees the prompt flash and "exit
|
||
directly" before they can answer, and the resulting
|
||
TimeoutExpired used to be swallowed silently.
|
||
|
||
Strategy: if root, plain systemctl. If not root, try
|
||
non-interactive sudo (``sudo -n``) — first a blanket probe,
|
||
then a targeted ``systemctl reset-failed`` probe so a
|
||
least-privilege sudoers entry scoped to
|
||
``systemctl ... hermes-gateway*`` also qualifies
|
||
(``reset-failed`` is an idempotent no-op we run before every
|
||
privileged restart anyway). If neither works, return None —
|
||
the caller must SKIP the restart (without draining the
|
||
gateway first!) and tell the user how to restart manually.
|
||
``--no-ask-password`` guarantees polkit can never hang a
|
||
captured subprocess on this path.
|
||
"""
|
||
if scope_ in _manage_cmd_cache:
|
||
return _manage_cmd_cache[scope_]
|
||
cmd = scope_cmd_ + ["--no-ask-password"]
|
||
if (
|
||
scope_ == "system"
|
||
and hasattr(os, "geteuid")
|
||
and os.geteuid() != 0 # windows-footgun: ok — systemd path, Linux-only
|
||
):
|
||
sudo_cmd = ["sudo", "-n"] + scope_cmd_ + ["--no-ask-password"]
|
||
sudo_ok = False
|
||
try:
|
||
_probe = subprocess.run(
|
||
["sudo", "-n", "true"],
|
||
capture_output=True,
|
||
timeout=5,
|
||
)
|
||
sudo_ok = _probe.returncode == 0
|
||
if not sudo_ok:
|
||
# Blanket sudo refused — a targeted sudoers entry
|
||
# (NOPASSWD for systemctl ... hermes-gateway*)
|
||
# may still allow the exact commands we need.
|
||
_probe = subprocess.run(
|
||
sudo_cmd + ["reset-failed", svc_name_],
|
||
capture_output=True,
|
||
timeout=5,
|
||
)
|
||
sudo_ok = _probe.returncode == 0
|
||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||
sudo_ok = False
|
||
cmd = sudo_cmd if sudo_ok else None
|
||
_manage_cmd_cache[scope_] = cmd
|
||
return cmd
|
||
|
||
# Drain budget for graceful SIGUSR1 restarts. The gateway drains
|
||
# for up to ``agent.restart_drain_timeout`` (default 60s) before
|
||
# exiting with code 75; we wait slightly longer so the drain
|
||
# completes before we fall back to a hard restart. On older
|
||
# systemd units without SIGUSR1 wiring this wait just times out
|
||
# and we fall back to ``systemctl restart`` (the old behaviour).
|
||
try:
|
||
from hermes_constants import (
|
||
DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT as _DEFAULT_DRAIN,
|
||
)
|
||
except Exception:
|
||
_DEFAULT_DRAIN = 60.0
|
||
_cfg_drain = None
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
|
||
_cfg_agent = load_config().get("agent") or {}
|
||
_cfg_drain = _cfg_agent.get("restart_drain_timeout")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
_drain_budget = (
|
||
float(_cfg_drain)
|
||
if _cfg_drain is not None
|
||
else float(_DEFAULT_DRAIN)
|
||
)
|
||
except (TypeError, ValueError):
|
||
_drain_budget = float(_DEFAULT_DRAIN)
|
||
# Add a 15s margin so the drain loop + final exit finish before
|
||
# we escalate to ``systemctl restart`` / SIGTERM.
|
||
_drain_budget = max(_drain_budget, 30.0) + 15.0
|
||
|
||
restarted_services = []
|
||
failed_or_stale_units = []
|
||
killed_pids = set()
|
||
relaunched_profiles = []
|
||
externally_supervised_profiles = []
|
||
|
||
# --- Systemd services (Linux) ---
|
||
# Discover all hermes-gateway* units (default + profiles)
|
||
if supports_systemd_services():
|
||
try:
|
||
_ensure_user_systemd_env()
|
||
except Exception:
|
||
pass
|
||
|
||
for scope, scope_cmd in [
|
||
("user", ["systemctl", "--user"]),
|
||
("system", ["systemctl"]),
|
||
]:
|
||
try:
|
||
result = subprocess.run(
|
||
scope_cmd
|
||
+ [
|
||
"list-units",
|
||
"hermes-gateway*",
|
||
"--plain",
|
||
"--no-legend",
|
||
"--no-pager",
|
||
],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=10,
|
||
)
|
||
except FileNotFoundError:
|
||
continue
|
||
except subprocess.TimeoutExpired as exc:
|
||
# Discovery timeout — skip this scope, keep the other.
|
||
print(
|
||
f" ⚠ systemctl timed out listing {scope}-scope "
|
||
f"gateway units ({exc.cmd if exc.cmd else 'unknown command'}). "
|
||
f"Check the gateway with: hermes gateway status"
|
||
)
|
||
continue
|
||
|
||
def _restart_one_systemd_gateway_unit(svc_name: str) -> None:
|
||
# Check if active
|
||
check = subprocess.run(
|
||
scope_cmd + ["is-active", svc_name],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=5,
|
||
)
|
||
if check.stdout.strip() != "active":
|
||
return
|
||
|
||
# Resolve how we may run manage-units verbs
|
||
# (reset-failed/start/restart) for this scope.
|
||
# None ⇒ no non-interactive privilege path; we
|
||
# must avoid those verbs entirely or polkit will
|
||
# throw an interactive auth prompt inside our
|
||
# captured 10-15s subprocess (the user sees it
|
||
# flash and "exit directly" — reported June 2026).
|
||
_manage_cmd = _resolve_manage_cmd(
|
||
scope, scope_cmd, svc_name
|
||
)
|
||
|
||
# Prefer a graceful SIGUSR1 restart so in-flight
|
||
# agent runs drain instead of being SIGKILLed.
|
||
# The gateway's SIGUSR1 handler calls
|
||
# request_restart(via_service=True) → drain →
|
||
# exit; systemd's Restart=always respawns the unit.
|
||
_main_pid = 0
|
||
try:
|
||
_show = subprocess.run(
|
||
scope_cmd
|
||
+ [
|
||
"show",
|
||
svc_name,
|
||
"--property=MainPID",
|
||
"--value",
|
||
],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=5,
|
||
)
|
||
_main_pid = int((_show.stdout or "").strip() or 0)
|
||
except (
|
||
ValueError,
|
||
subprocess.TimeoutExpired,
|
||
FileNotFoundError,
|
||
):
|
||
_main_pid = 0
|
||
|
||
_graceful_ok = False
|
||
if _main_pid > 0:
|
||
print(
|
||
f" → {svc_name}: draining (up to {int(_drain_budget)}s)..."
|
||
)
|
||
_graceful_ok = _graceful_restart_via_sigusr1(
|
||
_main_pid,
|
||
drain_timeout=_drain_budget,
|
||
)
|
||
|
||
if _graceful_ok:
|
||
# Gateway exited after a planned restart.
|
||
# ``Restart=always`` means systemd WILL respawn
|
||
# the unit — but only after
|
||
# ``RestartSec`` (default 60s on our unit
|
||
# file). That 60s wait is a crash-loop guard,
|
||
# and is the right default when the gateway
|
||
# dies unexpectedly. For a voluntary restart
|
||
# on update, it's dead time the user watches.
|
||
#
|
||
# Shortcut it: ``reset-failed`` + ``start``
|
||
# skips RestartSec entirely (we're manually
|
||
# initiating the unit, not waiting for
|
||
# systemd's auto-restart logic). Takes about
|
||
# as long as the process takes to come up
|
||
# (~1-3s on a warm box).
|
||
#
|
||
# If the unit is already active because
|
||
# RestartSec elapsed while we were draining,
|
||
# ``start`` is a no-op and we fall through to
|
||
# the poll below. Either way we collapse the
|
||
# 60s+ delay to a ~5s one.
|
||
#
|
||
# The shortcut needs manage-units privileges.
|
||
# Without them (system service, non-root, no
|
||
# passwordless sudo) skip it — systemd's own
|
||
# auto-restart still relaunches the unit after
|
||
# RestartSec, no privileges required.
|
||
if _manage_cmd is not None:
|
||
subprocess.run(
|
||
_manage_cmd + ["reset-failed", svc_name],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=10,
|
||
)
|
||
subprocess.run(
|
||
_manage_cmd + ["start", svc_name],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=15,
|
||
)
|
||
# Short poll: the gateway should be up
|
||
# within a few seconds now that we
|
||
# bypassed RestartSec.
|
||
if _wait_for_service_active(
|
||
scope_cmd,
|
||
svc_name,
|
||
timeout=10.0,
|
||
):
|
||
restarted_services.append(svc_name)
|
||
return
|
||
# Passive poll: systemd's auto-restart fires
|
||
# after RestartSec regardless of privileges.
|
||
# This is the primary path when _manage_cmd is
|
||
# None, and the fallback when the explicit
|
||
# start didn't take.
|
||
_restart_sec = _service_restart_sec(
|
||
scope_cmd,
|
||
svc_name,
|
||
default=0.0,
|
||
)
|
||
_post_drain_timeout = max(
|
||
10.0,
|
||
_restart_sec + 10.0,
|
||
)
|
||
if _manage_cmd is None and _restart_sec > 5.0:
|
||
print(
|
||
f" → {svc_name}: waiting for systemd "
|
||
f"auto-restart (~{int(_restart_sec)}s; "
|
||
"no root for an immediate restart)..."
|
||
)
|
||
if _wait_for_service_active(
|
||
scope_cmd,
|
||
svc_name,
|
||
timeout=_post_drain_timeout,
|
||
):
|
||
restarted_services.append(svc_name)
|
||
return
|
||
# Process exited but wasn't respawned (older
|
||
# unit without Restart=on-failure or
|
||
# RestartForceExitStatus=75). Fall through
|
||
# to systemctl start/restart.
|
||
print(
|
||
f" ⚠ {svc_name} drained but didn't relaunch — forcing restart"
|
||
)
|
||
|
||
# Forcing a restart requires manage-units
|
||
# privileges. Without a non-interactive path,
|
||
# running systemctl here would spawn a polkit
|
||
# auth prompt inside a captured 10-15s subprocess
|
||
# — it flashes and dies before the user can
|
||
# answer. Skip with clear instructions instead.
|
||
if _manage_cmd is None:
|
||
failed_or_stale_units.append(svc_name)
|
||
print(
|
||
f" ⚠ {svc_name} is a system service and restarting it needs root.\n"
|
||
f" Restart it manually to load the new version:\n"
|
||
f" sudo systemctl restart {svc_name}\n"
|
||
f" To let `hermes update` restart it automatically, allow\n"
|
||
f" passwordless sudo for systemctl, or run updates with sudo."
|
||
)
|
||
return
|
||
|
||
# Fallback: blunt systemctl restart. This is
|
||
# what the old code always did; we get here only
|
||
# when the graceful path failed (unit missing
|
||
# SIGUSR1 wiring, drain exceeded the budget,
|
||
# restart-policy mismatch).
|
||
#
|
||
# Always `reset-failed` first. If systemd's own
|
||
# auto-restart attempts already parked the unit
|
||
# in a failed state (transient CHDIR / OOM /
|
||
# filesystem race after our drain + exit-75),
|
||
# a plain `systemctl restart` can wedge against
|
||
# the RestartSec backoff and leave the unit
|
||
# dead. Clearing the failed state first makes
|
||
# the restart idempotent. Mirrors the recovery
|
||
# path in `hermes gateway restart`
|
||
# (`systemd_restart()`) as of PR #20949.
|
||
subprocess.run(
|
||
_manage_cmd + ["reset-failed", svc_name],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=10,
|
||
)
|
||
restart = subprocess.run(
|
||
_manage_cmd + ["restart", svc_name],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=15,
|
||
)
|
||
if restart.returncode == 0:
|
||
# Verify the service actually survived the
|
||
# restart. systemctl restart returns 0 even
|
||
# if the new process crashes immediately.
|
||
if _wait_for_service_active(
|
||
scope_cmd,
|
||
svc_name,
|
||
timeout=10.0,
|
||
):
|
||
restarted_services.append(svc_name)
|
||
else:
|
||
# Retry once — transient startup failures
|
||
# (stale module cache, import race) often
|
||
# resolve on the second attempt. Again
|
||
# clear any failed state first so the
|
||
# retry isn't blocked by the previous
|
||
# crash.
|
||
print(
|
||
f" ⚠ {svc_name} died after restart, retrying..."
|
||
)
|
||
subprocess.run(
|
||
_manage_cmd + ["reset-failed", svc_name],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=10,
|
||
)
|
||
subprocess.run(
|
||
_manage_cmd + ["restart", svc_name],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=15,
|
||
)
|
||
if _wait_for_service_active(
|
||
scope_cmd,
|
||
svc_name,
|
||
timeout=10.0,
|
||
):
|
||
restarted_services.append(svc_name)
|
||
print(f" ✓ {svc_name} recovered on retry")
|
||
else:
|
||
failed_or_stale_units.append(svc_name)
|
||
_scope_flag = "--user " if scope == "user" else ""
|
||
_sudo_hint = "sudo " if scope == "system" else ""
|
||
print(
|
||
f" ✗ {svc_name} failed to stay running after restart.\n"
|
||
f" Check logs: {_sudo_hint}journalctl {_scope_flag}-u {svc_name} --since '2 min ago'\n"
|
||
f" Recover manually:\n"
|
||
f" {_sudo_hint}systemctl {_scope_flag}reset-failed {svc_name}\n"
|
||
f" {_sudo_hint}systemctl {_scope_flag}restart {svc_name}"
|
||
)
|
||
else:
|
||
failed_or_stale_units.append(svc_name)
|
||
print(
|
||
f" ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}"
|
||
)
|
||
|
||
def _on_unit_timeout(svc_name: str, exc: subprocess.TimeoutExpired) -> None:
|
||
# Isolate the timeout to this unit and keep going
|
||
# (#68523). A scope-wide handler used to abort every
|
||
# later gateway and leave the fleet on mixed code.
|
||
failed_or_stale_units.append(svc_name)
|
||
print(
|
||
f" ⚠ systemctl timed out restarting {svc_name} "
|
||
f"({exc.cmd if exc.cmd else 'unknown command'}); "
|
||
f"continuing with remaining gateways"
|
||
)
|
||
|
||
_for_each_systemd_gateway_unit(
|
||
result.stdout,
|
||
process_unit=_restart_one_systemd_gateway_unit,
|
||
on_unit_timeout=_on_unit_timeout,
|
||
)
|
||
|
||
# --- Launchd services (macOS) ---
|
||
if is_macos():
|
||
try:
|
||
from hermes_cli.gateway import (
|
||
launchd_restart,
|
||
get_launchd_label,
|
||
get_launchd_plist_path,
|
||
)
|
||
|
||
plist_path = get_launchd_plist_path()
|
||
if plist_path.exists():
|
||
check = subprocess.run(
|
||
["launchctl", "list", get_launchd_label()],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=5,
|
||
)
|
||
if check.returncode == 0:
|
||
try:
|
||
launchd_restart()
|
||
restarted_services.append(get_launchd_label())
|
||
except subprocess.CalledProcessError as e:
|
||
stderr = (getattr(e, "stderr", "") or "").strip()
|
||
print(f" ⚠ Gateway restart failed: {stderr}")
|
||
except (FileNotFoundError, subprocess.TimeoutExpired, ImportError):
|
||
pass
|
||
|
||
# --- Manual (non-service) gateways ---
|
||
# Kill any remaining gateway processes not managed by a service.
|
||
# Exclude PIDs that belong to just-restarted services so we don't
|
||
# immediately kill the process that systemd/launchd just spawned.
|
||
service_pids = _get_service_pids()
|
||
manual_pids = find_gateway_pids(
|
||
exclude_pids=service_pids, all_profiles=True
|
||
)
|
||
profile_processes = {
|
||
proc.pid: proc
|
||
for proc in find_profile_gateway_processes(exclude_pids=service_pids)
|
||
if proc.pid in manual_pids
|
||
}
|
||
for pid, proc in profile_processes.items():
|
||
restart_mode = _prepare_profile_gateway_update_restart(
|
||
proc.profile, pid
|
||
)
|
||
if restart_mode is None:
|
||
continue
|
||
# Prefer a graceful SIGUSR1 drain so in-flight agent runs
|
||
# finish before the watcher respawns the gateway. If the
|
||
# gateway doesn't support SIGUSR1 or doesn't exit within
|
||
# the drain budget, fall back to SIGTERM — the watcher
|
||
# still sees the exit and relaunches either way.
|
||
# Announce the drain first: this wait can hold for the full
|
||
# budget per gateway with no other output, and on surfaces
|
||
# that stream update progress (the desktop updater most of
|
||
# all) the silence reads as a hung update (#44515).
|
||
print(
|
||
f" → {proc.profile}: draining gateway PID {pid} "
|
||
f"(up to {int(_drain_budget)}s)..."
|
||
)
|
||
drained = _graceful_restart_via_sigusr1(
|
||
pid,
|
||
drain_timeout=_drain_budget,
|
||
)
|
||
if not drained:
|
||
try:
|
||
os.kill(pid, _signal.SIGTERM)
|
||
except (ProcessLookupError, PermissionError):
|
||
pass
|
||
# Wait for the old process to fully exit before the watcher
|
||
# spawns the new gateway. Telegram holds the previous
|
||
# getUpdates long-poll session open on its servers for up to
|
||
# ~30s after the client disconnects. If the new gateway
|
||
# connects before that window expires it receives a 409
|
||
# Conflict, which _handle_polling_conflict() recovers from
|
||
# via back-off retries — but a brief wait here reduces the
|
||
# chance of hitting that path at all, especially on fast
|
||
# machines where the watcher loop restarts in < 1s.
|
||
# We wait up to 5s for the process to exit (the OS-level
|
||
# close, not the Telegram server-side expiry), then let the
|
||
# watcher take over. The Telegram adapter's retry logic
|
||
# handles any remaining 409s if the server session is still
|
||
# live when the new gateway polls.
|
||
_wait_for_gateway_exit(timeout=5.0, force_after=None)
|
||
killed_pids.add(pid)
|
||
if restart_mode == "external-supervisor":
|
||
externally_supervised_profiles.append(proc.profile)
|
||
else:
|
||
relaunched_profiles.append(proc.profile)
|
||
|
||
for pid in manual_pids:
|
||
if pid in profile_processes:
|
||
continue
|
||
try:
|
||
os.kill(pid, _signal.SIGTERM)
|
||
killed_pids.add(pid)
|
||
except (ProcessLookupError, PermissionError):
|
||
pass
|
||
|
||
if restarted_services or killed_pids:
|
||
print()
|
||
for svc in restarted_services:
|
||
print(f" ✓ Restarted {svc}")
|
||
if relaunched_profiles:
|
||
names = ", ".join(relaunched_profiles)
|
||
print(f" ✓ Restarting manual gateway profile(s): {names}")
|
||
if externally_supervised_profiles:
|
||
names = ", ".join(externally_supervised_profiles)
|
||
print(
|
||
" ✓ Handed gateway profile(s) back to their external "
|
||
f"supervisor: {names}"
|
||
)
|
||
unmapped_count = (
|
||
len(killed_pids)
|
||
- len(relaunched_profiles)
|
||
- len(externally_supervised_profiles)
|
||
)
|
||
if unmapped_count:
|
||
print(f" → Stopped {unmapped_count} manual gateway process(es)")
|
||
print(" Restart manually: hermes gateway run")
|
||
if unmapped_count > 1:
|
||
print(
|
||
" (or: hermes -p <profile> gateway run for each profile)"
|
||
)
|
||
|
||
if failed_or_stale_units:
|
||
gateway_fleet_restart_incomplete = True
|
||
if gateway_mode:
|
||
_exit_code_path = get_hermes_home() / ".update_exit_code"
|
||
try:
|
||
_exit_code_path.write_text("1", encoding="utf-8")
|
||
except OSError:
|
||
pass
|
||
_warn_incomplete_gateway_fleet_restart(failed_or_stale_units)
|
||
|
||
if not restarted_services and not killed_pids:
|
||
# No gateways were running — nothing to do
|
||
pass
|
||
|
||
# --- Post-restart survivor sweep -----------------------------
|
||
# Issue #17648: some gateways ignore SIGTERM (stuck drain,
|
||
# blocked I/O, PID dead but zombie). The detached profile
|
||
# watchers wait 120s for the old PID to exit — if it never
|
||
# does, no respawn happens and the user keeps hitting
|
||
# ImportError against a stale sys.modules. Give the
|
||
# graceful paths a brief window to complete, then SIGKILL
|
||
# any remaining pre-update PIDs so the watcher / service
|
||
# manager can relaunch with fresh code.
|
||
try:
|
||
_time.sleep(3.0)
|
||
_service_pids_after = _get_service_pids()
|
||
_surviving = find_gateway_pids(
|
||
exclude_pids=_service_pids_after,
|
||
all_profiles=True,
|
||
)
|
||
# Scope to PIDs we already tried to kill during this
|
||
# update (killed_pids). Anything new is a gateway that
|
||
# started AFTER our restart attempt — respecting user
|
||
# intent, we don't kill those.
|
||
_stuck = [pid for pid in _surviving if pid in killed_pids]
|
||
if _stuck:
|
||
print()
|
||
print(
|
||
f" ⚠ {len(_stuck)} gateway process(es) ignored SIGTERM — force-killing"
|
||
)
|
||
from gateway.status import terminate_pid as _terminate_pid
|
||
for pid in _stuck:
|
||
try:
|
||
# Routes through taskkill /T /F on Windows,
|
||
# SIGKILL on POSIX — _signal.SIGKILL doesn't
|
||
# exist on Windows so the old raw os.kill call
|
||
# used to crash the entire update path.
|
||
_terminate_pid(pid, force=True)
|
||
except (ProcessLookupError, PermissionError, OSError):
|
||
pass
|
||
# Give the OS a beat to reap the processes so the
|
||
# watchers see them exit and respawn.
|
||
_time.sleep(1.5)
|
||
except Exception as _sweep_exc:
|
||
logger.debug("Post-restart survivor sweep failed: %s", _sweep_exc)
|
||
|
||
except Exception as e:
|
||
logger.debug("Gateway restart during update failed: %s", e)
|
||
|
||
_m()._resume_windows_gateways_after_update(_windows_gateway_resume)
|
||
|
||
# Warn if legacy Hermes gateway unit files are still installed.
|
||
# When both hermes.service (from a pre-rename install) and the
|
||
# current hermes-gateway.service are enabled, they SIGTERM-fight
|
||
# for the same bot token (see PR #11909). Flagging here means
|
||
# every `hermes update` surfaces the issue until the user migrates.
|
||
try:
|
||
from hermes_cli.gateway import (
|
||
has_legacy_hermes_units,
|
||
_find_legacy_hermes_units,
|
||
supports_systemd_services,
|
||
)
|
||
|
||
if supports_systemd_services() and has_legacy_hermes_units():
|
||
print()
|
||
print("⚠ Legacy Hermes gateway unit(s) detected:")
|
||
for name, path, is_sys in _find_legacy_hermes_units():
|
||
scope = "system" if is_sys else "user"
|
||
print(f" {path} ({scope} scope)")
|
||
print()
|
||
print(" These pre-rename units (hermes.service) fight the current")
|
||
print(" hermes-gateway.service for the bot token and cause SIGTERM")
|
||
print(" flap loops. Remove them with:")
|
||
print()
|
||
print(" hermes gateway migrate-legacy")
|
||
print()
|
||
print(" (add `sudo` if any are in system scope)")
|
||
except Exception as e:
|
||
logger.debug("Legacy unit check during update failed: %s", e)
|
||
|
||
# Restart a managed dashboard through systemd, or stop stale manual
|
||
# dashboard processes. Raw-killing a systemd-owned dashboard PID makes
|
||
# systemd treat it as a clean stop, leaving the Cloudflare origin dead.
|
||
# Preserve the safety rule above: a failed Node refresh leaves the
|
||
# currently running dashboard untouched.
|
||
_finish_dashboard_update_cleanup(node_failures)
|
||
|
||
print()
|
||
print("Tip: You can now select a provider and model:")
|
||
print(" hermes model # Select provider and model")
|
||
|
||
if gateway_fleet_restart_incomplete:
|
||
# Code update itself succeeded, but at least one gateway still
|
||
# runs pre-update modules — surface that as a failed update so
|
||
# automation / operators do not treat the fleet as healthy.
|
||
sys.exit(1)
|
||
|
||
except subprocess.CalledProcessError as e:
|
||
if sys.platform == "win32":
|
||
print(f"⚠ Git update failed: {e}")
|
||
print("→ Falling back to ZIP download...")
|
||
print()
|
||
_update_via_zip(args)
|
||
else:
|
||
print(f"✗ Update failed: {e}")
|
||
sys.exit(1)
|
||
|
||
# --- Hoisted from the body of _cmd_update_impl (self-contained, no closure state) ---
|
||
|
||
def _print_items(items, label, key, fallback_key=None):
|
||
if not items:
|
||
return
|
||
print(f" {label}:")
|
||
shown = items[:8]
|
||
for it in shown:
|
||
if isinstance(it, dict):
|
||
name = it.get(key) or (fallback_key and it.get(fallback_key)) or "?"
|
||
desc = (it.get("description") or "").strip()
|
||
else:
|
||
# Defensive: some callers/mocks pass bare name strings.
|
||
name = str(it)
|
||
desc = ""
|
||
if desc:
|
||
print(f" • {name} — {desc}")
|
||
else:
|
||
print(f" • {name}")
|
||
extra = len(items) - len(shown)
|
||
if extra > 0:
|
||
print(f" … and {extra} more")
|
||
|
||
def _wait_for_service_active(
|
||
scope_cmd_: list,
|
||
svc_name_: str,
|
||
timeout: float = 10.0,
|
||
) -> bool:
|
||
"""Poll ``systemctl is-active`` until the unit reports active.
|
||
|
||
systemd's Stopped -> Started transition after a graceful exit
|
||
(or a hard restart) is not instantaneous; a one-shot check
|
||
races that window and falsely reports the unit as down.
|
||
Poll every 0.5s up to ``timeout`` seconds before giving up.
|
||
"""
|
||
deadline = _time.monotonic() + max(timeout, 0.5)
|
||
while True:
|
||
try:
|
||
_verify = subprocess.run(
|
||
scope_cmd_ + ["is-active", svc_name_],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=5,
|
||
)
|
||
if _verify.stdout.strip() == "active":
|
||
return True
|
||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||
pass
|
||
if _time.monotonic() >= deadline:
|
||
return False
|
||
_time.sleep(0.5)
|
||
|
||
def _service_restart_sec(
|
||
scope_cmd_: list,
|
||
svc_name_: str,
|
||
default: float = 0.0,
|
||
) -> float:
|
||
"""Read the unit's ``RestartUSec`` (RestartSec) in seconds.
|
||
|
||
After a graceful exit-75, systemd waits ``RestartSec`` before
|
||
respawning the unit. Callers that poll for ``is-active``
|
||
must use a timeout >= ``RestartSec`` + transition slack, or
|
||
they'll give up *during* the cooldown window and wrongly
|
||
conclude the unit didn't relaunch.
|
||
"""
|
||
try:
|
||
_show = subprocess.run(
|
||
scope_cmd_
|
||
+ [
|
||
"show",
|
||
svc_name_,
|
||
"--property=RestartUSec",
|
||
"--value",
|
||
],
|
||
capture_output=True,
|
||
text=True, encoding="utf-8", errors="replace",
|
||
timeout=5,
|
||
)
|
||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||
return default
|
||
raw = (_show.stdout or "").strip()
|
||
# systemd emits values like "30s", "100ms", "1min 30s", or
|
||
# "infinity". Parse conservatively; on any miss return default.
|
||
if not raw or raw == "infinity":
|
||
return default
|
||
total = 0.0
|
||
matched = False
|
||
for part in raw.split():
|
||
for _suf, _mult in (
|
||
("ms", 0.001),
|
||
("us", 0.000001),
|
||
("min", 60.0),
|
||
("s", 1.0),
|
||
):
|
||
if part.endswith(_suf):
|
||
try:
|
||
total += float(part[: -len(_suf)]) * _mult
|
||
matched = True
|
||
except ValueError:
|
||
pass
|
||
break
|
||
return total if matched else default
|