From d1f376006a523b274f7ff8e55ceb2da48fc6b9fa Mon Sep 17 00:00:00 2001 From: webtecnica Date: Tue, 21 Jul 2026 22:34:31 -0300 Subject: [PATCH] fix(update): restart systemd-supervised backends after stale kill --- hermes_cli/main.py | 145 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 142 insertions(+), 3 deletions(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index be6b7fce0452..34ff61702e40 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -7041,12 +7041,109 @@ def _restart_managed_dashboard_service( return True +def _get_systemd_service_for_pid(pid: int) -> str | None: + """If *pid* belongs to a systemd service unit, return the unit name. + + Reads ``/proc//cgroup`` and extracts the service name (e.g. + ``hermes-serve.service``). Returns ``None`` when the PID is not + part of a systemd service, when the file is unreadable, or on + non-Linux platforms. + """ + try: + cgroup_path = Path(f"/proc/{pid}/cgroup") + if not cgroup_path.is_file(): + return None + text = cgroup_path.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + line = line.strip() + # Format: 0::/system.slice/hermes-serve.service + # 0::/user.slice/user-1000.slice/session-42.scope + parts = line.split("::", 1) + if len(parts) != 2: + continue + cg_path = parts[1] + if cg_path.endswith(".service"): + svc_name = cg_path.rsplit("/", 1)[-1] + if svc_name: + return svc_name + except (OSError, PermissionError): + pass + return None + + +def _extract_scope_from_cgroup(cgroup_entry: str) -> str | None: + """Extract the systemd scope (``user`` or ``system``) from a cgroup path. + + The cgroup path format is ``/system.slice/.service`` for system + services and ``/user.slice/user-.slice/.service`` for user + services. Returns ``None`` when the scope cannot be determined. + """ + if "/system.slice/" in cgroup_entry: + return "system" + if "/user.slice/" in cgroup_entry: + return "user" + return None + + +def _get_pid_cgroup_path(pid: int) -> str | None: + """Return the cgroup path from ``/proc//cgroup``, or ``None``. + + Only the unified (``0::``) hierarchy cgroup entry is examined. + """ + try: + cgroup_path = Path(f"/proc/{pid}/cgroup") + if not cgroup_path.is_file(): + return None + text = cgroup_path.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + line = line.strip() + parts = line.split("::", 1) + if len(parts) == 2: + return parts[1] + except (OSError, PermissionError): + pass + return None + + +def _try_restart_systemd_service(svc_name: str, cgroup_path: str | None = None) -> bool: + """Attempt to restart *svc_name* via systemctl. + + Uses ``systemctl --user`` for user-scope services and ``systemctl`` + for system-scope services. Returns ``True`` on success. + """ + scope = _extract_scope_from_cgroup(cgroup_path) if cgroup_path else None + if scope == "user": + cmd = ["systemctl", "--user", "restart", svc_name] + elif scope == "system": + cmd = ["systemctl", "restart", svc_name] + else: + # Unknown scope — try system first, then user + cmd = None + for candidate in ( + ["systemctl", "restart", svc_name], + ["systemctl", "--user", "restart", svc_name], + ): + try: + r = subprocess.run(candidate, capture_output=True, text=True, timeout=15) + if r.returncode == 0: + return True + except (FileNotFoundError, subprocess.TimeoutExpired, OSError): + continue + return False + + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=15) + return r.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired, OSError): + return False + + def _kill_stale_dashboard_processes( reason: str = "the running backend no longer matches the updated frontend", *, restart_managed: bool = False, ) -> None: - """Kill running ``hermes dashboard`` processes. + """Kill running ``hermes dashboard`` / ``hermes serve`` processes. Called at the end of ``hermes update`` (default ``reason``) and also from ``hermes dashboard --stop`` (which overrides ``reason``). The @@ -7063,8 +7160,11 @@ def _kill_stale_dashboard_processes( Manually-started dashboards are not auto-restarted because we don't know the original launch args (--host, --port, --insecure, --tui, --no-open). When ``restart_managed`` is true (the ``hermes update`` path), a detected - ``hermes-dashboard.service`` is restarted through systemd instead of - raw-killing its main PID. + ``hermes-dashboard.service`` is restarted through systemd; any OTHER + killed PID that was supervised by a systemd unit (custom unit names — + e.g. a remote backend's ``hermes-serve.service``) has its owning unit + restarted after the kill, because systemd treats our SIGTERM as a clean + stop and ``Restart=on-failure`` would never fire (#68934). """ if restart_managed and _restart_managed_dashboard_service(reason): return @@ -7096,6 +7196,17 @@ def _kill_stale_dashboard_processes( print() print(f"⟲ Stopping {len(pids)} dashboard process(es) ({reason})") + # Before killing, snapshot systemd cgroup info for each PID so we can + # restart supervised services after the kill (the cgroup disappears + # along with the process). Only meaningful on Linux. + pid_cgroup: dict[int, str | None] = {} + pid_service: dict[int, str | None] = {} + if sys.platform != "win32": + for pid in pids: + cg_path = _get_pid_cgroup_path(pid) + pid_cgroup[pid] = cg_path + pid_service[pid] = _get_systemd_service_for_pid(pid) + killed: list[int] = [] failed: list[tuple[int, str]] = [] @@ -7162,7 +7273,35 @@ def _kill_stale_dashboard_processes( for pid, err_msg in failed: print(f" ✗ failed to stop PID {pid}: {err_msg}") + # Restart systemd-supervised processes that we just killed. + # Without this, a remote backend (hermes serve) brought down by + # systemctl kill / Restart=on-failure won't auto-relaunch after + # the clean SIGTERM, and the Desktop never reconnects (#68934). + restarted_services: list[str] = [] if killed: + failed_restarts: list[tuple[str, str]] = [] + seen_services: set[str] = set() + for pid in killed: + cg_path = pid_cgroup.get(pid) + if not cg_path: + continue + svc_name = pid_service.get(pid) + if not svc_name or svc_name in seen_services: + continue + seen_services.add(svc_name) + if _try_restart_systemd_service(svc_name, cg_path): + restarted_services.append(svc_name) + else: + failed_restarts.append((svc_name, "systemctl restart returned non-zero")) + + if restarted_services: + for svc in restarted_services: + print(f" ✓ restarted systemd service {svc}") + if failed_restarts: + for svc, err in failed_restarts: + print(f" ⚠ {svc}: {err}") + + if killed and not restarted_services: print(" Restart the dashboard when you're ready:") print(" hermes dashboard --port ")