mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-23 16:36:23 +00:00
1927b60775
The salvaged deferred-finalize wiring (#65670) covered cli.py, the gateway slash command, and the three in-process tui_gateway/server.py compress sites — but the dashboard compute-host isolated session.compress / slash.compress routes run the compress mirror inside the host child, where a control-handler exception after compress_context() queued the boundary notification would leak it: never fired on success paths, or worse, fired by a LATER compress against a boundary the host had rejected. - tui_gateway/compute_host.py: _handle_control discards any pending context-engine compression notification (committed=False) when a session.compress / slash.compress control frame errors. finalize is exactly-once, so this is a no-op when the mirror already emitted or discarded it. - tui_gateway/server.py: _mirror_slash_side_effects normalizes the /compact alias onto the compress branch so isolated-session compacts actually compress and hit the same finalize/discard wiring instead of silently no-oping. - tests/tui_gateway/test_compute_host_phase1.py: compute-host route tests — commit-then-notify ordering, discard on host commit failure, /compact alias routing.
703 lines
28 KiB
Python
703 lines
28 KiB
Python
"""Persistent dashboard compute-host process.
|
|
|
|
Phase 0 used this module as a deterministic line-JSON spike. Phase 1 keeps the
|
|
same transport and turns it into the long-lived child that owns live AIAgent
|
|
objects when ``dashboard.turn_isolation`` is enabled.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
|
|
def now_ns() -> int:
|
|
return time.perf_counter_ns()
|
|
|
|
|
|
@dataclass
|
|
class SpikeAgent:
|
|
"""A deterministic AIAgent-shaped object for pipe/interrupt measurements."""
|
|
|
|
session_id: str
|
|
history: list[dict[str, str]] = field(default_factory=list)
|
|
_interrupt: threading.Event = field(default_factory=threading.Event)
|
|
|
|
def clear_interrupt(self) -> None:
|
|
self._interrupt.clear()
|
|
|
|
def interrupt(self) -> None:
|
|
self._interrupt.set()
|
|
|
|
def run_conversation(
|
|
self,
|
|
prompt: str,
|
|
*,
|
|
conversation_history: list[dict[str, str]] | None = None,
|
|
stream_callback: Callable[[str], None] | None = None,
|
|
delta_count: int = 24,
|
|
delay_s: float = 0.001,
|
|
) -> dict[str, Any]:
|
|
base_history = list(conversation_history if conversation_history is not None else self.history)
|
|
chunks: list[str] = []
|
|
interrupted = False
|
|
for index in range(max(0, int(delta_count))):
|
|
if self._interrupt.is_set():
|
|
interrupted = True
|
|
break
|
|
chunk = f"{self.session_id}:{prompt}:{index:04d} "
|
|
chunks.append(chunk)
|
|
if stream_callback is not None:
|
|
stream_callback(chunk)
|
|
if delay_s > 0:
|
|
time.sleep(delay_s)
|
|
if self._interrupt.is_set():
|
|
interrupted = True
|
|
final = "".join(chunks)
|
|
if interrupted:
|
|
final += "[interrupted]"
|
|
messages = [
|
|
*base_history,
|
|
{"role": "user", "content": prompt},
|
|
{"role": "assistant", "content": final},
|
|
]
|
|
self.history = messages
|
|
return {"final_response": final, "messages": messages, "interrupted": interrupted}
|
|
|
|
|
|
@dataclass
|
|
class HostSession:
|
|
sid: str
|
|
agent: SpikeAgent
|
|
history_version: int = 0
|
|
running: bool = False
|
|
lock: threading.Lock = field(default_factory=threading.Lock)
|
|
|
|
|
|
class _HostTransport:
|
|
def __init__(self, emit: Callable[[dict[str, Any]], None]) -> None:
|
|
self._emit = emit
|
|
|
|
def write(self, obj: dict) -> bool:
|
|
sid = ""
|
|
try:
|
|
if obj.get("method") == "event":
|
|
sid = str(((obj.get("params") or {}).get("session_id")) or "")
|
|
except Exception:
|
|
sid = ""
|
|
self._emit({"type": "rpc", "sid": sid, "message": obj})
|
|
return True
|
|
|
|
def close(self) -> None:
|
|
return None
|
|
|
|
|
|
def _repo_root() -> Path:
|
|
return Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def _build_sha() -> str:
|
|
try:
|
|
return subprocess.check_output(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=str(_repo_root()),
|
|
text=True,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=2,
|
|
).strip()
|
|
except Exception:
|
|
return "unknown"
|
|
|
|
|
|
class ComputeHost:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
stdout: Any = None,
|
|
max_workers: int | None = None,
|
|
heartbeat_secs: int | float | None = None,
|
|
) -> None:
|
|
self._stdout = stdout or sys.stdout
|
|
self._write_lock = threading.Lock()
|
|
self._sessions: dict[str, HostSession] = {}
|
|
self._executor = concurrent.futures.ThreadPoolExecutor(
|
|
max_workers=max_workers or _default_workers(),
|
|
thread_name_prefix="compute-host-turn",
|
|
)
|
|
self._closed = threading.Event()
|
|
self._parent_pid = os.getppid()
|
|
self._boot_id = uuid.uuid4().hex
|
|
self._progress_counter = 0
|
|
self._progress_lock = threading.Lock()
|
|
self._turn_futures: set[concurrent.futures.Future] = set()
|
|
self._turn_futures_lock = threading.Lock()
|
|
self._transport = _HostTransport(self.emit)
|
|
self._heartbeat_secs = (
|
|
float(heartbeat_secs)
|
|
if heartbeat_secs is not None
|
|
else float(os.environ.get("HERMES_COMPUTE_HOST_HEARTBEAT_SECS") or "15")
|
|
)
|
|
if self._heartbeat_secs > 0:
|
|
threading.Thread(target=self._heartbeat_loop, name="compute-host-heartbeat", daemon=True).start()
|
|
threading.Thread(target=self._parent_guard_loop, name="compute-host-ppid-guard", daemon=True).start()
|
|
|
|
def emit(self, frame: dict[str, Any]) -> None:
|
|
frame.setdefault("host_ns", now_ns())
|
|
data = json.dumps(frame, separators=(",", ":"), ensure_ascii=False)
|
|
with self._write_lock:
|
|
print(data, file=self._stdout, flush=True)
|
|
|
|
def close(self) -> None:
|
|
self._closed.set()
|
|
self._executor.shutdown(wait=False, cancel_futures=True)
|
|
|
|
def shutdown(self, *, reason: str = "shutdown", wait: float = 10.0) -> None:
|
|
self._closed.set()
|
|
self.flush_all_sessions(reason=reason)
|
|
deadline = time.monotonic() + max(0.0, wait)
|
|
while time.monotonic() < deadline:
|
|
with self._turn_futures_lock:
|
|
pending = [f for f in self._turn_futures if not f.done()]
|
|
if not pending:
|
|
break
|
|
time.sleep(0.05)
|
|
self._executor.shutdown(wait=False, cancel_futures=True)
|
|
|
|
def flush_all_sessions(self, *, reason: str = "shutdown") -> None:
|
|
try:
|
|
from tui_gateway import server
|
|
except Exception:
|
|
return
|
|
for session in list(getattr(server, "_sessions", {}).values()):
|
|
try:
|
|
server._finalize_session(session, end_reason=f"compute_host_{reason}")
|
|
except Exception:
|
|
pass
|
|
|
|
def handle_frame(self, frame: dict[str, Any]) -> None:
|
|
kind = str(frame.get("type") or "")
|
|
if kind == "session.seed":
|
|
self._handle_seed(frame)
|
|
elif kind == "turn.start":
|
|
self._handle_turn_start(frame)
|
|
elif kind == "interrupt":
|
|
self._handle_interrupt(frame)
|
|
elif kind == "reload_mcp":
|
|
self._handle_reload_mcp(frame)
|
|
elif kind == "control":
|
|
self._handle_control(frame)
|
|
elif kind == "shutdown":
|
|
self.emit({"type": "shutdown.ack", "request_id": frame.get("request_id")})
|
|
# Explicit supervisor/test shutdown is a clean child-process close;
|
|
# SIGTERM and orphan paths are the durability flush paths.
|
|
self._closed.set()
|
|
self._executor.shutdown(wait=False, cancel_futures=True)
|
|
else:
|
|
self.emit(
|
|
{
|
|
"type": "error",
|
|
"request_id": frame.get("request_id"),
|
|
"message": f"unknown frame type: {kind}",
|
|
}
|
|
)
|
|
|
|
# ── Phase-0 deterministic spike frames ─────────────────────────────
|
|
|
|
def _handle_seed(self, frame: dict[str, Any]) -> None:
|
|
sid = str(frame.get("sid") or "")
|
|
if not sid:
|
|
self.emit({"type": "error", "request_id": frame.get("request_id"), "message": "sid required"})
|
|
return
|
|
history = frame.get("history")
|
|
if not isinstance(history, list):
|
|
history = []
|
|
self._sessions[sid] = HostSession(sid=sid, agent=SpikeAgent(sid, list(history)))
|
|
self.emit({"type": "session.seeded", "sid": sid, "request_id": frame.get("request_id")})
|
|
|
|
def _handle_turn_start(self, frame: dict[str, Any]) -> None:
|
|
sid = str(frame.get("sid") or "")
|
|
if sid in self._sessions:
|
|
self._handle_spike_turn_start(frame)
|
|
return
|
|
future = self._executor.submit(self._run_real_turn, dict(frame))
|
|
with self._turn_futures_lock:
|
|
self._turn_futures.add(future)
|
|
future.add_done_callback(self._turn_futures.discard)
|
|
|
|
def _handle_spike_turn_start(self, frame: dict[str, Any]) -> None:
|
|
sid = str(frame.get("sid") or "")
|
|
session = self._sessions.get(sid)
|
|
if session is None:
|
|
self.emit({"type": "turn.error", "sid": sid, "request_id": frame.get("request_id"), "message": "unknown session"})
|
|
return
|
|
with session.lock:
|
|
if session.running:
|
|
self.emit({"type": "turn.error", "sid": sid, "request_id": frame.get("request_id"), "message": "session busy"})
|
|
return
|
|
session.running = True
|
|
future = self._executor.submit(self._run_spike_turn, session, dict(frame))
|
|
with self._turn_futures_lock:
|
|
self._turn_futures.add(future)
|
|
future.add_done_callback(self._turn_futures.discard)
|
|
|
|
def _handle_interrupt(self, frame: dict[str, Any]) -> None:
|
|
sid = str(frame.get("sid") or "")
|
|
spike = self._sessions.get(sid)
|
|
if spike is not None:
|
|
spike.agent.interrupt()
|
|
self.emit(
|
|
{
|
|
"type": "interrupt.ack",
|
|
"sid": sid,
|
|
"request_id": frame.get("request_id"),
|
|
"applied": True,
|
|
"applied_ns": now_ns(),
|
|
}
|
|
)
|
|
return
|
|
try:
|
|
from tui_gateway import server
|
|
|
|
session = server._sessions.get(sid)
|
|
if session is None:
|
|
self.emit({"type": "interrupt.ack", "sid": sid, "request_id": frame.get("request_id"), "applied": False})
|
|
return
|
|
agent = session.get("agent")
|
|
if agent is not None and hasattr(agent, "interrupt"):
|
|
agent.interrupt()
|
|
with session.get("history_lock", threading.Lock()):
|
|
session["_turn_cancel_requested"] = True
|
|
session["queued_prompt"] = None
|
|
self.emit({"type": "interrupt.ack", "sid": sid, "request_id": frame.get("request_id"), "applied": True, "applied_ns": now_ns()})
|
|
except Exception as exc:
|
|
self.emit({"type": "interrupt.ack", "sid": sid, "request_id": frame.get("request_id"), "applied": False, "message": str(exc)})
|
|
|
|
def _run_spike_turn(self, session: HostSession, frame: dict[str, Any]) -> None:
|
|
request_id = frame.get("request_id") or uuid.uuid4().hex
|
|
prompt = str(frame.get("prompt") or frame.get("text") or "")
|
|
try:
|
|
delta_count = int(frame.get("delta_count", 24))
|
|
except (TypeError, ValueError):
|
|
delta_count = 24
|
|
try:
|
|
delay_s = float(frame.get("delay_s", 0.001))
|
|
except (TypeError, ValueError):
|
|
delay_s = 0.001
|
|
with session.lock:
|
|
history = list(session.agent.history)
|
|
session.agent.clear_interrupt()
|
|
self.emit({"type": "turn.started", "sid": session.sid, "request_id": request_id, "started_ns": now_ns()})
|
|
|
|
def stream(delta: str) -> None:
|
|
self._bump_progress()
|
|
self.emit(
|
|
{
|
|
"type": "delta",
|
|
"sid": session.sid,
|
|
"request_id": request_id,
|
|
"text": delta,
|
|
"emitted_ns": now_ns(),
|
|
}
|
|
)
|
|
|
|
try:
|
|
result = session.agent.run_conversation(
|
|
prompt,
|
|
conversation_history=history,
|
|
stream_callback=stream,
|
|
delta_count=delta_count,
|
|
delay_s=delay_s,
|
|
)
|
|
with session.lock:
|
|
session.history_version += 1
|
|
session.running = False
|
|
history_version = session.history_version
|
|
self._bump_progress()
|
|
self.emit(
|
|
{
|
|
"type": "turn.end",
|
|
"sid": session.sid,
|
|
"request_id": request_id,
|
|
"history_version": history_version,
|
|
"message_count": len(result.get("messages") or []),
|
|
"interrupted": bool(result.get("interrupted")),
|
|
"ended_ns": now_ns(),
|
|
}
|
|
)
|
|
except Exception as exc: # pragma: no cover - defensive host boundary
|
|
with session.lock:
|
|
session.running = False
|
|
self.emit({"type": "turn.error", "sid": session.sid, "request_id": request_id, "message": str(exc)})
|
|
|
|
# ── Real dashboard turn path ───────────────────────────────────────
|
|
|
|
def _run_real_turn(self, frame: dict[str, Any]) -> None:
|
|
sid = str(frame.get("sid") or "")
|
|
request_id = str(frame.get("request_id") or uuid.uuid4().hex)
|
|
if not sid:
|
|
self.emit({"type": "turn.error", "sid": sid, "request_id": request_id, "message": "sid required"})
|
|
return
|
|
try:
|
|
from tui_gateway import server
|
|
|
|
session = self._ensure_server_session(server, frame)
|
|
with session["history_lock"]:
|
|
if session.get("running"):
|
|
self.emit({"type": "turn.error", "sid": sid, "request_id": request_id, "message": "session busy"})
|
|
return
|
|
session["running"] = True
|
|
session["_turn_cancel_requested"] = False
|
|
session["last_active"] = time.time()
|
|
server._start_inflight_turn(session, frame.get("text") if "text" in frame else frame.get("prompt"))
|
|
self.emit({"type": "turn.started", "sid": sid, "request_id": request_id, "started_ns": now_ns()})
|
|
try:
|
|
server._ensure_session_db_row(session)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
import hermes_undo
|
|
|
|
hermes_undo.on_user_message_appended(session["session_key"])
|
|
except Exception:
|
|
pass
|
|
try:
|
|
server._persist_branch_seed(session)
|
|
except Exception:
|
|
pass
|
|
text = frame.get("text") if "text" in frame else frame.get("prompt", "")
|
|
server._run_prompt_submit(request_id, sid, session, text)
|
|
run_thread = session.get("_run_thread")
|
|
if run_thread is not None and hasattr(run_thread, "join"):
|
|
run_thread.join()
|
|
with session["history_lock"]:
|
|
history_version = int(session.get("history_version", 0))
|
|
message_count = len(session.get("history") or [])
|
|
interrupted = bool(session.get("_turn_cancel_requested"))
|
|
session_key = str(session.get("session_key") or "")
|
|
session_info = server._session_info(session.get("agent"), session)
|
|
self._bump_progress()
|
|
self.emit(
|
|
{
|
|
"type": "turn.end",
|
|
"sid": sid,
|
|
"request_id": request_id,
|
|
"history_version": history_version,
|
|
"session_key": session_key,
|
|
"message_count": message_count,
|
|
"interrupted": interrupted,
|
|
"ended_ns": now_ns(),
|
|
"session_info": session_info,
|
|
"session_info_emitted": True,
|
|
}
|
|
)
|
|
except Exception as exc:
|
|
try:
|
|
from tui_gateway import server
|
|
|
|
session = server._sessions.get(sid)
|
|
if session is not None:
|
|
with session.get("history_lock", threading.Lock()):
|
|
session["running"] = False
|
|
server._clear_inflight_turn(session)
|
|
except Exception:
|
|
pass
|
|
self.emit({"type": "turn.error", "sid": sid, "request_id": request_id, "reason": "exception", "message": str(exc)})
|
|
|
|
def _ensure_server_session(self, server: Any, frame: dict[str, Any]) -> dict:
|
|
sid = str(frame.get("sid") or "")
|
|
key = str(frame.get("session_key") or sid)
|
|
session = server._sessions.get(sid)
|
|
if session is not None:
|
|
session["transport"] = self._transport
|
|
if frame.get("cols") is not None:
|
|
session["cols"] = int(frame.get("cols") or 80)
|
|
if frame.get("cwd"):
|
|
session["cwd"] = str(frame.get("cwd"))
|
|
if frame.get("profile_home"):
|
|
session["profile_home"] = str(frame.get("profile_home"))
|
|
if isinstance(frame.get("attached_images"), list):
|
|
session["attached_images"] = list(frame.get("attached_images") or [])
|
|
return session
|
|
|
|
history = frame.get("history") if isinstance(frame.get("history"), list) else []
|
|
profile_home = str(frame.get("profile_home") or "")
|
|
session_db = None
|
|
home_token = None
|
|
try:
|
|
if profile_home:
|
|
from hermes_constants import set_hermes_home_override
|
|
from hermes_state import SessionDB
|
|
|
|
home_token = set_hermes_home_override(profile_home)
|
|
session_db = SessionDB(db_path=Path(profile_home) / "state.db")
|
|
agent = server._make_agent(
|
|
sid,
|
|
key,
|
|
session_id=key,
|
|
model_override=frame.get("model_override"),
|
|
reasoning_config_override=frame.get("reasoning_config_override"),
|
|
service_tier_override=frame.get("service_tier_override"),
|
|
platform_override=frame.get("source"),
|
|
session_db=session_db,
|
|
)
|
|
finally:
|
|
if home_token is not None:
|
|
try:
|
|
from hermes_constants import reset_hermes_home_override
|
|
|
|
reset_hermes_home_override(home_token)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
from tui_gateway.transport import bind_transport, reset_transport
|
|
|
|
token = bind_transport(self._transport)
|
|
try:
|
|
server._init_session(
|
|
sid,
|
|
key,
|
|
agent,
|
|
list(history),
|
|
cols=int(frame.get("cols") or 80),
|
|
cwd=str(frame.get("cwd") or "") or None,
|
|
session_db=session_db,
|
|
source=frame.get("source"),
|
|
)
|
|
finally:
|
|
reset_transport(token)
|
|
except Exception:
|
|
# If _init_session's side machinery (slash worker, approval notify) is
|
|
# unavailable, keep a minimal host-owned session rather than failing
|
|
# the turn after the expensive agent build succeeded.
|
|
server._sessions[sid] = {
|
|
"agent": agent,
|
|
"session_key": key,
|
|
"history": list(history),
|
|
"history_lock": threading.Lock(),
|
|
"history_version": int(frame.get("history_version") or 0),
|
|
"inflight_turn": None,
|
|
"created_at": time.time(),
|
|
"last_active": time.time(),
|
|
"running": False,
|
|
"attached_images": [],
|
|
"image_counter": 0,
|
|
"cwd": str(frame.get("cwd") or os.getcwd()),
|
|
"cols": int(frame.get("cols") or 80),
|
|
"slash_worker": None,
|
|
"show_reasoning": server._load_show_reasoning(),
|
|
"tool_progress_mode": server._load_tool_progress_mode(),
|
|
"edit_snapshots": {},
|
|
"tool_started_at": {},
|
|
"model_override": frame.get("model_override"),
|
|
"source": server._sanitize_client_source(frame.get("source")),
|
|
"transport": self._transport,
|
|
}
|
|
session = server._sessions[sid]
|
|
session["transport"] = self._transport
|
|
session["profile_home"] = profile_home or session.get("profile_home")
|
|
if isinstance(frame.get("attached_images"), list):
|
|
session["attached_images"] = list(frame.get("attached_images") or [])
|
|
if frame.get("model_override") is not None:
|
|
session["model_override"] = frame.get("model_override")
|
|
return session
|
|
|
|
def _handle_reload_mcp(self, frame: dict[str, Any]) -> None:
|
|
sid = str(frame.get("sid") or "")
|
|
request_id = frame.get("request_id")
|
|
try:
|
|
from tui_gateway import server
|
|
|
|
resp = server.handle_request({"id": request_id, "method": "reload.mcp", "params": {"session_id": sid, "confirm": True}})
|
|
self.emit({"type": "reload_mcp.ack", "sid": sid, "request_id": request_id, "response": resp})
|
|
except Exception as exc:
|
|
self.emit({"type": "control.error", "sid": sid, "request_id": request_id, "message": str(exc)})
|
|
|
|
def _handle_control(self, frame: dict[str, Any]) -> None:
|
|
sid = str(frame.get("sid") or "")
|
|
request_id = frame.get("request_id")
|
|
route_name = str(frame.get("route_name") or "")
|
|
try:
|
|
from tui_gateway import server
|
|
from tui_gateway.host_supervisor import MUTATOR_ROUTE_TABLE
|
|
|
|
route = MUTATOR_ROUTE_TABLE.get(route_name)
|
|
if route is None:
|
|
self.emit({"type": "control.error", "sid": sid, "request_id": request_id, "message": f"unclassified route: {route_name}"})
|
|
return
|
|
session = server._sessions.get(sid)
|
|
if session is None:
|
|
self.emit({"type": "control.error", "sid": sid, "request_id": request_id, "message": "session not found"})
|
|
return
|
|
if route == "idle-gated" and session.get("running"):
|
|
self.emit({"type": "control.error", "sid": sid, "request_id": request_id, "message": "session busy"})
|
|
return
|
|
if route_name == "reload.mcp":
|
|
self._handle_reload_mcp({**frame, "type": "reload_mcp"})
|
|
return
|
|
command = str(frame.get("command") or "")
|
|
output = ""
|
|
if command:
|
|
output = server._mirror_slash_side_effects(sid, session, command)
|
|
with session["history_lock"]:
|
|
history_version = int(session.get("history_version", 0))
|
|
message_count = len(session.get("history") or [])
|
|
session_key = str(session.get("session_key") or "")
|
|
self.emit(
|
|
{
|
|
"type": "control.ack",
|
|
"sid": sid,
|
|
"request_id": request_id,
|
|
"route_name": route_name,
|
|
"output": output,
|
|
"session_key": session_key,
|
|
"history_version": history_version,
|
|
"message_count": message_count,
|
|
"session_info": server._session_info(session.get("agent"), session),
|
|
}
|
|
)
|
|
except Exception as exc:
|
|
if route_name in {"session.compress", "slash.compress"}:
|
|
# The compress mirror defers the context-engine boundary
|
|
# notification until the host commits. If anything raises
|
|
# between queueing and finalize (e.g. building the ack's
|
|
# session_info), discard the pending notification so it can't
|
|
# leak onto the agent and fire against a rejected boundary on
|
|
# a later compress. finalize is exactly-once, so this is a
|
|
# no-op when the mirror already emitted or discarded it.
|
|
try:
|
|
from tui_gateway import server as _server
|
|
from agent.conversation_compression import (
|
|
finalize_context_engine_compression_notification,
|
|
)
|
|
|
|
_agent = (_server._sessions.get(sid) or {}).get("agent")
|
|
if _agent is not None:
|
|
finalize_context_engine_compression_notification(
|
|
_agent,
|
|
committed=False,
|
|
)
|
|
except Exception:
|
|
pass
|
|
self.emit({"type": "control.error", "sid": sid, "request_id": request_id, "message": str(exc)})
|
|
|
|
def _bump_progress(self) -> None:
|
|
with self._progress_lock:
|
|
self._progress_counter += 1
|
|
|
|
def _heartbeat_loop(self) -> None:
|
|
while not self._closed.wait(self._heartbeat_secs):
|
|
with self._turn_futures_lock:
|
|
active_turns = sum(1 for f in self._turn_futures if not f.done())
|
|
with self._progress_lock:
|
|
counter = self._progress_counter
|
|
self.emit(
|
|
{
|
|
"type": "hb",
|
|
"active_turns": active_turns,
|
|
"progress_counter": counter,
|
|
"rss_mb": _rss_mb(os.getpid()),
|
|
}
|
|
)
|
|
|
|
def _parent_guard_loop(self) -> None:
|
|
while not self._closed.wait(1.0):
|
|
ppid = os.getppid()
|
|
if ppid in {0, 1} or (self._parent_pid and ppid != self._parent_pid):
|
|
self.emit({"type": "orphan", "old_ppid": self._parent_pid, "ppid": ppid})
|
|
self.shutdown(reason="orphan")
|
|
os._exit(0)
|
|
|
|
|
|
def _rss_mb(pid: int) -> float:
|
|
try:
|
|
out = subprocess.check_output(["ps", "-o", "rss=", "-p", str(pid)], text=True, stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2).strip()
|
|
return int(out.splitlines()[-1].strip()) / 1024.0 if out else 0.0
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
def _default_workers() -> int:
|
|
try:
|
|
return max(2, int(os.environ.get("HERMES_TUI_RPC_POOL_WORKERS") or "8"))
|
|
except (TypeError, ValueError):
|
|
return 8
|
|
|
|
|
|
def run_host(stdin: Any = None, stdout: Any = None) -> None:
|
|
os.environ["HERMES_COMPUTE_HOST_CHILD"] = "1"
|
|
stdin = stdin or sys.stdin
|
|
host = ComputeHost(stdout=stdout or sys.stdout)
|
|
shutting_down = threading.Event()
|
|
|
|
def _signal_handler(_signum, _frame) -> None:
|
|
if shutting_down.is_set():
|
|
return
|
|
shutting_down.set()
|
|
host.shutdown(reason="sigterm")
|
|
raise SystemExit(0)
|
|
|
|
try:
|
|
signal.signal(signal.SIGTERM, _signal_handler)
|
|
signal.signal(signal.SIGINT, _signal_handler)
|
|
except Exception:
|
|
pass
|
|
|
|
host.emit(
|
|
{
|
|
"type": "hello",
|
|
"host_pid": os.getpid(),
|
|
"boot_id": host._boot_id,
|
|
"build_sha": _build_sha(),
|
|
"cwd": os.getcwd(),
|
|
"hermes_home": os.environ.get("HERMES_HOME", ""),
|
|
}
|
|
)
|
|
|
|
def _reader() -> None:
|
|
for raw in stdin:
|
|
if host._closed.is_set():
|
|
break
|
|
try:
|
|
frame = json.loads(raw)
|
|
except json.JSONDecodeError as exc:
|
|
host.emit({"type": "error", "message": f"invalid json: {exc}"})
|
|
continue
|
|
if not isinstance(frame, dict):
|
|
host.emit({"type": "error", "message": "frame must be an object"})
|
|
continue
|
|
host.handle_frame(frame)
|
|
if frame.get("type") == "shutdown":
|
|
os._exit(0)
|
|
if host._closed.is_set():
|
|
break
|
|
|
|
reader = threading.Thread(target=_reader, name="compute-host-control-reader", daemon=True)
|
|
reader.start()
|
|
try:
|
|
while not host._closed.wait(0.2):
|
|
if not reader.is_alive():
|
|
break
|
|
finally:
|
|
host.shutdown(reason="stdin_closed", wait=2.0)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Dashboard compute-host process")
|
|
parser.parse_args(argv)
|
|
run_host()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover
|
|
raise SystemExit(main())
|