mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-23 16:36:23 +00:00
start_background_mcp_discovery() sets _mcp_discovery_started once and never resets it. If the first background run exits without connecting any MCP server (startup cancellation, OOM restart, transient network failure), every later call returns immediately and the process is permanently stuck with zero MCP tools until a full restart. Fix: when discovery is marked started but the thread is dead and no server is connected, reset the flag and spawn a fresh discovery thread. Also log a WARNING when a discovery run completes with zero connected servers, so the condition is visible instead of silent. Caught in production on a long-running gateway fleet where a gateway restarted under memory pressure and came back with all MCP tools missing.
166 lines
6.3 KiB
Python
166 lines
6.3 KiB
Python
"""Shared CLI/TUI-safe helpers for background MCP discovery."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
from contextlib import nullcontext
|
|
from typing import Optional
|
|
|
|
_mcp_discovery_lock = threading.Lock()
|
|
_mcp_discovery_started = False
|
|
_mcp_discovery_thread: Optional[threading.Thread] = None
|
|
|
|
|
|
def _has_configured_mcp_servers() -> bool:
|
|
"""Cheap config probe so non-MCP users avoid importing the MCP stack."""
|
|
try:
|
|
from hermes_cli.config import read_raw_config
|
|
|
|
mcp_servers = (read_raw_config() or {}).get("mcp_servers")
|
|
return isinstance(mcp_servers, dict) and len(mcp_servers) > 0
|
|
except Exception:
|
|
# Be conservative: if config probing fails, try discovery in the
|
|
# background so startup still can't block.
|
|
return True
|
|
|
|
|
|
def start_background_mcp_discovery(*, logger, thread_name: str) -> None:
|
|
"""Spawn one shared background MCP discovery thread for this process.
|
|
|
|
If the first background discovery run exits without connecting any MCP
|
|
server (for example after startup cancellation / OOM restart), later calls
|
|
are allowed to retry instead of permanently pinning the process in a
|
|
"discovery already started" state with zero MCP tools.
|
|
"""
|
|
global _mcp_discovery_started, _mcp_discovery_thread
|
|
|
|
with _mcp_discovery_lock:
|
|
if _mcp_discovery_started:
|
|
thread = _mcp_discovery_thread
|
|
if thread is not None and thread.is_alive():
|
|
return
|
|
try:
|
|
from tools.mcp_tool import get_mcp_status
|
|
|
|
status = get_mcp_status() or []
|
|
if any(entry.get("connected") for entry in status):
|
|
return
|
|
except Exception:
|
|
return
|
|
logger.warning(
|
|
"Background MCP discovery previously exited with no connected "
|
|
"servers; retrying discovery thread"
|
|
)
|
|
_mcp_discovery_started = False
|
|
_mcp_discovery_thread = None
|
|
|
|
_mcp_discovery_started = True
|
|
if not _has_configured_mcp_servers():
|
|
return
|
|
|
|
def _discover() -> None:
|
|
try:
|
|
_discover_mcp_tools_without_interactive_oauth()
|
|
try:
|
|
from tools.mcp_tool import get_mcp_status
|
|
status = get_mcp_status() or []
|
|
if not any(entry.get("connected") for entry in status):
|
|
logger.warning(
|
|
"Background MCP discovery completed with zero connected servers"
|
|
)
|
|
except Exception:
|
|
logger.debug("Failed to inspect MCP status after background discovery", exc_info=True)
|
|
except Exception:
|
|
logger.debug("Background MCP tool discovery failed", exc_info=True)
|
|
finally:
|
|
with _mcp_discovery_lock:
|
|
global _mcp_discovery_thread, _mcp_discovery_started
|
|
_mcp_discovery_thread = None
|
|
|
|
thread = threading.Thread(
|
|
target=_discover,
|
|
name=thread_name,
|
|
daemon=True,
|
|
)
|
|
_mcp_discovery_thread = thread
|
|
thread.start()
|
|
|
|
|
|
def _resolve_discovery_timeout(explicit: "float | None") -> float:
|
|
"""Resolve the MCP discovery wait bound: explicit arg > config > default.
|
|
|
|
Reads ``mcp_discovery_timeout`` from config.yaml, defaulting to the value in
|
|
``DEFAULT_CONFIG`` (single source of truth) when the key is absent. Kept lazy
|
|
and fail-safe — a missing/invalid value or a broken config falls back to a
|
|
short safe bound so startup can never hang or crash.
|
|
"""
|
|
if explicit is not None:
|
|
return explicit
|
|
try:
|
|
from hermes_cli.config import load_config, DEFAULT_CONFIG
|
|
|
|
default = float(DEFAULT_CONFIG.get("mcp_discovery_timeout", 1.5))
|
|
raw = (load_config() or {}).get("mcp_discovery_timeout", default)
|
|
val = float(raw)
|
|
return val if val > 0 else default
|
|
except Exception:
|
|
return 1.5
|
|
|
|
|
|
def _discover_mcp_tools_without_interactive_oauth() -> None:
|
|
"""Run MCP discovery without letting OAuth read from the user's stdin."""
|
|
try:
|
|
from tools.mcp_oauth import suppress_interactive_oauth
|
|
except Exception:
|
|
suppress_interactive_oauth = nullcontext
|
|
|
|
with suppress_interactive_oauth():
|
|
from tools.mcp_tool import discover_mcp_tools
|
|
|
|
discover_mcp_tools()
|
|
|
|
|
|
def wait_for_mcp_discovery(timeout: "float | None" = None) -> None:
|
|
"""Wait for background MCP discovery before the first tool snapshot.
|
|
|
|
``thread.join(timeout)`` returns the INSTANT discovery completes, so this
|
|
only ever blocks for the real connect time of a still-pending server —
|
|
users with no MCP servers or fast servers pay ~0s. The bound (from
|
|
``mcp_discovery_timeout`` in config) just caps the wait so a dead server
|
|
can't freeze startup; servers that miss it are picked up by the automatic
|
|
late-binding refresh.
|
|
"""
|
|
thread = _mcp_discovery_thread
|
|
if thread is None or not thread.is_alive():
|
|
return
|
|
thread.join(timeout=_resolve_discovery_timeout(timeout))
|
|
|
|
|
|
def mcp_discovery_in_flight() -> bool:
|
|
"""Return True if THIS module's background discovery thread is still running.
|
|
|
|
Mirrors ``tui_gateway.entry.mcp_discovery_in_flight`` for the surfaces that
|
|
start discovery through ``start_background_mcp_discovery`` here (the desktop
|
|
app + dashboard WebSocket sidecar via ``tui_gateway/ws.py``, and
|
|
``hermes dashboard``). Those processes populate THIS module's
|
|
``_mcp_discovery_thread``, not ``tui_gateway.entry``'s, so the late-refresh
|
|
scheduler must consult both to decide whether a slow server's tools are
|
|
still pending (see #51587).
|
|
"""
|
|
thread = _mcp_discovery_thread
|
|
return thread is not None and thread.is_alive()
|
|
|
|
|
|
def join_mcp_discovery(timeout: "float | None" = None) -> bool:
|
|
"""Block until THIS module's background discovery finishes, up to ``timeout``.
|
|
|
|
Returns True if discovery has completed (thread absent or no longer alive),
|
|
False if it is still running after the timeout. Unlike
|
|
``wait_for_mcp_discovery`` this accepts an unbounded/long wait and reports
|
|
the outcome, for the off-critical-path late-refresh waiter.
|
|
"""
|
|
thread = _mcp_discovery_thread
|
|
if thread is None:
|
|
return True
|
|
thread.join(timeout=timeout)
|
|
return not thread.is_alive()
|