From a124d16764a23a0ac29cfbfe14ef22310c4959d8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:37:33 -0700 Subject: [PATCH] perf: cut first-turn time-to-first-token by ~80% (all platforms) (#59332) Four independent pre-request stalls sat on the critical path between prompt submission and the first streamed token, measured with cProfile against a live process: 1. Discord capability detection (~2.0s, worst 5s): get_tool_definitions -> _get_dynamic_schema made a BLOCKING https call to discord.com inside AIAgent.__init__ for any user with DISCORD_BOT_TOKEN set, on every platform, every cold process. Now non-blocking: memory cache -> 24h disk cache -> permissive default + one background detection that seeds the disk cache for the next process. The permissive default is pinned per-process so tool schemas never flip mid-conversation (prompt-cache safety); it mirrors the existing detection-failure fallback (all actions exposed, 403s enriched at call time). 2. Ollama /api/show probe (~0.3s): get_model_context_length step 5e POSTed to /api/show for KNOWN providers (openrouter etc.), got a 404, and never cached the miss - so every fresh process paid a full HTTP round-trip. Known non-Ollama providers now skip the probe; local/custom/unknown endpoints keep the exact previous behavior. 3. env_probe subprocess sweep (~0.5s): the Python-toolchain probe ran 4-8 subprocess calls inside the FIRST system prompt build. Now warmed off-thread during agent init; the prompt build hits the cache (same lock, so a mid-flight warm just joins instead of recomputing). 4. tools.mcp_tool import (~0.4s): the between-turns MCP refresh in build_turn_context imported the whole mcp package even with zero MCP servers configured. MCP tools can only exist if tools.mcp_tool was already imported (discovery/reload paths), so gate the import on sys.modules membership - no behavior change for MCP users. CLI additionally pre-imports run_agent + openai off-thread during the idle banner window (same pattern as the /model picker prewarm), hiding the remaining ~1.5s of module imports while the user types. Fixes 1-4 apply to every interaction layer (CLI, gateway, TUI, desktop, cron). Measured cold first turn (submit -> request dispatched, openrouter, discord token set): 4.3s before -> 0.9s after CLI prewarm (~80%); the agent-side non-import cost drops 2.9s -> 0.36s (init) + 0.27s (turn prologue). --- agent/agent_init.py | 11 ++ agent/model_metadata.py | 28 ++++-- agent/turn_context.py | 16 ++- cli.py | 23 +++++ tests/tools/test_discord_tool.py | 119 ++++++++++++++++++++++ tools/discord_tool.py | 167 ++++++++++++++++++++++++++++--- tools/env_probe.py | 29 +++++- 7 files changed, 365 insertions(+), 28 deletions(-) diff --git a/agent/agent_init.py b/agent/agent_init.py index b3b499a9934..5d65c4176e3 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1363,6 +1363,17 @@ def init_agent( # line). Useful for users on exotic setups where the probe heuristics # are noisy. agent._environment_probe = bool(_agent_section.get("environment_probe", True)) + # Warm the probe off-thread: it shells out to python3/pip (~0.5s of + # subprocess round-trips) and its result lands in the FIRST system + # prompt build, which sits on the time-to-first-token critical path. + # The warm runs during agent init (network/credential setup dominates), + # so by the time the first prompt is built the line is already cached. + if agent._environment_probe: + try: + from tools.env_probe import warm_environment_probe_async + warm_environment_probe_async() + except Exception: + pass # Per-platform prompt-hint overrides (config.yaml → platform_hints). # Lets an enterprise admin append to or replace Hermes' built-in diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 726c3300a90..ba87a6dc496 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -2188,21 +2188,29 @@ def get_model_context_length( ctx = _resolve_endpoint_context_length(model, base_url, api_key=api_key) if ctx is not None: return ctx - # 5e. Ollama native /api/show probe — runs for ANY provider with a - # base_url, not just ollama-cloud. Ollama-compatible servers expose + # 5e. Ollama native /api/show probe — runs for providers whose base_url + # is NOT a known non-Ollama provider. Ollama-compatible servers expose # this endpoint regardless of hostname (local Ollama, Ollama Cloud, # custom Ollama hosting). The OpenAI-compat /v1/models endpoint # correctly omits context_length per the OpenAI schema, but /api/show # returns the authoritative GGUF model_info.context_length. - # For non-Ollama servers (OpenAI, Anthropic, etc.), the POST returns - # 404/405 quickly. Results are cached, so the hit is per-model+URL, - # once per hour. + # Known hosted providers (OpenRouter, Anthropic, OpenAI, …) are skipped: + # they are definitively not Ollama, the POST always 404s, and the result + # is never cached for them — so every fresh process used to pay a + # ~300ms blocking HTTP round-trip on the first-turn critical path + # (measured against openrouter.ai; worse on slow DNS). if base_url: - ctx = _query_ollama_api_show(model, base_url, api_key=api_key) - if ctx is not None: - if not _skip_persistent_context_cache(base_url, provider): - save_context_length(model, base_url, ctx) - return ctx + _inferred_for_probe = _infer_provider_from_url(base_url) + _skip_ollama_probe = ( + _inferred_for_probe is not None + and "ollama" not in _inferred_for_probe + ) + if not _skip_ollama_probe: + ctx = _query_ollama_api_show(model, base_url, api_key=api_key) + if ctx is not None: + if not _skip_persistent_context_cache(base_url, provider): + save_context_length(model, base_url, ctx) + return ctx # 5f. OpenRouter live /models metadata — authoritative for OpenRouter-routed # models. OpenRouter's catalog carries per-model context_length (e.g. # anthropic/claude-fable-5 -> 1M) and refreshes as new slugs ship, so it diff --git a/agent/turn_context.py b/agent/turn_context.py index 36a3db987f9..89b00819c2c 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -185,9 +185,19 @@ def build_turn_context( # name and leaves the snapshot untouched on no-change). try: if not getattr(agent, "_skip_mcp_refresh", False): - from tools.mcp_tool import has_registered_mcp_tools, refresh_agent_mcp_tools - if has_registered_mcp_tools(): - refresh_agent_mcp_tools(agent, quiet_mode=True) + # Import-cost gate: ``tools.mcp_tool`` pulls in the whole ``mcp`` + # package (~0.4s measured) even when the user has zero MCP servers + # configured. MCP tools can only be registered by code that has + # already imported ``tools.mcp_tool`` (discovery, /reload-mcp, + # late-binding refresh) — so if it isn't in sys.modules yet, there + # is nothing to refresh and the import can be skipped outright. + # This keeps the no-MCP first turn off the heavy import path + # without changing behavior for MCP users. + import sys as _sys + if "tools.mcp_tool" in _sys.modules: + from tools.mcp_tool import has_registered_mcp_tools, refresh_agent_mcp_tools + if has_registered_mcp_tools(): + refresh_agent_mcp_tools(agent, quiet_mode=True) except Exception: logger.debug("between-turns MCP tool refresh skipped", exc_info=True) diff --git a/cli.py b/cli.py index 5a52b29f40d..56a5b796d14 100644 --- a/cli.py +++ b/cli.py @@ -13055,6 +13055,29 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): except Exception: pass + # Pre-import the agent runtime off-thread during the same idle window. + # The first turn otherwise pays ~1.5s of module imports on the + # time-to-first-token critical path: `import run_agent` (~0.9s, + # deferred by the lazy AIAgent wrapper above) plus the OpenAI SDK + # (~0.6s, deferred until client construction). Python's import lock + # makes this safe: if the user submits before the warm finishes, the + # main thread simply blocks on the remaining import work instead of + # redoing it. Skipped when agent startup is explicitly deferred + # (Termux) — that path defers heavy work on purpose. + if os.environ.get("HERMES_DEFER_AGENT_STARTUP") != "1": + def _prewarm_agent_runtime() -> None: + try: + import run_agent # noqa: F401 (imports model_tools + tool registry) + import openai # noqa: F401 + except Exception: + logger.debug("agent runtime pre-import failed", exc_info=True) + + threading.Thread( + target=_prewarm_agent_runtime, + name="agent-runtime-prewarm", + daemon=True, + ).start() + # Redaction opt-out warning (#17691): ON by default, loud when off. # The redactor snapshots its state at import time so any toggle now # won't affect the running process — we just want the operator to diff --git a/tests/tools/test_discord_tool.py b/tests/tools/test_discord_tool.py index ac94ce5e751..4bc960f7440 100644 --- a/tests/tools/test_discord_tool.py +++ b/tests/tools/test_discord_tool.py @@ -711,6 +711,119 @@ class TestCapabilityDetection: _detect_capabilities("tok", force=True) assert mock_req.call_count == 2 + +class TestNonBlockingCapabilityDetection: + """The schema-build path must never block on a discord.com HTTP call. + + ``_detect_capabilities_nonblocking`` resolves memory cache → disk cache → + permissive default (+ background detect), keeping the ~2s blocking + detection off the first-token critical path (TTFT fix, July 2026). + """ + + def setup_method(self): + _reset_capability_cache() + + def teardown_method(self): + _reset_capability_cache() + + def test_memory_cache_hit_no_network(self): + from tools.discord_tool import _capability_cache, _detect_capabilities_nonblocking + caps_in = {"has_members_intent": False, "has_message_content": True, "detected": True} + _capability_cache["tok"] = caps_in + with patch("tools.discord_tool._discord_request") as mock_req: + caps = _detect_capabilities_nonblocking("tok") + assert caps == caps_in + mock_req.assert_not_called() + + def test_cold_start_returns_permissive_default_immediately(self): + from tools.discord_tool import _detect_capabilities_nonblocking + with patch("tools.discord_tool._load_caps_from_disk", return_value=None), \ + patch("tools.discord_tool.threading.Thread") as mock_thread: + caps = _detect_capabilities_nonblocking("tok") + assert caps["has_members_intent"] is True + assert caps["has_message_content"] is True + assert caps["detected"] is False + # Background detection was scheduled exactly once + assert mock_thread.call_count == 1 + + def test_cold_start_pins_default_for_process_schema_stability(self): + """Within one process the schema must not flip mid-conversation: + the permissive default is pinned in the memory cache so later + schema builds see the same caps even after bg detection lands + on disk.""" + from tools.discord_tool import _capability_cache, _detect_capabilities_nonblocking + with patch("tools.discord_tool._load_caps_from_disk", return_value=None), \ + patch("tools.discord_tool.threading.Thread"): + first = _detect_capabilities_nonblocking("tok") + # Even if the disk now has restrictive caps, the pinned entry wins. + with patch( + "tools.discord_tool._load_caps_from_disk", + return_value={"has_members_intent": False, "has_message_content": False, "detected": True}, + ): + second = _detect_capabilities_nonblocking("tok") + assert first == second + assert _capability_cache["tok"] is first + + def test_bg_detection_scheduled_once_per_token(self): + from tools.discord_tool import _detect_capabilities_nonblocking + with patch("tools.discord_tool._load_caps_from_disk", return_value=None), \ + patch("tools.discord_tool.threading.Thread") as mock_thread: + _detect_capabilities_nonblocking("tok") + # Second cold call for the same token in the same process + from tools.discord_tool import _capability_cache + _capability_cache.pop("tok", None) # simulate another cold path + _detect_capabilities_nonblocking("tok") + # bg started set persists → only one thread scheduled + assert mock_thread.call_count == 1 + + def test_disk_cache_round_trip(self, tmp_path, monkeypatch): + import tools.discord_tool as dt + monkeypatch.setattr( + dt, "_capability_disk_cache_path", + lambda: tmp_path / "discord_capabilities.json", + ) + caps_in = {"has_members_intent": True, "has_message_content": False, "detected": True} + dt._save_caps_to_disk("tok", caps_in) + assert dt._load_caps_from_disk("tok") == caps_in + # Wrong token → miss + assert dt._load_caps_from_disk("other") is None + + def test_disk_cache_expires(self, tmp_path, monkeypatch): + import time as _time + + import tools.discord_tool as dt + monkeypatch.setattr( + dt, "_capability_disk_cache_path", + lambda: tmp_path / "discord_capabilities.json", + ) + caps_in = {"has_members_intent": True, "has_message_content": True, "detected": True} + dt._save_caps_to_disk("tok", caps_in) + # Rewrite timestamp to be stale + import json as _json + p = tmp_path / "discord_capabilities.json" + data = _json.loads(p.read_text()) + for entry in data.values(): + entry["ts"] = _time.time() - dt._CAPABILITY_DISK_TTL_SECONDS - 10 + p.write_text(_json.dumps(data)) + assert dt._load_caps_from_disk("tok") is None + + def test_schema_build_uses_nonblocking_path(self, monkeypatch): + """get_dynamic_schema_core must not call the blocking detection.""" + monkeypatch.setenv("DISCORD_BOT_TOKEN", "tok") + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: {"discord": {"server_actions": ""}}, + ) + with patch("tools.discord_tool._load_caps_from_disk", return_value=None), \ + patch("tools.discord_tool.threading.Thread"), \ + patch("tools.discord_tool._discord_request") as mock_req: + schema = get_dynamic_schema_core() + # No blocking HTTP call happened on the schema-build path + mock_req.assert_not_called() + assert schema is not None + actions = set(schema["parameters"]["properties"]["action"]["enum"]) + assert actions == set(_CORE_ACTIONS.keys()) # permissive default + @patch("tools.discord_tool._discord_request") def test_cache_is_keyed_by_token(self, mock_req): """Regression: token A's capabilities must not leak to token B. @@ -932,6 +1045,10 @@ class TestDynamicSchema: lambda: {"discord": {"server_actions": ""}}, ) mock_req.return_value = {"flags": 1 << 18} # only MESSAGE_CONTENT + # Warm the capability cache — schema builds are non-blocking and use + # the permissive default until detection has completed (background + # thread + disk cache); filtering applies once caps are known. + _detect_capabilities("tok") schema = get_dynamic_schema_admin() actions = schema["parameters"]["properties"]["action"]["enum"] assert "member_info" not in actions @@ -948,6 +1065,7 @@ class TestDynamicSchema: lambda: {"discord": {"server_actions": ""}}, ) mock_req.return_value = {"flags": 1 << 18} # only MESSAGE_CONTENT + _detect_capabilities("tok") # warm cache — schema builds are non-blocking schema = get_dynamic_schema_core() actions = schema["parameters"]["properties"]["action"]["enum"] assert "search_members" not in actions @@ -960,6 +1078,7 @@ class TestDynamicSchema: lambda: {"discord": {"server_actions": ""}}, ) mock_req.return_value = {"flags": 1 << 14} # only GUILD_MEMBERS + _detect_capabilities("tok") # warm cache — schema builds are non-blocking schema = get_dynamic_schema_core() assert "MESSAGE_CONTENT" in schema["description"] # But fetch_messages is still available diff --git a/tools/discord_tool.py b/tools/discord_tool.py index 1da43ac9140..9dd397ccba4 100644 --- a/tools/discord_tool.py +++ b/tools/discord_tool.py @@ -28,13 +28,17 @@ actionable guidance the model can relay to the user. import json import logging import os +import threading import urllib.error import urllib.parse import urllib.request -from typing import Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from tools.registry import registry +if TYPE_CHECKING: + from pathlib import Path + logger = logging.getLogger(__name__) DISCORD_API_BASE = "https://discord.com/api/v10" @@ -134,23 +138,136 @@ def _channel_type_name(type_id: int) -> str: # Module-level cache so the app/me endpoint is hit at most once per process. _capability_cache: Dict[str, Dict[str, Any]] = {} +# Disk-cache TTL for detected capabilities. Privileged intents change only +# when the user flips them in the Discord Developer Portal, so 24h staleness +# is harmless — and a stale value only affects which actions appear in the +# schema (a hidden action re-appears on the next refresh; an exposed action +# the bot lost fails at call time with an enriched 403). +_CAPABILITY_DISK_TTL_SECONDS = 24 * 3600 -def _detect_capabilities(token: str, *, force: bool = False) -> Dict[str, Any]: - """Detect the bot's app-wide capabilities via GET /applications/@me. +# One background detection per process at most. +_capability_bg_started: set = set() +_capability_bg_lock = threading.Lock() - Returns a dict with keys: - - ``has_members_intent``: GUILD_MEMBERS intent is enabled - - ``has_message_content``: MESSAGE_CONTENT intent is enabled - - ``detected``: detection succeeded (False means exposing everything - and letting runtime errors handle it) +def _capability_disk_cache_path() -> "Path": + from pathlib import Path - Cached in a module-global. Pass ``force=True`` to re-fetch. + from hermes_constants import get_hermes_home + + return get_hermes_home() / "cache" / "discord_capabilities.json" + + +def _token_cache_key(token: str) -> str: + """Stable non-reversible cache key for a bot token.""" + import hashlib + + return hashlib.sha256(token.encode("utf-8")).hexdigest()[:16] + + +def _load_caps_from_disk(token: str) -> Optional[Dict[str, Any]]: + """Return fresh disk-cached capabilities for *token*, or None.""" + import time + + try: + path = _capability_disk_cache_path() + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + entry = data.get(_token_cache_key(token)) + if not isinstance(entry, dict): + return None + if time.time() - float(entry.get("ts", 0)) > _CAPABILITY_DISK_TTL_SECONDS: + return None + caps = entry.get("caps") + if isinstance(caps, dict) and "has_members_intent" in caps: + return caps + except Exception: + pass + return None + + +def _save_caps_to_disk(token: str, caps: Dict[str, Any]) -> None: + import time + + try: + path = _capability_disk_cache_path() + path.parent.mkdir(parents=True, exist_ok=True) + try: + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + data = {} + except Exception: + data = {} + data[_token_cache_key(token)] = {"caps": caps, "ts": time.time()} + tmp = path.with_suffix(".json.tmp") + with tmp.open("w", encoding="utf-8") as f: + json.dump(data, f) + tmp.replace(path) + except Exception: + logger.debug("discord capability disk-cache write failed", exc_info=True) + + +def _detect_capabilities_nonblocking(token: str) -> Dict[str, Any]: + """Non-blocking capability lookup for schema builds. + + Resolution order: + 1. In-process memory cache (populated by a previous sync/bg detection). + 2. Fresh disk cache (populated by a previous process). + 3. Permissive default + fire-and-forget background detection that + populates both caches for the next schema build / process. + + Rationale: ``_detect_capabilities`` makes a blocking HTTPS call to + discord.com (measured ~2s, up to 5s on the timeout) and used to run + inside ``get_tool_definitions`` → ``AIAgent.__init__`` — i.e. on the + critical path of the FIRST TOKEN of every cold process for any user + with DISCORD_BOT_TOKEN set, on every platform. The permissive default + mirrors the existing detection-failure fallback: all actions exposed, + call-time 403s mapped to guidance by ``_enrich_403``. """ - global _capability_cache - if token in _capability_cache and not force: - return _capability_cache[token] + cached = _capability_cache.get(token) + if cached is not None: + return cached + disk = _load_caps_from_disk(token) + if disk is not None: + _capability_cache[token] = disk + return disk + + # Cold start — pin the permissive default for THIS process (schema + # stability: tool schemas must not change between agent inits within a + # live process, or the per-conversation prompt cache breaks) and detect + # in the background for the NEXT process via the disk cache. + caps_default = { + "has_members_intent": True, + "has_message_content": True, + "detected": False, + } + _capability_cache[token] = caps_default + + with _capability_bg_lock: + if token not in _capability_bg_started: + _capability_bg_started.add(token) + + def _bg_detect() -> None: + try: + caps = _fetch_capabilities(token) + if caps.get("detected"): + _save_caps_to_disk(token, caps) + except Exception: + logger.debug("background discord capability detection failed", exc_info=True) + + threading.Thread( + target=_bg_detect, name="discord-caps-detect", daemon=True + ).start() + + return caps_default + + +def _fetch_capabilities(token: str) -> Dict[str, Any]: + """Fetch capabilities from GET /applications/@me. Pure network fetch — + does NOT read or write the in-process cache (background detection must + not mutate schemas mid-process).""" caps: Dict[str, Any] = { "has_members_intent": True, "has_message_content": True, @@ -172,14 +289,36 @@ def _detect_capabilities(token: str, *, force: bool = False) -> Dict[str, Any]: "Discord capability detection failed (%s); exposing all actions.", exc, ) + return caps + + +def _detect_capabilities(token: str, *, force: bool = False) -> Dict[str, Any]: + """Detect the bot's app-wide capabilities via GET /applications/@me. + + Returns a dict with keys: + + - ``has_members_intent``: GUILD_MEMBERS intent is enabled + - ``has_message_content``: MESSAGE_CONTENT intent is enabled + - ``detected``: detection succeeded (False means exposing everything + and letting runtime errors handle it) + + Cached in a module-global. Pass ``force=True`` to re-fetch. + """ + global _capability_cache + if token in _capability_cache and not force: + return _capability_cache[token] + + caps = _fetch_capabilities(token) _capability_cache[token] = caps return caps def _reset_capability_cache() -> None: """Test hook: clear the detection cache.""" - global _capability_cache + global _capability_cache, _capability_bg_started _capability_cache = {} + with _capability_bg_lock: + _capability_bg_started = set() # --------------------------------------------------------------------------- @@ -733,7 +872,7 @@ def _get_dynamic_schema( token = _get_bot_token() if not token: return None - caps = _detect_capabilities(token) + caps = _detect_capabilities_nonblocking(token) allowlist = _load_allowed_actions_config() actions = [a for a in _available_actions(caps, allowlist) if a in action_subset] if not actions: diff --git a/tools/env_probe.py b/tools/env_probe.py index 4b156b06edb..71a1c8116cf 100644 --- a/tools/env_probe.py +++ b/tools/env_probe.py @@ -241,8 +241,35 @@ def get_environment_probe_line(*, force_refresh: bool = False) -> str: return line +_warm_started = False + + +def warm_environment_probe_async() -> None: + """Kick off the probe in a background thread so the first + system-prompt build doesn't pay the ~0.5s of subprocess calls + (python3/pip/PEP-668 version checks) on the time-to-first-token + critical path. + + Idempotent and fail-safe. The prompt-build call to + ``get_environment_probe_line`` takes the same ``_CACHE_LOCK``, so it + blocks only for whatever remains of an in-flight warm instead of + recomputing. Called from agent init (all platforms); safe to call + from anywhere. + """ + global _warm_started + if _warm_started or _CACHED_LINE is not None: + return + _warm_started = True + threading.Thread( + target=get_environment_probe_line, + name="env-probe-warm", + daemon=True, + ).start() + + def _reset_cache_for_tests() -> None: """Test helper — clear the cache between probe scenarios.""" - global _CACHED_LINE + global _CACHED_LINE, _warm_started with _CACHE_LOCK: _CACHED_LINE = None + _warm_started = False