diff --git a/agent/agent_init.py b/agent/agent_init.py index 045fcfc1eeb..41ed85e998d 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1721,6 +1721,33 @@ def init_agent( f"(this must be at least {MINIMUM_CONTEXT_LENGTH // 1000}K)." ) + # Nous Hermes 3/4 are chat models, not tool-call-tuned. The interactive + # CLI already warns via cli.py show_banner() (richer output + /model hint), + # so skip platform=="cli" here to avoid emitting the warning twice per + # startup. (Gateway/TUI/cron construct with quiet_mode=True and are already + # gated off by the `not agent.quiet_mode` check above; this guard's active + # job is the CLI dedup, and it leaves the door open for any non-quiet + # non-CLI surface to still surface the warning.) + if not agent.quiet_mode and (agent.platform or "cli") != "cli": + try: + from hermes_cli.model_switch import _check_hermes_model_warning + + _hermes_warn = _check_hermes_model_warning(agent.model or "") + if _hermes_warn: + _user_msg = ( + "⚠ Nous Research Hermes 3 & 4 models are NOT agentic — they " + "lack reliable tool-calling for agent workflows (delegation, " + "cron, proactive tools). Consider an agentic model instead " + "(Claude, GPT, Gemini, Qwen-Coder, etc.)." + ) + if hasattr(agent, "_emit_warning"): + agent._emit_warning(_user_msg) + else: + print(f"\n{_user_msg}\n", file=sys.stderr) + _ra().logger.warning(_hermes_warn) + except Exception: + pass + # Inject context engine tool schemas (e.g. lcm_grep, lcm_describe, lcm_expand). # Skip names that are already present — the _ra().get_tool_definitions() # quiet_mode cache returned a shared list pre-#17335, so a stray diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 547fe2f5791..726c3300a90 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -184,6 +184,15 @@ DEFAULT_FALLBACK_CONTEXT = CONTEXT_PROBE_TIERS[0] # Sessions, model switches, and cron jobs should reject models below this. MINIMUM_CONTEXT_LENGTH = 64_000 +# Short-lived in-process cache for local-server context probes. Bounds the +# probe rate when the new local-endpoint live-probe paths (reconcile-on-hit + +# pre-defaults step 7) resolve the same model several times during one startup +# (banner, /model switch, compressor update_model). Keyed by (model, base_url); +# values are (result, monotonic_timestamp). Not persisted to disk — cross- +# restart freshness is handled by the reconcile logic re-probing after expiry. +_LOCAL_CTX_PROBE_TTL_SECONDS = 30.0 +_LOCAL_CTX_PROBE_CACHE: Dict[tuple, tuple] = {} + # Thin fallback defaults — only broad model family patterns. # These fire only when provider is unknown AND models.dev/OpenRouter/Anthropic # all miss. Replaced the previous 80+ entry dict. @@ -496,6 +505,68 @@ def _is_known_provider_base_url(base_url: str) -> bool: return _infer_provider_from_url(base_url) is not None +def _skip_persistent_context_cache(base_url: str, provider: str) -> bool: + """Return True when the on-disk context cache must not short-circuit probing. + + LM Studio excludes caching because loaded context is transient — the user + can reload the model with a different context_length at any time. + """ + return provider == "lmstudio" + + +def _maybe_cache_local_context_length( + model: str, + base_url: str, + length: int, +) -> None: + """Persist a locally probed context length only when it meets Hermes minimum. + + Sub-minimum live windows (e.g. vLLM ``--max-model-len 32768``) are still + returned to callers so ``agent_init`` can fail with the existing + minimum-context guidance — they must not be normalized into the on-disk cache + as if they were valid operating limits. + """ + if length >= MINIMUM_CONTEXT_LENGTH: + save_context_length(model, base_url, length) + + +def _reconcile_local_cached_context_length( + model: str, + base_url: str, + cached: int, + api_key: str = "", +) -> int: + """Return *cached* unless a live local probe reports a different limit. + + vLLM/Ollama operators can restart with a new ``--max-model-len`` / ``num_ctx`` + without changing the model id. When the server is reachable, prefer its + reported window over a stale disk entry; when the probe fails (offline tests, + network blip), keep the cached value. + + Live probes below :data:`MINIMUM_CONTEXT_LENGTH` invalidate stale cache + entries but are not persisted — startup should reject them, not bless a + sub-64K window as config. + """ + live_ctx = _query_local_context_length(model, base_url, api_key=api_key) + if live_ctx and live_ctx > 0 and live_ctx != cached: + if live_ctx < MINIMUM_CONTEXT_LENGTH: + logger.info( + "Live local probe for %s@%s reports %s (< minimum %s); " + "invalidating stale cache — agent init should reject", + model, base_url, f"{live_ctx:,}", f"{MINIMUM_CONTEXT_LENGTH:,}", + ) + _invalidate_cached_context_length(model, base_url) + return live_ctx + logger.info( + "Reconciling stale local cache entry %s@%s: %s -> %s (live probe)", + model, base_url, f"{cached:,}", f"{live_ctx:,}", + ) + _invalidate_cached_context_length(model, base_url) + _maybe_cache_local_context_length(model, base_url, live_ctx) + return live_ctx + return cached + + def is_local_endpoint(base_url: str) -> bool: """Return True if base_url points to a local machine. @@ -1006,6 +1077,8 @@ def parse_context_limit_from_error(error_msg: str) -> Optional[int]: error_lower = error_msg.lower() # Pattern: look for numbers near context-related keywords patterns = [ + r'max_model_len\s*(?:is\s*)?[:=(]?\s*(\d{4,})', # vLLM: "max_model_len 32768", "=32768", ": 32768", "(32768)", "is 32768" + r'maximum model length\s*(?:is\s*)?[:=(]?\s*(\d{4,})', # vLLM alt: "maximum model length 131072", "... is 131072" r'(?:max(?:imum)?|limit)\s*(?:context\s*)?(?:length|size|window)?\s*(?:is|of|:)?\s*(\d{4,})', r'context\s*(?:length|size|window)\s*(?:is|of|:)?\s*(\d{4,})', r'(\d{4,})\s*(?:token)?\s*(?:context|limit)', @@ -1451,6 +1524,40 @@ def _model_name_suggests_grok_4_3(model: str) -> bool: def _query_local_context_length(model: str, base_url: str, api_key: str = "") -> Optional[int]: + """Query a local server for the model's context length (short-TTL cached). + + The live-probe paths added for local endpoints (reconcile-on-hit and the + pre-defaults step-7 probe) can fire this function several times in quick + succession during one startup — banner display, ``/model`` switch, + compressor ``update_model`` all resolve the same model. Each raw probe + issues synchronous ``detect_local_server_type`` + query HTTP calls (bounded + by the 3s httpx timeout), so an unreachable/slow local server would pay + that cost repeatedly. A tiny in-process TTL cache collapses back-to-back + probes for the same (model, base_url) into one network round-trip without + persisting anything to disk (freshness across restarts is still handled by + the reconcile logic, which probes again once the TTL expires). + """ + import time as _time + + cache_key = (_strip_provider_prefix(model), base_url.rstrip("/")) + now = _time.monotonic() + cached = _LOCAL_CTX_PROBE_CACHE.get(cache_key) + if cached is not None and (now - cached[1]) < _LOCAL_CTX_PROBE_TTL_SECONDS: + return cached[0] + + result = _query_local_context_length_uncached(model, base_url, api_key=api_key) + # Cache only positive results. A None/failure (server not up yet, + # connection refused, timeout) must NOT be memoized — otherwise a probe + # that fails during a startup race would suppress a legit retry seconds + # later once the server is reachable. Positive-only caching still fully + # bounds the hot-path probe rate (a reachable server returns a value and + # gets cached); an unreachable one simply re-probes on the next call. + if result: + _LOCAL_CTX_PROBE_CACHE[cache_key] = (result, now) + return result + + +def _query_local_context_length_uncached(model: str, base_url: str, api_key: str = "") -> Optional[int]: """Query a local server for the model's context length.""" import httpx @@ -1805,8 +1912,8 @@ def get_model_context_length( e. Ollama native /api/show probe (any base_url, provider-agnostic) f. models.dev registry lookup (with :cloud/-cloud suffix fallback) 6. OpenRouter live API metadata (Kimi-family 32k guard) - 7. Hardcoded defaults (broad family patterns, longest-key-first) - 8. Local server query (last resort) + 7. Local server query (before hardcoded defaults for local endpoints) + 8. Hardcoded defaults (broad family patterns, longest-key-first) 9. Default fallback (256K)""" # 0. Explicit config override — user knows best if config_context_length is not None and isinstance(config_context_length, int) and config_context_length > 0: @@ -1866,7 +1973,7 @@ def get_model_context_length( # LM Studio is excluded — its loaded context length is transient (the # user can reload the model with a different context_length at any time # via /api/v1/models/load), so a stale cached value would mask reloads. - if base_url and provider != "lmstudio": + if base_url and not _skip_persistent_context_cache(base_url, provider): cached = get_cached_context_length(model, base_url) if cached is not None: # Invalidate stale Codex OAuth cache entries: pre-PR #14935 builds @@ -1931,6 +2038,10 @@ def get_model_context_length( ) # Fall through; step 5b reconciles and overwrites if portal responds. else: + if is_local_endpoint(base_url): + return _reconcile_local_cached_context_length( + model, base_url, cached, api_key=api_key, + ) return cached # 1b. AWS Bedrock — use static context length table. @@ -1975,14 +2086,15 @@ def get_model_context_length( # 404/405 quickly. Fall through on failure. ctx = _query_ollama_api_show(model, base_url, api_key=api_key) if ctx is not None: - save_context_length(model, base_url, ctx) + if not _skip_persistent_context_cache(base_url, provider): + save_context_length(model, base_url, ctx) return ctx # 3. Try querying local server directly if is_local_endpoint(base_url): local_ctx = _query_local_context_length(model, base_url, api_key=api_key) if local_ctx and local_ctx > 0: - if provider != "lmstudio": - save_context_length(model, base_url, local_ctx) + if not _skip_persistent_context_cache(base_url, provider): + _maybe_cache_local_context_length(model, base_url, local_ctx) return local_ctx logger.info( "Could not detect context length for model %r at %s — " @@ -2088,7 +2200,8 @@ def get_model_context_length( if base_url: ctx = _query_ollama_api_show(model, base_url, api_key=api_key) if ctx is not None: - save_context_length(model, base_url, ctx) + if not _skip_persistent_context_cache(base_url, provider): + save_context_length(model, base_url, ctx) return ctx # 5f. OpenRouter live /models metadata — authoritative for OpenRouter-routed # models. OpenRouter's catalog carries per-model context_length (e.g. @@ -2147,7 +2260,15 @@ def get_model_context_length( else: return or_ctx - # 7. (reserved) + # 7. Query local server before hardcoded defaults — model names like + # ``Hermes-3-Llama-3.1-70B`` substring-match ``llama`` (131072) even when + # vLLM is running at a lower ``--max-model-len`` (e.g. 32768 on limited VRAM). + if base_url and is_local_endpoint(base_url): + local_ctx = _query_local_context_length(model, base_url, api_key=api_key) + if local_ctx and local_ctx > 0: + if not _skip_persistent_context_cache(base_url, provider): + _maybe_cache_local_context_length(model, base_url, local_ctx) + return local_ctx # 8. Hardcoded defaults (fuzzy match — longest key first for specificity) # Only check `default_model in model` (is the key a substring of the input). @@ -2160,15 +2281,7 @@ def get_model_context_length( if default_model in model_lower: return length - # 9. Query local server as last resort - if base_url and is_local_endpoint(base_url): - local_ctx = _query_local_context_length(model, base_url, api_key=api_key) - if local_ctx and local_ctx > 0: - if provider != "lmstudio": - save_context_length(model, base_url, local_ctx) - return local_ctx - - # 10. Default fallback — 256K + # 9. Default fallback — 256K return DEFAULT_FALLBACK_CONTEXT diff --git a/scripts/release.py b/scripts/release.py index 2218f3fb7d7..1b3d3a672fe 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -45,6 +45,7 @@ ACP_REGISTRY_MANIFEST = REPO_ROOT / "acp_registry" / "agent.json" # Auto-extracted from noreply emails + manual overrides AUTHOR_MAP = { + "infinitycrew39@gmail.com": "infinitycrew39", # PR #56431 salvage (honor live vLLM context limits on local endpoints) "hermes.wanderer@yahoo.com": "trismegistus-wanderer", # PR #31856 salvage (gateway: defer idle-TTL agent-cache eviction until the session store says the session actually expired, so the expiry watcher can still fire MemoryProvider.on_session_end with the live transcript; #11205) "louis@letsfive.io": "Mibayy", # PR #3243 salvage (/compact alias + preview/aggressive flags for /compress) "louis@letsfive.io": "Mibayy", # PR #3176 salvage (api-server: per-client model routing via model_routes) diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index a18d9fb4618..d450580cd3c 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -12,6 +12,7 @@ Coverage levels: import time +import pytest import yaml from unittest.mock import patch, MagicMock @@ -1373,6 +1374,43 @@ class TestParseContextLimitFromError: msg = "Error: context window of 4096 tokens exceeded" assert parse_context_limit_from_error(msg) == 4096 + def test_vllm_max_model_len_format(self): + msg = ( + "The engine prompt length 1327246 exceeds the max_model_len 32768. " + "Please reduce prompt." + ) + assert parse_context_limit_from_error(msg) == 32768 + + def test_vllm_maximum_model_length_format(self): + msg = "prompt length 200000 exceeds maximum model length 131072" + assert parse_context_limit_from_error(msg) == 131072 + + @pytest.mark.parametrize("msg,expected", [ + ("max_model_len 32768", 32768), + ("max_model_len: 32768", 32768), + ("max_model_len=32768", 32768), + ("max_model_len (32768)", 32768), + ("max_model_len is 32768", 32768), + ("maximum model length 131072", 131072), + ("maximum model length is 131072", 131072), + ("maximum model length: 131072", 131072), + ]) + def test_vllm_delimiter_variants(self, msg, expected): + """vLLM emits the limit with various delimiters (space/colon/equals/ + paren/'is'). The parser must catch all of them — the original + space-only patterns silently missed ':', '=', '(' and 'is' forms and + fell through to None.""" + assert parse_context_limit_from_error(msg) == expected + + def test_get_context_length_from_vllm_max_model_len_error(self): + from agent.model_metadata import get_context_length_from_provider_error + + msg = ( + "The engine prompt length 90000 exceeds the max_model_len 32768. " + "Please reduce prompt." + ) + assert get_context_length_from_provider_error(msg, 131072) == 32768 + def test_minimax_delta_only_message_returns_none(self): msg = "invalid params, context window exceeds limit (2013)" assert parse_context_limit_from_error(msg) is None diff --git a/tests/agent/test_model_metadata_local_ctx.py b/tests/agent/test_model_metadata_local_ctx.py index 9b0268bda0f..2c069aaf6a4 100644 --- a/tests/agent/test_model_metadata_local_ctx.py +++ b/tests/agent/test_model_metadata_local_ctx.py @@ -8,9 +8,27 @@ import sys import os from unittest.mock import MagicMock, patch +import pytest + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +@pytest.fixture(autouse=True) +def _clear_local_ctx_probe_cache(): + """Reset the in-process local-probe TTL cache around every test. + + _query_local_context_length memoizes probes per (model, base_url) for a + short TTL to bound the probe rate on hot paths. In tests that mock httpx + to return different responses for the same (model, base_url), a stale + cache entry would leak across cases — clear it before and after each test. + """ + import agent.model_metadata as _mm + + _mm._LOCAL_CTX_PROBE_CACHE.clear() + yield + _mm._LOCAL_CTX_PROBE_CACHE.clear() + + # --------------------------------------------------------------------------- # _query_local_context_length — unit tests with mocked httpx @@ -615,6 +633,70 @@ class TestGetModelContextLengthLocalFallback: mock_save.assert_called_once_with("omnicoder-9b", "http://localhost:11434/v1", 131072) + def test_local_endpoint_stale_cache_reconciled_from_live_probe(self): + """Stale disk cache must yield to a live local max_model_len probe.""" + from agent.model_metadata import get_model_context_length + + model = "NousResearch/Hermes-3-Llama-3.1-70B" + base = "http://192.168.1.50:8000/v1" + + with patch("agent.model_metadata.get_cached_context_length", return_value=131072), \ + patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \ + patch("agent.model_metadata.fetch_model_metadata", return_value={}), \ + patch("agent.model_metadata._query_ollama_api_show", return_value=None), \ + patch("agent.model_metadata._is_custom_endpoint", return_value=False), \ + patch("agent.model_metadata.is_local_endpoint", return_value=True), \ + patch("agent.model_metadata._query_local_context_length", return_value=32768), \ + patch("agent.model_metadata._invalidate_cached_context_length") as mock_invalidate, \ + patch("agent.model_metadata.save_context_length") as mock_save: + result = get_model_context_length(model, base, provider="custom") + + assert result == 32768 + mock_invalidate.assert_called_once_with(model, base) + mock_save.assert_not_called() + + def test_local_endpoint_stale_cache_reconciled_to_valid_live_probe(self): + """Live probes at or above the 64K minimum are persisted.""" + from agent.model_metadata import get_model_context_length + + model = "NousResearch/Hermes-3-Llama-3.1-70B" + base = "http://192.168.1.50:8000/v1" + + with patch("agent.model_metadata.get_cached_context_length", return_value=131072), \ + patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \ + patch("agent.model_metadata.fetch_model_metadata", return_value={}), \ + patch("agent.model_metadata._query_ollama_api_show", return_value=None), \ + patch("agent.model_metadata._is_custom_endpoint", return_value=False), \ + patch("agent.model_metadata.is_local_endpoint", return_value=True), \ + patch("agent.model_metadata._query_local_context_length", return_value=65536), \ + patch("agent.model_metadata._invalidate_cached_context_length") as mock_invalidate, \ + patch("agent.model_metadata.save_context_length") as mock_save: + result = get_model_context_length(model, base, provider="custom") + + assert result == 65536 + mock_invalidate.assert_called_once_with(model, base) + mock_save.assert_called_once_with(model, base, 65536) + + def test_local_endpoint_bypasses_stale_persistent_cache(self): + """Hermes-3-Llama names must not inherit the generic llama 131072 default.""" + from agent.model_metadata import get_model_context_length + + model = "NousResearch/Hermes-3-Llama-3.1-70B" + base = "http://spark1:8000/v1" + + with patch("agent.model_metadata.get_cached_context_length", return_value=None), \ + patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \ + patch("agent.model_metadata.fetch_model_metadata", return_value={}), \ + patch("agent.model_metadata._query_ollama_api_show", return_value=None), \ + patch("agent.model_metadata._is_custom_endpoint", return_value=False), \ + patch("agent.model_metadata.is_local_endpoint", return_value=True), \ + patch("agent.model_metadata._query_local_context_length", return_value=32768), \ + patch("agent.model_metadata.save_context_length") as mock_save: + result = get_model_context_length(model, base, provider="custom") + + assert result == 32768 + mock_save.assert_not_called() + def test_local_endpoint_server_returns_none_falls_back_to_2m(self): """When local server returns None, still falls back to 2M probe tier.""" from agent.model_metadata import get_model_context_length, CONTEXT_PROBE_TIERS @@ -648,8 +730,11 @@ class TestGetModelContextLengthLocalFallback: from agent.model_metadata import get_model_context_length with patch("agent.model_metadata.get_cached_context_length", return_value=65536), \ + patch("agent.model_metadata.is_local_endpoint", return_value=False), \ patch("agent.model_metadata._query_local_context_length") as mock_query: - result = get_model_context_length("omnicoder-9b", "http://localhost:11434/v1") + result = get_model_context_length( + "omnicoder-9b", "https://api.example.com/v1" + ) assert result == 65536 mock_query.assert_not_called() @@ -665,3 +750,79 @@ class TestGetModelContextLengthLocalFallback: result = get_model_context_length("unknown-xyz-model", "") mock_query.assert_not_called() + + +class TestLocalContextProbeTTLCache: + """The in-process TTL cache collapses back-to-back probes for the same + (model, base_url) into one network round-trip (bounds probe rate on hot + paths like banner + /model switch + compressor update within one startup), + while a different key still probes.""" + + def _make_resp(self, status_code, body): + resp = MagicMock() + resp.status_code = status_code + resp.json.return_value = body + return resp + + def test_second_call_within_ttl_does_not_reprobe(self): + from agent.model_metadata import _query_local_context_length + + show_resp = self._make_resp(200, {"model_info": {"llama.context_length": 32768}}) + models_resp = self._make_resp(404, {}) + client_mock = MagicMock() + client_mock.__enter__ = lambda s: client_mock + client_mock.__exit__ = MagicMock(return_value=False) + client_mock.post.return_value = show_resp + client_mock.get.return_value = models_resp + + with patch("agent.model_metadata.detect_local_server_type", return_value="ollama") as detect, \ + patch("httpx.Client", return_value=client_mock): + first = _query_local_context_length("m", "http://localhost:11434/v1") + second = _query_local_context_length("m", "http://localhost:11434/v1") + + assert first == 32768 + assert second == 32768 + # Only the first call hits the network; the second is served from cache. + assert detect.call_count == 1 + + def test_different_key_still_probes(self): + from agent.model_metadata import _query_local_context_length + + show_resp = self._make_resp(200, {"model_info": {"llama.context_length": 32768}}) + models_resp = self._make_resp(404, {}) + client_mock = MagicMock() + client_mock.__enter__ = lambda s: client_mock + client_mock.__exit__ = MagicMock(return_value=False) + client_mock.post.return_value = show_resp + client_mock.get.return_value = models_resp + + with patch("agent.model_metadata.detect_local_server_type", return_value="ollama") as detect, \ + patch("httpx.Client", return_value=client_mock): + _query_local_context_length("m1", "http://localhost:11434/v1") + _query_local_context_length("m2", "http://localhost:11434/v1") + + assert detect.call_count == 2 + + + def test_none_result_not_cached(self): + """A failed probe (None) must NOT be memoized — a retry within the TTL + window must re-probe so a server that comes up mid-startup is caught.""" + from agent.model_metadata import _query_local_context_length + + # First probe: server unreachable -> detect returns None, all queries miss -> None. + fail_resp = self._make_resp(404, {}) + client_mock = MagicMock() + client_mock.__enter__ = lambda s: client_mock + client_mock.__exit__ = MagicMock(return_value=False) + client_mock.post.return_value = fail_resp + client_mock.get.return_value = fail_resp + + with patch("agent.model_metadata.detect_local_server_type", return_value=None) as detect, \ + patch("httpx.Client", return_value=client_mock): + first = _query_local_context_length("m", "http://localhost:11434/v1") + # Retry within TTL must re-probe (None was not cached). + second = _query_local_context_length("m", "http://localhost:11434/v1") + + assert first is None + assert second is None + assert detect.call_count == 2, "None result was wrongly cached; retry did not re-probe"