mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-09 13:21:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge pull request #57379 from kshitijk4poor/salvage/vllm-local-context

Browse source
This commit is contained in:
kshitij 2026-07-03 03:58:27 +05:30 • committed by GitHub
commit 80a774f972
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 358 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

27
agent/agent_init.py
View file

@ -1721,6 +1721,33 @@ def init_agent(
f"(this must be at least {MINIMUM_CONTEXT_LENGTH // 1000}K)."
)
# Nous Hermes 3/4 are chat models, not tool-call-tuned. The interactive
# CLI already warns via cli.py show_banner() (richer output + /model hint),
# so skip platform=="cli" here to avoid emitting the warning twice per
# startup. (Gateway/TUI/cron construct with quiet_mode=True and are already
# gated off by the `not agent.quiet_mode` check above; this guard's active
# job is the CLI dedup, and it leaves the door open for any non-quiet
# non-CLI surface to still surface the warning.)
if not agent.quiet_mode and (agent.platform or "cli") != "cli":
try:
from hermes_cli.model_switch import _check_hermes_model_warning
_hermes_warn = _check_hermes_model_warning(agent.model or "")
if _hermes_warn:
_user_msg = (
"⚠ Nous Research Hermes 3 & 4 models are NOT agentic — they "
"lack reliable tool-calling for agent workflows (delegation, "
"cron, proactive tools). Consider an agentic model instead "
"(Claude, GPT, Gemini, Qwen-Coder, etc.)."
)
if hasattr(agent, "_emit_warning"):
agent._emit_warning(_user_msg)
else:
print(f"\n{_user_msg}\n", file=sys.stderr)
_ra().logger.warning(_hermes_warn)
except Exception:
pass
# Inject context engine tool schemas (e.g. lcm_grep, lcm_describe, lcm_expand).
# Skip names that are already present — the _ra().get_tool_definitions()
# quiet_mode cache returned a shared list pre-#17335, so a stray

147
agent/model_metadata.py
View file

@ -184,6 +184,15 @@ DEFAULT_FALLBACK_CONTEXT = CONTEXT_PROBE_TIERS[0]
# Sessions, model switches, and cron jobs should reject models below this.
MINIMUM_CONTEXT_LENGTH = 64_000
# Short-lived in-process cache for local-server context probes. Bounds the
# probe rate when the new local-endpoint live-probe paths (reconcile-on-hit +
# pre-defaults step 7) resolve the same model several times during one startup
# (banner, /model switch, compressor update_model). Keyed by (model, base_url);
# values are (result, monotonic_timestamp). Not persisted to disk — cross-
# restart freshness is handled by the reconcile logic re-probing after expiry.
_LOCAL_CTX_PROBE_TTL_SECONDS = 30.0
_LOCAL_CTX_PROBE_CACHE: Dict[tuple, tuple] = {}
# Thin fallback defaults — only broad model family patterns.
# These fire only when provider is unknown AND models.dev/OpenRouter/Anthropic
# all miss. Replaced the previous 80+ entry dict.
@ -496,6 +505,68 @@ def _is_known_provider_base_url(base_url: str) -> bool:
return _infer_provider_from_url(base_url) is not None
def _skip_persistent_context_cache(base_url: str, provider: str) -> bool:
"""Return True when the on-disk context cache must not short-circuit probing.
LM Studio excludes caching because loaded context is transient — the user
can reload the model with a different context_length at any time.
"""
return provider == "lmstudio"
def _maybe_cache_local_context_length(
model: str,
base_url: str,
length: int,
) -> None:
"""Persist a locally probed context length only when it meets Hermes minimum.
Sub-minimum live windows (e.g. vLLM ``--max-model-len 32768``) are still
returned to callers so ``agent_init`` can fail with the existing
minimum-context guidance — they must not be normalized into the on-disk cache
as if they were valid operating limits.
"""
if length >= MINIMUM_CONTEXT_LENGTH:
save_context_length(model, base_url, length)
def _reconcile_local_cached_context_length(
model: str,
base_url: str,
cached: int,
api_key: str = "",
) -> int:
"""Return *cached* unless a live local probe reports a different limit.
vLLM/Ollama operators can restart with a new ``--max-model-len`` / ``num_ctx``
without changing the model id. When the server is reachable, prefer its
reported window over a stale disk entry; when the probe fails (offline tests,
network blip), keep the cached value.
Live probes below :data:`MINIMUM_CONTEXT_LENGTH` invalidate stale cache
entries but are not persisted — startup should reject them, not bless a
sub-64K window as config.
"""
live_ctx = _query_local_context_length(model, base_url, api_key=api_key)
if live_ctx and live_ctx > 0 and live_ctx != cached:
if live_ctx < MINIMUM_CONTEXT_LENGTH:
logger.info(
"Live local probe for %s@%s reports %s (< minimum %s); "
"invalidating stale cache — agent init should reject",
model, base_url, f"{live_ctx:,}", f"{MINIMUM_CONTEXT_LENGTH:,}",
)
_invalidate_cached_context_length(model, base_url)
return live_ctx
logger.info(
"Reconciling stale local cache entry %s@%s: %s -> %s (live probe)",
model, base_url, f"{cached:,}", f"{live_ctx:,}",
)
_invalidate_cached_context_length(model, base_url)
_maybe_cache_local_context_length(model, base_url, live_ctx)
return live_ctx
return cached
def is_local_endpoint(base_url: str) -> bool:
"""Return True if base_url points to a local machine.
@ -1006,6 +1077,8 @@ def parse_context_limit_from_error(error_msg: str) -> Optional[int]:
error_lower = error_msg.lower()
# Pattern: look for numbers near context-related keywords
patterns = [
r'max_model_len\s*(?:is\s*)?[:=(]?\s*(\d{4,})', # vLLM: "max_model_len 32768", "=32768", ": 32768", "(32768)", "is 32768"
r'maximum model length\s*(?:is\s*)?[:=(]?\s*(\d{4,})', # vLLM alt: "maximum model length 131072", "... is 131072"
r'(?:max(?:imum)?|limit)\s*(?:context\s*)?(?:length|size|window)?\s*(?:is|of|:)?\s*(\d{4,})',
r'context\s*(?:length|size|window)\s*(?:is|of|:)?\s*(\d{4,})',
r'(\d{4,})\s*(?:token)?\s*(?:context|limit)',
@ -1451,6 +1524,40 @@ def _model_name_suggests_grok_4_3(model: str) -> bool:
def _query_local_context_length(model: str, base_url: str, api_key: str = "") -> Optional[int]:
"""Query a local server for the model's context length (short-TTL cached).
The live-probe paths added for local endpoints (reconcile-on-hit and the
pre-defaults step-7 probe) can fire this function several times in quick
succession during one startup — banner display, ``/model`` switch,
compressor ``update_model`` all resolve the same model. Each raw probe
issues synchronous ``detect_local_server_type`` + query HTTP calls (bounded
by the 3s httpx timeout), so an unreachable/slow local server would pay
that cost repeatedly. A tiny in-process TTL cache collapses back-to-back
probes for the same (model, base_url) into one network round-trip without
persisting anything to disk (freshness across restarts is still handled by
the reconcile logic, which probes again once the TTL expires).
"""
import time as _time
cache_key = (_strip_provider_prefix(model), base_url.rstrip("/"))
now = _time.monotonic()
cached = _LOCAL_CTX_PROBE_CACHE.get(cache_key)
if cached is not None and (now - cached[1]) < _LOCAL_CTX_PROBE_TTL_SECONDS:
return cached[0]
result = _query_local_context_length_uncached(model, base_url, api_key=api_key)
# Cache only positive results. A None/failure (server not up yet,
# connection refused, timeout) must NOT be memoized — otherwise a probe
# that fails during a startup race would suppress a legit retry seconds
# later once the server is reachable. Positive-only caching still fully
# bounds the hot-path probe rate (a reachable server returns a value and
# gets cached); an unreachable one simply re-probes on the next call.
if result:
_LOCAL_CTX_PROBE_CACHE[cache_key] = (result, now)
return result
def _query_local_context_length_uncached(model: str, base_url: str, api_key: str = "") -> Optional[int]:
"""Query a local server for the model's context length."""
import httpx
@ -1805,8 +1912,8 @@ def get_model_context_length(
e. Ollama native /api/show probe (any base_url, provider-agnostic)
f. models.dev registry lookup (with :cloud/-cloud suffix fallback)
6. OpenRouter live API metadata (Kimi-family 32k guard)
7. Hardcoded defaults (broad family patterns, longest-key-first)
8. Local server query (last resort)
7. Local server query (before hardcoded defaults for local endpoints)
8. Hardcoded defaults (broad family patterns, longest-key-first)
9. Default fallback (256K)"""
# 0. Explicit config override — user knows best
if config_context_length is not None and isinstance(config_context_length, int) and config_context_length > 0:
@ -1866,7 +1973,7 @@ def get_model_context_length(
# LM Studio is excluded — its loaded context length is transient (the
# user can reload the model with a different context_length at any time
# via /api/v1/models/load), so a stale cached value would mask reloads.
if base_url and provider != "lmstudio":
if base_url and not _skip_persistent_context_cache(base_url, provider):
cached = get_cached_context_length(model, base_url)
if cached is not None:
# Invalidate stale Codex OAuth cache entries: pre-PR #14935 builds
@ -1931,6 +2038,10 @@ def get_model_context_length(
)
# Fall through; step 5b reconciles and overwrites if portal responds.
else:
if is_local_endpoint(base_url):
return _reconcile_local_cached_context_length(
model, base_url, cached, api_key=api_key,
)
return cached
# 1b. AWS Bedrock — use static context length table.
@ -1975,14 +2086,15 @@ def get_model_context_length(
# 404/405 quickly. Fall through on failure.
ctx = _query_ollama_api_show(model, base_url, api_key=api_key)
if ctx is not None:
save_context_length(model, base_url, ctx)
if not _skip_persistent_context_cache(base_url, provider):
save_context_length(model, base_url, ctx)
return ctx
# 3. Try querying local server directly
if is_local_endpoint(base_url):
local_ctx = _query_local_context_length(model, base_url, api_key=api_key)
if local_ctx and local_ctx > 0:
if provider != "lmstudio":
save_context_length(model, base_url, local_ctx)
if not _skip_persistent_context_cache(base_url, provider):
_maybe_cache_local_context_length(model, base_url, local_ctx)
return local_ctx
logger.info(
"Could not detect context length for model %r at %s — "
@ -2088,7 +2200,8 @@ def get_model_context_length(
if base_url:
ctx = _query_ollama_api_show(model, base_url, api_key=api_key)
if ctx is not None:
save_context_length(model, base_url, ctx)
if not _skip_persistent_context_cache(base_url, provider):
save_context_length(model, base_url, ctx)
return ctx
# 5f. OpenRouter live /models metadata — authoritative for OpenRouter-routed
# models. OpenRouter's catalog carries per-model context_length (e.g.
@ -2147,7 +2260,15 @@ def get_model_context_length(
else:
return or_ctx
# 7. (reserved)
# 7. Query local server before hardcoded defaults — model names like
# ``Hermes-3-Llama-3.1-70B`` substring-match ``llama`` (131072) even when
# vLLM is running at a lower ``--max-model-len`` (e.g. 32768 on limited VRAM).
if base_url and is_local_endpoint(base_url):
local_ctx = _query_local_context_length(model, base_url, api_key=api_key)
if local_ctx and local_ctx > 0:
if not _skip_persistent_context_cache(base_url, provider):
_maybe_cache_local_context_length(model, base_url, local_ctx)
return local_ctx
# 8. Hardcoded defaults (fuzzy match — longest key first for specificity)
# Only check `default_model in model` (is the key a substring of the input).
@ -2160,15 +2281,7 @@ def get_model_context_length(
if default_model in model_lower:
return length
# 9. Query local server as last resort
if base_url and is_local_endpoint(base_url):
local_ctx = _query_local_context_length(model, base_url, api_key=api_key)
if local_ctx and local_ctx > 0:
if provider != "lmstudio":
save_context_length(model, base_url, local_ctx)
return local_ctx
# 10. Default fallback — 256K
# 9. Default fallback — 256K
return DEFAULT_FALLBACK_CONTEXT

1
scripts/release.py
View file

@ -45,6 +45,7 @@ ACP_REGISTRY_MANIFEST = REPO_ROOT / "acp_registry" / "agent.json"
# Auto-extracted from noreply emails + manual overrides
AUTHOR_MAP = {
"infinitycrew39@gmail.com": "infinitycrew39", # PR #56431 salvage (honor live vLLM context limits on local endpoints)
"hermes.wanderer@yahoo.com": "trismegistus-wanderer", # PR #31856 salvage (gateway: defer idle-TTL agent-cache eviction until the session store says the session actually expired, so the expiry watcher can still fire MemoryProvider.on_session_end with the live transcript; #11205)
"louis@letsfive.io": "Mibayy", # PR #3243 salvage (/compact alias + preview/aggressive flags for /compress)
"louis@letsfive.io": "Mibayy", # PR #3176 salvage (api-server: per-client model routing via model_routes)

38
tests/agent/test_model_metadata.py
View file

@ -12,6 +12,7 @@ Coverage levels:
import time
import pytest
import yaml
from unittest.mock import patch, MagicMock
@ -1373,6 +1374,43 @@ class TestParseContextLimitFromError:
msg = "Error: context window of 4096 tokens exceeded"
assert parse_context_limit_from_error(msg) == 4096
def test_vllm_max_model_len_format(self):
msg = (
"The engine prompt length 1327246 exceeds the max_model_len 32768. "
"Please reduce prompt."
)
assert parse_context_limit_from_error(msg) == 32768
def test_vllm_maximum_model_length_format(self):
msg = "prompt length 200000 exceeds maximum model length 131072"
assert parse_context_limit_from_error(msg) == 131072
@pytest.mark.parametrize("msg,expected", [
("max_model_len 32768", 32768),
("max_model_len: 32768", 32768),
("max_model_len=32768", 32768),
("max_model_len (32768)", 32768),
("max_model_len is 32768", 32768),
("maximum model length 131072", 131072),
("maximum model length is 131072", 131072),
("maximum model length: 131072", 131072),
])
def test_vllm_delimiter_variants(self, msg, expected):
"""vLLM emits the limit with various delimiters (space/colon/equals/
paren/'is'). The parser must catch all of them — the original
space-only patterns silently missed ':', '=', '(' and 'is' forms and
fell through to None."""
assert parse_context_limit_from_error(msg) == expected
def test_get_context_length_from_vllm_max_model_len_error(self):
from agent.model_metadata import get_context_length_from_provider_error
msg = (
"The engine prompt length 90000 exceeds the max_model_len 32768. "
"Please reduce prompt."
)
assert get_context_length_from_provider_error(msg, 131072) == 32768
def test_minimax_delta_only_message_returns_none(self):
msg = "invalid params, context window exceeds limit (2013)"
assert parse_context_limit_from_error(msg) is None

163
tests/agent/test_model_metadata_local_ctx.py
View file

@ -8,9 +8,27 @@ import sys
import os
from unittest.mock import MagicMock, patch
import pytest
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
@pytest.fixture(autouse=True)
def _clear_local_ctx_probe_cache():
"""Reset the in-process local-probe TTL cache around every test.
_query_local_context_length memoizes probes per (model, base_url) for a
short TTL to bound the probe rate on hot paths. In tests that mock httpx
to return different responses for the same (model, base_url), a stale
cache entry would leak across cases — clear it before and after each test.
"""
import agent.model_metadata as _mm
_mm._LOCAL_CTX_PROBE_CACHE.clear()
yield
_mm._LOCAL_CTX_PROBE_CACHE.clear()
# ---------------------------------------------------------------------------
# _query_local_context_length — unit tests with mocked httpx
@ -615,6 +633,70 @@ class TestGetModelContextLengthLocalFallback:
mock_save.assert_called_once_with("omnicoder-9b", "http://localhost:11434/v1", 131072)
def test_local_endpoint_stale_cache_reconciled_from_live_probe(self):
"""Stale disk cache must yield to a live local max_model_len probe."""
from agent.model_metadata import get_model_context_length
model = "NousResearch/Hermes-3-Llama-3.1-70B"
base = "http://192.168.1.50:8000/v1"
with patch("agent.model_metadata.get_cached_context_length", return_value=131072), \
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \
patch("agent.model_metadata.fetch_model_metadata", return_value={}), \
patch("agent.model_metadata._query_ollama_api_show", return_value=None), \
patch("agent.model_metadata._is_custom_endpoint", return_value=False), \
patch("agent.model_metadata.is_local_endpoint", return_value=True), \
patch("agent.model_metadata._query_local_context_length", return_value=32768), \
patch("agent.model_metadata._invalidate_cached_context_length") as mock_invalidate, \
patch("agent.model_metadata.save_context_length") as mock_save:
result = get_model_context_length(model, base, provider="custom")
assert result == 32768
mock_invalidate.assert_called_once_with(model, base)
mock_save.assert_not_called()
def test_local_endpoint_stale_cache_reconciled_to_valid_live_probe(self):
"""Live probes at or above the 64K minimum are persisted."""
from agent.model_metadata import get_model_context_length
model = "NousResearch/Hermes-3-Llama-3.1-70B"
base = "http://192.168.1.50:8000/v1"
with patch("agent.model_metadata.get_cached_context_length", return_value=131072), \
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \
patch("agent.model_metadata.fetch_model_metadata", return_value={}), \
patch("agent.model_metadata._query_ollama_api_show", return_value=None), \
patch("agent.model_metadata._is_custom_endpoint", return_value=False), \
patch("agent.model_metadata.is_local_endpoint", return_value=True), \
patch("agent.model_metadata._query_local_context_length", return_value=65536), \
patch("agent.model_metadata._invalidate_cached_context_length") as mock_invalidate, \
patch("agent.model_metadata.save_context_length") as mock_save:
result = get_model_context_length(model, base, provider="custom")
assert result == 65536
mock_invalidate.assert_called_once_with(model, base)
mock_save.assert_called_once_with(model, base, 65536)
def test_local_endpoint_bypasses_stale_persistent_cache(self):
"""Hermes-3-Llama names must not inherit the generic llama 131072 default."""
from agent.model_metadata import get_model_context_length
model = "NousResearch/Hermes-3-Llama-3.1-70B"
base = "http://spark1:8000/v1"
with patch("agent.model_metadata.get_cached_context_length", return_value=None), \
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \
patch("agent.model_metadata.fetch_model_metadata", return_value={}), \
patch("agent.model_metadata._query_ollama_api_show", return_value=None), \
patch("agent.model_metadata._is_custom_endpoint", return_value=False), \
patch("agent.model_metadata.is_local_endpoint", return_value=True), \
patch("agent.model_metadata._query_local_context_length", return_value=32768), \
patch("agent.model_metadata.save_context_length") as mock_save:
result = get_model_context_length(model, base, provider="custom")
assert result == 32768
mock_save.assert_not_called()
def test_local_endpoint_server_returns_none_falls_back_to_2m(self):
"""When local server returns None, still falls back to 2M probe tier."""
from agent.model_metadata import get_model_context_length, CONTEXT_PROBE_TIERS
@ -648,8 +730,11 @@ class TestGetModelContextLengthLocalFallback:
from agent.model_metadata import get_model_context_length
with patch("agent.model_metadata.get_cached_context_length", return_value=65536), \
patch("agent.model_metadata.is_local_endpoint", return_value=False), \
patch("agent.model_metadata._query_local_context_length") as mock_query:
result = get_model_context_length("omnicoder-9b", "http://localhost:11434/v1")
result = get_model_context_length(
"omnicoder-9b", "https://api.example.com/v1"
)
assert result == 65536
mock_query.assert_not_called()
@ -665,3 +750,79 @@ class TestGetModelContextLengthLocalFallback:
result = get_model_context_length("unknown-xyz-model", "")
mock_query.assert_not_called()
class TestLocalContextProbeTTLCache:
"""The in-process TTL cache collapses back-to-back probes for the same
(model, base_url) into one network round-trip (bounds probe rate on hot
paths like banner + /model switch + compressor update within one startup),
while a different key still probes."""
def _make_resp(self, status_code, body):
resp = MagicMock()
resp.status_code = status_code
resp.json.return_value = body
return resp
def test_second_call_within_ttl_does_not_reprobe(self):
from agent.model_metadata import _query_local_context_length
show_resp = self._make_resp(200, {"model_info": {"llama.context_length": 32768}})
models_resp = self._make_resp(404, {})
client_mock = MagicMock()
client_mock.__enter__ = lambda s: client_mock
client_mock.__exit__ = MagicMock(return_value=False)
client_mock.post.return_value = show_resp
client_mock.get.return_value = models_resp
with patch("agent.model_metadata.detect_local_server_type", return_value="ollama") as detect, \
patch("httpx.Client", return_value=client_mock):
first = _query_local_context_length("m", "http://localhost:11434/v1")
second = _query_local_context_length("m", "http://localhost:11434/v1")
assert first == 32768
assert second == 32768
# Only the first call hits the network; the second is served from cache.
assert detect.call_count == 1
def test_different_key_still_probes(self):
from agent.model_metadata import _query_local_context_length
show_resp = self._make_resp(200, {"model_info": {"llama.context_length": 32768}})
models_resp = self._make_resp(404, {})
client_mock = MagicMock()
client_mock.__enter__ = lambda s: client_mock
client_mock.__exit__ = MagicMock(return_value=False)
client_mock.post.return_value = show_resp
client_mock.get.return_value = models_resp
with patch("agent.model_metadata.detect_local_server_type", return_value="ollama") as detect, \
patch("httpx.Client", return_value=client_mock):
_query_local_context_length("m1", "http://localhost:11434/v1")
_query_local_context_length("m2", "http://localhost:11434/v1")
assert detect.call_count == 2
def test_none_result_not_cached(self):
"""A failed probe (None) must NOT be memoized — a retry within the TTL
window must re-probe so a server that comes up mid-startup is caught."""
from agent.model_metadata import _query_local_context_length
# First probe: server unreachable -> detect returns None, all queries miss -> None.
fail_resp = self._make_resp(404, {})
client_mock = MagicMock()
client_mock.__enter__ = lambda s: client_mock
client_mock.__exit__ = MagicMock(return_value=False)
client_mock.post.return_value = fail_resp
client_mock.get.return_value = fail_resp
with patch("agent.model_metadata.detect_local_server_type", return_value=None) as detect, \
patch("httpx.Client", return_value=client_mock):
first = _query_local_context_length("m", "http://localhost:11434/v1")
# Retry within TTL must re-probe (None was not cached).
second = _query_local_context_length("m", "http://localhost:11434/v1")
assert first is None
assert second is None
assert detect.call_count == 2, "None result was wrongly cached; retry did not re-probe"