mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-15 14:22:43 +00:00
fix(model_metadata): cache detect_local_server_type result for process lifetime
Every 5 minutes fetch_endpoint_model_metadata() re-runs the full server-type waterfall (LM Studio -> Ollama -> llama.cpp -> vLLM), spraying 404s at endpoints the server never exposes (e.g. /api/v1/models and /api/tags on a vllm backend). Add _endpoint_probe_path_cache (base_url -> server type) so the first successful probe's result is reused for the lifetime of the process. Subsequent refreshes skip straight to the known-good path. Fixes #29971.
This commit is contained in:
parent
ba9964ff0d
commit
f3d7a8960a
2 changed files with 52 additions and 33 deletions
|
|
@ -110,6 +110,10 @@ _MODEL_CACHE_TTL = 3600
|
|||
_endpoint_model_metadata_cache: Dict[str, Dict[str, Dict[str, Any]]] = {}
|
||||
_endpoint_model_metadata_cache_time: Dict[str, float] = {}
|
||||
_ENDPOINT_MODEL_CACHE_TTL = 300
|
||||
# Process-lifetime cache: after the first successful probe we remember the
|
||||
# server type so subsequent refreshes skip the full waterfall (no more 404
|
||||
# spam every 5 minutes on non-matching endpoints like /api/v1/models on vllm).
|
||||
_endpoint_probe_path_cache: Dict[str, str] = {}
|
||||
|
||||
# Descending tiers for context length probing when the model is unknown.
|
||||
# We start at 256K (covers GPT-5.x, many current large-context models) and
|
||||
|
|
@ -467,6 +471,10 @@ def detect_local_server_type(base_url: str, api_key: str = "") -> Optional[str]:
|
|||
"""Detect which local server is running at base_url by probing known endpoints.
|
||||
|
||||
Returns one of: "ollama", "lm-studio", "vllm", "llamacpp", or None.
|
||||
|
||||
The result is cached for the lifetime of the process so that repeated
|
||||
calls (e.g. every 5-minute metadata refresh) never re-run the waterfall
|
||||
and never spray 404s at endpoints the server does not expose.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
|
|
@ -475,53 +483,63 @@ def detect_local_server_type(base_url: str, api_key: str = "") -> Optional[str]:
|
|||
if server_url.endswith("/v1"):
|
||||
server_url = server_url[:-3]
|
||||
|
||||
cached = _endpoint_probe_path_cache.get(server_url)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
headers = _auth_headers(api_key)
|
||||
|
||||
result: Optional[str] = None
|
||||
try:
|
||||
with httpx.Client(timeout=2.0, headers=headers) as client:
|
||||
# LM Studio exposes /api/v1/models — check first (most specific)
|
||||
try:
|
||||
r = client.get(f"{server_url}/api/v1/models")
|
||||
if r.status_code == 200:
|
||||
return "lm-studio"
|
||||
result = "lm-studio"
|
||||
except Exception:
|
||||
pass
|
||||
# Ollama exposes /api/tags and responds with {"models": [...]}
|
||||
# LM Studio returns {"error": "Unexpected endpoint"} with status 200
|
||||
# on this path, so we must verify the response contains "models".
|
||||
try:
|
||||
r = client.get(f"{server_url}/api/tags")
|
||||
if r.status_code == 200:
|
||||
try:
|
||||
if result is None:
|
||||
# Ollama exposes /api/tags and responds with {"models": [...]}
|
||||
# LM Studio returns {"error": "Unexpected endpoint"} with status 200
|
||||
# on this path, so we must verify the response contains "models".
|
||||
try:
|
||||
r = client.get(f"{server_url}/api/tags")
|
||||
if r.status_code == 200:
|
||||
try:
|
||||
data = r.json()
|
||||
if "models" in data:
|
||||
result = "ollama"
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
if result is None:
|
||||
# llama.cpp exposes /v1/props (older builds used /props without the /v1 prefix)
|
||||
try:
|
||||
r = client.get(f"{server_url}/v1/props")
|
||||
if r.status_code != 200:
|
||||
r = client.get(f"{server_url}/props") # fallback for older builds
|
||||
if r.status_code == 200 and "default_generation_settings" in r.text:
|
||||
result = "llamacpp"
|
||||
except Exception:
|
||||
pass
|
||||
if result is None:
|
||||
# vLLM: /version
|
||||
try:
|
||||
r = client.get(f"{server_url}/version")
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
if "models" in data:
|
||||
return "ollama"
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
# llama.cpp exposes /v1/props (older builds used /props without the /v1 prefix)
|
||||
try:
|
||||
r = client.get(f"{server_url}/v1/props")
|
||||
if r.status_code != 200:
|
||||
r = client.get(f"{server_url}/props") # fallback for older builds
|
||||
if r.status_code == 200 and "default_generation_settings" in r.text:
|
||||
return "llamacpp"
|
||||
except Exception:
|
||||
pass
|
||||
# vLLM: /version
|
||||
try:
|
||||
r = client.get(f"{server_url}/version")
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
if "version" in data:
|
||||
return "vllm"
|
||||
except Exception:
|
||||
pass
|
||||
if "version" in data:
|
||||
result = "vllm"
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
if result is not None:
|
||||
_endpoint_probe_path_cache[server_url] = result
|
||||
return result
|
||||
|
||||
|
||||
def _iter_nested_dicts(value: Any):
|
||||
|
|
|
|||
|
|
@ -1261,6 +1261,7 @@ AUTHOR_MAP = {
|
|||
"120500656+oooindefatigable@users.noreply.github.com": "ooovenenoso",
|
||||
"vanthinh6886@gmail.com": "vanthinh6886", # PR #28018 salvage (yaml/flock/atomic write guards)
|
||||
"erik.engervall@gmail.com": "erikengervall", # PR #28774 (firecrawl integration tag)
|
||||
"1torhan@protonmail.com": "uzaylisak",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue