From 2a3a374c78ec083e4367577008d0d065168e3bf1 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 19 Mar 2026 05:57:02 -0700 Subject: [PATCH] fix: auto-detect local model name and context length for local servers Cherry-picked from PR #2043 by sudoingX. - Auto-detect model name from local server's /v1/models when only one model is loaded (no manual model name config needed) - Add n_ctx_train and n_ctx to context length detection keys for llama.cpp - Query llama.cpp /props endpoint for actual allocated context (not just training context from GGUF metadata) - Strip .gguf suffix from display in banner and status bar - _auto_detect_local_model() in runtime_provider.py for CLI init Co-authored-by: sudo --- agent/model_metadata.py | 21 +++++++++++++++++++++ cli.py | 10 ++++++++++ hermes_cli/banner.py | 2 ++ hermes_cli/runtime_provider.py | 32 +++++++++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 3ca34bcf2442..0a448990d32c 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -136,6 +136,8 @@ _CONTEXT_LENGTH_KEYS = ( "max_input_tokens", "max_sequence_length", "max_seq_len", + "n_ctx_train", + "n_ctx", ) _MAX_COMPLETION_KEYS = ( @@ -342,6 +344,25 @@ def fetch_endpoint_model_metadata( entry["pricing"] = pricing _add_model_aliases(cache, model_id, entry) + # If this is a llama.cpp server, query /props for actual allocated context + is_llamacpp = any( + m.get("owned_by") == "llamacpp" + for m in payload.get("data", []) if isinstance(m, dict) + ) + if is_llamacpp: + try: + props_url = candidate.rstrip("/").replace("/v1", "") + "/props" + props_resp = requests.get(props_url, headers=headers, timeout=5) + if props_resp.ok: + props = props_resp.json() + gen_settings = props.get("default_generation_settings", {}) + n_ctx = gen_settings.get("n_ctx") + model_alias = props.get("model_alias", "") + if n_ctx and model_alias and model_alias in cache: + cache[model_alias]["context_length"] = n_ctx + except Exception: + pass + _endpoint_model_metadata_cache[normalized] = cache _endpoint_model_metadata_cache_time[normalized] = time.time() return cache diff --git a/cli.py b/cli.py index 24f12f3866c6..2be21b902fa6 100755 --- a/cli.py +++ b/cli.py @@ -1046,6 +1046,14 @@ class HermesCLI: _config_model = _model_config.get("default", "") if isinstance(_model_config, dict) else (_model_config or "") _FALLBACK_MODEL = "anthropic/claude-opus-4.6" self.model = model or _config_model or _FALLBACK_MODEL + # Auto-detect model from local server if still on fallback + if self.model == _FALLBACK_MODEL: + _base_url = _model_config.get("base_url", "") if isinstance(_model_config, dict) else "" + if "localhost" in _base_url or "127.0.0.1" in _base_url: + from hermes_cli.runtime_provider import _auto_detect_local_model + _detected = _auto_detect_local_model(_base_url) + if _detected: + self.model = _detected # Track whether model was explicitly chosen by the user or fell back # to the global default. Provider-specific normalisation may override # the default silently but should warn when overriding an explicit choice. @@ -1251,6 +1259,8 @@ class HermesCLI: def _get_status_bar_snapshot(self) -> Dict[str, Any]: model_name = self.model or "unknown" model_short = model_name.split("/")[-1] if "/" in model_name else model_name + if model_short.endswith(".gguf"): + model_short = model_short[:-5] if len(model_short) > 26: model_short = f"{model_short[:23]}..." diff --git a/hermes_cli/banner.py b/hermes_cli/banner.py index 21c577dd36e5..e457a0a83f1a 100644 --- a/hermes_cli/banner.py +++ b/hermes_cli/banner.py @@ -289,6 +289,8 @@ def build_welcome_banner(console: Console, model: str, cwd: str, _hero = HERMES_CADUCEUS left_lines = ["", _hero, ""] model_short = model.split("/")[-1] if "/" in model else model + if model_short.endswith(".gguf"): + model_short = model_short[:-5] if len(model_short) > 28: model_short = model_short[:25] + "..." ctx_str = f" [dim {dim}]ยท[/] [dim {dim}]{_format_context_length(context_length)} context[/]" if context_length else "" diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index 6bfa5ae8a6e9..ed9e2854044f 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -24,11 +24,41 @@ def _normalize_custom_provider_name(value: str) -> str: return value.strip().lower().replace(" ", "-") +def _auto_detect_local_model(base_url: str) -> str: + """Query a local server for its model name when only one model is loaded.""" + if not base_url: + return "" + try: + import requests + url = base_url.rstrip("/") + if not url.endswith("/v1"): + url += "/v1" + resp = requests.get(url + "/models", timeout=5) + if resp.ok: + models = resp.json().get("data", []) + if len(models) == 1: + model_id = models[0].get("id", "") + if model_id: + return model_id + except Exception: + pass + return "" + + def _get_model_config() -> Dict[str, Any]: config = load_config() model_cfg = config.get("model") if isinstance(model_cfg, dict): - return dict(model_cfg) + cfg = dict(model_cfg) + default = cfg.get("default", "").strip() + base_url = cfg.get("base_url", "").strip() + is_local = "localhost" in base_url or "127.0.0.1" in base_url + is_fallback = not default or default == "anthropic/claude-opus-4.6" + if is_local and is_fallback and base_url: + detected = _auto_detect_local_model(base_url) + if detected: + cfg["default"] = detected + return cfg if isinstance(model_cfg, str) and model_cfg.strip(): return {"default": model_cfg.strip()} return {}