mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-09 13:21:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6
hermes-agent/tests/hermes_cli/test_model_switch_custom_providers.py

1115 lines
42 KiB
Python
Raw Blame History

"""Regression tests for /model support of config.yaml custom_providers.
The terminal `hermes model` flow already exposes `custom_providers`, but the
shared slash-command pipeline (`/model` in CLI/gateway/Telegram) historically
only looked at `providers:`.
"""
import hermes_cli.providers as providers_mod
from hermes_cli.model_switch import list_authenticated_providers, switch_model
from hermes_cli.providers import resolve_provider_full
_MOCK_VALIDATION = {
"accepted": True,
"persist": True,
"recognized": True,
"message": None,
}
def test_list_authenticated_providers_includes_custom_providers(monkeypatch):
"""No-args /model menus should include saved custom_providers entries."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="openai-codex",
user_providers={},
custom_providers=[
{
"name": "Local (127.0.0.1:4141)",
"base_url": "http://127.0.0.1:4141/v1",
"model": "rotator-openrouter-coding",
}
],
max_models=50,
)
assert any(
p["slug"] == "custom:local-(127.0.0.1:4141)"
and p["name"] == "Local (127.0.0.1:4141)"
and p["models"] == ["rotator-openrouter-coding"]
and p["api_url"] == "http://127.0.0.1:4141/v1"
for p in providers
)
def test_list_authenticated_providers_can_skip_custom_provider_live_probe(monkeypatch):
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
fetch = lambda *a, **k: (_ for _ in ()).throw(AssertionError("unexpected probe"))
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
providers = list_authenticated_providers(
user_providers={},
custom_providers=[
{
"name": "Slow Local",
"base_url": "http://127.0.0.1:8080/v1",
"api_key": "sk-local",
"model": "local-model",
}
],
probe_custom_providers=False,
)
row = next(p for p in providers if p["slug"] == "custom:slow-local")
assert row["models"] == ["local-model"]
assert row["total_models"] == 1
def test_list_authenticated_providers_can_probe_only_current_custom_provider(monkeypatch):
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
calls = []
def fetch(api_key, api_url, **kwargs):
calls.append(api_url)
if api_url == "http://active.local/v1":
return ["active-a", "active-b"]
raise AssertionError(f"unexpected probe for {api_url}")
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
providers = list_authenticated_providers(
current_provider="custom:active-proxy",
user_providers={},
custom_providers=[
{
"name": "Active Proxy",
"base_url": "http://active.local/v1",
"api_key": "sk-active",
"model": "configured-active",
},
{
"name": "Offline Proxy",
"base_url": "http://offline.local/v1",
"api_key": "sk-offline",
"model": "configured-offline",
},
],
probe_custom_providers=False,
probe_current_custom_provider=True,
)
active = next(p for p in providers if p["slug"] == "custom:active-proxy")
offline = next(p for p in providers if p["slug"] == "custom:offline-proxy")
assert calls == ["http://active.local/v1"]
assert active["is_current"] is True
assert active["models"] == ["active-a", "active-b"]
assert offline["models"] == ["configured-offline"]
def test_resolve_provider_full_finds_named_custom_provider():
"""Explicit /model --provider should resolve saved custom_providers entries."""
resolved = resolve_provider_full(
"custom:local-(127.0.0.1:4141)",
user_providers={},
custom_providers=[
{
"name": "Local (127.0.0.1:4141)",
"base_url": "http://127.0.0.1:4141/v1",
}
],
)
assert resolved is not None
assert resolved.id == "custom:local-(127.0.0.1:4141)"
assert resolved.name == "Local (127.0.0.1:4141)"
assert resolved.base_url == "http://127.0.0.1:4141/v1"
assert resolved.source == "user-config"
def test_list_authenticated_providers_includes_active_bare_custom_endpoint(monkeypatch):
"""Bare model.provider=custom + model.base_url should still populate /model.
Users can configure a one-off OpenAI-compatible endpoint directly under
``model:`` without a named ``providers:`` or ``custom_providers:`` row.
The gateway picker receives only the current model/base_url slice, so it
must surface that active endpoint rather than looking like config was
ignored.
"""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="custom",
current_base_url="https://www.ccsub.net/v1",
current_model="gpt-4o",
user_providers={},
custom_providers=[],
max_models=50,
)
bare_custom = next((p for p in providers if p["slug"] == "custom"), None)
assert bare_custom is not None
assert bare_custom["name"] == "Custom endpoint"
assert bare_custom["is_current"] is True
assert bare_custom["is_user_defined"] is True
assert bare_custom["models"] == ["gpt-4o"]
assert bare_custom["api_url"] == "https://www.ccsub.net/v1"
def test_list_authenticated_providers_can_probe_active_bare_custom_endpoint(monkeypatch):
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
monkeypatch.setattr(
"hermes_cli.models.fetch_api_models",
lambda api_key, api_url, **kwargs: ["gpt-4o", "gpt-4o-mini"],
)
providers = list_authenticated_providers(
current_provider="custom",
current_base_url="https://www.ccsub.net/v1",
current_model="gpt-4o",
user_providers={},
custom_providers=[],
probe_custom_providers=False,
probe_current_custom_provider=True,
)
bare_custom = next(p for p in providers if p["slug"] == "custom")
assert bare_custom["is_current"] is True
assert bare_custom["models"] == ["gpt-4o", "gpt-4o-mini"]
def test_switch_model_accepts_explicit_bare_custom_current_endpoint(monkeypatch):
"""Picker selections for bare custom endpoints should route to current base_url."""
monkeypatch.setattr("hermes_cli.models.validate_requested_model", lambda *a, **k: _MOCK_VALIDATION)
monkeypatch.setattr("hermes_cli.model_switch.get_model_info", lambda *a, **k: None)
monkeypatch.setattr("hermes_cli.model_switch.get_model_capabilities", lambda *a, **k: None)
result = switch_model(
raw_input="gpt-4o-mini",
current_provider="custom",
current_model="gpt-4o",
current_base_url="https://www.ccsub.net/v1",
current_api_key="sk-test",
explicit_provider="custom",
user_providers={},
custom_providers=[],
)
assert result.success is True
assert result.target_provider == "custom"
assert result.provider_label == "Custom endpoint"
assert result.new_model == "gpt-4o-mini"
assert result.base_url == "https://www.ccsub.net/v1"
assert result.api_key == "sk-test"
def test_is_aggregator_recognizes_named_custom_provider():
assert providers_mod.is_aggregator("custom:hpc-ai") is True
assert providers_mod.is_aggregator("custom:litellm") is True
def test_is_aggregator_leaves_unknown_provider_non_aggregator():
assert providers_mod.is_aggregator("not-a-provider") is False
def test_is_routing_aggregator_excludes_flat_namespace_resellers():
"""opencode-go / opencode-zen stay ``is_aggregator=True`` (model-switch
relies on it to search their flat bare-name catalog), but they are NOT
routing aggregators — their models are first-party, so the picker dedup
must not strip them. (#47077)"""
# Still aggregators for model-switch flat-catalog resolution.
assert providers_mod.is_aggregator("opencode-go") is True
assert providers_mod.is_aggregator("opencode-zen") is True
# But NOT routing aggregators for picker-dedup purposes.
assert providers_mod.is_routing_aggregator("opencode-go") is False
assert providers_mod.is_routing_aggregator("opencode-zen") is False
# True routers and custom proxies remain routing aggregators.
assert providers_mod.is_routing_aggregator("openrouter") is True
assert providers_mod.is_routing_aggregator("custom:litellm") is True
assert providers_mod.is_routing_aggregator("not-a-provider") is False
def test_switch_model_accepts_explicit_named_custom_provider(monkeypatch):
"""Shared /model switch pipeline should accept --provider for custom_providers."""
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kwargs: {
"api_key": "no-key-required",
"base_url": "http://127.0.0.1:4141/v1",
"api_mode": "chat_completions",
},
)
monkeypatch.setattr("hermes_cli.models.validate_requested_model", lambda *a, **k: _MOCK_VALIDATION)
monkeypatch.setattr("hermes_cli.model_switch.get_model_info", lambda *a, **k: None)
monkeypatch.setattr("hermes_cli.model_switch.get_model_capabilities", lambda *a, **k: None)
result = switch_model(
raw_input="rotator-openrouter-coding",
current_provider="openai-codex",
current_model="gpt-5.4",
current_base_url="https://chatgpt.com/backend-api/codex",
current_api_key="",
explicit_provider="custom:local-(127.0.0.1:4141)",
user_providers={},
custom_providers=[
{
"name": "Local (127.0.0.1:4141)",
"base_url": "http://127.0.0.1:4141/v1",
"model": "rotator-openrouter-coding",
}
],
)
assert result.success is True
assert result.target_provider == "custom:local-(127.0.0.1:4141)"
assert result.provider_label == "Local (127.0.0.1:4141)"
assert result.new_model == "rotator-openrouter-coding"
assert result.base_url == "http://127.0.0.1:4141/v1"
assert result.api_key == "no-key-required"
def test_list_groups_same_name_custom_providers_into_one_row(monkeypatch):
"""Multiple custom_providers entries sharing a name should produce one row
with all models collected, not N duplicate rows."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="openrouter",
user_providers={},
custom_providers=[
{"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "qwen3-coder:480b-cloud"},
{"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "glm-5.1:cloud"},
{"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "kimi-k2.5"},
{"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "minimax-m2.7:cloud"},
{"name": "Moonshot", "base_url": "https://api.moonshot.ai/v1", "model": "kimi-k2-thinking"},
],
max_models=50,
)
ollama_rows = [p for p in providers if p["name"] == "Ollama Cloud"]
assert len(ollama_rows) == 1, f"Expected 1 Ollama Cloud row, got {len(ollama_rows)}"
assert ollama_rows[0]["models"] == [
"qwen3-coder:480b-cloud", "glm-5.1:cloud", "kimi-k2.5", "minimax-m2.7:cloud"
]
assert ollama_rows[0]["total_models"] == 4
moonshot_rows = [p for p in providers if p["name"] == "Moonshot"]
assert len(moonshot_rows) == 1
assert moonshot_rows[0]["models"] == ["kimi-k2-thinking"]
def test_list_deduplicates_same_model_in_group(monkeypatch):
"""Duplicate model entries under the same provider name should not produce
duplicate entries in the models list."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="openrouter",
user_providers={},
custom_providers=[
{"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "llama3"},
{"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "llama3"},
{"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "mistral"},
],
max_models=50,
)
my_rows = [p for p in providers if p["name"] == "MyProvider"]
assert len(my_rows) == 1
assert my_rows[0]["models"] == ["llama3", "mistral"]
assert my_rows[0]["total_models"] == 2
def test_list_enumerates_dict_format_models_alongside_default(monkeypatch):
"""custom_providers entry with dict-format ``models:`` plus singular
``model:`` should surface the default and every dict key.
Regression: Hermes's own writer stores configured models as a dict
keyed by model id, but the /model picker previously only honored the
singular ``model:`` field, so multi-model custom providers appeared
to have only the active model.
"""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="openai-codex",
user_providers={},
custom_providers=[
{
"name": "DeepSeek",
"base_url": "https://api.deepseek.com",
"api_mode": "chat_completions",
"model": "deepseek-chat",
"models": {
"deepseek-chat": {"context_length": 128000},
"deepseek-reasoner": {"context_length": 128000},
},
}
],
max_models=50,
)
ds_rows = [p for p in providers if p["name"] == "DeepSeek"]
assert len(ds_rows) == 1
assert ds_rows[0]["models"] == ["deepseek-chat", "deepseek-reasoner"]
assert ds_rows[0]["total_models"] == 2
def test_list_enumerates_dict_format_models_without_singular_model(monkeypatch):
"""Dict-format ``models:`` with no singular ``model:`` should still
enumerate every dict key (previously the picker reported 0 models)."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="openai-codex",
user_providers={},
custom_providers=[
{
"name": "Thor",
"base_url": "http://thor.lab:8337/v1",
"models": {
"gemma-4-26B-A4B-it-MXFP4_MOE": {"context_length": 262144},
"Qwen3.5-35B-A3B-MXFP4_MOE": {"context_length": 262144},
"gemma-4-31B-it-Q4_K_M": {"context_length": 262144},
},
}
],
max_models=50,
)
thor_rows = [p for p in providers if p["name"] == "Thor"]
assert len(thor_rows) == 1
assert set(thor_rows[0]["models"]) == {
"gemma-4-26B-A4B-it-MXFP4_MOE",
"Qwen3.5-35B-A3B-MXFP4_MOE",
"gemma-4-31B-it-Q4_K_M",
}
assert thor_rows[0]["total_models"] == 3
def test_list_dedupes_dict_model_matching_singular_default(monkeypatch):
"""When the singular ``model:`` is also a key in the ``models:`` dict,
it must appear exactly once in the picker."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="openai-codex",
user_providers={},
custom_providers=[
{
"name": "DeepSeek",
"base_url": "https://api.deepseek.com",
"model": "deepseek-chat",
"models": {
"deepseek-chat": {"context_length": 128000},
"deepseek-reasoner": {"context_length": 128000},
},
}
],
max_models=50,
)
ds_rows = [p for p in providers if p["name"] == "DeepSeek"]
assert ds_rows[0]["models"].count("deepseek-chat") == 1
assert ds_rows[0]["models"] == ["deepseek-chat", "deepseek-reasoner"]
# ─────────────────────────────────────────────────────────────────────────────
# #9210: group custom_providers by (base_url, api_key) in /model picker
# ─────────────────────────────────────────────────────────────────────────────
def test_list_authenticated_providers_groups_same_endpoint(monkeypatch):
"""Multiple custom_providers entries sharing a base_url+api_key must be
returned as a single picker row with all their models merged."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="custom",
current_base_url="http://localhost:11434/v1",
user_providers={},
custom_providers=[
{"name": "Ollama — MiniMax M2.7", "base_url": "http://localhost:11434/v1",
"api_key": "ollama", "model": "minimax-m2.7"},
{"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1",
"api_key": "ollama", "model": "glm-5.1"},
{"name": "Ollama — Qwen3-coder", "base_url": "http://localhost:11434/v1",
"api_key": "ollama", "model": "qwen3-coder"},
],
max_models=50,
)
custom_groups = [p for p in providers if p.get("is_user_defined")]
assert len(custom_groups) == 1, (
"Expected 1 group for shared endpoint, got "
f"{[p['slug'] for p in custom_groups]}"
)
group = custom_groups[0]
assert set(group["models"]) == {"minimax-m2.7", "glm-5.1", "qwen3-coder"}
assert group["total_models"] == 3
# Per-model suffix stripped from display name
assert group["name"] == "Ollama"
def test_list_authenticated_providers_current_endpoint_uses_current_slug(monkeypatch):
"""When current_base_url matches the grouped endpoint, the slug must
equal current_provider so picker selection routes through the live
credential pipeline — provided current_provider is a real slug, not
the corrupt bare "custom" (see #17478)."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="custom:ollama",
current_base_url="http://localhost:11434/v1",
user_providers={},
custom_providers=[
{"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1",
"api_key": "ollama", "model": "glm-5.1"},
],
max_models=50,
)
matches = [p for p in providers if p.get("is_user_defined")]
assert len(matches) == 1
group = matches[0]
assert group["slug"] == "custom:ollama"
assert group["is_current"] is True
def test_list_authenticated_providers_bare_custom_slug_recovers(monkeypatch):
"""Regression for #17478: when a prior failed switch left the bare
literal "custom" in model.provider, the picker must NOT propagate
that broken slug. It must fall back to the canonical
``custom:<name>`` form so the picker stays usable."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="custom",
current_base_url="http://localhost:11434/v1",
user_providers={},
custom_providers=[
{"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1",
"api_key": "ollama", "model": "glm-5.1"},
],
max_models=50,
)
matches = [p for p in providers if p.get("is_user_defined")]
assert len(matches) == 1
group = matches[0]
# Canonical slug, NOT the bare "custom" that caused #17478
assert group["slug"] == "custom:ollama"
assert group["is_current"] is True
def test_list_authenticated_providers_distinct_endpoints_stay_separate(monkeypatch):
"""Entries with different base_urls must produce separate picker rows
even if some display names happen to be similar."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
user_providers={},
custom_providers=[
{"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1",
"api_key": "ollama", "model": "glm-5.1"},
{"name": "Moonshot", "base_url": "https://api.moonshot.cn/v1",
"api_key": "sk-m", "model": "moonshot-v1"},
{"name": "Ollama — Qwen3-coder", "base_url": "http://localhost:11434/v1",
"api_key": "ollama", "model": "qwen3-coder"},
],
max_models=50,
)
custom_groups = [p for p in providers if p.get("is_user_defined")]
assert len(custom_groups) == 2
# Ollama endpoint collapses to one row with both models
ollama = next(p for p in custom_groups if p["name"] == "Ollama")
assert set(ollama["models"]) == {"glm-5.1", "qwen3-coder"}
moonshot = next(p for p in custom_groups if p["name"] == "Moonshot")
assert moonshot["models"] == ["moonshot-v1"]
def test_list_authenticated_providers_same_url_different_keys_disambiguated(monkeypatch):
"""Two custom_providers entries with the same base_url but different
api_keys (and identical cleaned names) must both stay visible in the
picker — slug is suffixed to disambiguate."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
user_providers={},
custom_providers=[
{"name": "OpenAI — key A", "base_url": "https://api.openai.com/v1",
"api_key": "sk-AAA", "model": "gpt-5.4"},
{"name": "OpenAI — key B", "base_url": "https://api.openai.com/v1",
"api_key": "sk-BBB", "model": "gpt-4.6"},
],
max_models=50,
)
custom_groups = [p for p in providers if p.get("is_user_defined")]
assert len(custom_groups) == 2
slugs = sorted(p["slug"] for p in custom_groups)
# First group keeps the base slug, second gets a numeric suffix
assert slugs == ["custom:openai", "custom:openai-2"]
# Each row has a distinct model
models = {p["slug"]: p["models"] for p in custom_groups}
assert models["custom:openai"] == ["gpt-5.4"]
assert models["custom:openai-2"] == ["gpt-4.6"]
def test_list_authenticated_providers_same_url_different_key_env_and_api_mode_stay_separate(monkeypatch):
"""Same gateway host but different key_env/api_mode entries are distinct providers."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="custom:gpt",
current_base_url="https://gateway.example.com",
user_providers={},
custom_providers=[
{
"name": "gpt",
"base_url": "https://gateway.example.com",
"key_env": "GPT_KEY",
"api_mode": "codex_responses",
"model": "gpt-5.5",
},
{
"name": "claude",
"base_url": "https://gateway.example.com",
"key_env": "CLAUDE_KEY",
"api_mode": "anthropic_messages",
"model": "claude-opus-4-8",
},
],
max_models=50,
)
custom = [p for p in providers if p.get("is_user_defined")]
by_slug = {p["slug"]: p for p in custom}
assert set(by_slug) == {"custom:gpt", "custom:claude"}
assert by_slug["custom:gpt"]["models"] == ["gpt-5.5"]
assert by_slug["custom:claude"]["models"] == ["claude-opus-4-8"]
assert by_slug["custom:gpt"]["is_current"] is True
assert by_slug["custom:claude"]["is_current"] is False
def test_list_authenticated_providers_total_models_reflects_grouped_count(monkeypatch):
"""After grouping six entries into one row, total_models must reflect
the full count, and every grouped model appears in the list."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
entries = [
{"name": f"Ollama \u2014 Model {i}", "base_url": "http://localhost:11434/v1",
"api_key": "ollama", "model": f"model-{i}"}
for i in range(6)
]
providers = list_authenticated_providers(
user_providers={},
custom_providers=entries,
max_models=4,
)
groups = [p for p in providers if p.get("is_user_defined")]
assert len(groups) == 1
group = groups[0]
assert group["total_models"] == 6
# All six models are preserved in the grouped row.
assert sorted(group["models"]) == sorted(f"model-{i}" for i in range(6))
def test_lmstudio_picker_probes_active_config_base_url(monkeypatch):
"""When `provider: lmstudio` is saved with a remote base_url and no
LM_BASE_URL env var, the picker must probe the saved base_url — not
127.0.0.1. Regression: prior behavior always probed localhost, so users
with LM Studio on a lab box saw the wrong (or empty) model list.
"""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
monkeypatch.delenv("LM_BASE_URL", raising=False)
monkeypatch.delenv("LM_API_KEY", raising=False)
captured: dict = {}
def _fake_fetch(api_key=None, base_url=None, timeout=5.0):
captured["base_url"] = base_url
captured["api_key"] = api_key
return ["qwen/qwen3-coder-30b"]
monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch)
list_authenticated_providers(
current_provider="lmstudio",
current_base_url="http://192.168.1.10:1234/v1",
current_model="qwen/qwen3-coder-30b",
)
assert captured["base_url"] == "http://192.168.1.10:1234/v1"
def test_lmstudio_picker_lm_base_url_env_wins_over_active_config(monkeypatch):
"""LM_BASE_URL env var must still take precedence over the saved
base_url so users can temporarily redirect the picker without editing
config.yaml.
"""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
monkeypatch.setenv("LM_BASE_URL", "http://override.local:9999/v1")
monkeypatch.delenv("LM_API_KEY", raising=False)
captured: dict = {}
def _fake_fetch(api_key=None, base_url=None, timeout=5.0):
captured["base_url"] = base_url
return []
monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch)
list_authenticated_providers(
current_provider="lmstudio",
current_base_url="http://192.168.1.10:1234/v1",
)
assert captured["base_url"] == "http://override.local:9999/v1"
def test_lmstudio_picker_skips_probe_when_not_configured(monkeypatch):
"""If the user has never configured LM Studio (no LM_API_KEY / LM_BASE_URL
and not on lmstudio), the picker must not pay the localhost probe cost
just to discover LM Studio is unavailable.
"""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
monkeypatch.delenv("LM_BASE_URL", raising=False)
monkeypatch.delenv("LM_API_KEY", raising=False)
captured: dict = {}
def _fake_fetch(api_key=None, base_url=None, timeout=5.0):
captured["base_url"] = base_url
return []
monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch)
list_authenticated_providers(
current_provider="openrouter",
current_base_url="https://openrouter.ai/api/v1",
)
assert "base_url" not in captured
def test_custom_providers_uses_live_models_for_multi_model_endpoint(monkeypatch):
"""Custom providers with api_key + base_url should prefer live /models.
Custom providers (section 4 of list_authenticated_providers) point at
gateways like Bifrost that expose hundreds of models. Reading only the
static ``models:`` dict from config.yaml leaves the /model picker with
a stale subset. Live discovery fills the picker with all available
models from the endpoint.
"""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
calls = []
def fake_fetch_api_models(api_key, base_url, **kwargs):
calls.append((api_key, base_url, kwargs))
return ["gateway-model-a", "gateway-model-b", "gateway-model-c"]
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
custom_providers = [
{
"name": "my-gateway",
"api_key": "sk-gateway-key",
"base_url": "https://gateway.example.com/v1",
"model": "gateway-model-a",
"models": {
"gateway-model-a": {"context_length": 128000},
"gateway-model-b": {"context_length": 128000},
},
}
]
providers = list_authenticated_providers(
current_provider="openrouter",
current_base_url="https://openrouter.ai/api/v1",
custom_providers=custom_providers,
max_models=50,
)
gateway_prov = next(
(
p
for p in providers
if p.get("api_url") == "https://gateway.example.com/v1"
),
None,
)
assert gateway_prov is not None, "Custom provider group not found in results"
assert calls == [
("sk-gateway-key", "https://gateway.example.com/v1", {"headers": None})
], "fetch_api_models must be called with the custom provider's credentials"
assert gateway_prov["models"] == [
"gateway-model-a",
"gateway-model-b",
"gateway-model-c",
], "Live models must replace the static subset"
assert gateway_prov["total_models"] == 3
def test_custom_provider_live_model_probe_uses_extra_headers(monkeypatch):
"""custom_providers[].extra_headers must apply to live /models probes."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
calls = []
def fake_fetch_api_models(api_key, base_url, **kwargs):
calls.append((api_key, base_url, kwargs))
return ["gateway-model"]
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
providers = list_authenticated_providers(
current_provider="openrouter",
current_base_url="https://openrouter.ai/api/v1",
custom_providers=[
{
"name": "LLM Proxy",
"api_key": "local-key",
"base_url": "http://localhost:8081/v1",
"extra_headers": {
"sleeve-harness": "hermes",
"sleeve-base-url": "http://localhost:8081/v1",
},
}
],
max_models=50,
)
gateway_prov = next(
(
p
for p in providers
if p.get("api_url") == "http://localhost:8081/v1"
),
None,
)
assert gateway_prov is not None
assert calls == [
(
"local-key",
"http://localhost:8081/v1",
{
"headers": {
"sleeve-harness": "hermes",
"sleeve-base-url": "http://localhost:8081/v1",
}
},
)
]
assert gateway_prov["models"] == ["gateway-model"]
def test_same_endpoint_different_extra_headers_not_collapsed(monkeypatch):
"""Entries sharing (api_url, credential, api_mode) but declaring different
extra_headers must NOT collapse into one picker row — each is a distinct
header-authenticated endpoint (e.g. per-tenant routing behind one proxy)
and must probe /models with its own headers."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
calls = []
def fake_fetch_api_models(api_key, base_url, **kwargs):
calls.append((api_key, base_url, kwargs.get("headers")))
# Return a per-tenant model list keyed by the routing header so we can
# assert each row got its OWN probe rather than a shared one.
tenant = (kwargs.get("headers") or {}).get("X-Tenant", "none")
return [f"model-{tenant}"]
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
providers = list_authenticated_providers(
current_provider="openrouter",
current_base_url="https://openrouter.ai/api/v1",
custom_providers=[
{
"name": "Proxy Tenant A",
"api_key": "shared-key",
"base_url": "http://localhost:8081/v1",
"extra_headers": {"X-Tenant": "a"},
},
{
"name": "Proxy Tenant B",
"api_key": "shared-key",
"base_url": "http://localhost:8081/v1",
"extra_headers": {"X-Tenant": "b"},
},
],
max_models=50,
)
rows = [
p for p in providers if p.get("api_url") == "http://localhost:8081/v1"
]
# Two distinct rows, not one collapsed row.
assert len(rows) == 2, f"expected 2 rows, got {len(rows)}: {rows}"
# Each tenant was probed with its OWN header set (order-independent).
assert ("shared-key", "http://localhost:8081/v1", {"X-Tenant": "a"}) in calls
assert ("shared-key", "http://localhost:8081/v1", {"X-Tenant": "b"}) in calls
# Each row surfaces the model list its own headers unlocked.
models_by_row = {tuple(r["models"]) for r in rows}
assert models_by_row == {("model-a",), ("model-b",)}
def test_custom_providers_discover_models_false_keeps_explicit_subset(monkeypatch):
"""Custom providers (section 4) with ``discover_models: false`` must keep
their explicit ``models:`` subset instead of replacing it with live
/models, even when an api_key is present.
This mirrors section 3 (user ``providers:``) behaviour and supports
endpoints that expose a full aggregator catalog via /models but only
serve a configured subset.
"""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
calls = []
def fake_fetch_api_models(api_key, base_url, **kwargs):
calls.append((api_key, base_url, kwargs))
return ["gateway-model-a", "gateway-model-b", "gateway-model-c"]
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
custom_providers = [
{
"name": "my-gateway",
"api_key": "***",
"base_url": "https://gateway.example.com/v1",
"discover_models": False,
"model": "gateway-model-a",
"models": {
"gateway-model-a": {"context_length": 128000},
"gateway-model-b": {"context_length": 128000},
},
}
]
providers = list_authenticated_providers(
current_provider="openrouter",
current_base_url="https://openrouter.ai/api/v1",
custom_providers=custom_providers,
max_models=50,
)
gateway_prov = next(
(
p
for p in providers
if p.get("api_url") == "https://gateway.example.com/v1"
),
None,
)
assert gateway_prov is not None, "Custom provider group not found in results"
assert calls == [], (
"fetch_api_models must NOT be called when discover_models is false"
)
assert gateway_prov["models"] == [
"gateway-model-a",
"gateway-model-b",
], "Explicit models: subset must be preserved when discovery is disabled"
assert gateway_prov["total_models"] == 2
def test_custom_providers_discover_models_false_string_is_normalised(monkeypatch):
"""String ``discover_models: "false"`` (hand-edited / env-style configs)
must be treated as a disable, same as the boolean ``False`` and section 3.
"""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
calls = []
def fake_fetch_api_models(api_key, base_url, **kwargs):
calls.append((api_key, base_url, kwargs))
return ["live-a", "live-b"]
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
custom_providers = [
{
"name": "my-gateway",
"api_key": "***",
"base_url": "https://gateway.example.com/v1",
"discover_models": "false",
"model": "only-model",
"models": {"only-model": {"context_length": 128000}},
}
]
providers = list_authenticated_providers(
current_provider="openrouter",
current_base_url="https://openrouter.ai/api/v1",
custom_providers=custom_providers,
max_models=50,
)
gateway_prov = next(
(p for p in providers if p.get("api_url") == "https://gateway.example.com/v1"),
None,
)
assert gateway_prov is not None
assert calls == [], "string 'false' must disable live discovery"
assert gateway_prov["models"] == ["only-model"]
def test_custom_providers_discover_models_false_list_of_dict_ids(monkeypatch):
"""List-of-dicts ``models: [{id: ...}]`` must be preserved as configured
model IDs when discovery is disabled."""
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
calls = []
def fake_fetch_api_models(api_key, base_url, **kwargs):
calls.append((api_key, base_url, kwargs))
return ["live-a", "live-b"]
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
custom_providers = [
{
"name": "static-gateway",
"api_key": "***",
"base_url": "https://router.example.com/v1",
"discover_models": False,
"model": "claude-3-7-sonnet",
"models": [
{"id": "claude-3-7-sonnet"},
{"id": "claude-sonnet-4"},
],
}
]
providers = list_authenticated_providers(
current_provider="openrouter",
current_base_url="https://openrouter.ai/api/v1",
custom_providers=custom_providers,
max_models=50,
)
gateway_prov = next(
(p for p in providers if p.get("api_url") == "https://router.example.com/v1"),
None,
)
assert gateway_prov is not None
assert calls == [], "discover_models: false must skip live discovery"
assert gateway_prov["models"] == ["claude-3-7-sonnet", "claude-sonnet-4"]
assert gateway_prov["total_models"] == 2
def test_list_of_dict_models_prefers_id_over_label(monkeypatch):
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
providers = list_authenticated_providers(
current_provider="openrouter",
current_base_url="https://openrouter.ai/api/v1",
custom_providers=[
{
"name": "static-gateway",
"base_url": "https://router.example.com/v1",
"discover_models": False,
"models": [{"id": "real-model-id", "name": "Friendly Label"}],
}
],
max_models=50,
)
gateway_prov = next(
(p for p in providers if p.get("api_url") == "https://router.example.com/v1"),
None,
)
assert gateway_prov is not None
assert gateway_prov["models"] == ["real-model-id"]
def test_resolve_custom_provider_passes_key_env():
"""resolve_custom_provider should propagate key_env into api_key_env_vars.
Regression: previously api_key_env_vars was always (), silently dropping
the configured env var and causing 401s on every request.
"""
from hermes_cli.providers import resolve_custom_provider
resolved = resolve_custom_provider(
"custom:token-plan",
custom_providers=[
{
"name": "token-plan",
"base_url": "https://token-plan-sgp.xiaomimimo.com/v1",
"key_env": "XIAOMI_MIMO_API_KEY",
"model": "mimo-v2-pro",
}
],
)
assert resolved is not None
assert resolved.api_key_env_vars == ("XIAOMI_MIMO_API_KEY",)
assert resolved.base_url == "https://token-plan-sgp.xiaomimimo.com/v1"
def test_resolve_custom_provider_bare_custom_self_heal_passes_key_env():
"""The bare-'custom' self-heal path must also propagate key_env.
A corrupt stored provider of the bare string 'custom' falls back to the
first valid entry; that fallback previously hardcoded api_key_env_vars=(),
dropping the env var just like the named-match path did.
"""
from hermes_cli.providers import resolve_custom_provider
resolved = resolve_custom_provider(
"custom",
custom_providers=[
{
"name": "token-plan",
"base_url": "https://token-plan-sgp.xiaomimimo.com/v1",
"key_env": "XIAOMI_MIMO_API_KEY",
}
],
)
assert resolved is not None
assert resolved.api_key_env_vars == ("XIAOMI_MIMO_API_KEY",)
Reference in a new issue View git blame Copy permalink