mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-28 01:21:43 +00:00
8bbeaea6c7
The raw-template lookup added in PR #15817 went through `get_compatible_custom_providers(read_raw_config())`, which calls `_normalize_custom_provider_entry` → `urlparse(base_url)`. Any entry whose `base_url` is itself an env-ref (`${NEURALWATT_API_BASE}`) was dropped as 'not a valid URL', so `api_key_ref` stayed empty and the resolved secret was still written to `model.api_key` — the exact case the original Discord report described. Replace the normalizer-gated lookup with a direct read of `raw['custom_providers']` and `raw['providers']`, indexed by name (case-insensitive, optionally qualified by model) so the loaded (expanded) entry can be matched regardless of how `base_url` is written. Add an integration regression test driving the real `select_provider_and_model` entry point with the Discord-reported NeuralWatt config (`${VAR}` in both `base_url` and `api_key`). This test fails on the PR-only fix and passes with the broadened lookup.
324 lines
13 KiB
Python
324 lines
13 KiB
Python
"""Tests that `hermes model` always shows the model selection menu for custom
|
|
providers, even when a model is already saved.
|
|
|
|
Regression test for the bug where _model_flow_named_custom() returned
|
|
immediately when provider_info had a saved ``model`` field, making it
|
|
impossible to switch models on multi-model endpoints.
|
|
"""
|
|
|
|
import os
|
|
from unittest.mock import patch, MagicMock, call
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def config_home(tmp_path, monkeypatch):
|
|
"""Isolated HERMES_HOME with a minimal config."""
|
|
home = tmp_path / "hermes"
|
|
home.mkdir()
|
|
config_yaml = home / "config.yaml"
|
|
config_yaml.write_text("model: old-model\ncustom_providers: []\n")
|
|
env_file = home / ".env"
|
|
env_file.write_text("")
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
monkeypatch.delenv("HERMES_MODEL", raising=False)
|
|
monkeypatch.delenv("LLM_MODEL", raising=False)
|
|
monkeypatch.delenv("HERMES_INFERENCE_PROVIDER", raising=False)
|
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
|
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
|
return home
|
|
|
|
|
|
class TestCustomProviderModelSwitch:
|
|
"""Ensure _model_flow_named_custom always probes and shows menu."""
|
|
|
|
def test_saved_model_still_probes_endpoint(self, config_home):
|
|
"""When a model is already saved, the function must still call
|
|
fetch_api_models to probe the endpoint — not skip with early return."""
|
|
from hermes_cli.main import _model_flow_named_custom
|
|
|
|
provider_info = {
|
|
"name": "My vLLM",
|
|
"base_url": "https://vllm.example.com/v1",
|
|
"api_key": "sk-test",
|
|
"model": "model-A", # already saved
|
|
}
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["model-A", "model-B"]) as mock_fetch, \
|
|
patch.dict("sys.modules", {"simple_term_menu": None}), \
|
|
patch("builtins.input", return_value="2"), \
|
|
patch("builtins.print"):
|
|
_model_flow_named_custom({}, provider_info)
|
|
|
|
# fetch_api_models MUST be called even though model was saved
|
|
mock_fetch.assert_called_once_with(
|
|
"sk-test",
|
|
"https://vllm.example.com/v1",
|
|
timeout=8.0,
|
|
api_mode=None,
|
|
)
|
|
|
|
def test_can_switch_to_different_model(self, config_home):
|
|
"""User selects a different model than the saved one."""
|
|
import yaml
|
|
from hermes_cli.main import _model_flow_named_custom
|
|
|
|
provider_info = {
|
|
"name": "My vLLM",
|
|
"base_url": "https://vllm.example.com/v1",
|
|
"api_key": "sk-test",
|
|
"model": "model-A",
|
|
}
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["model-A", "model-B"]), \
|
|
patch.dict("sys.modules", {"simple_term_menu": None}), \
|
|
patch("builtins.input", return_value="2"), \
|
|
patch("builtins.print"):
|
|
_model_flow_named_custom({}, provider_info)
|
|
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict)
|
|
assert model["default"] == "model-B"
|
|
|
|
def test_probe_failure_falls_back_to_saved(self, config_home):
|
|
"""When endpoint probe fails and user presses Enter, saved model is used."""
|
|
import yaml
|
|
from hermes_cli.main import _model_flow_named_custom
|
|
|
|
provider_info = {
|
|
"name": "My vLLM",
|
|
"base_url": "https://vllm.example.com/v1",
|
|
"api_key": "sk-test",
|
|
"model": "model-A",
|
|
}
|
|
|
|
# fetch returns empty list (probe failed), user presses Enter (empty input)
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=[]), \
|
|
patch("builtins.input", return_value=""), \
|
|
patch("builtins.print"):
|
|
_model_flow_named_custom({}, provider_info)
|
|
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict)
|
|
assert model["default"] == "model-A"
|
|
|
|
def test_no_saved_model_still_works(self, config_home):
|
|
"""First-time flow (no saved model) still works as before."""
|
|
import yaml
|
|
from hermes_cli.main import _model_flow_named_custom
|
|
|
|
provider_info = {
|
|
"name": "My vLLM",
|
|
"base_url": "https://vllm.example.com/v1",
|
|
"api_key": "sk-test",
|
|
# no "model" key
|
|
}
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["model-X"]), \
|
|
patch.dict("sys.modules", {"simple_term_menu": None}), \
|
|
patch("builtins.input", return_value="1"), \
|
|
patch("builtins.print"):
|
|
_model_flow_named_custom({}, provider_info)
|
|
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict)
|
|
assert model["default"] == "model-X"
|
|
|
|
def test_api_mode_set_from_provider_info(self, config_home):
|
|
"""When custom_providers entry has api_mode, it should be applied."""
|
|
import yaml
|
|
from hermes_cli.main import _model_flow_named_custom
|
|
|
|
provider_info = {
|
|
"name": "Anthropic Proxy",
|
|
"base_url": "https://proxy.example.com/anthropic",
|
|
"api_key": "***",
|
|
"model": "claude-3",
|
|
"api_mode": "anthropic_messages",
|
|
}
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["claude-3"]), \
|
|
patch.dict("sys.modules", {"simple_term_menu": None}), \
|
|
patch("builtins.input", return_value="1"), \
|
|
patch("builtins.print"):
|
|
_model_flow_named_custom({}, provider_info)
|
|
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict)
|
|
assert model.get("api_mode") == "anthropic_messages"
|
|
|
|
def test_api_mode_cleared_when_not_specified(self, config_home):
|
|
"""When custom_providers entry has no api_mode, stale api_mode is removed."""
|
|
import yaml
|
|
from hermes_cli.main import _model_flow_named_custom
|
|
|
|
# Pre-seed a stale api_mode in config
|
|
config_path = config_home / "config.yaml"
|
|
config_path.write_text(yaml.dump({"model": {"api_mode": "anthropic_messages"}}))
|
|
|
|
provider_info = {
|
|
"name": "My vLLM",
|
|
"base_url": "https://vllm.example.com/v1",
|
|
"api_key": "***",
|
|
"model": "llama-3",
|
|
}
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["llama-3"]), \
|
|
patch.dict("sys.modules", {"simple_term_menu": None}), \
|
|
patch("builtins.input", return_value="1"), \
|
|
patch("builtins.print"):
|
|
_model_flow_named_custom({}, provider_info)
|
|
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict)
|
|
assert "api_mode" not in model, "Stale api_mode should be removed"
|
|
|
|
def test_env_template_api_key_is_preserved_in_model_config(self, config_home, monkeypatch):
|
|
"""Selecting an env-backed custom provider must not inline the secret."""
|
|
import yaml
|
|
from hermes_cli.main import _model_flow_named_custom
|
|
|
|
config_path = config_home / "config.yaml"
|
|
config_path.write_text(
|
|
"model:\n"
|
|
" default: old-model\n"
|
|
" provider: openrouter\n"
|
|
"custom_providers:\n"
|
|
"- name: Example Provider\n"
|
|
" base_url: https://api.example-provider.test/v1\n"
|
|
" api_key: ${EXAMPLE_PROVIDER_API_KEY}\n"
|
|
" model: qwen3.6-35b-fast\n"
|
|
)
|
|
monkeypatch.setenv("EXAMPLE_PROVIDER_API_KEY", "sk-live-example-provider")
|
|
|
|
provider_info = {
|
|
"name": "Example Provider",
|
|
"base_url": "https://api.example-provider.test/v1",
|
|
"api_key": "sk-live-example-provider",
|
|
"api_key_ref": "${EXAMPLE_PROVIDER_API_KEY}",
|
|
"model": "qwen3.6-35b-fast",
|
|
}
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["qwen3.6-35b-fast"]) as mock_fetch, \
|
|
patch.dict("sys.modules", {"simple_term_menu": None}), \
|
|
patch("builtins.input", return_value="1"), \
|
|
patch("builtins.print"):
|
|
_model_flow_named_custom({}, provider_info)
|
|
|
|
mock_fetch.assert_called_once_with(
|
|
"sk-live-example-provider",
|
|
"https://api.example-provider.test/v1",
|
|
timeout=8.0,
|
|
api_mode=None,
|
|
)
|
|
config = yaml.safe_load(config_path.read_text()) or {}
|
|
assert config["model"]["api_key"] == "${EXAMPLE_PROVIDER_API_KEY}"
|
|
assert config["custom_providers"][0]["api_key"] == "${EXAMPLE_PROVIDER_API_KEY}"
|
|
assert "sk-live-example-provider" not in config_path.read_text()
|
|
|
|
def test_key_env_custom_provider_persists_reference_not_secret(self, config_home, monkeypatch):
|
|
"""key_env custom providers should also avoid writing plaintext keys."""
|
|
import yaml
|
|
from hermes_cli.main import _model_flow_named_custom
|
|
|
|
config_path = config_home / "config.yaml"
|
|
config_path.write_text(
|
|
"model:\n"
|
|
" default: old-model\n"
|
|
"custom_providers:\n"
|
|
"- name: Example Provider\n"
|
|
" base_url: https://api.example-provider.test/v1\n"
|
|
" key_env: EXAMPLE_PROVIDER_API_KEY\n"
|
|
" model: qwen3.6-35b-fast\n"
|
|
)
|
|
monkeypatch.setenv("EXAMPLE_PROVIDER_API_KEY", "sk-live-example-provider")
|
|
|
|
provider_info = {
|
|
"name": "Example Provider",
|
|
"base_url": "https://api.example-provider.test/v1",
|
|
"api_key": "",
|
|
"key_env": "EXAMPLE_PROVIDER_API_KEY",
|
|
"model": "qwen3.6-35b-fast",
|
|
}
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["qwen3.6-35b-fast"]), \
|
|
patch.dict("sys.modules", {"simple_term_menu": None}), \
|
|
patch("builtins.input", return_value="1"), \
|
|
patch("builtins.print"):
|
|
_model_flow_named_custom({}, provider_info)
|
|
|
|
config = yaml.safe_load(config_path.read_text()) or {}
|
|
assert config["model"]["api_key"] == "${EXAMPLE_PROVIDER_API_KEY}"
|
|
assert config["custom_providers"][0]["key_env"] == "EXAMPLE_PROVIDER_API_KEY"
|
|
assert "sk-live-example-provider" not in config_path.read_text()
|
|
|
|
def test_env_ref_base_url_preserves_api_key_ref_through_picker(
|
|
self, config_home, monkeypatch
|
|
):
|
|
"""Integration regression: when BOTH ``base_url`` and ``api_key`` use
|
|
``${VAR}`` templates (the Discord-reported NeuralWatt case), the picker
|
|
must still preserve the env reference in ``model.api_key``.
|
|
|
|
The earlier lookup went through ``get_compatible_custom_providers``
|
|
which dropped entries whose ``base_url`` was an env-ref template
|
|
(``urlparse("${NEURALWATT_API_BASE}")`` has no scheme/netloc), causing
|
|
``api_key_ref`` to stay empty and the resolved secret to be written to
|
|
``config.yaml``. This test drives the real picker-callsite code path.
|
|
"""
|
|
import yaml
|
|
from hermes_cli.main import select_provider_and_model
|
|
|
|
config_path = config_home / "config.yaml"
|
|
config_path.write_text(
|
|
"model:\n"
|
|
" default: old-model\n"
|
|
" provider: openrouter\n"
|
|
"custom_providers:\n"
|
|
"- name: NeuralWatt\n"
|
|
" base_url: ${NEURALWATT_API_BASE}\n"
|
|
" api_key: ${NEURALWATT_API_KEY}\n"
|
|
" model: qwen3.6-35b-fast\n"
|
|
" models: []\n"
|
|
)
|
|
monkeypatch.setenv("NEURALWATT_API_BASE", "https://api.neuralwatt.com/v1")
|
|
monkeypatch.setenv("NEURALWATT_API_KEY", "sk-live-neuralwatt-secret")
|
|
|
|
# Exercise the real picker: select "custom:neuralwatt" from the
|
|
# provider menu. ``select_provider_and_model`` prompts for a provider
|
|
# choice (returns an index), then hands off to
|
|
# ``_model_flow_named_custom`` with the provider_info built by
|
|
# ``_named_custom_provider_map``.
|
|
def _pick_neuralwatt(labels, default=0):
|
|
for i, label in enumerate(labels):
|
|
if "NeuralWatt" in label:
|
|
return i
|
|
raise AssertionError(
|
|
f"NeuralWatt entry missing from provider menu: {labels}"
|
|
)
|
|
|
|
with patch("hermes_cli.main._prompt_provider_choice",
|
|
side_effect=_pick_neuralwatt), \
|
|
patch("hermes_cli.models.fetch_api_models",
|
|
return_value=["qwen3.6-35b-fast"]) as mock_fetch, \
|
|
patch.dict("sys.modules", {"simple_term_menu": None}), \
|
|
patch("builtins.input", return_value="1"), \
|
|
patch("builtins.print"):
|
|
select_provider_and_model()
|
|
|
|
# The live probe must still use the resolved secret.
|
|
mock_fetch.assert_called_once()
|
|
probe_args, probe_kwargs = mock_fetch.call_args
|
|
assert probe_args[0] == "sk-live-neuralwatt-secret"
|
|
|
|
# But config.yaml must keep the env reference, not the plaintext secret.
|
|
saved = config_path.read_text()
|
|
config = yaml.safe_load(saved) or {}
|
|
assert config["model"]["api_key"] == "${NEURALWATT_API_KEY}"
|
|
assert config["custom_providers"][0]["api_key"] == "${NEURALWATT_API_KEY}"
|
|
assert "sk-live-neuralwatt-secret" not in saved
|