hermes-agent/tests/hermes_cli/test_custom_provider_model_switch.py

"""Tests that `hermes model` always shows the model selection menu for custom
providers, even when a model is already saved.

Regression test for the bug where _model_flow_named_custom() returned
immediately when provider_info had a saved ``model`` field, making it
impossible to switch models on multi-model endpoints.
"""

from unittest.mock import patch

import pytest


@pytest.fixture
def config_home(tmp_path, monkeypatch):
    """Isolated HERMES_HOME with a minimal config."""
    home = tmp_path / "hermes"
    home.mkdir()
    config_yaml = home / "config.yaml"
    config_yaml.write_text("model: old-model\ncustom_providers: []\n")
    env_file = home / ".env"
    env_file.write_text("")
    monkeypatch.setenv("HERMES_HOME", str(home))
    monkeypatch.delenv("HERMES_MODEL", raising=False)
    monkeypatch.delenv("LLM_MODEL", raising=False)
    monkeypatch.delenv("HERMES_INFERENCE_PROVIDER", raising=False)
    monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
    return home


class TestCustomProviderModelSwitch:
    """Ensure _model_flow_named_custom always probes and shows menu."""

    def test_custom_endpoint_switch_prunes_stale_model_config_pool_entry(
        self,
        config_home,
    ):
        """Switching custom endpoints must not leave the old model.api_key
        credential selectable from the previous endpoint's pool."""
        import yaml
        from agent.credential_pool import load_pool
        from hermes_cli.auth import read_credential_pool, write_credential_pool
        from hermes_cli.main import _model_flow_custom

        config_path = config_home / "config.yaml"
        config_path.write_text(
            "model:\n"
            "  default: old-model\n"
            "  provider: custom\n"
            "  base_url: https://old.example.test/v1\n"
            "  api_key: sk-old-model-config\n"
            "custom_providers:\n"
            "- name: Old Endpoint\n"
            "  base_url: https://old.example.test/v1\n"
            "  api_key: sk-old-config\n"
            "  model: old-model\n"
        )
        write_credential_pool(
            "custom:old-endpoint",
            [
                {
                    "id": "old-model-config",
                    "source": "model_config",
                    "auth_type": "api_key",
                    "access_token": "sk-old-model-config",
                    "base_url": "https://old.example.test/v1",
                    "label": "model_config",
                },
                {
                    "id": "old-manual",
                    "source": "manual",
                    "auth_type": "api_key",
                    "access_token": "sk-old-manual",
                    "base_url": "https://old.example.test/v1",
                    "label": "manual",
                },
            ],
        )

        with patch(
            "hermes_cli.models.probe_api_models",
            return_value={
                "models": ["new-model"],
                "used_fallback": False,
                "probed_url": "https://new.example.test/v1/models",
            },
        ), \
             patch("hermes_cli.secret_prompt.masked_secret_prompt", return_value="sk-new"), \
             patch("hermes_cli.main._prompt_custom_api_mode_selection", return_value=""), \
             patch(
                 "builtins.input",
                 side_effect=[
                     "https://new.example.test/v1",
                     "",
                     "",
                     "New Endpoint",
                 ],
             ), \
             patch("builtins.print"):
            _model_flow_custom({})

        auth = read_credential_pool(None)
        old_sources = [
            entry.get("source")
            for entry in auth.get("custom:old-endpoint", [])
            if isinstance(entry, dict)
        ]
        assert old_sources == ["manual"]

        new_pool = load_pool("custom:new-endpoint")
        selected = new_pool.select()
        assert selected is not None
        assert selected.access_token == "sk-new"

        config = yaml.safe_load(config_path.read_text()) or {}
        assert config["model"]["base_url"] == "https://new.example.test/v1"




    def test_env_template_api_key_is_preserved_in_model_config(self, config_home, monkeypatch):
        """Selecting an env-backed custom provider must not inline the secret."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        config_path = config_home / "config.yaml"
        config_path.write_text(
            "model:\n"
            "  default: old-model\n"
            "  provider: openrouter\n"
            "custom_providers:\n"
            "- name: Example Provider\n"
            "  base_url: https://api.example-provider.test/v1\n"
            "  api_key: ${EXAMPLE_PROVIDER_API_KEY}\n"
            "  model: qwen3.6-35b-fast\n"
        )
        monkeypatch.setenv("EXAMPLE_PROVIDER_API_KEY", "sk-live-example-provider")

        provider_info = {
            "name": "Example Provider",
            "base_url": "https://api.example-provider.test/v1",
            "api_key": "sk-live-example-provider",
            "api_key_ref": "${EXAMPLE_PROVIDER_API_KEY}",
            "model": "qwen3.6-35b-fast",
        }

        with patch("hermes_cli.models.fetch_api_models", return_value=["qwen3.6-35b-fast"]) as mock_fetch, \
             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        mock_fetch.assert_called_once_with(
            "sk-live-example-provider",
            "https://api.example-provider.test/v1",
            timeout=8.0,
        )
        config = yaml.safe_load(config_path.read_text()) or {}
        assert config["model"]["api_key"] == "${EXAMPLE_PROVIDER_API_KEY}"
        assert config["custom_providers"][0]["api_key"] == "${EXAMPLE_PROVIDER_API_KEY}"
        assert "sk-live-example-provider" not in config_path.read_text()

    def test_key_env_custom_provider_persists_reference_not_secret(self, config_home, monkeypatch):
        """key_env custom providers should also avoid writing plaintext keys."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        config_path = config_home / "config.yaml"
        config_path.write_text(
            "model:\n"
            "  default: old-model\n"
            "custom_providers:\n"
            "- name: Example Provider\n"
            "  base_url: https://api.example-provider.test/v1\n"
            "  key_env: EXAMPLE_PROVIDER_API_KEY\n"
            "  model: qwen3.6-35b-fast\n"
        )
        monkeypatch.setenv("EXAMPLE_PROVIDER_API_KEY", "sk-live-example-provider")

        provider_info = {
            "name": "Example Provider",
            "base_url": "https://api.example-provider.test/v1",
            "api_key": "",
            "key_env": "EXAMPLE_PROVIDER_API_KEY",
            "model": "qwen3.6-35b-fast",
        }

        with patch("hermes_cli.models.fetch_api_models", return_value=["qwen3.6-35b-fast"]), \
             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        config = yaml.safe_load(config_path.read_text()) or {}
        assert config["model"]["api_key"] == "${EXAMPLE_PROVIDER_API_KEY}"
        assert config["custom_providers"][0]["key_env"] == "EXAMPLE_PROVIDER_API_KEY"
        assert "sk-live-example-provider" not in config_path.read_text()

    def test_env_ref_base_url_preserves_api_key_ref_through_picker(
        self, config_home, monkeypatch
    ):
        """Integration regression: when BOTH ``base_url`` and ``api_key`` use
        ``${VAR}`` templates (the Discord-reported NeuralWatt case), the picker
        must still preserve the env reference in ``model.api_key``.

        The earlier lookup went through ``get_compatible_custom_providers``
        which dropped entries whose ``base_url`` was an env-ref template
        (``urlparse("${NEURALWATT_API_BASE}")`` has no scheme/netloc), causing
        ``api_key_ref`` to stay empty and the resolved secret to be written to
        ``config.yaml``. This test drives the real picker-callsite code path.
        """
        import yaml
        from hermes_cli.main import select_provider_and_model

        config_path = config_home / "config.yaml"
        config_path.write_text(
            "model:\n"
            "  default: old-model\n"
            "  provider: openrouter\n"
            "custom_providers:\n"
            "- name: NeuralWatt\n"
            "  base_url: ${NEURALWATT_API_BASE}\n"
            "  api_key: ${NEURALWATT_API_KEY}\n"
            "  model: qwen3.6-35b-fast\n"
            "  models: []\n"
        )
        monkeypatch.setenv("NEURALWATT_API_BASE", "https://api.neuralwatt.com/v1")
        monkeypatch.setenv("NEURALWATT_API_KEY", "sk-live-neuralwatt-secret")

        # Exercise the real picker: select "custom:neuralwatt" from the
        # provider menu. ``select_provider_and_model`` prompts for a provider
        # choice (returns an index), then hands off to
        # ``_model_flow_named_custom`` with the provider_info built by
        # ``_named_custom_provider_map``.
        def _pick_neuralwatt(labels, default=0):
            for i, label in enumerate(labels):
                if "NeuralWatt" in label:
                    return i
            raise AssertionError(
                f"NeuralWatt entry missing from provider menu: {labels}"
            )

        with patch("hermes_cli.main._prompt_provider_choice",
                   side_effect=_pick_neuralwatt), \
             patch("hermes_cli.models.fetch_api_models",
                   return_value=["qwen3.6-35b-fast"]) as mock_fetch, \
             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            select_provider_and_model()

        # The live probe must still use the resolved secret.
        mock_fetch.assert_called_once()
        probe_args, probe_kwargs = mock_fetch.call_args
        assert probe_args[0] == "sk-live-neuralwatt-secret"

        # But config.yaml must keep the env reference, not the plaintext secret.
        saved = config_path.read_text()
        config = yaml.safe_load(saved) or {}
        assert config["model"]["api_key"] == "${NEURALWATT_API_KEY}"
        assert config["custom_providers"][0]["api_key"] == "${NEURALWATT_API_KEY}"
        assert "sk-live-neuralwatt-secret" not in saved



    def test_key_env_providers_dict_entry_does_not_add_api_key(
        self, config_home, monkeypatch
    ):
        """Regression for #15803: a ``providers:`` (keyed-schema) entry that
        relies on ``key_env`` must not gain an ``api_key`` field after the
        model picker runs.

        Before the fix, ``_model_flow_named_custom`` synthesized
        ``api_key: ${KEY_ENV}`` from the resolved secret and wrote it to the
        ``providers.<key>`` entry, cluttering configs that intentionally keep
        credentials out of ``config.yaml``. The entry already carries
        ``key_env``; the runtime resolves it directly, so no inline
        ``api_key`` belongs on disk.
        """
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        config_path = config_home / "config.yaml"
        config_path.write_text(
            "providers:\n"
            "  crs-henkee:\n"
            "    name: CRS Henkee\n"
            "    base_url: http://127.0.0.1:3000/api/v1\n"
            "    key_env: HERMES_CRS_HENKEE_KEY\n"
            "    transport: anthropic_messages\n"
            "    model: claude-opus-4-7\n"
            "    default_model: claude-opus-4-7\n"
            "custom_providers: []\n"
        )
        monkeypatch.setenv("HERMES_CRS_HENKEE_KEY", "cr_live_secret_xyz")

        # provider_info as built by _named_custom_provider_map for a
        # ``providers:`` entry that has key_env but no inline api_key.
        provider_info = {
            "name": "CRS Henkee",
            "base_url": "http://127.0.0.1:3000/api/v1",
            "api_key": "",
            "key_env": "HERMES_CRS_HENKEE_KEY",
            "model": "claude-opus-4-7",
            "api_mode": "anthropic_messages",
            "provider_key": "crs-henkee",
            "api_key_ref": "",
        }

        with patch(
            "hermes_cli.models.fetch_api_models",
            return_value=["claude-opus-4-7"],
        ) as mock_fetch, \
             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        # The /models probe must resolve the secret from the env var.
        mock_fetch.assert_called_once()
        probe_args, _ = mock_fetch.call_args
        assert probe_args[0] == "cr_live_secret_xyz"

        # The providers entry must NOT gain an api_key field — neither the
        # plaintext secret nor a synthesized ${KEY_ENV} template.
        saved_text = config_path.read_text()
        saved = yaml.safe_load(saved_text) or {}
        entry = saved["providers"]["crs-henkee"]
        assert "api_key" not in entry, (
            f"providers.crs-henkee gained an api_key field: {entry.get('api_key')!r}"
        )
        assert entry["key_env"] == "HERMES_CRS_HENKEE_KEY"
        assert entry["default_model"] == "claude-opus-4-7"

        # And the plaintext secret must never appear anywhere on disk.
        assert "cr_live_secret_xyz" not in saved_text
        # The synthesized template is also redundant here — key_env owns it.
        assert "${HERMES_CRS_HENKEE_KEY}" not in saved_text

    def test_key_env_providers_dict_preserves_existing_api_key(
        self, config_home, monkeypatch
    ):
        """A ``providers:`` entry that already has an inline ``api_key``
        template must keep it untouched. Only entries that never declared
        an ``api_key`` should skip the write."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        config_path = config_home / "config.yaml"
        config_path.write_text(
            "providers:\n"
            "  crs-henkee:\n"
            "    name: CRS Henkee\n"
            "    base_url: http://127.0.0.1:3000/api/v1\n"
            "    api_key: ${HERMES_CRS_HENKEE_KEY}\n"
            "    key_env: HERMES_CRS_HENKEE_KEY\n"
            "    transport: anthropic_messages\n"
            "    model: claude-opus-4-7\n"
            "    default_model: claude-opus-4-7\n"
            "custom_providers: []\n"
        )
        monkeypatch.setenv("HERMES_CRS_HENKEE_KEY", "cr_live_secret_xyz")

        provider_info = {
            "name": "CRS Henkee",
            "base_url": "http://127.0.0.1:3000/api/v1",
            "api_key": "cr_live_secret_xyz",  # expanded by load_config
            "key_env": "HERMES_CRS_HENKEE_KEY",
            "model": "claude-opus-4-7",
            "api_mode": "anthropic_messages",
            "provider_key": "crs-henkee",
            "api_key_ref": "${HERMES_CRS_HENKEE_KEY}",  # raw template preserved
        }

        with patch(
            "hermes_cli.models.fetch_api_models",
            return_value=["claude-opus-4-7"],
        ), \
             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        saved_text = config_path.read_text()
        saved = yaml.safe_load(saved_text) or {}
        entry = saved["providers"]["crs-henkee"]
        # Existing api_key template must survive (the resolved secret must not
        # clobber it via _preserve_env_ref_templates).
        assert entry["api_key"] == "${HERMES_CRS_HENKEE_KEY}"
        assert "cr_live_secret_xyz" not in saved_text


class TestCustomProviderDiscoverModels:
    """#18726: honor ``discover_models: false`` in the terminal ``hermes model``
    named-custom flow so the picker shows the configured ``models:`` subset
    instead of the endpoint's full live catalog."""


    def test_discover_false_saves_choice_from_configured_list(self, config_home):
        """User picks the 2nd configured model; it persists, list-driven."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        provider_info = {
            "name": "Baidu Coding",
            "base_url": "https://qianfan.baidubce.com/v2/coding",
            "api_key": "sk-test",
            "discover_models": False,
            "models": {"kimi-k2.5": {}, "glm-5": {}},
            "model": "kimi-k2.5",
        }

        with patch("hermes_cli.models.fetch_api_models") as mock_fetch, \
             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
             patch("builtins.input", return_value="2"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        mock_fetch.assert_not_called()
        config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
        model = config.get("model")
        assert isinstance(model, dict)
        assert model["default"] == "glm-5"


    def test_probe_empty_falls_back_to_configured_list(self, config_home):
        """When discovery is on but the probe returns nothing, fall back to the
        configured models: list instead of forcing manual entry."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        provider_info = {
            "name": "My Gateway",
            "base_url": "https://gw.example.com/v1",
            "api_key": "sk-test",
            "models": {"fallback-a": {}, "fallback-b": {}},
            "model": "fallback-a",
        }

        with patch("hermes_cli.models.fetch_api_models", return_value=[]), \
             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
             patch("builtins.input", return_value="2"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
        model = config.get("model")
        assert isinstance(model, dict)
        assert model["default"] == "fallback-b"

    def test_discover_false_string_is_normalised(self, config_home):
        """String 'false' (hand-edited configs) disables discovery too."""
        from hermes_cli.main import _model_flow_named_custom

        provider_info = {
            "name": "Baidu Coding",
            "base_url": "https://qianfan.baidubce.com/v2/coding",
            "api_key": "sk-test",
            "discover_models": "false",
            "models": {"kimi-k2.5": {}, "glm-5": {}},
            "model": "kimi-k2.5",
        }

        with patch("hermes_cli.models.fetch_api_models") as mock_fetch, \
             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        mock_fetch.assert_not_called()