"""Regression tests for the ``auto`` → main-model-first policy. Prior to this change, aggregator users (OpenRouter / Nous Portal) had aux tasks routed through a cheap provider-side default (Gemini Flash) while non-aggregator users got their main model. This made behavior inconsistent and surprising — users picked Claude but got Gemini Flash summaries. The current policy: ``auto`` means "use my main chat model" for every user, regardless of provider type. Explicit per-task overrides in ``config.yaml`` (``auxiliary..provider``) still win. The cheap fallback chain only runs when the main provider has no working client. """ from __future__ import annotations from unittest.mock import MagicMock, patch import pytest # ── Text aux tasks — _resolve_auto ────────────────────────────────────────── class TestResolveAutoMainFirst: """_resolve_auto() must prefer main provider + main model for every user.""" def test_openrouter_main_uses_main_model_for_aux(self, monkeypatch): """OpenRouter main user → aux uses their picked OR model, not Gemini Flash.""" monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key") with patch( "agent.auxiliary_client._read_main_provider", return_value="openrouter", ), patch( "agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4.6", ), patch( "agent.auxiliary_client.resolve_provider_client" ) as mock_resolve: mock_client = MagicMock() mock_resolve.return_value = (mock_client, "anthropic/claude-sonnet-4.6") from agent.auxiliary_client import _resolve_auto client, model = _resolve_auto() assert client is mock_client assert model == "anthropic/claude-sonnet-4.6" # Verify it asked resolve_provider_client for the MAIN provider+model, # not a fallback-chain provider mock_resolve.assert_called_once() assert mock_resolve.call_args.args[0] == "openrouter" assert mock_resolve.call_args.args[1] == "anthropic/claude-sonnet-4.6" def test_nous_main_uses_main_model_for_aux(self, monkeypatch): """Nous Portal main user → aux uses their picked Nous model, not free-tier MiMo.""" # No OPENROUTER_API_KEY → ensures if main failed we'd fall to chain with patch( "agent.auxiliary_client._read_main_provider", return_value="nous", ), patch( "agent.auxiliary_client._read_main_model", return_value="anthropic/claude-opus-4.6", ), patch( "agent.auxiliary_client.resolve_provider_client" ) as mock_resolve: mock_client = MagicMock() mock_resolve.return_value = (mock_client, "anthropic/claude-opus-4.6") from agent.auxiliary_client import _resolve_auto client, model = _resolve_auto() assert client is mock_client assert model == "anthropic/claude-opus-4.6" assert mock_resolve.call_args.args[0] == "nous" def test_non_aggregator_main_still_uses_main(self, monkeypatch): """Non-aggregator main (DeepSeek) → unchanged behavior, main model used.""" monkeypatch.setenv("DEEPSEEK_API_KEY", "ds-test") with patch( "agent.auxiliary_client._read_main_provider", return_value="deepseek", ), patch( "agent.auxiliary_client._read_main_model", return_value="deepseek-chat", ), patch( "agent.auxiliary_client.resolve_provider_client" ) as mock_resolve: mock_client = MagicMock() mock_resolve.return_value = (mock_client, "deepseek-chat") from agent.auxiliary_client import _resolve_auto client, model = _resolve_auto() assert client is mock_client assert model == "deepseek-chat" assert mock_resolve.call_args.args[0] == "deepseek" def test_main_unavailable_falls_through_to_chain(self, monkeypatch): """Main provider with no working client → fall back to aux chain.""" monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") chain_client = MagicMock() with patch( "agent.auxiliary_client._read_main_provider", return_value="anthropic", ), patch( "agent.auxiliary_client._read_main_model", return_value="claude-opus", ), patch( "agent.auxiliary_client.resolve_provider_client", return_value=(None, None), # main provider has no client ), patch( "agent.auxiliary_client._try_openrouter", return_value=(chain_client, "google/gemini-3-flash-preview"), ): from agent.auxiliary_client import _resolve_auto client, model = _resolve_auto() assert client is chain_client assert model == "google/gemini-3-flash-preview" def test_no_main_config_uses_chain_directly(self): """No main provider configured → skip step 1, use chain (no regression).""" chain_client = MagicMock() with patch( "agent.auxiliary_client._read_main_provider", return_value="", ), patch( "agent.auxiliary_client._read_main_model", return_value="", ), patch( "agent.auxiliary_client._try_openrouter", return_value=(chain_client, "google/gemini-3-flash-preview"), ): from agent.auxiliary_client import _resolve_auto client, model = _resolve_auto() assert client is chain_client def test_runtime_override_wins_over_config(self, monkeypatch): """main_runtime kwarg overrides config-read main provider/model.""" with patch( "agent.auxiliary_client._read_main_provider", return_value="openrouter", ), patch( "agent.auxiliary_client._read_main_model", return_value="config-model", ), patch( "agent.auxiliary_client.resolve_provider_client" ) as mock_resolve: mock_resolve.return_value = (MagicMock(), "runtime-model") from agent.auxiliary_client import _resolve_auto _resolve_auto(main_runtime={ "provider": "anthropic", "model": "runtime-model", "base_url": "", "api_key": "", "api_mode": "", }) # Runtime override wins assert mock_resolve.call_args.args[0] == "anthropic" assert mock_resolve.call_args.args[1] == "runtime-model" # ── Vision — resolve_vision_provider_client ───────────────────────────────── class TestResolveVisionMainFirst: """Vision auto-detection prefers main provider + main model first.""" def test_openrouter_main_vision_uses_main_model(self, monkeypatch): """OpenRouter main with vision-capable model → aux vision uses main model.""" monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") with patch( "agent.auxiliary_client._read_main_provider", return_value="openrouter", ), patch( "agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4.6", ), patch( "agent.auxiliary_client.resolve_provider_client" ) as mock_resolve, patch( "agent.auxiliary_client._resolve_task_provider_model", return_value=("auto", None, None, None, None), ): mock_client = MagicMock() mock_resolve.return_value = (mock_client, "anthropic/claude-sonnet-4.6") from agent.auxiliary_client import resolve_vision_provider_client provider, client, model = resolve_vision_provider_client() assert provider == "openrouter" assert client is mock_client assert model == "anthropic/claude-sonnet-4.6" # Verify it did NOT call the strict vision backend for OpenRouter # (which would have used a cheap gemini-flash-preview default) mock_resolve.assert_called_once() assert mock_resolve.call_args.args[0] == "openrouter" assert mock_resolve.call_args.args[1] == "anthropic/claude-sonnet-4.6" def test_nous_main_vision_uses_main_model(self): """Nous Portal main → aux vision uses main model, not free-tier MiMo-V2-Omni.""" with patch( "agent.auxiliary_client._read_main_provider", return_value="nous", ), patch( "agent.auxiliary_client._read_main_model", return_value="openai/gpt-5", ), patch( "agent.auxiliary_client.resolve_provider_client" ) as mock_resolve, patch( "agent.auxiliary_client._resolve_task_provider_model", return_value=("auto", None, None, None, None), ): mock_client = MagicMock() mock_resolve.return_value = (mock_client, "openai/gpt-5") from agent.auxiliary_client import resolve_vision_provider_client provider, client, model = resolve_vision_provider_client() assert provider == "nous" assert model == "openai/gpt-5" def test_exotic_provider_with_vision_override_preserved(self): """xiaomi → mimo-v2-omni override still wins over main_model.""" with patch( "agent.auxiliary_client._read_main_provider", return_value="xiaomi", ), patch( "agent.auxiliary_client._read_main_model", return_value="mimo-v2-pro", # text model ), patch( "agent.auxiliary_client.resolve_provider_client" ) as mock_resolve, patch( "agent.auxiliary_client._resolve_task_provider_model", return_value=("auto", None, None, None, None), ): mock_resolve.return_value = (MagicMock(), "mimo-v2-omni") from agent.auxiliary_client import resolve_vision_provider_client provider, client, model = resolve_vision_provider_client() assert provider == "xiaomi" # Should use mimo-v2-omni (vision override), not mimo-v2-pro (text main) assert mock_resolve.call_args.args[1] == "mimo-v2-omni" def test_main_unavailable_vision_falls_through_to_aggregators(self): """Main provider fails → fall back to OpenRouter/Nous strict backends.""" fallback_client = MagicMock() with patch( "agent.auxiliary_client._read_main_provider", return_value="deepseek", ), patch( "agent.auxiliary_client._read_main_model", return_value="deepseek-chat", ), patch( "agent.auxiliary_client.resolve_provider_client", return_value=(None, None), ), patch( "agent.auxiliary_client._resolve_strict_vision_backend", return_value=(fallback_client, "google/gemini-3-flash-preview"), ), patch( "agent.auxiliary_client._resolve_task_provider_model", return_value=("auto", None, None, None, None), ): from agent.auxiliary_client import resolve_vision_provider_client provider, client, model = resolve_vision_provider_client() assert client is fallback_client assert provider in ("openrouter", "nous") def test_explicit_provider_override_still_wins(self): """Explicit config override bypasses main-first policy.""" with patch( "agent.auxiliary_client._read_main_provider", return_value="openrouter", ), patch( "agent.auxiliary_client._read_main_model", return_value="anthropic/claude-opus-4.6", ), patch( "agent.auxiliary_client._resolve_task_provider_model", return_value=("nous", None, None, None, None), # explicit override ), patch( "agent.auxiliary_client._resolve_strict_vision_backend" ) as mock_strict: mock_strict.return_value = (MagicMock(), "nous-default-model") from agent.auxiliary_client import resolve_vision_provider_client provider, client, model = resolve_vision_provider_client() # Explicit "nous" override → uses strict backend, NOT main model path assert provider == "nous" mock_strict.assert_called_once_with("nous") # ── Constant cleanup ──────────────────────────────────────────────────────── def test_aggregator_providers_constant_removed(): """The dead _AGGREGATOR_PROVIDERS constant should no longer live in the module. Removed when the main-first policy made the aggregator-skip guard obsolete. """ import agent.auxiliary_client as aux_mod assert not hasattr(aux_mod, "_AGGREGATOR_PROVIDERS"), ( "_AGGREGATOR_PROVIDERS was removed when _resolve_auto stopped " "treating aggregators specially. If you re-added it, the main-first " "policy may have regressed." )