hermes-agent/tests/providers/test_provider_profiles.py
Teknium 39975613b1
test: prune wave 2 + speed fixes — 28,106 → 19,757 test functions, suite wall 315s → 294s
Second, deeper pass over tools/gateway/hermes_cli plus first pass over
the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker,
dashboard, conformance, monitoring, secret_sources, hermes_state,
providers). Same rubric as wave 1 (AGENTS.md test policy); security,
alternation/caching invariants, issue-number regressions, and E2E kept.

Real test-quality fixes found and rooted out along the way:
- tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls
  (DEFAULT_CONFIG smart-approval leaked in) — pinned approval
  mode=manual via autouse fixture: 17.4s → 0.4s.
- test_model_switch_custom_providers.py / test_user_providers_model_switch.py
  silently probed live provider catalogs (~2s/test) — stubbed
  cached_provider_model_ids/provider_model_ids/fetch_api_models.
- test_telegram_noise_filter.py: 15-platform copy-paste matrix over
  shared gateway.run logic → 3 representative platforms (55s → 3.9s).
- test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on
  MagicMock agents — interrupt.side_effect now clears _running_agents
  (22s → 1.0s).
- test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x
  (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps
  patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait
  5s → 0.5s.
- test_telegram_init_deadline.py: loop-block margin restored to 1.0s
  with rationale comment — the watchdog-dump assertion needs the loop
  blocked well past deadline+grace under parallel load (flaked once in
  the 40-worker verification run at a 0.2s margin).

Verification: full hermetic suite via scripts/run_tests.sh —
2,438 files, 21,718 tests passed, 0 failed, 293.9s wall.
Suite totals vs original baseline: 46,820 → 19,757 test functions
(−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
2026-07-29 13:39:40 -07:00

191 lines
5.4 KiB
Python

"""Tests for the provider module registry and profiles."""
from providers import get_provider_profile, _REGISTRY
from providers.base import ProviderProfile, OMIT_TEMPERATURE
class TestRegistry:
def test_discovery_populates_registry(self):
p = get_provider_profile("nvidia")
assert p is not None
assert p.name == "nvidia"
class TestNvidiaProfile:
def test_max_tokens(self):
p = get_provider_profile("nvidia")
assert p.default_max_tokens == 16384
def test_base_url(self):
p = get_provider_profile("nvidia")
assert "nvidia.com" in p.base_url
class TestKimiProfile:
def test_temperature_omit(self):
p = get_provider_profile("kimi")
assert p.fixed_temperature is OMIT_TEMPERATURE
def test_thinking_enabled(self):
# xor contract (fix ce4e74b3): an explicit recognized effort sends
# reasoning_effort ONLY — never paired with extra_body.thinking.
p = get_provider_profile("kimi")
eb, tl = p.build_api_kwargs_extras(reasoning_config={"enabled": True, "effort": "high"})
assert tl["reasoning_effort"] == "high"
assert "thinking" not in eb
class TestOpenRouterProfile:
def test_extra_body_with_prefs(self):
p = get_provider_profile("openrouter")
body = p.build_extra_body(provider_preferences={"allow": ["anthropic"]})
assert body["provider"] == {"allow": ["anthropic"]}
def test_pareto_min_coding_score_emitted_for_pareto_model(self):
"""min_coding_score → plugins block when model is openrouter/pareto-code."""
p = get_provider_profile("openrouter")
body = p.build_extra_body(
model="openrouter/pareto-code",
openrouter_min_coding_score=0.65,
)
assert body["plugins"] == [
{"id": "pareto-router", "min_coding_score": 0.65}
]
def test_grok_session_id_sets_cache_affinity_header(self):
"""OpenRouter + Grok model + session_id => x-grok-conv-id header."""
p = get_provider_profile("openrouter")
_, tl = p.build_api_kwargs_extras(
model="x-ai/grok-4",
session_id="sess-abc123",
)
assert tl["extra_headers"]["x-grok-conv-id"] == "sess-abc123"
# --- reasoning-mandatory Anthropic effort → top-level verbosity (#43432) ---
#
# These models (Claude 4.6+ / fable / mythos-class) ignore
# ``reasoning.effort`` and use adaptive thinking. OpenRouter honors the
# requested effort on the top-level ``verbosity`` field instead (maps to
# Anthropic ``output_config.effort``). The profile must route the existing
# ``reasoning_config["effort"]`` there while still NEVER emitting a
# ``reasoning`` field (which would 400 — see #42991). Gate every fixture on
# the real predicate so this stays a behavior contract, not a name snapshot.
@staticmethod
def _is_mandatory(model):
import inspect
p = get_provider_profile("openrouter")
mod = inspect.getmodule(type(p))
return mod._anthropic_reasoning_is_mandatory(model)
def test_mandatory_anthropic_verbosity_coexists_with_grok_header(self):
"""A reasoning-mandatory Anthropic model is never a Grok model, but the
top-level dict must remain a single merged dict — verify the verbosity
path doesn't clobber the extra_headers slot used by Grok affinity."""
p = get_provider_profile("openrouter")
# mandatory anthropic + effort → verbosity, no extra_headers
_, tl = p.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
supports_reasoning=True,
model="anthropic/claude-fable-5",
)
assert tl == {"verbosity": "high"}
class TestNousProfile:
def test_tags(self):
from agent.portal_tags import nous_portal_tags
p = get_provider_profile("nous")
body = p.build_extra_body()
assert body["tags"] == nous_portal_tags()
def test_auth_type(self):
p = get_provider_profile("nous")
assert p.auth_type == "oauth_device_code"
class TestQwenProfile:
def test_prepare_messages_protects_nested_image_url_retry_mutation(self):
qwen = get_provider_profile("qwen-oauth")
image_url = {"url": "data:image/png;base64,original"}
msgs = [
{"role": "system", "content": "Be helpful"},
{
"role": "user",
"content": [
{"type": "text", "text": "see image"},
{"type": "image_url", "image_url": image_url},
],
},
]
qwen_result = qwen.prepare_messages(msgs)
assert qwen_result[1] is not msgs[1]
assert qwen_result[1]["content"] is not msgs[1]["content"]
assert qwen_result[1]["content"][1] is not msgs[1]["content"][1]
assert qwen_result[1]["content"][1]["image_url"] is not image_url
qwen_result[1]["content"][1]["image_url"]["url"] = (
"data:image/png;base64,shrunk"
)
assert msgs[1]["content"][1]["image_url"]["url"] == (
"data:image/png;base64,original"
)
def test_metadata_top_level(self):
p = get_provider_profile("qwen-oauth")
meta = {"sessionId": "s123", "promptId": "p456"}
eb, tl = p.build_api_kwargs_extras(qwen_session_metadata=meta)
assert tl["metadata"] == meta
assert "metadata" not in eb