feat(/model): merge models.dev entries for lesser-loved providers (#14221)

New and newer models from models.dev now surface automatically in
/model (both hermes model CLI and the gateway Telegram/Discord picker)
for a curated set of secondary providers — no Hermes release required
when the registry publishes a new model.

Primary user-visible fix: on OpenCode Go, typing '/model mimo-v2.5-pro'
no longer silently fuzzy-corrects to 'mimo-v2-pro'. The exact match
against the merged models.dev catalog wins.

Scope (opt-in frozenset _MODELS_DEV_PREFERRED in hermes_cli/models.py):
  opencode-go, opencode-zen, deepseek, kilocode, fireworks, mistral,
  togetherai, cohere, perplexity, groq, nvidia, huggingface, zai,
  gemini, google.

Explicitly NOT merged:
  - openrouter and nous (never): curated list is already a hand-picked
    subset / Portal is source of truth.
  - xai, xiaomi, minimax, minimax-cn, kimi-coding, kimi-coding-cn,
    alibaba, qwen-oauth (per-project decision to keep curated-only).
  - providers with dedicated live-endpoint paths (copilot, anthropic,
    ai-gateway, ollama-cloud, custom, stepfun, openai-codex) — those
    paths already handle freshness themselves.

Changes:
  - hermes_cli/models.py: add _MODELS_DEV_PREFERRED + _merge_with_models_dev
    helper. provider_model_ids() branches on the set at its curated-fallback
    return. Merge is models.dev-first, curated-only extras appended,
    case-insensitive dedup, graceful fallback when models.dev is offline.
  - hermes_cli/model_switch.py: list_authenticated_providers() calls the
    same merge in both its code paths (PROVIDER_TO_MODELS_DEV loop +
    HERMES_OVERLAYS loop). Picker AND validation-fallback both see
    fresh entries.
  - tests/hermes_cli/test_models_dev_preferred_merge.py (new): 13 tests —
    merge-helper unit tests (empty/raise/order/dedup), opencode-go/zen
    behavior, openrouter+nous explicitly guarded from merge.
  - tests/hermes_cli/test_opencode_go_in_model_list.py: converted from
    snapshot-style assertion to a behavior-based floor check, so it
    doesn't break when models.dev publishes additional opencode-go
    entries.

Addresses a report from @pfanis via Telegram: newer Xiaomi variants
on OpenCode Go weren't appearing in the /model picker, and /model
was silently routing requests for new variants to older ones.
This commit is contained in:
Teknium 2026-04-22 17:33:42 -07:00 committed by GitHub
parent ea0e4c267d
commit 9eb543cafe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 246 additions and 10 deletions

View file

@ -0,0 +1,124 @@
"""Tests for the models.dev-preferred merge behavior in provider_model_ids
and list_authenticated_providers.
These guard the contract:
* For providers in ``_MODELS_DEV_PREFERRED`` (opencode-go, opencode-zen,
xiaomi, deepseek, smaller inference providers), both the CLI model
picker path (``provider_model_ids``) and the gateway ``/model`` picker
path (``list_authenticated_providers``) merge fresh models.dev entries
on top of the curated static list.
* OpenRouter and Nous Portal are NEVER merged they keep their curated
(OpenRouter) or live-Portal (Nous) semantics.
* If models.dev is unreachable (offline / CI), the curated list is the
fallback no crash, no empty list.
Merging is what lets new models (e.g. ``mimo-v2.5-pro`` on opencode-go)
appear in ``/model`` without a Hermes release.
"""
import os
from unittest.mock import patch
import pytest
from hermes_cli.models import (
_MODELS_DEV_PREFERRED,
_merge_with_models_dev,
provider_model_ids,
)
class TestMergeHelper:
def test_merge_empty_mdev_returns_curated(self):
"""When models.dev returns nothing, curated list is preserved verbatim."""
with patch("agent.models_dev.list_agentic_models", return_value=[]):
out = _merge_with_models_dev("opencode-go", ["mimo-v2-pro", "kimi-k2.6"])
assert out == ["mimo-v2-pro", "kimi-k2.6"]
def test_merge_mdev_raises_returns_curated(self):
"""Offline / broken models.dev must not break the catalog path."""
def boom(_provider):
raise RuntimeError("network down")
with patch("agent.models_dev.list_agentic_models", side_effect=boom):
out = _merge_with_models_dev("opencode-go", ["mimo-v2-pro"])
assert out == ["mimo-v2-pro"]
def test_merge_mdev_first_then_curated_extras(self):
"""models.dev entries come first; curated-only entries are appended."""
mdev = ["mimo-v2.5-pro", "mimo-v2-pro", "kimi-k2.6"]
curated = ["kimi-k2.6", "kimi-k2.5", "mimo-v2-pro"] # kimi-k2.5 is curated-only
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
out = _merge_with_models_dev("opencode-go", curated)
# models.dev entries first (in order), then curated-only entries
assert out == ["mimo-v2.5-pro", "mimo-v2-pro", "kimi-k2.6", "kimi-k2.5"]
def test_merge_case_insensitive_dedup(self):
"""Dedup is case-insensitive but preserves the first occurrence's casing."""
mdev = ["MiniMax-M2.7"]
curated = ["minimax-m2.7", "minimax-m2.5"]
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
out = _merge_with_models_dev("minimax", curated)
# models.dev casing wins since it came first
assert out == ["MiniMax-M2.7", "minimax-m2.5"]
class TestProviderModelIdsPreferred:
def test_opencode_go_is_preferred(self):
assert "opencode-go" in _MODELS_DEV_PREFERRED
def test_opencode_go_includes_fresh_models_dev_entries(self):
"""provider_model_ids('opencode-go') adds models.dev entries on top."""
mdev = ["mimo-v2.5-pro", "mimo-v2.5", "mimo-v2-pro", "kimi-k2.6"]
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
out = provider_model_ids("opencode-go")
# Fresh models must surface (this is exactly the reported bug fix:
# mimo-v2.5-pro should be pickable on opencode-go).
assert "mimo-v2.5-pro" in out
assert "mimo-v2.5" in out
# Curated entries are still present.
assert "mimo-v2-pro" in out
assert "kimi-k2.6" in out
def test_opencode_go_offline_falls_back_to_curated(self):
"""Offline models.dev → curated-only list, no crash."""
with patch("agent.models_dev.list_agentic_models", return_value=[]):
out = provider_model_ids("opencode-go")
# Curated floor (see hermes_cli/models.py _PROVIDER_MODELS["opencode-go"])
assert "mimo-v2-pro" in out
assert "kimi-k2.6" in out
def test_opencode_zen_includes_fresh_models(self):
"""opencode-zen follows the same pattern as opencode-go."""
assert "opencode-zen" in _MODELS_DEV_PREFERRED
mdev = ["claude-opus-4-7", "kimi-k2.6", "glm-5.1"]
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
out = provider_model_ids("opencode-zen")
assert "claude-opus-4-7" in out
assert "kimi-k2.6" in out
class TestOpenRouterAndNousUnchanged:
"""Per Teknium: openrouter and nous are NEVER merged with models.dev."""
def test_openrouter_not_in_preferred_set(self):
assert "openrouter" not in _MODELS_DEV_PREFERRED
def test_nous_not_in_preferred_set(self):
assert "nous" not in _MODELS_DEV_PREFERRED
def test_openrouter_does_not_call_merge(self):
"""openrouter takes its own live path — merge helper must NOT run."""
with patch(
"hermes_cli.models._merge_with_models_dev",
side_effect=AssertionError("merge should not be called for openrouter"),
):
# Even if model_ids() fails for some other reason, we just care
# that the merge path isn't invoked.
try:
provider_model_ids("openrouter")
except AssertionError:
raise
except Exception:
pass # model_ids() may fail in the hermetic test env — that's fine.