mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-25 00:51:20 +00:00
feat(/model): merge models.dev entries for lesser-loved providers (#14221)
New and newer models from models.dev now surface automatically in
/model (both hermes model CLI and the gateway Telegram/Discord picker)
for a curated set of secondary providers — no Hermes release required
when the registry publishes a new model.
Primary user-visible fix: on OpenCode Go, typing '/model mimo-v2.5-pro'
no longer silently fuzzy-corrects to 'mimo-v2-pro'. The exact match
against the merged models.dev catalog wins.
Scope (opt-in frozenset _MODELS_DEV_PREFERRED in hermes_cli/models.py):
opencode-go, opencode-zen, deepseek, kilocode, fireworks, mistral,
togetherai, cohere, perplexity, groq, nvidia, huggingface, zai,
gemini, google.
Explicitly NOT merged:
- openrouter and nous (never): curated list is already a hand-picked
subset / Portal is source of truth.
- xai, xiaomi, minimax, minimax-cn, kimi-coding, kimi-coding-cn,
alibaba, qwen-oauth (per-project decision to keep curated-only).
- providers with dedicated live-endpoint paths (copilot, anthropic,
ai-gateway, ollama-cloud, custom, stepfun, openai-codex) — those
paths already handle freshness themselves.
Changes:
- hermes_cli/models.py: add _MODELS_DEV_PREFERRED + _merge_with_models_dev
helper. provider_model_ids() branches on the set at its curated-fallback
return. Merge is models.dev-first, curated-only extras appended,
case-insensitive dedup, graceful fallback when models.dev is offline.
- hermes_cli/model_switch.py: list_authenticated_providers() calls the
same merge in both its code paths (PROVIDER_TO_MODELS_DEV loop +
HERMES_OVERLAYS loop). Picker AND validation-fallback both see
fresh entries.
- tests/hermes_cli/test_models_dev_preferred_merge.py (new): 13 tests —
merge-helper unit tests (empty/raise/order/dedup), opencode-go/zen
behavior, openrouter+nous explicitly guarded from merge.
- tests/hermes_cli/test_opencode_go_in_model_list.py: converted from
snapshot-style assertion to a behavior-based floor check, so it
doesn't break when models.dev publishes additional opencode-go
entries.
Addresses a report from @pfanis via Telegram: newer Xiaomi variants
on OpenCode Go weren't appearing in the /model picker, and /model
was silently routing requests for new variants to older ones.
This commit is contained in:
parent
ea0e4c267d
commit
9eb543cafe
4 changed files with 246 additions and 10 deletions
|
|
@ -810,7 +810,10 @@ def list_authenticated_providers(
|
|||
get_provider_info as _mdev_pinfo,
|
||||
)
|
||||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||||
from hermes_cli.models import OPENROUTER_MODELS, _PROVIDER_MODELS
|
||||
from hermes_cli.models import (
|
||||
OPENROUTER_MODELS, _PROVIDER_MODELS,
|
||||
_MODELS_DEV_PREFERRED, _merge_with_models_dev,
|
||||
)
|
||||
|
||||
results: List[dict] = []
|
||||
seen_slugs: set = set() # lowercase-normalized to catch case variants (#9545)
|
||||
|
|
@ -856,8 +859,13 @@ def list_authenticated_providers(
|
|||
if not has_creds:
|
||||
continue
|
||||
|
||||
# Use curated list, falling back to models.dev if no curated list
|
||||
# Use curated list, falling back to models.dev if no curated list.
|
||||
# For preferred providers, merge models.dev entries into the curated
|
||||
# catalog so newly released models (e.g. mimo-v2.5-pro on opencode-go)
|
||||
# show up in the picker without requiring a Hermes release.
|
||||
model_ids = curated.get(hermes_id, [])
|
||||
if hermes_id in _MODELS_DEV_PREFERRED:
|
||||
model_ids = _merge_with_models_dev(hermes_id, model_ids)
|
||||
total = len(model_ids)
|
||||
top = model_ids[:max_models]
|
||||
|
||||
|
|
@ -961,6 +969,9 @@ def list_authenticated_providers(
|
|||
|
||||
# Use curated list — look up by Hermes slug, fall back to overlay key
|
||||
model_ids = curated.get(hermes_slug, []) or curated.get(pid, [])
|
||||
# Merge with models.dev for preferred providers (same rationale as above).
|
||||
if hermes_slug in _MODELS_DEV_PREFERRED:
|
||||
model_ids = _merge_with_models_dev(hermes_slug, model_ids)
|
||||
total = len(model_ids)
|
||||
top = model_ids[:max_models]
|
||||
|
||||
|
|
|
|||
|
|
@ -1589,11 +1589,84 @@ def _resolve_copilot_catalog_api_key() -> str:
|
|||
return ""
|
||||
|
||||
|
||||
# Providers where models.dev is treated as authoritative: curated static
|
||||
# lists are kept only as an offline fallback and to capture custom additions
|
||||
# the registry doesn't publish yet. Adding a provider here causes its
|
||||
# curated list to be merged with fresh models.dev entries (fresh first, any
|
||||
# curated-only names appended) for both the CLI and the gateway /model picker.
|
||||
#
|
||||
# DELIBERATELY EXCLUDED:
|
||||
# - "openrouter": curated list is already a hand-picked agentic subset of
|
||||
# OpenRouter's 400+ catalog. Blindly merging would dump everything.
|
||||
# - "nous": curated list and Portal /models endpoint are the source of
|
||||
# truth for the subscription tier.
|
||||
# Also excluded: providers that already have dedicated live-endpoint
|
||||
# branches below (copilot, anthropic, ai-gateway, ollama-cloud, custom,
|
||||
# stepfun, openai-codex) — those paths handle freshness themselves.
|
||||
_MODELS_DEV_PREFERRED: frozenset[str] = frozenset({
|
||||
"opencode-go",
|
||||
"opencode-zen",
|
||||
"deepseek",
|
||||
"kilocode",
|
||||
"fireworks",
|
||||
"mistral",
|
||||
"togetherai",
|
||||
"cohere",
|
||||
"perplexity",
|
||||
"groq",
|
||||
"nvidia",
|
||||
"huggingface",
|
||||
"zai",
|
||||
"gemini",
|
||||
"google",
|
||||
})
|
||||
|
||||
|
||||
def _merge_with_models_dev(provider: str, curated: list[str]) -> list[str]:
|
||||
"""Merge curated list with fresh models.dev entries for a preferred provider.
|
||||
|
||||
Returns models.dev entries first (in models.dev order), then any
|
||||
curated-only entries appended. Preserves case for curated fallbacks
|
||||
(e.g. ``MiniMax-M2.7``) while trusting models.dev for newer variants.
|
||||
|
||||
If models.dev is unreachable or returns nothing, the curated list is
|
||||
returned unchanged — this is the offline/CI fallback path.
|
||||
"""
|
||||
try:
|
||||
from agent.models_dev import list_agentic_models
|
||||
mdev = list_agentic_models(provider)
|
||||
except Exception:
|
||||
mdev = []
|
||||
|
||||
if not mdev:
|
||||
return list(curated)
|
||||
|
||||
# Case-insensitive dedup while preserving order and curated casing.
|
||||
seen_lower: set[str] = set()
|
||||
merged: list[str] = []
|
||||
for mid in mdev:
|
||||
key = str(mid).lower()
|
||||
if key in seen_lower:
|
||||
continue
|
||||
seen_lower.add(key)
|
||||
merged.append(mid)
|
||||
for mid in curated:
|
||||
key = str(mid).lower()
|
||||
if key in seen_lower:
|
||||
continue
|
||||
seen_lower.add(key)
|
||||
merged.append(mid)
|
||||
return merged
|
||||
|
||||
|
||||
def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False) -> list[str]:
|
||||
"""Return the best known model catalog for a provider.
|
||||
|
||||
Tries live API endpoints for providers that support them (Codex, Nous),
|
||||
falling back to static lists.
|
||||
falling back to static lists. For providers in ``_MODELS_DEV_PREFERRED``
|
||||
(opencode-go/zen, xiaomi, deepseek, smaller inference providers, etc.),
|
||||
models.dev entries are merged on top of curated so new models released
|
||||
on the platform appear in ``/model`` without a Hermes release.
|
||||
"""
|
||||
normalized = normalize_provider(provider)
|
||||
if normalized == "openrouter":
|
||||
|
|
@ -1659,7 +1732,10 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)
|
|||
live = fetch_api_models(api_key, base_url)
|
||||
if live:
|
||||
return live
|
||||
return list(_PROVIDER_MODELS.get(normalized, []))
|
||||
curated_static = list(_PROVIDER_MODELS.get(normalized, []))
|
||||
if normalized in _MODELS_DEV_PREFERRED:
|
||||
return _merge_with_models_dev(normalized, curated_static)
|
||||
return curated_static
|
||||
|
||||
|
||||
def _fetch_anthropic_models(timeout: float = 5.0) -> Optional[list[str]]:
|
||||
|
|
|
|||
124
tests/hermes_cli/test_models_dev_preferred_merge.py
Normal file
124
tests/hermes_cli/test_models_dev_preferred_merge.py
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
"""Tests for the models.dev-preferred merge behavior in provider_model_ids
|
||||
and list_authenticated_providers.
|
||||
|
||||
These guard the contract:
|
||||
|
||||
* For providers in ``_MODELS_DEV_PREFERRED`` (opencode-go, opencode-zen,
|
||||
xiaomi, deepseek, smaller inference providers), both the CLI model
|
||||
picker path (``provider_model_ids``) and the gateway ``/model`` picker
|
||||
path (``list_authenticated_providers``) merge fresh models.dev entries
|
||||
on top of the curated static list.
|
||||
* OpenRouter and Nous Portal are NEVER merged — they keep their curated
|
||||
(OpenRouter) or live-Portal (Nous) semantics.
|
||||
* If models.dev is unreachable (offline / CI), the curated list is the
|
||||
fallback — no crash, no empty list.
|
||||
|
||||
Merging is what lets new models (e.g. ``mimo-v2.5-pro`` on opencode-go)
|
||||
appear in ``/model`` without a Hermes release.
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.models import (
|
||||
_MODELS_DEV_PREFERRED,
|
||||
_merge_with_models_dev,
|
||||
provider_model_ids,
|
||||
)
|
||||
|
||||
|
||||
class TestMergeHelper:
|
||||
def test_merge_empty_mdev_returns_curated(self):
|
||||
"""When models.dev returns nothing, curated list is preserved verbatim."""
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=[]):
|
||||
out = _merge_with_models_dev("opencode-go", ["mimo-v2-pro", "kimi-k2.6"])
|
||||
assert out == ["mimo-v2-pro", "kimi-k2.6"]
|
||||
|
||||
def test_merge_mdev_raises_returns_curated(self):
|
||||
"""Offline / broken models.dev must not break the catalog path."""
|
||||
def boom(_provider):
|
||||
raise RuntimeError("network down")
|
||||
|
||||
with patch("agent.models_dev.list_agentic_models", side_effect=boom):
|
||||
out = _merge_with_models_dev("opencode-go", ["mimo-v2-pro"])
|
||||
assert out == ["mimo-v2-pro"]
|
||||
|
||||
def test_merge_mdev_first_then_curated_extras(self):
|
||||
"""models.dev entries come first; curated-only entries are appended."""
|
||||
mdev = ["mimo-v2.5-pro", "mimo-v2-pro", "kimi-k2.6"]
|
||||
curated = ["kimi-k2.6", "kimi-k2.5", "mimo-v2-pro"] # kimi-k2.5 is curated-only
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
|
||||
out = _merge_with_models_dev("opencode-go", curated)
|
||||
# models.dev entries first (in order), then curated-only entries
|
||||
assert out == ["mimo-v2.5-pro", "mimo-v2-pro", "kimi-k2.6", "kimi-k2.5"]
|
||||
|
||||
def test_merge_case_insensitive_dedup(self):
|
||||
"""Dedup is case-insensitive but preserves the first occurrence's casing."""
|
||||
mdev = ["MiniMax-M2.7"]
|
||||
curated = ["minimax-m2.7", "minimax-m2.5"]
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
|
||||
out = _merge_with_models_dev("minimax", curated)
|
||||
# models.dev casing wins since it came first
|
||||
assert out == ["MiniMax-M2.7", "minimax-m2.5"]
|
||||
|
||||
|
||||
class TestProviderModelIdsPreferred:
|
||||
def test_opencode_go_is_preferred(self):
|
||||
assert "opencode-go" in _MODELS_DEV_PREFERRED
|
||||
|
||||
def test_opencode_go_includes_fresh_models_dev_entries(self):
|
||||
"""provider_model_ids('opencode-go') adds models.dev entries on top."""
|
||||
mdev = ["mimo-v2.5-pro", "mimo-v2.5", "mimo-v2-pro", "kimi-k2.6"]
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
|
||||
out = provider_model_ids("opencode-go")
|
||||
# Fresh models must surface (this is exactly the reported bug fix:
|
||||
# mimo-v2.5-pro should be pickable on opencode-go).
|
||||
assert "mimo-v2.5-pro" in out
|
||||
assert "mimo-v2.5" in out
|
||||
# Curated entries are still present.
|
||||
assert "mimo-v2-pro" in out
|
||||
assert "kimi-k2.6" in out
|
||||
|
||||
def test_opencode_go_offline_falls_back_to_curated(self):
|
||||
"""Offline models.dev → curated-only list, no crash."""
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=[]):
|
||||
out = provider_model_ids("opencode-go")
|
||||
# Curated floor (see hermes_cli/models.py _PROVIDER_MODELS["opencode-go"])
|
||||
assert "mimo-v2-pro" in out
|
||||
assert "kimi-k2.6" in out
|
||||
|
||||
def test_opencode_zen_includes_fresh_models(self):
|
||||
"""opencode-zen follows the same pattern as opencode-go."""
|
||||
assert "opencode-zen" in _MODELS_DEV_PREFERRED
|
||||
mdev = ["claude-opus-4-7", "kimi-k2.6", "glm-5.1"]
|
||||
with patch("agent.models_dev.list_agentic_models", return_value=mdev):
|
||||
out = provider_model_ids("opencode-zen")
|
||||
assert "claude-opus-4-7" in out
|
||||
assert "kimi-k2.6" in out
|
||||
|
||||
|
||||
class TestOpenRouterAndNousUnchanged:
|
||||
"""Per Teknium: openrouter and nous are NEVER merged with models.dev."""
|
||||
|
||||
def test_openrouter_not_in_preferred_set(self):
|
||||
assert "openrouter" not in _MODELS_DEV_PREFERRED
|
||||
|
||||
def test_nous_not_in_preferred_set(self):
|
||||
assert "nous" not in _MODELS_DEV_PREFERRED
|
||||
|
||||
def test_openrouter_does_not_call_merge(self):
|
||||
"""openrouter takes its own live path — merge helper must NOT run."""
|
||||
with patch(
|
||||
"hermes_cli.models._merge_with_models_dev",
|
||||
side_effect=AssertionError("merge should not be called for openrouter"),
|
||||
):
|
||||
# Even if model_ids() fails for some other reason, we just care
|
||||
# that the merge path isn't invoked.
|
||||
try:
|
||||
provider_model_ids("openrouter")
|
||||
except AssertionError:
|
||||
raise
|
||||
except Exception:
|
||||
pass # model_ids() may fail in the hermetic test env — that's fine.
|
||||
|
|
@ -6,16 +6,41 @@ from unittest.mock import patch
|
|||
from hermes_cli.model_switch import list_authenticated_providers
|
||||
|
||||
|
||||
# Minimum set of models that must be present for opencode-go no matter
|
||||
# whether the picker sourced its list from curated-only or curated+models.dev.
|
||||
# The curated list in hermes_cli/models.py defines the floor; models.dev only
|
||||
# ever adds names on top of it via _merge_with_models_dev.
|
||||
_OPENCODE_GO_REQUIRED = {
|
||||
"kimi-k2.6",
|
||||
"kimi-k2.5",
|
||||
"glm-5.1",
|
||||
"glm-5",
|
||||
"mimo-v2-pro",
|
||||
"mimo-v2-omni",
|
||||
"minimax-m2.7",
|
||||
"minimax-m2.5",
|
||||
}
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENCODE_GO_API_KEY": "test-key"}, clear=False)
|
||||
def test_opencode_go_appears_when_api_key_set():
|
||||
"""opencode-go should appear in list_authenticated_providers when OPENCODE_GO_API_KEY is set."""
|
||||
providers = list_authenticated_providers(current_provider="openrouter")
|
||||
|
||||
providers = list_authenticated_providers(current_provider="openrouter", max_models=50)
|
||||
|
||||
# Find opencode-go in results
|
||||
opencode_go = next((p for p in providers if p["slug"] == "opencode-go"), None)
|
||||
|
||||
|
||||
assert opencode_go is not None, "opencode-go should appear when OPENCODE_GO_API_KEY is set"
|
||||
assert opencode_go["models"] == ["kimi-k2.6", "kimi-k2.5", "glm-5.1", "glm-5", "mimo-v2-pro", "mimo-v2-omni", "minimax-m2.7", "minimax-m2.5"]
|
||||
# Behavior check: the curated floor must be present. The list may also
|
||||
# include extra models.dev entries (e.g. mimo-v2.5-pro) when the registry
|
||||
# is reachable — that's the whole point of the models.dev-preferred merge
|
||||
# introduced for opencode-go, so don't pin to an exact list here.
|
||||
present = set(opencode_go["models"])
|
||||
missing = _OPENCODE_GO_REQUIRED - present
|
||||
assert not missing, (
|
||||
f"opencode-go picker should include the curated floor; missing: {sorted(missing)}. "
|
||||
f"Got: {opencode_go['models']}"
|
||||
)
|
||||
# opencode-go can appear as "built-in" (from PROVIDER_TO_MODELS_DEV when
|
||||
# models.dev is reachable) or "hermes" (from HERMES_OVERLAYS fallback when
|
||||
# the API is unavailable, e.g. in CI).
|
||||
|
|
@ -26,10 +51,10 @@ def test_opencode_go_not_appears_when_no_creds():
|
|||
"""opencode-go should NOT appear when no credentials are set."""
|
||||
# Ensure OPENCODE_GO_API_KEY is not set
|
||||
env_without_key = {k: v for k, v in os.environ.items() if k != "OPENCODE_GO_API_KEY"}
|
||||
|
||||
|
||||
with patch.dict(os.environ, env_without_key, clear=True):
|
||||
providers = list_authenticated_providers(current_provider="openrouter")
|
||||
|
||||
|
||||
# opencode-go should not be in results
|
||||
opencode_go = next((p for p in providers if p["slug"] == "opencode-go"), None)
|
||||
assert opencode_go is None, "opencode-go should not appear without credentials"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue