mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-29 18:46:59 +00:00
feat(picker): derive a newest-per-lab featured shortlist from models.dev
Aggregator providers (nous, openrouter) serve dozens of models across many labs. Add a featured=True enricher to build_models_payload that attaches a featured_models shortlist to each row: within every lab keep the newest _FEATURED_PER_LAB models by models.dev release_date, ranked among that row's own models — never against the current date, so the choice is stable as models age. Same-date ties fall back to curated list order. Single-lab providers get an empty list. Derived live from the models.dev catalog already loaded on this path; no hand-maintained allowlist.
This commit is contained in:
parent
5771a6ebe6
commit
d4449c47e2
2 changed files with 199 additions and 0 deletions
|
|
@ -120,6 +120,7 @@ def build_models_payload(
|
|||
canonical_order: bool = False,
|
||||
pricing: bool = False,
|
||||
capabilities: bool = False,
|
||||
featured: bool = False,
|
||||
force_fresh_nous_tier: bool = False,
|
||||
refresh: bool = False,
|
||||
probe_custom_providers: bool = True,
|
||||
|
|
@ -152,6 +153,13 @@ def build_models_payload(
|
|||
``{model: {fast, reasoning}}`` so pickers can gate the model-options
|
||||
controls (fast toggle / reasoning) to what each model actually
|
||||
supports, instead of offering knobs the backend would reject.
|
||||
- ``featured``: add a per-row ``featured_models`` list — the newest few
|
||||
models per lab (by models.dev release_date, ranked within the row's own
|
||||
models; see ``_FEATURED_PER_LAB``) for aggregator providers that serve
|
||||
dozens of models across many labs. Pickers default their visible set to
|
||||
these; the rest of ``models`` stays reachable via search / show-all. Empty
|
||||
for single-lab providers (callers fall back to top-N). Derived live from
|
||||
models.dev — no allowlist.
|
||||
- ``force_fresh_nous_tier``: bypass the short Nous free-tier cache when
|
||||
selecting Portal-recommended Nous models and applying tier gating. Keep
|
||||
this false for UI picker opens; explicit auth/model flows can opt in
|
||||
|
|
@ -262,6 +270,8 @@ def build_models_payload(
|
|||
_apply_pricing(rows, force_fresh_nous_tier=force_fresh_nous_tier)
|
||||
if capabilities:
|
||||
_apply_capabilities(rows)
|
||||
if featured:
|
||||
_apply_featured(rows)
|
||||
|
||||
return {
|
||||
"providers": rows,
|
||||
|
|
@ -296,6 +306,7 @@ def build_model_options_payload(
|
|||
canonical_order=True,
|
||||
pricing=True,
|
||||
capabilities=True,
|
||||
featured=True,
|
||||
refresh=refresh,
|
||||
probe_custom_providers=refresh,
|
||||
probe_current_custom_provider=not refresh,
|
||||
|
|
@ -428,6 +439,72 @@ def _apply_capabilities(rows: list[dict]) -> None:
|
|||
row["capabilities"] = caps
|
||||
|
||||
|
||||
# How many models per lab the picker features by default. Aggregator rows keep
|
||||
# the newest N of each lab (by models.dev release_date) and hide the older tail
|
||||
# behind search / show-all. 5 keeps a lab's current headliners without letting a
|
||||
# prolific vendor (OpenAI's gpt-5.6-* family) flood the default view.
|
||||
_FEATURED_PER_LAB = 5
|
||||
|
||||
|
||||
def _apply_featured(rows: list[dict]) -> None:
|
||||
"""Attach a ``featured_models`` shortlist to each aggregator provider row.
|
||||
|
||||
Aggregator providers (nous, openrouter) serve dozens of models across many
|
||||
labs, so a flat "top-N" default would drop whole labs from the picker.
|
||||
Instead we surface the ``_FEATURED_PER_LAB`` newest models per lab (the
|
||||
vendor segment of a ``vendor/model`` id), ranked by models.dev
|
||||
``release_date`` among that row's OWN models — never against the current
|
||||
date, so the choice is stable as models age. Same-date ties (and labs whose
|
||||
models lack a date) fall back to the row's curated order, which is already
|
||||
flagship-first, so a lab keeps its headliners rather than an arbitrary slice.
|
||||
|
||||
Derived live from the models.dev catalog already loaded on this path (same
|
||||
source as pricing/capabilities) — there is no hand-maintained allowlist to
|
||||
keep in sync. Non-aggregator providers (a single lab, local endpoints,
|
||||
custom proxies) get an empty list and callers fall back to their existing
|
||||
top-N behaviour; splitting one lab into a shortlist would just hide models.
|
||||
"""
|
||||
try:
|
||||
from agent.models_dev import get_model_info
|
||||
except Exception:
|
||||
get_model_info = None # type: ignore[assignment]
|
||||
|
||||
for row in rows:
|
||||
slug = str(row.get("slug") or "").strip().lower()
|
||||
models = row.get("models") or []
|
||||
|
||||
# Group models by lab; only multi-lab aggregators get a shortlist.
|
||||
by_lab: dict[str, list[tuple[int, str, str]]] = {}
|
||||
for pos, model in enumerate(models):
|
||||
lab = model.split("/", 1)[0] if "/" in model else ""
|
||||
if not lab:
|
||||
# No vendor prefix → single-namespace provider, not an
|
||||
# aggregator. Bail on the whole row (see below).
|
||||
by_lab = {}
|
||||
break
|
||||
date = ""
|
||||
if get_model_info is not None:
|
||||
info = get_model_info(slug, model) or get_model_info("openrouter", model)
|
||||
date = getattr(info, "release_date", "") if info else ""
|
||||
by_lab.setdefault(lab, []).append((pos, date, model))
|
||||
|
||||
# A shortlist only makes sense when the row spans several labs.
|
||||
if len(by_lab) < 2:
|
||||
row["featured_models"] = []
|
||||
continue
|
||||
|
||||
featured: list[str] = []
|
||||
for entries in by_lab.values():
|
||||
# Newest release_date first; earlier list position breaks ties and
|
||||
# is the sole key when a lab has no dated models (all ""). Keep the
|
||||
# newest _FEATURED_PER_LAB of each lab.
|
||||
ranked = sorted(entries, key=lambda e: (e[1], -e[0]), reverse=True)
|
||||
featured.extend(model for _pos, _date, model in ranked[:_FEATURED_PER_LAB])
|
||||
# Preserve the row's model order for stable rendering.
|
||||
order = {m: i for i, m in enumerate(models)}
|
||||
row["featured_models"] = sorted(featured, key=lambda m: order[m])
|
||||
|
||||
|
||||
# ─── Internal: row post-processing ──────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1031,3 +1031,125 @@ def test_list_authenticated_providers_refresh_busts_cache():
|
|||
assert clear.call_count == 0
|
||||
model_switch.list_authenticated_providers(refresh=True)
|
||||
assert clear.call_count == 1
|
||||
|
||||
|
||||
# ─── _apply_featured (one-flagship-per-lab shortlist) ──────────────────
|
||||
|
||||
|
||||
class _FakeInfo:
|
||||
def __init__(self, release_date: str) -> None:
|
||||
self.release_date = release_date
|
||||
|
||||
|
||||
def _apply_featured_with_dates(rows, dates: dict[str, str]):
|
||||
"""Run _apply_featured with a deterministic models.dev stub."""
|
||||
from hermes_cli import inventory
|
||||
|
||||
def _fake_get_model_info(provider, model):
|
||||
return _FakeInfo(dates[model]) if model in dates else None
|
||||
|
||||
with patch("agent.models_dev.get_model_info", side_effect=_fake_get_model_info):
|
||||
inventory._apply_featured(rows)
|
||||
|
||||
|
||||
def test_apply_featured_keeps_newest_n_per_lab():
|
||||
"""Each lab keeps its newest _FEATURED_PER_LAB models by release_date; the
|
||||
older tail is dropped. Uses a lab with more than N models to exercise the
|
||||
cut."""
|
||||
from hermes_cli.inventory import _FEATURED_PER_LAB
|
||||
|
||||
# One lab ("a") with N+2 dated models, plus a second lab so the row counts
|
||||
# as a multi-lab aggregator.
|
||||
a_models = [f"a/m{i}" for i in range(_FEATURED_PER_LAB + 2)]
|
||||
rows = [{"slug": "nous", "models": [*a_models, "b/solo"]}]
|
||||
# m0 newest … m{N+1} oldest (descending dates), b/solo dated in the middle.
|
||||
dates = {f"a/m{i}": f"2026-{12 - i:02d}-01" for i in range(_FEATURED_PER_LAB + 2)}
|
||||
dates["b/solo"] = "2026-01-01"
|
||||
_apply_featured_with_dates(rows, dates)
|
||||
|
||||
featured = rows[0]["featured_models"]
|
||||
# Lab "a" keeps its newest N (m0..m{N-1}); the two oldest drop. "b" keeps its one.
|
||||
assert featured == [*a_models[:_FEATURED_PER_LAB], "b/solo"]
|
||||
assert f"a/m{_FEATURED_PER_LAB}" not in featured
|
||||
assert f"a/m{_FEATURED_PER_LAB + 1}" not in featured
|
||||
|
||||
|
||||
def test_apply_featured_keeps_whole_lab_when_under_the_cap():
|
||||
"""A lab with <= _FEATURED_PER_LAB models keeps all of them."""
|
||||
rows = [
|
||||
{
|
||||
"slug": "nous",
|
||||
"models": ["anthropic/opus", "anthropic/haiku", "google/gemini"],
|
||||
}
|
||||
]
|
||||
_apply_featured_with_dates(
|
||||
rows,
|
||||
{
|
||||
"anthropic/opus": "2026-07-01",
|
||||
"anthropic/haiku": "2026-03-01",
|
||||
"google/gemini": "2026-05-01",
|
||||
},
|
||||
)
|
||||
# Both anthropic models (2 <= 5) and the single google model survive, in
|
||||
# the row's original order.
|
||||
assert rows[0]["featured_models"] == [
|
||||
"anthropic/opus",
|
||||
"anthropic/haiku",
|
||||
"google/gemini",
|
||||
]
|
||||
|
||||
|
||||
def test_apply_featured_ranks_within_list_not_against_now():
|
||||
"""The kept models are the newest *in the list*, even if every model is old
|
||||
— the current date never enters the comparison."""
|
||||
rows = [{"slug": "nous", "models": ["a/one", "a/two", "b/three"]}]
|
||||
_apply_featured_with_dates(
|
||||
rows,
|
||||
{"a/one": "2019-01-01", "a/two": "2020-01-01", "b/three": "2018-06-01"},
|
||||
)
|
||||
# Both "a" models kept (2 <= 5), ordered by the row; "b" kept.
|
||||
assert rows[0]["featured_models"] == ["a/one", "a/two", "b/three"]
|
||||
|
||||
|
||||
def test_apply_featured_tie_breaks_on_list_order():
|
||||
"""Same release_date within a lab falls back to curated (earliest) order
|
||||
when the cut has to choose."""
|
||||
from hermes_cli.inventory import _FEATURED_PER_LAB
|
||||
|
||||
# N+1 same-dated models in lab "x" so exactly one must be dropped; the LAST
|
||||
# one in list order loses the tie.
|
||||
x_models = [f"x/m{i}" for i in range(_FEATURED_PER_LAB + 1)]
|
||||
rows = [{"slug": "nous", "models": [*x_models, "y/solo"]}]
|
||||
dates = {m: "2026-07-09" for m in x_models}
|
||||
dates["y/solo"] = "2026-01-01"
|
||||
_apply_featured_with_dates(rows, dates)
|
||||
|
||||
featured = rows[0]["featured_models"]
|
||||
# Earliest N in list order survive the tie; the last is dropped.
|
||||
assert featured == [*x_models[:_FEATURED_PER_LAB], "y/solo"]
|
||||
assert x_models[_FEATURED_PER_LAB] not in featured
|
||||
|
||||
|
||||
def test_apply_featured_undated_lab_falls_back_to_list_order():
|
||||
"""A lab whose models have no models.dev date keeps them in list order
|
||||
(undated sorts last, ties broken by position), up to the per-lab cap."""
|
||||
rows = [{"slug": "nous", "models": ["a/first", "a/second", "b/only"]}]
|
||||
_apply_featured_with_dates(rows, {"b/only": "2026-01-01"}) # a/* undated
|
||||
# Both undated "a" models kept (2 <= 5), in list order; "b" kept.
|
||||
assert rows[0]["featured_models"] == ["a/first", "a/second", "b/only"]
|
||||
|
||||
|
||||
def test_apply_featured_empty_for_single_lab_provider():
|
||||
"""A provider serving one lab is not an aggregator — no shortlist, so the
|
||||
caller falls back to top-N instead of hiding models."""
|
||||
rows = [{"slug": "deepseek", "models": ["deepseek-v4-pro", "deepseek-v4-flash"]}]
|
||||
_apply_featured_with_dates(rows, {})
|
||||
assert rows[0]["featured_models"] == []
|
||||
|
||||
|
||||
def test_apply_featured_empty_for_prefixless_models():
|
||||
"""Models with no vendor/ prefix (ollama, custom endpoints) get no
|
||||
shortlist — there are no labs to split on."""
|
||||
rows = [{"slug": "ollama", "models": ["qwen3:latest", "llama3.2:latest"]}]
|
||||
_apply_featured_with_dates(rows, {})
|
||||
assert rows[0]["featured_models"] == []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue