feat(picker): derive a newest-per-lab featured shortlist from models.dev

Aggregator providers (nous, openrouter) serve dozens of models across many
labs. Add a featured=True enricher to build_models_payload that attaches a
featured_models shortlist to each row: within every lab keep the newest
_FEATURED_PER_LAB models by models.dev release_date, ranked among that row's
own models — never against the current date, so the choice is stable as
models age. Same-date ties fall back to curated list order. Single-lab
providers get an empty list. Derived live from the models.dev catalog already
loaded on this path; no hand-maintained allowlist.
This commit is contained in:
Brooklyn Nicholson 2026-07-28 03:52:33 -05:00
parent 5771a6ebe6
commit d4449c47e2
2 changed files with 199 additions and 0 deletions

View file

@ -120,6 +120,7 @@ def build_models_payload(
canonical_order: bool = False,
pricing: bool = False,
capabilities: bool = False,
featured: bool = False,
force_fresh_nous_tier: bool = False,
refresh: bool = False,
probe_custom_providers: bool = True,
@ -152,6 +153,13 @@ def build_models_payload(
``{model: {fast, reasoning}}`` so pickers can gate the model-options
controls (fast toggle / reasoning) to what each model actually
supports, instead of offering knobs the backend would reject.
- ``featured``: add a per-row ``featured_models`` list the newest few
models per lab (by models.dev release_date, ranked within the row's own
models; see ``_FEATURED_PER_LAB``) for aggregator providers that serve
dozens of models across many labs. Pickers default their visible set to
these; the rest of ``models`` stays reachable via search / show-all. Empty
for single-lab providers (callers fall back to top-N). Derived live from
models.dev no allowlist.
- ``force_fresh_nous_tier``: bypass the short Nous free-tier cache when
selecting Portal-recommended Nous models and applying tier gating. Keep
this false for UI picker opens; explicit auth/model flows can opt in
@ -262,6 +270,8 @@ def build_models_payload(
_apply_pricing(rows, force_fresh_nous_tier=force_fresh_nous_tier)
if capabilities:
_apply_capabilities(rows)
if featured:
_apply_featured(rows)
return {
"providers": rows,
@ -296,6 +306,7 @@ def build_model_options_payload(
canonical_order=True,
pricing=True,
capabilities=True,
featured=True,
refresh=refresh,
probe_custom_providers=refresh,
probe_current_custom_provider=not refresh,
@ -428,6 +439,72 @@ def _apply_capabilities(rows: list[dict]) -> None:
row["capabilities"] = caps
# How many models per lab the picker features by default. Aggregator rows keep
# the newest N of each lab (by models.dev release_date) and hide the older tail
# behind search / show-all. 5 keeps a lab's current headliners without letting a
# prolific vendor (OpenAI's gpt-5.6-* family) flood the default view.
_FEATURED_PER_LAB = 5
def _apply_featured(rows: list[dict]) -> None:
"""Attach a ``featured_models`` shortlist to each aggregator provider row.
Aggregator providers (nous, openrouter) serve dozens of models across many
labs, so a flat "top-N" default would drop whole labs from the picker.
Instead we surface the ``_FEATURED_PER_LAB`` newest models per lab (the
vendor segment of a ``vendor/model`` id), ranked by models.dev
``release_date`` among that row's OWN models — never against the current
date, so the choice is stable as models age. Same-date ties (and labs whose
models lack a date) fall back to the row's curated order, which is already
flagship-first, so a lab keeps its headliners rather than an arbitrary slice.
Derived live from the models.dev catalog already loaded on this path (same
source as pricing/capabilities) there is no hand-maintained allowlist to
keep in sync. Non-aggregator providers (a single lab, local endpoints,
custom proxies) get an empty list and callers fall back to their existing
top-N behaviour; splitting one lab into a shortlist would just hide models.
"""
try:
from agent.models_dev import get_model_info
except Exception:
get_model_info = None # type: ignore[assignment]
for row in rows:
slug = str(row.get("slug") or "").strip().lower()
models = row.get("models") or []
# Group models by lab; only multi-lab aggregators get a shortlist.
by_lab: dict[str, list[tuple[int, str, str]]] = {}
for pos, model in enumerate(models):
lab = model.split("/", 1)[0] if "/" in model else ""
if not lab:
# No vendor prefix → single-namespace provider, not an
# aggregator. Bail on the whole row (see below).
by_lab = {}
break
date = ""
if get_model_info is not None:
info = get_model_info(slug, model) or get_model_info("openrouter", model)
date = getattr(info, "release_date", "") if info else ""
by_lab.setdefault(lab, []).append((pos, date, model))
# A shortlist only makes sense when the row spans several labs.
if len(by_lab) < 2:
row["featured_models"] = []
continue
featured: list[str] = []
for entries in by_lab.values():
# Newest release_date first; earlier list position breaks ties and
# is the sole key when a lab has no dated models (all ""). Keep the
# newest _FEATURED_PER_LAB of each lab.
ranked = sorted(entries, key=lambda e: (e[1], -e[0]), reverse=True)
featured.extend(model for _pos, _date, model in ranked[:_FEATURED_PER_LAB])
# Preserve the row's model order for stable rendering.
order = {m: i for i, m in enumerate(models)}
row["featured_models"] = sorted(featured, key=lambda m: order[m])
# ─── Internal: row post-processing ──────────────────────────────────────

View file

@ -1031,3 +1031,125 @@ def test_list_authenticated_providers_refresh_busts_cache():
assert clear.call_count == 0
model_switch.list_authenticated_providers(refresh=True)
assert clear.call_count == 1
# ─── _apply_featured (one-flagship-per-lab shortlist) ──────────────────
class _FakeInfo:
def __init__(self, release_date: str) -> None:
self.release_date = release_date
def _apply_featured_with_dates(rows, dates: dict[str, str]):
"""Run _apply_featured with a deterministic models.dev stub."""
from hermes_cli import inventory
def _fake_get_model_info(provider, model):
return _FakeInfo(dates[model]) if model in dates else None
with patch("agent.models_dev.get_model_info", side_effect=_fake_get_model_info):
inventory._apply_featured(rows)
def test_apply_featured_keeps_newest_n_per_lab():
"""Each lab keeps its newest _FEATURED_PER_LAB models by release_date; the
older tail is dropped. Uses a lab with more than N models to exercise the
cut."""
from hermes_cli.inventory import _FEATURED_PER_LAB
# One lab ("a") with N+2 dated models, plus a second lab so the row counts
# as a multi-lab aggregator.
a_models = [f"a/m{i}" for i in range(_FEATURED_PER_LAB + 2)]
rows = [{"slug": "nous", "models": [*a_models, "b/solo"]}]
# m0 newest … m{N+1} oldest (descending dates), b/solo dated in the middle.
dates = {f"a/m{i}": f"2026-{12 - i:02d}-01" for i in range(_FEATURED_PER_LAB + 2)}
dates["b/solo"] = "2026-01-01"
_apply_featured_with_dates(rows, dates)
featured = rows[0]["featured_models"]
# Lab "a" keeps its newest N (m0..m{N-1}); the two oldest drop. "b" keeps its one.
assert featured == [*a_models[:_FEATURED_PER_LAB], "b/solo"]
assert f"a/m{_FEATURED_PER_LAB}" not in featured
assert f"a/m{_FEATURED_PER_LAB + 1}" not in featured
def test_apply_featured_keeps_whole_lab_when_under_the_cap():
"""A lab with <= _FEATURED_PER_LAB models keeps all of them."""
rows = [
{
"slug": "nous",
"models": ["anthropic/opus", "anthropic/haiku", "google/gemini"],
}
]
_apply_featured_with_dates(
rows,
{
"anthropic/opus": "2026-07-01",
"anthropic/haiku": "2026-03-01",
"google/gemini": "2026-05-01",
},
)
# Both anthropic models (2 <= 5) and the single google model survive, in
# the row's original order.
assert rows[0]["featured_models"] == [
"anthropic/opus",
"anthropic/haiku",
"google/gemini",
]
def test_apply_featured_ranks_within_list_not_against_now():
"""The kept models are the newest *in the list*, even if every model is old
the current date never enters the comparison."""
rows = [{"slug": "nous", "models": ["a/one", "a/two", "b/three"]}]
_apply_featured_with_dates(
rows,
{"a/one": "2019-01-01", "a/two": "2020-01-01", "b/three": "2018-06-01"},
)
# Both "a" models kept (2 <= 5), ordered by the row; "b" kept.
assert rows[0]["featured_models"] == ["a/one", "a/two", "b/three"]
def test_apply_featured_tie_breaks_on_list_order():
"""Same release_date within a lab falls back to curated (earliest) order
when the cut has to choose."""
from hermes_cli.inventory import _FEATURED_PER_LAB
# N+1 same-dated models in lab "x" so exactly one must be dropped; the LAST
# one in list order loses the tie.
x_models = [f"x/m{i}" for i in range(_FEATURED_PER_LAB + 1)]
rows = [{"slug": "nous", "models": [*x_models, "y/solo"]}]
dates = {m: "2026-07-09" for m in x_models}
dates["y/solo"] = "2026-01-01"
_apply_featured_with_dates(rows, dates)
featured = rows[0]["featured_models"]
# Earliest N in list order survive the tie; the last is dropped.
assert featured == [*x_models[:_FEATURED_PER_LAB], "y/solo"]
assert x_models[_FEATURED_PER_LAB] not in featured
def test_apply_featured_undated_lab_falls_back_to_list_order():
"""A lab whose models have no models.dev date keeps them in list order
(undated sorts last, ties broken by position), up to the per-lab cap."""
rows = [{"slug": "nous", "models": ["a/first", "a/second", "b/only"]}]
_apply_featured_with_dates(rows, {"b/only": "2026-01-01"}) # a/* undated
# Both undated "a" models kept (2 <= 5), in list order; "b" kept.
assert rows[0]["featured_models"] == ["a/first", "a/second", "b/only"]
def test_apply_featured_empty_for_single_lab_provider():
"""A provider serving one lab is not an aggregator — no shortlist, so the
caller falls back to top-N instead of hiding models."""
rows = [{"slug": "deepseek", "models": ["deepseek-v4-pro", "deepseek-v4-flash"]}]
_apply_featured_with_dates(rows, {})
assert rows[0]["featured_models"] == []
def test_apply_featured_empty_for_prefixless_models():
"""Models with no vendor/ prefix (ollama, custom endpoints) get no
shortlist there are no labs to split on."""
rows = [{"slug": "ollama", "models": ["qwen3:latest", "llama3.2:latest"]}]
_apply_featured_with_dates(rows, {})
assert rows[0]["featured_models"] == []