From d4449c47e264166071cdb339d01fcc80bba502dc Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Tue, 28 Jul 2026 03:52:33 -0500 Subject: [PATCH] feat(picker): derive a newest-per-lab featured shortlist from models.dev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregator providers (nous, openrouter) serve dozens of models across many labs. Add a featured=True enricher to build_models_payload that attaches a featured_models shortlist to each row: within every lab keep the newest _FEATURED_PER_LAB models by models.dev release_date, ranked among that row's own models — never against the current date, so the choice is stable as models age. Same-date ties fall back to curated list order. Single-lab providers get an empty list. Derived live from the models.dev catalog already loaded on this path; no hand-maintained allowlist. --- hermes_cli/inventory.py | 77 ++++++++++++++++++ tests/hermes_cli/test_inventory.py | 122 +++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) diff --git a/hermes_cli/inventory.py b/hermes_cli/inventory.py index e1b9a6616666..4e95665d481f 100644 --- a/hermes_cli/inventory.py +++ b/hermes_cli/inventory.py @@ -120,6 +120,7 @@ def build_models_payload( canonical_order: bool = False, pricing: bool = False, capabilities: bool = False, + featured: bool = False, force_fresh_nous_tier: bool = False, refresh: bool = False, probe_custom_providers: bool = True, @@ -152,6 +153,13 @@ def build_models_payload( ``{model: {fast, reasoning}}`` so pickers can gate the model-options controls (fast toggle / reasoning) to what each model actually supports, instead of offering knobs the backend would reject. + - ``featured``: add a per-row ``featured_models`` list — the newest few + models per lab (by models.dev release_date, ranked within the row's own + models; see ``_FEATURED_PER_LAB``) for aggregator providers that serve + dozens of models across many labs. Pickers default their visible set to + these; the rest of ``models`` stays reachable via search / show-all. Empty + for single-lab providers (callers fall back to top-N). Derived live from + models.dev — no allowlist. - ``force_fresh_nous_tier``: bypass the short Nous free-tier cache when selecting Portal-recommended Nous models and applying tier gating. Keep this false for UI picker opens; explicit auth/model flows can opt in @@ -262,6 +270,8 @@ def build_models_payload( _apply_pricing(rows, force_fresh_nous_tier=force_fresh_nous_tier) if capabilities: _apply_capabilities(rows) + if featured: + _apply_featured(rows) return { "providers": rows, @@ -296,6 +306,7 @@ def build_model_options_payload( canonical_order=True, pricing=True, capabilities=True, + featured=True, refresh=refresh, probe_custom_providers=refresh, probe_current_custom_provider=not refresh, @@ -428,6 +439,72 @@ def _apply_capabilities(rows: list[dict]) -> None: row["capabilities"] = caps +# How many models per lab the picker features by default. Aggregator rows keep +# the newest N of each lab (by models.dev release_date) and hide the older tail +# behind search / show-all. 5 keeps a lab's current headliners without letting a +# prolific vendor (OpenAI's gpt-5.6-* family) flood the default view. +_FEATURED_PER_LAB = 5 + + +def _apply_featured(rows: list[dict]) -> None: + """Attach a ``featured_models`` shortlist to each aggregator provider row. + + Aggregator providers (nous, openrouter) serve dozens of models across many + labs, so a flat "top-N" default would drop whole labs from the picker. + Instead we surface the ``_FEATURED_PER_LAB`` newest models per lab (the + vendor segment of a ``vendor/model`` id), ranked by models.dev + ``release_date`` among that row's OWN models — never against the current + date, so the choice is stable as models age. Same-date ties (and labs whose + models lack a date) fall back to the row's curated order, which is already + flagship-first, so a lab keeps its headliners rather than an arbitrary slice. + + Derived live from the models.dev catalog already loaded on this path (same + source as pricing/capabilities) — there is no hand-maintained allowlist to + keep in sync. Non-aggregator providers (a single lab, local endpoints, + custom proxies) get an empty list and callers fall back to their existing + top-N behaviour; splitting one lab into a shortlist would just hide models. + """ + try: + from agent.models_dev import get_model_info + except Exception: + get_model_info = None # type: ignore[assignment] + + for row in rows: + slug = str(row.get("slug") or "").strip().lower() + models = row.get("models") or [] + + # Group models by lab; only multi-lab aggregators get a shortlist. + by_lab: dict[str, list[tuple[int, str, str]]] = {} + for pos, model in enumerate(models): + lab = model.split("/", 1)[0] if "/" in model else "" + if not lab: + # No vendor prefix → single-namespace provider, not an + # aggregator. Bail on the whole row (see below). + by_lab = {} + break + date = "" + if get_model_info is not None: + info = get_model_info(slug, model) or get_model_info("openrouter", model) + date = getattr(info, "release_date", "") if info else "" + by_lab.setdefault(lab, []).append((pos, date, model)) + + # A shortlist only makes sense when the row spans several labs. + if len(by_lab) < 2: + row["featured_models"] = [] + continue + + featured: list[str] = [] + for entries in by_lab.values(): + # Newest release_date first; earlier list position breaks ties and + # is the sole key when a lab has no dated models (all ""). Keep the + # newest _FEATURED_PER_LAB of each lab. + ranked = sorted(entries, key=lambda e: (e[1], -e[0]), reverse=True) + featured.extend(model for _pos, _date, model in ranked[:_FEATURED_PER_LAB]) + # Preserve the row's model order for stable rendering. + order = {m: i for i, m in enumerate(models)} + row["featured_models"] = sorted(featured, key=lambda m: order[m]) + + # ─── Internal: row post-processing ────────────────────────────────────── diff --git a/tests/hermes_cli/test_inventory.py b/tests/hermes_cli/test_inventory.py index 6c262180b9d8..0a1314c426c3 100644 --- a/tests/hermes_cli/test_inventory.py +++ b/tests/hermes_cli/test_inventory.py @@ -1031,3 +1031,125 @@ def test_list_authenticated_providers_refresh_busts_cache(): assert clear.call_count == 0 model_switch.list_authenticated_providers(refresh=True) assert clear.call_count == 1 + + +# ─── _apply_featured (one-flagship-per-lab shortlist) ────────────────── + + +class _FakeInfo: + def __init__(self, release_date: str) -> None: + self.release_date = release_date + + +def _apply_featured_with_dates(rows, dates: dict[str, str]): + """Run _apply_featured with a deterministic models.dev stub.""" + from hermes_cli import inventory + + def _fake_get_model_info(provider, model): + return _FakeInfo(dates[model]) if model in dates else None + + with patch("agent.models_dev.get_model_info", side_effect=_fake_get_model_info): + inventory._apply_featured(rows) + + +def test_apply_featured_keeps_newest_n_per_lab(): + """Each lab keeps its newest _FEATURED_PER_LAB models by release_date; the + older tail is dropped. Uses a lab with more than N models to exercise the + cut.""" + from hermes_cli.inventory import _FEATURED_PER_LAB + + # One lab ("a") with N+2 dated models, plus a second lab so the row counts + # as a multi-lab aggregator. + a_models = [f"a/m{i}" for i in range(_FEATURED_PER_LAB + 2)] + rows = [{"slug": "nous", "models": [*a_models, "b/solo"]}] + # m0 newest … m{N+1} oldest (descending dates), b/solo dated in the middle. + dates = {f"a/m{i}": f"2026-{12 - i:02d}-01" for i in range(_FEATURED_PER_LAB + 2)} + dates["b/solo"] = "2026-01-01" + _apply_featured_with_dates(rows, dates) + + featured = rows[0]["featured_models"] + # Lab "a" keeps its newest N (m0..m{N-1}); the two oldest drop. "b" keeps its one. + assert featured == [*a_models[:_FEATURED_PER_LAB], "b/solo"] + assert f"a/m{_FEATURED_PER_LAB}" not in featured + assert f"a/m{_FEATURED_PER_LAB + 1}" not in featured + + +def test_apply_featured_keeps_whole_lab_when_under_the_cap(): + """A lab with <= _FEATURED_PER_LAB models keeps all of them.""" + rows = [ + { + "slug": "nous", + "models": ["anthropic/opus", "anthropic/haiku", "google/gemini"], + } + ] + _apply_featured_with_dates( + rows, + { + "anthropic/opus": "2026-07-01", + "anthropic/haiku": "2026-03-01", + "google/gemini": "2026-05-01", + }, + ) + # Both anthropic models (2 <= 5) and the single google model survive, in + # the row's original order. + assert rows[0]["featured_models"] == [ + "anthropic/opus", + "anthropic/haiku", + "google/gemini", + ] + + +def test_apply_featured_ranks_within_list_not_against_now(): + """The kept models are the newest *in the list*, even if every model is old + — the current date never enters the comparison.""" + rows = [{"slug": "nous", "models": ["a/one", "a/two", "b/three"]}] + _apply_featured_with_dates( + rows, + {"a/one": "2019-01-01", "a/two": "2020-01-01", "b/three": "2018-06-01"}, + ) + # Both "a" models kept (2 <= 5), ordered by the row; "b" kept. + assert rows[0]["featured_models"] == ["a/one", "a/two", "b/three"] + + +def test_apply_featured_tie_breaks_on_list_order(): + """Same release_date within a lab falls back to curated (earliest) order + when the cut has to choose.""" + from hermes_cli.inventory import _FEATURED_PER_LAB + + # N+1 same-dated models in lab "x" so exactly one must be dropped; the LAST + # one in list order loses the tie. + x_models = [f"x/m{i}" for i in range(_FEATURED_PER_LAB + 1)] + rows = [{"slug": "nous", "models": [*x_models, "y/solo"]}] + dates = {m: "2026-07-09" for m in x_models} + dates["y/solo"] = "2026-01-01" + _apply_featured_with_dates(rows, dates) + + featured = rows[0]["featured_models"] + # Earliest N in list order survive the tie; the last is dropped. + assert featured == [*x_models[:_FEATURED_PER_LAB], "y/solo"] + assert x_models[_FEATURED_PER_LAB] not in featured + + +def test_apply_featured_undated_lab_falls_back_to_list_order(): + """A lab whose models have no models.dev date keeps them in list order + (undated sorts last, ties broken by position), up to the per-lab cap.""" + rows = [{"slug": "nous", "models": ["a/first", "a/second", "b/only"]}] + _apply_featured_with_dates(rows, {"b/only": "2026-01-01"}) # a/* undated + # Both undated "a" models kept (2 <= 5), in list order; "b" kept. + assert rows[0]["featured_models"] == ["a/first", "a/second", "b/only"] + + +def test_apply_featured_empty_for_single_lab_provider(): + """A provider serving one lab is not an aggregator — no shortlist, so the + caller falls back to top-N instead of hiding models.""" + rows = [{"slug": "deepseek", "models": ["deepseek-v4-pro", "deepseek-v4-flash"]}] + _apply_featured_with_dates(rows, {}) + assert rows[0]["featured_models"] == [] + + +def test_apply_featured_empty_for_prefixless_models(): + """Models with no vendor/ prefix (ollama, custom endpoints) get no + shortlist — there are no labs to split on.""" + rows = [{"slug": "ollama", "models": ["qwen3:latest", "llama3.2:latest"]}] + _apply_featured_with_dates(rows, {}) + assert rows[0]["featured_models"] == []