mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-28 01:21:43 +00:00
855366909f
OpenRouter and Nous Portal curated picker lists now resolve via a JSON manifest served by the docs site, falling back to the in-repo snapshot when unreachable. Lets us update model lists without shipping a release. Live URL: https://hermes-agent.nousresearch.com/docs/api/model-catalog.json (source at website/static/api/model-catalog.json; auto-deploys via the existing deploy-site.yml GitHub Pages pipeline on every merge to main). Schema (v1) carries id + optional description + free-form metadata at manifest, provider, and model levels. Pricing and context length stay live-fetched via existing machinery (/v1/models endpoints, models.dev). Config (new model_catalog section, default enabled): model_catalog.url master manifest URL model_catalog.ttl_hours disk cache TTL (default 24h) model_catalog.providers.<name>.url optional per-provider override Fetch pipeline: in-process cache -> disk cache (fresh < TTL) -> HTTP fetch -> disk-cache-on-failure fallback -> in-repo snapshot as last resort. Never raises to callers; at worst returns the bundled list. Changes: - website/static/api/model-catalog.json initial manifest (35 OR + 31 Nous) - scripts/build_model_catalog.py regenerator from in-repo lists - hermes_cli/model_catalog.py fetch + validate + cache module - hermes_cli/models.py fetch_openrouter_models() + new get_curated_nous_model_ids() - hermes_cli/main.py, hermes_cli/auth.py Nous flows use the helper - hermes_cli/config.py model_catalog defaults - website/docs/reference/model-catalog.md + sidebars.ts - tests/hermes_cli/test_model_catalog.py 21 tests (validation, fetch success/failure, accessors, disabled, overrides, integration)
284 lines
10 KiB
Python
284 lines
10 KiB
Python
"""Tests for hermes_cli.model_catalog — remote manifest fetch + cache + fallback."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_home(tmp_path, monkeypatch):
|
|
"""Isolate HERMES_HOME + reset any module-level catalog cache per test."""
|
|
home = tmp_path / ".hermes"
|
|
home.mkdir()
|
|
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
|
|
# Force a fresh catalog module state for each test.
|
|
import importlib
|
|
from hermes_cli import model_catalog
|
|
importlib.reload(model_catalog)
|
|
yield home
|
|
model_catalog.reset_cache()
|
|
|
|
|
|
def _valid_manifest() -> dict:
|
|
return {
|
|
"version": 1,
|
|
"updated_at": "2026-04-25T22:00:00Z",
|
|
"metadata": {"source": "test"},
|
|
"providers": {
|
|
"openrouter": {
|
|
"metadata": {"display_name": "OpenRouter"},
|
|
"models": [
|
|
{"id": "anthropic/claude-opus-4.7", "description": "recommended"},
|
|
{"id": "openai/gpt-5.4", "description": ""},
|
|
{"id": "openrouter/elephant-alpha", "description": "free"},
|
|
],
|
|
},
|
|
"nous": {
|
|
"metadata": {"display_name": "Nous Portal"},
|
|
"models": [
|
|
{"id": "anthropic/claude-opus-4.7"},
|
|
{"id": "moonshotai/kimi-k2.6"},
|
|
],
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
class TestValidation:
|
|
def test_accepts_well_formed_manifest(self, isolated_home):
|
|
from hermes_cli.model_catalog import _validate_manifest
|
|
assert _validate_manifest(_valid_manifest()) is True
|
|
|
|
def test_rejects_non_dict(self, isolated_home):
|
|
from hermes_cli.model_catalog import _validate_manifest
|
|
assert _validate_manifest("string") is False
|
|
assert _validate_manifest([]) is False
|
|
assert _validate_manifest(None) is False
|
|
|
|
def test_rejects_missing_version(self, isolated_home):
|
|
from hermes_cli.model_catalog import _validate_manifest
|
|
m = _valid_manifest()
|
|
del m["version"]
|
|
assert _validate_manifest(m) is False
|
|
|
|
def test_rejects_future_version(self, isolated_home):
|
|
from hermes_cli.model_catalog import _validate_manifest
|
|
m = _valid_manifest()
|
|
m["version"] = 999
|
|
assert _validate_manifest(m) is False
|
|
|
|
def test_rejects_missing_providers(self, isolated_home):
|
|
from hermes_cli.model_catalog import _validate_manifest
|
|
m = _valid_manifest()
|
|
del m["providers"]
|
|
assert _validate_manifest(m) is False
|
|
|
|
def test_rejects_malformed_model_entry(self, isolated_home):
|
|
from hermes_cli.model_catalog import _validate_manifest
|
|
m = _valid_manifest()
|
|
m["providers"]["openrouter"]["models"][0] = {"id": ""} # empty id
|
|
assert _validate_manifest(m) is False
|
|
|
|
def test_rejects_non_string_model_id(self, isolated_home):
|
|
from hermes_cli.model_catalog import _validate_manifest
|
|
m = _valid_manifest()
|
|
m["providers"]["openrouter"]["models"][0] = {"id": 42}
|
|
assert _validate_manifest(m) is False
|
|
|
|
|
|
class TestFetchSuccess:
|
|
def test_fetch_and_cache_writes_disk(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
manifest = _valid_manifest()
|
|
with patch.object(
|
|
model_catalog, "_fetch_manifest", return_value=manifest
|
|
) as fetch:
|
|
result = model_catalog.get_catalog(force_refresh=True)
|
|
|
|
assert result == manifest
|
|
assert fetch.called
|
|
|
|
cache_file = model_catalog._cache_path()
|
|
assert cache_file.exists()
|
|
with open(cache_file) as fh:
|
|
assert json.load(fh) == manifest
|
|
|
|
def test_second_call_uses_in_process_cache(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
manifest = _valid_manifest()
|
|
with patch.object(
|
|
model_catalog, "_fetch_manifest", return_value=manifest
|
|
) as fetch:
|
|
model_catalog.get_catalog(force_refresh=True)
|
|
model_catalog.get_catalog() # should not hit network again
|
|
assert fetch.call_count == 1
|
|
|
|
def test_force_refresh_always_refetches(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
manifest = _valid_manifest()
|
|
with patch.object(
|
|
model_catalog, "_fetch_manifest", return_value=manifest
|
|
) as fetch:
|
|
model_catalog.get_catalog(force_refresh=True)
|
|
model_catalog.get_catalog(force_refresh=True)
|
|
assert fetch.call_count == 2
|
|
|
|
|
|
class TestFetchFailure:
|
|
def test_network_failure_returns_empty_when_no_cache(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
with patch.object(model_catalog, "_fetch_manifest", return_value=None):
|
|
result = model_catalog.get_catalog(force_refresh=True)
|
|
assert result == {}
|
|
|
|
def test_network_failure_falls_back_to_disk_cache(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
# Prime disk cache with a fresh copy.
|
|
manifest = _valid_manifest()
|
|
with patch.object(model_catalog, "_fetch_manifest", return_value=manifest):
|
|
model_catalog.get_catalog(force_refresh=True)
|
|
|
|
# Now wipe in-process cache and simulate network failure on refetch.
|
|
model_catalog.reset_cache()
|
|
with patch.object(model_catalog, "_fetch_manifest", return_value=None):
|
|
result = model_catalog.get_catalog(force_refresh=True)
|
|
|
|
assert result == manifest
|
|
|
|
def test_fetch_failure_falls_back_to_stale_cache(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
manifest = _valid_manifest()
|
|
# Write stale cache directly (mtime in the past).
|
|
cache = model_catalog._cache_path()
|
|
cache.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(cache, "w") as fh:
|
|
json.dump(manifest, fh)
|
|
old = time.time() - 30 * 24 * 3600 # 30 days ago
|
|
import os as _os
|
|
_os.utime(cache, (old, old))
|
|
|
|
with patch.object(model_catalog, "_fetch_manifest", return_value=None):
|
|
result = model_catalog.get_catalog()
|
|
|
|
# Stale cache is better than nothing.
|
|
assert result == manifest
|
|
|
|
|
|
class TestCuratedAccessors:
|
|
def test_openrouter_returns_tuples(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
with patch.object(
|
|
model_catalog, "_fetch_manifest", return_value=_valid_manifest()
|
|
):
|
|
result = model_catalog.get_curated_openrouter_models()
|
|
assert result == [
|
|
("anthropic/claude-opus-4.7", "recommended"),
|
|
("openai/gpt-5.4", ""),
|
|
("openrouter/elephant-alpha", "free"),
|
|
]
|
|
|
|
def test_nous_returns_ids(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
with patch.object(
|
|
model_catalog, "_fetch_manifest", return_value=_valid_manifest()
|
|
):
|
|
result = model_catalog.get_curated_nous_models()
|
|
assert result == ["anthropic/claude-opus-4.7", "moonshotai/kimi-k2.6"]
|
|
|
|
def test_openrouter_returns_none_when_catalog_empty(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
with patch.object(model_catalog, "_fetch_manifest", return_value=None):
|
|
assert model_catalog.get_curated_openrouter_models() is None
|
|
|
|
def test_nous_returns_none_when_catalog_empty(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
with patch.object(model_catalog, "_fetch_manifest", return_value=None):
|
|
assert model_catalog.get_curated_nous_models() is None
|
|
|
|
|
|
class TestDisabled:
|
|
def test_disabled_config_short_circuits(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
with patch.object(
|
|
model_catalog,
|
|
"_load_catalog_config",
|
|
return_value={
|
|
"enabled": False,
|
|
"url": "http://ignored",
|
|
"ttl_hours": 24.0,
|
|
"providers": {},
|
|
},
|
|
):
|
|
with patch.object(model_catalog, "_fetch_manifest") as fetch:
|
|
result = model_catalog.get_catalog()
|
|
assert result == {}
|
|
fetch.assert_not_called()
|
|
|
|
|
|
class TestProviderOverride:
|
|
def test_override_url_takes_precedence(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
|
|
override_payload = {
|
|
"version": 1,
|
|
"providers": {
|
|
"openrouter": {
|
|
"models": [
|
|
{"id": "override/model", "description": "custom"},
|
|
]
|
|
}
|
|
},
|
|
}
|
|
|
|
def fake_fetch(url, timeout):
|
|
if "override" in url:
|
|
return override_payload
|
|
return _valid_manifest()
|
|
|
|
with patch.object(
|
|
model_catalog,
|
|
"_load_catalog_config",
|
|
return_value={
|
|
"enabled": True,
|
|
"url": "http://master",
|
|
"ttl_hours": 24.0,
|
|
"providers": {"openrouter": {"url": "http://override"}},
|
|
},
|
|
):
|
|
with patch.object(model_catalog, "_fetch_manifest", side_effect=fake_fetch):
|
|
result = model_catalog.get_curated_openrouter_models()
|
|
|
|
assert result == [("override/model", "custom")]
|
|
|
|
|
|
class TestIntegrationWithModelsModule:
|
|
"""Exercise the fallback paths via the real callers in hermes_cli.models."""
|
|
|
|
def test_curated_nous_ids_falls_back_to_hardcoded_on_empty_catalog(
|
|
self, isolated_home
|
|
):
|
|
from hermes_cli import model_catalog
|
|
from hermes_cli.models import get_curated_nous_model_ids, _PROVIDER_MODELS
|
|
|
|
with patch.object(model_catalog, "_fetch_manifest", return_value=None):
|
|
result = get_curated_nous_model_ids()
|
|
|
|
assert result == list(_PROVIDER_MODELS["nous"])
|
|
|
|
def test_curated_nous_ids_prefers_manifest(self, isolated_home):
|
|
from hermes_cli import model_catalog
|
|
from hermes_cli.models import get_curated_nous_model_ids
|
|
|
|
with patch.object(
|
|
model_catalog, "_fetch_manifest", return_value=_valid_manifest()
|
|
):
|
|
result = get_curated_nous_model_ids()
|
|
|
|
assert result == ["anthropic/claude-opus-4.7", "moonshotai/kimi-k2.6"]
|