mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
8a5f8379ed
_custom_provider_model_matches() only compared the session model
against the entry's single 'model' field. A custom provider declaring
a multi-model catalog (providers.<name>.models mapping / models list)
whose default model differed from the session model silently failed to
match — dropping the entry's extra_body entirely. Real impact: an
OpenAI custom provider pinning service_tier=flex via extra_body ran
every request at STANDARD tier (~2.3x billing) with zero signal.
- Model matching now accepts the session model when it appears in the
entry's models catalog (dict keys or list), case-insensitive;
single-model 'model' field behavior unchanged; entries with neither
still match everything.
- Usage report ('hermes -z --usage-file') now carries service_tier
(the tier requested via request_overrides.extra_body) so batch
pipelines can audit the billed tier per run.
Validation: 8 new tests; live E2E via real 'hermes -p sweeper -z'
with httpx-level wire capture — service_tier=flex present in the
outgoing /v1/responses body and in the usage report.
78 lines
2.8 KiB
Python
78 lines
2.8 KiB
Python
"""Tests for custom-provider model matching (extra_body / service_tier drop bug).
|
|
|
|
July 2026 incident: a custom provider with a multi-model catalog
|
|
(``models: {gpt-5.5: {}, gpt-5.6-terra: {}, ...}``) and a ``model``/
|
|
``default_model`` differing from the session model failed
|
|
``_custom_provider_model_matches``, so ``extra_body: {service_tier: flex}``
|
|
was silently dropped — every request billed at standard tier (~2.3x).
|
|
"""
|
|
|
|
from agent.agent_init import (
|
|
_custom_provider_extra_body_for_agent,
|
|
_custom_provider_model_matches,
|
|
)
|
|
|
|
BASE = "https://api.openai.com/v1"
|
|
|
|
|
|
def _entry(**over):
|
|
e = {
|
|
"name": "openai",
|
|
"base_url": BASE,
|
|
"extra_body": {"service_tier": "flex"},
|
|
}
|
|
e.update(over)
|
|
return e
|
|
|
|
|
|
class TestModelMatches:
|
|
def test_models_dict_catalog_matches_session_model(self):
|
|
e = _entry(model="gpt-5.5", models={"gpt-5.5": {}, "gpt-5.6-terra": {}})
|
|
assert _custom_provider_model_matches("gpt-5.6-terra", e)
|
|
|
|
def test_models_list_catalog_matches(self):
|
|
e = _entry(model="gpt-5.5", models=["gpt-5.5", "gpt-5.6-sol"])
|
|
assert _custom_provider_model_matches("gpt-5.6-sol", e)
|
|
|
|
def test_catalog_miss_falls_back_to_model_field(self):
|
|
e = _entry(model="gpt-5.5", models={"gpt-5.5": {}})
|
|
assert _custom_provider_model_matches("gpt-5.5", e)
|
|
assert not _custom_provider_model_matches("gpt-4o", e)
|
|
|
|
def test_no_model_no_catalog_matches_everything(self):
|
|
e = _entry()
|
|
assert _custom_provider_model_matches("anything", e)
|
|
|
|
def test_catalog_case_insensitive(self):
|
|
e = _entry(models={"GPT-5.6-Terra": {}})
|
|
assert _custom_provider_model_matches("gpt-5.6-terra", e)
|
|
|
|
|
|
class TestExtraBodyResolution:
|
|
def test_multi_model_provider_yields_extra_body(self):
|
|
# The exact sweeper-profile shape that failed in production.
|
|
entry = _entry(
|
|
model="gpt-5.5",
|
|
models={"gpt-5.5": {}, "gpt-5.6-sol": {}, "gpt-5.6-terra": {}},
|
|
)
|
|
got = _custom_provider_extra_body_for_agent(
|
|
provider="custom", model="gpt-5.6-terra",
|
|
base_url=BASE, custom_providers=[entry],
|
|
)
|
|
assert got == {"service_tier": "flex"}
|
|
|
|
def test_non_catalog_model_gets_no_override(self):
|
|
entry = _entry(model="gpt-5.5", models={"gpt-5.5": {}})
|
|
got = _custom_provider_extra_body_for_agent(
|
|
provider="custom", model="gpt-4o",
|
|
base_url=BASE, custom_providers=[entry],
|
|
)
|
|
assert got is None
|
|
|
|
def test_non_custom_provider_unaffected(self):
|
|
entry = _entry(models={"gpt-5.6-terra": {}})
|
|
got = _custom_provider_extra_body_for_agent(
|
|
provider="openrouter", model="gpt-5.6-terra",
|
|
base_url=BASE, custom_providers=[entry],
|
|
)
|
|
assert got is None
|