fix(agent): honor custom-provider extra_body for multi-model catalogs

_custom_provider_model_matches() only compared the session model
against the entry's single 'model' field. A custom provider declaring
a multi-model catalog (providers.<name>.models mapping / models list)
whose default model differed from the session model silently failed to
match — dropping the entry's extra_body entirely. Real impact: an
OpenAI custom provider pinning service_tier=flex via extra_body ran
every request at STANDARD tier (~2.3x billing) with zero signal.

- Model matching now accepts the session model when it appears in the
  entry's models catalog (dict keys or list), case-insensitive;
  single-model 'model' field behavior unchanged; entries with neither
  still match everything.
- Usage report ('hermes -z --usage-file') now carries service_tier
  (the tier requested via request_overrides.extra_body) so batch
  pipelines can audit the billed tier per run.

Validation: 8 new tests; live E2E via real 'hermes -p sweeper -z'
with httpx-level wire capture — service_tier=flex present in the
outgoing /v1/responses body and in the usage report.
This commit is contained in:
Teknium 2026-07-12 23:12:05 -07:00
parent 902379ea3e
commit 8a5f8379ed
4 changed files with 108 additions and 3 deletions

View file

@ -187,10 +187,26 @@ def _normalized_custom_base_url(value: Any) -> str:
def _custom_provider_model_matches(agent_model: str, entry: Dict[str, Any]) -> bool:
provider_model = str(entry.get("model", "") or "").strip().lower()
if not provider_model:
agent_model_norm = str(agent_model or "").strip().lower()
# Multi-model entries (v12+ `providers.<name>.models` mapping / legacy
# `models:` list): the agent's model matching ANY catalog entry counts.
# Without this, a provider whose `model`/`default_model` differs from the
# session model silently fails to match and per-provider request settings
# (extra_body, e.g. OpenAI service_tier) are dropped — billing the whole
# session at the wrong tier (July 2026 sweeper incident: flex config
# ignored, ~2.3x overbilling).
models = entry.get("models")
catalog: List[str] = []
if isinstance(models, dict):
catalog = [str(k).strip().lower() for k in models.keys()]
elif isinstance(models, (list, tuple)):
catalog = [str(m).strip().lower() for m in models]
if catalog and agent_model_norm in catalog:
return True
return provider_model == str(agent_model or "").strip().lower()
provider_model = str(entry.get("model", "") or "").strip().lower()
if not provider_model and not catalog:
return True
return provider_model == agent_model_norm
def _custom_provider_extra_body_for_agent(

View file

@ -449,6 +449,11 @@ def finalize_turn(
"estimated_cost_usd": agent.session_estimated_cost_usd,
"cost_status": agent.session_cost_status,
"cost_source": agent.session_cost_source,
# Requested service tier (from request_overrides.extra_body), for
# billing audits by callers like `hermes -z --usage-file`.
"service_tier": (
(getattr(agent, "request_overrides", {}) or {}).get("extra_body") or {}
).get("service_tier"),
"session_id": agent.session_id,
}
if agent._tool_guardrail_halt_decision is not None:

View file

@ -150,6 +150,12 @@ def _write_usage_file(path: Optional[str], result: dict, failure: Optional[str]
"session_id": result.get("session_id"),
"completed": result.get("completed"),
"failed": bool(result.get("failed")) or failure is not None,
# Billing-audit field: the service tier this run REQUESTED via
# request_overrides.extra_body (e.g. OpenAI "flex"). None when
# unset. Lets batch pipelines verify the tier they think they're
# paying for actually went out on the wire (July 2026 incident:
# a config-matching bug silently dropped flex -> 2.3x billing).
"service_tier": result.get("service_tier"),
}
if failure is not None:
report["failure"] = failure

View file

@ -0,0 +1,78 @@
"""Tests for custom-provider model matching (extra_body / service_tier drop bug).
July 2026 incident: a custom provider with a multi-model catalog
(``models: {gpt-5.5: {}, gpt-5.6-terra: {}, ...}``) and a ``model``/
``default_model`` differing from the session model failed
``_custom_provider_model_matches``, so ``extra_body: {service_tier: flex}``
was silently dropped every request billed at standard tier (~2.3x).
"""
from agent.agent_init import (
_custom_provider_extra_body_for_agent,
_custom_provider_model_matches,
)
BASE = "https://api.openai.com/v1"
def _entry(**over):
e = {
"name": "openai",
"base_url": BASE,
"extra_body": {"service_tier": "flex"},
}
e.update(over)
return e
class TestModelMatches:
def test_models_dict_catalog_matches_session_model(self):
e = _entry(model="gpt-5.5", models={"gpt-5.5": {}, "gpt-5.6-terra": {}})
assert _custom_provider_model_matches("gpt-5.6-terra", e)
def test_models_list_catalog_matches(self):
e = _entry(model="gpt-5.5", models=["gpt-5.5", "gpt-5.6-sol"])
assert _custom_provider_model_matches("gpt-5.6-sol", e)
def test_catalog_miss_falls_back_to_model_field(self):
e = _entry(model="gpt-5.5", models={"gpt-5.5": {}})
assert _custom_provider_model_matches("gpt-5.5", e)
assert not _custom_provider_model_matches("gpt-4o", e)
def test_no_model_no_catalog_matches_everything(self):
e = _entry()
assert _custom_provider_model_matches("anything", e)
def test_catalog_case_insensitive(self):
e = _entry(models={"GPT-5.6-Terra": {}})
assert _custom_provider_model_matches("gpt-5.6-terra", e)
class TestExtraBodyResolution:
def test_multi_model_provider_yields_extra_body(self):
# The exact sweeper-profile shape that failed in production.
entry = _entry(
model="gpt-5.5",
models={"gpt-5.5": {}, "gpt-5.6-sol": {}, "gpt-5.6-terra": {}},
)
got = _custom_provider_extra_body_for_agent(
provider="custom", model="gpt-5.6-terra",
base_url=BASE, custom_providers=[entry],
)
assert got == {"service_tier": "flex"}
def test_non_catalog_model_gets_no_override(self):
entry = _entry(model="gpt-5.5", models={"gpt-5.5": {}})
got = _custom_provider_extra_body_for_agent(
provider="custom", model="gpt-4o",
base_url=BASE, custom_providers=[entry],
)
assert got is None
def test_non_custom_provider_unaffected(self):
entry = _entry(models={"gpt-5.6-terra": {}})
got = _custom_provider_extra_body_for_agent(
provider="openrouter", model="gpt-5.6-terra",
base_url=BASE, custom_providers=[entry],
)
assert got is None