diff --git a/agent/agent_init.py b/agent/agent_init.py index d9207b0b3d94..c5826b4b9475 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -187,10 +187,26 @@ def _normalized_custom_base_url(value: Any) -> str: def _custom_provider_model_matches(agent_model: str, entry: Dict[str, Any]) -> bool: - provider_model = str(entry.get("model", "") or "").strip().lower() - if not provider_model: + agent_model_norm = str(agent_model or "").strip().lower() + # Multi-model entries (v12+ `providers..models` mapping / legacy + # `models:` list): the agent's model matching ANY catalog entry counts. + # Without this, a provider whose `model`/`default_model` differs from the + # session model silently fails to match and per-provider request settings + # (extra_body, e.g. OpenAI service_tier) are dropped — billing the whole + # session at the wrong tier (July 2026 sweeper incident: flex config + # ignored, ~2.3x overbilling). + models = entry.get("models") + catalog: List[str] = [] + if isinstance(models, dict): + catalog = [str(k).strip().lower() for k in models.keys()] + elif isinstance(models, (list, tuple)): + catalog = [str(m).strip().lower() for m in models] + if catalog and agent_model_norm in catalog: return True - return provider_model == str(agent_model or "").strip().lower() + provider_model = str(entry.get("model", "") or "").strip().lower() + if not provider_model and not catalog: + return True + return provider_model == agent_model_norm def _custom_provider_extra_body_for_agent( diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index b41f0fdd506f..1adc3c6c3632 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -449,6 +449,11 @@ def finalize_turn( "estimated_cost_usd": agent.session_estimated_cost_usd, "cost_status": agent.session_cost_status, "cost_source": agent.session_cost_source, + # Requested service tier (from request_overrides.extra_body), for + # billing audits by callers like `hermes -z --usage-file`. + "service_tier": ( + (getattr(agent, "request_overrides", {}) or {}).get("extra_body") or {} + ).get("service_tier"), "session_id": agent.session_id, } if agent._tool_guardrail_halt_decision is not None: diff --git a/hermes_cli/oneshot.py b/hermes_cli/oneshot.py index a4c6ca9cc256..e7db628a7c1d 100644 --- a/hermes_cli/oneshot.py +++ b/hermes_cli/oneshot.py @@ -150,6 +150,12 @@ def _write_usage_file(path: Optional[str], result: dict, failure: Optional[str] "session_id": result.get("session_id"), "completed": result.get("completed"), "failed": bool(result.get("failed")) or failure is not None, + # Billing-audit field: the service tier this run REQUESTED via + # request_overrides.extra_body (e.g. OpenAI "flex"). None when + # unset. Lets batch pipelines verify the tier they think they're + # paying for actually went out on the wire (July 2026 incident: + # a config-matching bug silently dropped flex -> 2.3x billing). + "service_tier": result.get("service_tier"), } if failure is not None: report["failure"] = failure diff --git a/tests/agent/test_custom_provider_extra_body_matching.py b/tests/agent/test_custom_provider_extra_body_matching.py new file mode 100644 index 000000000000..4a62f8b341e5 --- /dev/null +++ b/tests/agent/test_custom_provider_extra_body_matching.py @@ -0,0 +1,78 @@ +"""Tests for custom-provider model matching (extra_body / service_tier drop bug). + +July 2026 incident: a custom provider with a multi-model catalog +(``models: {gpt-5.5: {}, gpt-5.6-terra: {}, ...}``) and a ``model``/ +``default_model`` differing from the session model failed +``_custom_provider_model_matches``, so ``extra_body: {service_tier: flex}`` +was silently dropped — every request billed at standard tier (~2.3x). +""" + +from agent.agent_init import ( + _custom_provider_extra_body_for_agent, + _custom_provider_model_matches, +) + +BASE = "https://api.openai.com/v1" + + +def _entry(**over): + e = { + "name": "openai", + "base_url": BASE, + "extra_body": {"service_tier": "flex"}, + } + e.update(over) + return e + + +class TestModelMatches: + def test_models_dict_catalog_matches_session_model(self): + e = _entry(model="gpt-5.5", models={"gpt-5.5": {}, "gpt-5.6-terra": {}}) + assert _custom_provider_model_matches("gpt-5.6-terra", e) + + def test_models_list_catalog_matches(self): + e = _entry(model="gpt-5.5", models=["gpt-5.5", "gpt-5.6-sol"]) + assert _custom_provider_model_matches("gpt-5.6-sol", e) + + def test_catalog_miss_falls_back_to_model_field(self): + e = _entry(model="gpt-5.5", models={"gpt-5.5": {}}) + assert _custom_provider_model_matches("gpt-5.5", e) + assert not _custom_provider_model_matches("gpt-4o", e) + + def test_no_model_no_catalog_matches_everything(self): + e = _entry() + assert _custom_provider_model_matches("anything", e) + + def test_catalog_case_insensitive(self): + e = _entry(models={"GPT-5.6-Terra": {}}) + assert _custom_provider_model_matches("gpt-5.6-terra", e) + + +class TestExtraBodyResolution: + def test_multi_model_provider_yields_extra_body(self): + # The exact sweeper-profile shape that failed in production. + entry = _entry( + model="gpt-5.5", + models={"gpt-5.5": {}, "gpt-5.6-sol": {}, "gpt-5.6-terra": {}}, + ) + got = _custom_provider_extra_body_for_agent( + provider="custom", model="gpt-5.6-terra", + base_url=BASE, custom_providers=[entry], + ) + assert got == {"service_tier": "flex"} + + def test_non_catalog_model_gets_no_override(self): + entry = _entry(model="gpt-5.5", models={"gpt-5.5": {}}) + got = _custom_provider_extra_body_for_agent( + provider="custom", model="gpt-4o", + base_url=BASE, custom_providers=[entry], + ) + assert got is None + + def test_non_custom_provider_unaffected(self): + entry = _entry(models={"gpt-5.6-terra": {}}) + got = _custom_provider_extra_body_for_agent( + provider="openrouter", model="gpt-5.6-terra", + base_url=BASE, custom_providers=[entry], + ) + assert got is None