From 8a5f8379eda3d8b9b223c82194cd88cc4ad41c69 Mon Sep 17 00:00:00 2001
From: Teknium <127238744+teknium1@users.noreply.github.com>
Date: Sun, 12 Jul 2026 23:12:05 -0700
Subject: [PATCH] fix(agent): honor custom-provider extra_body for multi-model
 catalogs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

_custom_provider_model_matches() only compared the session model
against the entry's single 'model' field. A custom provider declaring
a multi-model catalog (providers.<name>.models mapping / models list)
whose default model differed from the session model silently failed to
match — dropping the entry's extra_body entirely. Real impact: an
OpenAI custom provider pinning service_tier=flex via extra_body ran
every request at STANDARD tier (~2.3x billing) with zero signal.

- Model matching now accepts the session model when it appears in the
  entry's models catalog (dict keys or list), case-insensitive;
  single-model 'model' field behavior unchanged; entries with neither
  still match everything.
- Usage report ('hermes -z --usage-file') now carries service_tier
  (the tier requested via request_overrides.extra_body) so batch
  pipelines can audit the billed tier per run.

Validation: 8 new tests; live E2E via real 'hermes -p sweeper -z'
with httpx-level wire capture — service_tier=flex present in the
outgoing /v1/responses body and in the usage report.
---
 agent/agent_init.py                           | 22 +++++-
 agent/turn_finalizer.py                       |  5 ++
 hermes_cli/oneshot.py                         |  6 ++
 ...est_custom_provider_extra_body_matching.py | 78 +++++++++++++++++++
 4 files changed, 108 insertions(+), 3 deletions(-)
 create mode 100644 tests/agent/test_custom_provider_extra_body_matching.py

diff --git a/agent/agent_init.py b/agent/agent_init.py
index d9207b0b3d94..c5826b4b9475 100644
--- a/agent/agent_init.py
+++ b/agent/agent_init.py
@@ -187,10 +187,26 @@ def _normalized_custom_base_url(value: Any) -> str:
 
 
 def _custom_provider_model_matches(agent_model: str, entry: Dict[str, Any]) -> bool:
-    provider_model = str(entry.get("model", "") or "").strip().lower()
-    if not provider_model:
+    agent_model_norm = str(agent_model or "").strip().lower()
+    # Multi-model entries (v12+ `providers.<name>.models` mapping / legacy
+    # `models:` list): the agent's model matching ANY catalog entry counts.
+    # Without this, a provider whose `model`/`default_model` differs from the
+    # session model silently fails to match and per-provider request settings
+    # (extra_body, e.g. OpenAI service_tier) are dropped — billing the whole
+    # session at the wrong tier (July 2026 sweeper incident: flex config
+    # ignored, ~2.3x overbilling).
+    models = entry.get("models")
+    catalog: List[str] = []
+    if isinstance(models, dict):
+        catalog = [str(k).strip().lower() for k in models.keys()]
+    elif isinstance(models, (list, tuple)):
+        catalog = [str(m).strip().lower() for m in models]
+    if catalog and agent_model_norm in catalog:
         return True
-    return provider_model == str(agent_model or "").strip().lower()
+    provider_model = str(entry.get("model", "") or "").strip().lower()
+    if not provider_model and not catalog:
+        return True
+    return provider_model == agent_model_norm
 
 
 def _custom_provider_extra_body_for_agent(
diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py
index b41f0fdd506f..1adc3c6c3632 100644
--- a/agent/turn_finalizer.py
+++ b/agent/turn_finalizer.py
@@ -449,6 +449,11 @@ def finalize_turn(
         "estimated_cost_usd": agent.session_estimated_cost_usd,
         "cost_status": agent.session_cost_status,
         "cost_source": agent.session_cost_source,
+        # Requested service tier (from request_overrides.extra_body), for
+        # billing audits by callers like `hermes -z --usage-file`.
+        "service_tier": (
+            (getattr(agent, "request_overrides", {}) or {}).get("extra_body") or {}
+        ).get("service_tier"),
         "session_id": agent.session_id,
     }
     if agent._tool_guardrail_halt_decision is not None:
diff --git a/hermes_cli/oneshot.py b/hermes_cli/oneshot.py
index a4c6ca9cc256..e7db628a7c1d 100644
--- a/hermes_cli/oneshot.py
+++ b/hermes_cli/oneshot.py
@@ -150,6 +150,12 @@ def _write_usage_file(path: Optional[str], result: dict, failure: Optional[str]
             "session_id": result.get("session_id"),
             "completed": result.get("completed"),
             "failed": bool(result.get("failed")) or failure is not None,
+            # Billing-audit field: the service tier this run REQUESTED via
+            # request_overrides.extra_body (e.g. OpenAI "flex"). None when
+            # unset. Lets batch pipelines verify the tier they think they're
+            # paying for actually went out on the wire (July 2026 incident:
+            # a config-matching bug silently dropped flex -> 2.3x billing).
+            "service_tier": result.get("service_tier"),
         }
         if failure is not None:
             report["failure"] = failure
diff --git a/tests/agent/test_custom_provider_extra_body_matching.py b/tests/agent/test_custom_provider_extra_body_matching.py
new file mode 100644
index 000000000000..4a62f8b341e5
--- /dev/null
+++ b/tests/agent/test_custom_provider_extra_body_matching.py
@@ -0,0 +1,78 @@
+"""Tests for custom-provider model matching (extra_body / service_tier drop bug).
+
+July 2026 incident: a custom provider with a multi-model catalog
+(``models: {gpt-5.5: {}, gpt-5.6-terra: {}, ...}``) and a ``model``/
+``default_model`` differing from the session model failed
+``_custom_provider_model_matches``, so ``extra_body: {service_tier: flex}``
+was silently dropped — every request billed at standard tier (~2.3x).
+"""
+
+from agent.agent_init import (
+    _custom_provider_extra_body_for_agent,
+    _custom_provider_model_matches,
+)
+
+BASE = "https://api.openai.com/v1"
+
+
+def _entry(**over):
+    e = {
+        "name": "openai",
+        "base_url": BASE,
+        "extra_body": {"service_tier": "flex"},
+    }
+    e.update(over)
+    return e
+
+
+class TestModelMatches:
+    def test_models_dict_catalog_matches_session_model(self):
+        e = _entry(model="gpt-5.5", models={"gpt-5.5": {}, "gpt-5.6-terra": {}})
+        assert _custom_provider_model_matches("gpt-5.6-terra", e)
+
+    def test_models_list_catalog_matches(self):
+        e = _entry(model="gpt-5.5", models=["gpt-5.5", "gpt-5.6-sol"])
+        assert _custom_provider_model_matches("gpt-5.6-sol", e)
+
+    def test_catalog_miss_falls_back_to_model_field(self):
+        e = _entry(model="gpt-5.5", models={"gpt-5.5": {}})
+        assert _custom_provider_model_matches("gpt-5.5", e)
+        assert not _custom_provider_model_matches("gpt-4o", e)
+
+    def test_no_model_no_catalog_matches_everything(self):
+        e = _entry()
+        assert _custom_provider_model_matches("anything", e)
+
+    def test_catalog_case_insensitive(self):
+        e = _entry(models={"GPT-5.6-Terra": {}})
+        assert _custom_provider_model_matches("gpt-5.6-terra", e)
+
+
+class TestExtraBodyResolution:
+    def test_multi_model_provider_yields_extra_body(self):
+        # The exact sweeper-profile shape that failed in production.
+        entry = _entry(
+            model="gpt-5.5",
+            models={"gpt-5.5": {}, "gpt-5.6-sol": {}, "gpt-5.6-terra": {}},
+        )
+        got = _custom_provider_extra_body_for_agent(
+            provider="custom", model="gpt-5.6-terra",
+            base_url=BASE, custom_providers=[entry],
+        )
+        assert got == {"service_tier": "flex"}
+
+    def test_non_catalog_model_gets_no_override(self):
+        entry = _entry(model="gpt-5.5", models={"gpt-5.5": {}})
+        got = _custom_provider_extra_body_for_agent(
+            provider="custom", model="gpt-4o",
+            base_url=BASE, custom_providers=[entry],
+        )
+        assert got is None
+
+    def test_non_custom_provider_unaffected(self):
+        entry = _entry(models={"gpt-5.6-terra": {}})
+        got = _custom_provider_extra_body_for_agent(
+            provider="openrouter", model="gpt-5.6-terra",
+            base_url=BASE, custom_providers=[entry],
+        )
+        assert got is None