From db117af4785f79d0adfafcea4d75ee556f4006dd Mon Sep 17 00:00:00 2001 From: Kshitij Kapoor Date: Thu, 9 Jul 2026 23:27:35 +0530 Subject: [PATCH] review fixes: openai-api pricing route normalization, GA pricing_version, invariant tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase-2 review findings addressed: - resolve_billing_route: normalize the "openai-api" picker slug to the "openai" billing provider — without this the ("openai", ) _OFFICIAL_DOCS_PRICING keys (incl. every pre-existing gpt-4o/gpt-4.1 entry, not just 5.6) were unreachable when the provider is openai-api. - pricing_version: drop the "preview" tag (GA 2026-07-09 at same rates). - model_metadata comment: dict order is cosmetic — lookups length-sort keys at match time; the old comment implied a positional invariant. - model_switch comment: note "sol" is a series codename, not a generic quality word. - tests/hermes_cli/test_gpt56_registration.py: behavior contracts (no list snapshots) — sol > terra/luna > 5.5 sort invariant, pricing reachability from both openai and openai-api routes, cache-write 1.25x / cache-read 0.10x input relation. --- agent/model_metadata.py | 2 +- agent/usage_pricing.py | 12 ++-- hermes_cli/model_switch.py | 4 +- tests/hermes_cli/test_gpt56_registration.py | 66 +++++++++++++++++++++ 4 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 tests/hermes_cli/test_gpt56_registration.py diff --git a/agent/model_metadata.py b/agent/model_metadata.py index ada9cd15536..dbbcf81b72f 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -231,7 +231,7 @@ DEFAULT_CONTEXT_LENGTHS = { # This hardcoded value is only reached when every probe misses. # GPT-5.6 series (Sol/Terra/Luna, GA 2026-07-09) — 1.05M on the direct # OpenAI API (same as gpt-5.5). Codex OAuth caps these at 272K. - # More-specific keys precede "gpt-5.5" for longest-substring matching. + # (Lookups length-sort keys at match time, so dict order is cosmetic.) "gpt-5.6-luna": 1050000, "gpt-5.6-terra": 1050000, "gpt-5.6-sol": 1050000, diff --git a/agent/usage_pricing.py b/agent/usage_pricing.py index 112ab21be71..99d593ce6b2 100644 --- a/agent/usage_pricing.py +++ b/agent/usage_pricing.py @@ -121,7 +121,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = { cache_write_cost_per_million=Decimal("6.25"), source="official_docs_snapshot", source_url="https://openai.com/index/previewing-gpt-5-6-sol/", - pricing_version="openai-gpt-5.6-preview-2026-07", + pricing_version="openai-gpt-5.6-2026-07", ), ( "openai", @@ -133,7 +133,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = { cache_write_cost_per_million=Decimal("3.125"), source="official_docs_snapshot", source_url="https://openai.com/index/previewing-gpt-5-6-sol/", - pricing_version="openai-gpt-5.6-preview-2026-07", + pricing_version="openai-gpt-5.6-2026-07", ), ( "openai", @@ -145,7 +145,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = { cache_write_cost_per_million=Decimal("1.25"), source="official_docs_snapshot", source_url="https://openai.com/index/previewing-gpt-5-6-sol/", - pricing_version="openai-gpt-5.6-preview-2026-07", + pricing_version="openai-gpt-5.6-2026-07", ), # ── Anthropic Claude 4.8 ───────────────────────────────────────────── # Same $5/$25 base pricing as 4.6/4.7. Fast-mode variant is a separate @@ -646,7 +646,11 @@ def resolve_billing_route( return BillingRoute(provider="nous", model=model, base_url=base_url or _NOUS_DEFAULT_BASE_URL, billing_mode="official_models_api") if provider_name == "anthropic": return BillingRoute(provider="anthropic", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") - if provider_name == "openai": + # "openai-api" is the picker/registry slug for direct api.openai.com; it + # bills identically to bare "openai", so normalize it here — otherwise the + # ("openai", ) _OFFICIAL_DOCS_PRICING keys are unreachable from the + # openai-api provider path. + if provider_name in {"openai", "openai-api"}: return BillingRoute(provider="openai", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") if provider_name in {"minimax", "minimax-cn"}: return BillingRoute(provider=provider_name, model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index 2767b088a7d..4bf75fe3117 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -550,7 +550,9 @@ def _model_sort_key(model_id: str, prefix: str) -> tuple: # Lower number = preferred # "sol" is the flagship tier of the GPT-5.6 series (sol > terra > luna); # without it, alias resolution would tiebreak alphabetically and pick - # luna (the cheapest) for `/model gpt`. + # luna (the cheapest) for `/model gpt`. Unlike pro/max/plus/turbo it is a + # series codename, not a generic quality word — revisit if another vendor + # ever ships a "-sol" suffix that isn't a flagship. _SUFFIX_RANK = {"pro": 0, "max": 0, "plus": 0, "turbo": 0, "sol": 0} suffix_rank = _SUFFIX_RANK.get(suffix, 1) diff --git a/tests/hermes_cli/test_gpt56_registration.py b/tests/hermes_cli/test_gpt56_registration.py new file mode 100644 index 00000000000..f88a07cb6c5 --- /dev/null +++ b/tests/hermes_cli/test_gpt56_registration.py @@ -0,0 +1,66 @@ +"""Behavior contracts for the GPT-5.6 (Sol/Terra/Luna) registration. + +Invariant tests only — no list snapshots (per the no-change-detector-tests +policy). These pin the two behaviors that would silently regress: + +1. Version/tier sorting: the flagship Sol must outrank Terra/Luna and the + whole 5.6 series must outrank 5.5, so `/model gpt` resolves to the + flagship rather than an alphabetical-first cheap tier. +2. Pricing reachability: the ("openai", ) official-docs pricing keys + must be reachable from BOTH the bare "openai" provider and the + "openai-api" picker slug (resolve_billing_route normalizes the latter). +""" + +from decimal import Decimal + +from agent.usage_pricing import ( + _OFFICIAL_DOCS_PRICING, + _lookup_official_docs_pricing, + resolve_billing_route, +) +from hermes_cli.model_switch import _model_sort_key + + +class TestGpt56SortInvariants: + def test_sol_outranks_terra_and_luna(self): + models = ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"] + models.sort(key=lambda m: _model_sort_key(m, "gpt")) + assert models[0] == "gpt-5.6-sol" + + def test_56_series_outranks_55(self): + models = ["gpt-5.5", "gpt-5.5-pro", "gpt-5.6-sol"] + models.sort(key=lambda m: _model_sort_key(m, "gpt")) + assert models[0] == "gpt-5.6-sol" + + def test_aggregator_prefix_form(self): + models = ["openai/gpt-5.5-pro", "openai/gpt-5.6-sol"] + models.sort(key=lambda m: _model_sort_key(m, "openai/gpt")) + assert models[0] == "openai/gpt-5.6-sol" + + +class TestGpt56PricingRoute: + def test_official_pricing_reachable_from_openai(self): + route = resolve_billing_route("gpt-5.6-sol", provider="openai") + entry = _lookup_official_docs_pricing(route) + assert entry is not None + assert entry.input_cost_per_million == Decimal("5.00") + + def test_official_pricing_reachable_from_openai_api_slug(self): + # "openai-api" is the picker slug for direct api.openai.com and must + # normalize to the "openai" pricing key space. + route = resolve_billing_route("gpt-5.6-sol", provider="openai-api") + assert route.provider == "openai" + entry = _lookup_official_docs_pricing(route) + assert entry is not None + assert entry.input_cost_per_million == Decimal("5.00") + + def test_cache_write_is_1_25x_input_for_56_series(self): + for slug in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"): + entry = _OFFICIAL_DOCS_PRICING[("openai", slug)] + assert entry.input_cost_per_million is not None, slug + assert entry.cache_write_cost_per_million == ( + entry.input_cost_per_million * Decimal("1.25") + ), slug + assert entry.cache_read_cost_per_million == ( + entry.input_cost_per_million * Decimal("0.10") + ), slug