review fixes: openai-api pricing route normalization, GA pricing_version, invariant tests

Phase-2 review findings addressed:
- resolve_billing_route: normalize the "openai-api" picker slug to the
  "openai" billing provider — without this the ("openai", <model>)
  _OFFICIAL_DOCS_PRICING keys (incl. every pre-existing gpt-4o/gpt-4.1
  entry, not just 5.6) were unreachable when the provider is openai-api.
- pricing_version: drop the "preview" tag (GA 2026-07-09 at same rates).
- model_metadata comment: dict order is cosmetic — lookups length-sort
  keys at match time; the old comment implied a positional invariant.
- model_switch comment: note "sol" is a series codename, not a generic
  quality word.
- tests/hermes_cli/test_gpt56_registration.py: behavior contracts (no
  list snapshots) — sol > terra/luna > 5.5 sort invariant, pricing
  reachability from both openai and openai-api routes, cache-write
  1.25x / cache-read 0.10x input relation.
This commit is contained in:
Kshitij Kapoor 2026-07-09 23:27:35 +05:30 committed by kshitij
parent bd767b574b
commit db117af478
4 changed files with 78 additions and 6 deletions

View file

@ -231,7 +231,7 @@ DEFAULT_CONTEXT_LENGTHS = {
# This hardcoded value is only reached when every probe misses.
# GPT-5.6 series (Sol/Terra/Luna, GA 2026-07-09) — 1.05M on the direct
# OpenAI API (same as gpt-5.5). Codex OAuth caps these at 272K.
# More-specific keys precede "gpt-5.5" for longest-substring matching.
# (Lookups length-sort keys at match time, so dict order is cosmetic.)
"gpt-5.6-luna": 1050000,
"gpt-5.6-terra": 1050000,
"gpt-5.6-sol": 1050000,

View file

@ -121,7 +121,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = {
cache_write_cost_per_million=Decimal("6.25"),
source="official_docs_snapshot",
source_url="https://openai.com/index/previewing-gpt-5-6-sol/",
pricing_version="openai-gpt-5.6-preview-2026-07",
pricing_version="openai-gpt-5.6-2026-07",
),
(
"openai",
@ -133,7 +133,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = {
cache_write_cost_per_million=Decimal("3.125"),
source="official_docs_snapshot",
source_url="https://openai.com/index/previewing-gpt-5-6-sol/",
pricing_version="openai-gpt-5.6-preview-2026-07",
pricing_version="openai-gpt-5.6-2026-07",
),
(
"openai",
@ -145,7 +145,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = {
cache_write_cost_per_million=Decimal("1.25"),
source="official_docs_snapshot",
source_url="https://openai.com/index/previewing-gpt-5-6-sol/",
pricing_version="openai-gpt-5.6-preview-2026-07",
pricing_version="openai-gpt-5.6-2026-07",
),
# ── Anthropic Claude 4.8 ─────────────────────────────────────────────
# Same $5/$25 base pricing as 4.6/4.7. Fast-mode variant is a separate
@ -646,7 +646,11 @@ def resolve_billing_route(
return BillingRoute(provider="nous", model=model, base_url=base_url or _NOUS_DEFAULT_BASE_URL, billing_mode="official_models_api")
if provider_name == "anthropic":
return BillingRoute(provider="anthropic", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot")
if provider_name == "openai":
# "openai-api" is the picker/registry slug for direct api.openai.com; it
# bills identically to bare "openai", so normalize it here — otherwise the
# ("openai", <model>) _OFFICIAL_DOCS_PRICING keys are unreachable from the
# openai-api provider path.
if provider_name in {"openai", "openai-api"}:
return BillingRoute(provider="openai", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot")
if provider_name in {"minimax", "minimax-cn"}:
return BillingRoute(provider=provider_name, model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot")

View file

@ -550,7 +550,9 @@ def _model_sort_key(model_id: str, prefix: str) -> tuple:
# Lower number = preferred
# "sol" is the flagship tier of the GPT-5.6 series (sol > terra > luna);
# without it, alias resolution would tiebreak alphabetically and pick
# luna (the cheapest) for `/model gpt`.
# luna (the cheapest) for `/model gpt`. Unlike pro/max/plus/turbo it is a
# series codename, not a generic quality word — revisit if another vendor
# ever ships a "-sol" suffix that isn't a flagship.
_SUFFIX_RANK = {"pro": 0, "max": 0, "plus": 0, "turbo": 0, "sol": 0}
suffix_rank = _SUFFIX_RANK.get(suffix, 1)

View file

@ -0,0 +1,66 @@
"""Behavior contracts for the GPT-5.6 (Sol/Terra/Luna) registration.
Invariant tests only no list snapshots (per the no-change-detector-tests
policy). These pin the two behaviors that would silently regress:
1. Version/tier sorting: the flagship Sol must outrank Terra/Luna and the
whole 5.6 series must outrank 5.5, so `/model gpt` resolves to the
flagship rather than an alphabetical-first cheap tier.
2. Pricing reachability: the ("openai", <model>) official-docs pricing keys
must be reachable from BOTH the bare "openai" provider and the
"openai-api" picker slug (resolve_billing_route normalizes the latter).
"""
from decimal import Decimal
from agent.usage_pricing import (
_OFFICIAL_DOCS_PRICING,
_lookup_official_docs_pricing,
resolve_billing_route,
)
from hermes_cli.model_switch import _model_sort_key
class TestGpt56SortInvariants:
def test_sol_outranks_terra_and_luna(self):
models = ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"]
models.sort(key=lambda m: _model_sort_key(m, "gpt"))
assert models[0] == "gpt-5.6-sol"
def test_56_series_outranks_55(self):
models = ["gpt-5.5", "gpt-5.5-pro", "gpt-5.6-sol"]
models.sort(key=lambda m: _model_sort_key(m, "gpt"))
assert models[0] == "gpt-5.6-sol"
def test_aggregator_prefix_form(self):
models = ["openai/gpt-5.5-pro", "openai/gpt-5.6-sol"]
models.sort(key=lambda m: _model_sort_key(m, "openai/gpt"))
assert models[0] == "openai/gpt-5.6-sol"
class TestGpt56PricingRoute:
def test_official_pricing_reachable_from_openai(self):
route = resolve_billing_route("gpt-5.6-sol", provider="openai")
entry = _lookup_official_docs_pricing(route)
assert entry is not None
assert entry.input_cost_per_million == Decimal("5.00")
def test_official_pricing_reachable_from_openai_api_slug(self):
# "openai-api" is the picker slug for direct api.openai.com and must
# normalize to the "openai" pricing key space.
route = resolve_billing_route("gpt-5.6-sol", provider="openai-api")
assert route.provider == "openai"
entry = _lookup_official_docs_pricing(route)
assert entry is not None
assert entry.input_cost_per_million == Decimal("5.00")
def test_cache_write_is_1_25x_input_for_56_series(self):
for slug in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"):
entry = _OFFICIAL_DOCS_PRICING[("openai", slug)]
assert entry.input_cost_per_million is not None, slug
assert entry.cache_write_cost_per_million == (
entry.input_cost_per_million * Decimal("1.25")
), slug
assert entry.cache_read_cost_per_million == (
entry.input_cost_per_million * Decimal("0.10")
), slug