mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-15 14:22:43 +00:00
review fixes: openai-api pricing route normalization, GA pricing_version, invariant tests
Phase-2 review findings addressed:
- resolve_billing_route: normalize the "openai-api" picker slug to the
"openai" billing provider — without this the ("openai", <model>)
_OFFICIAL_DOCS_PRICING keys (incl. every pre-existing gpt-4o/gpt-4.1
entry, not just 5.6) were unreachable when the provider is openai-api.
- pricing_version: drop the "preview" tag (GA 2026-07-09 at same rates).
- model_metadata comment: dict order is cosmetic — lookups length-sort
keys at match time; the old comment implied a positional invariant.
- model_switch comment: note "sol" is a series codename, not a generic
quality word.
- tests/hermes_cli/test_gpt56_registration.py: behavior contracts (no
list snapshots) — sol > terra/luna > 5.5 sort invariant, pricing
reachability from both openai and openai-api routes, cache-write
1.25x / cache-read 0.10x input relation.
This commit is contained in:
parent
bd767b574b
commit
db117af478
4 changed files with 78 additions and 6 deletions
|
|
@ -231,7 +231,7 @@ DEFAULT_CONTEXT_LENGTHS = {
|
|||
# This hardcoded value is only reached when every probe misses.
|
||||
# GPT-5.6 series (Sol/Terra/Luna, GA 2026-07-09) — 1.05M on the direct
|
||||
# OpenAI API (same as gpt-5.5). Codex OAuth caps these at 272K.
|
||||
# More-specific keys precede "gpt-5.5" for longest-substring matching.
|
||||
# (Lookups length-sort keys at match time, so dict order is cosmetic.)
|
||||
"gpt-5.6-luna": 1050000,
|
||||
"gpt-5.6-terra": 1050000,
|
||||
"gpt-5.6-sol": 1050000,
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = {
|
|||
cache_write_cost_per_million=Decimal("6.25"),
|
||||
source="official_docs_snapshot",
|
||||
source_url="https://openai.com/index/previewing-gpt-5-6-sol/",
|
||||
pricing_version="openai-gpt-5.6-preview-2026-07",
|
||||
pricing_version="openai-gpt-5.6-2026-07",
|
||||
),
|
||||
(
|
||||
"openai",
|
||||
|
|
@ -133,7 +133,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = {
|
|||
cache_write_cost_per_million=Decimal("3.125"),
|
||||
source="official_docs_snapshot",
|
||||
source_url="https://openai.com/index/previewing-gpt-5-6-sol/",
|
||||
pricing_version="openai-gpt-5.6-preview-2026-07",
|
||||
pricing_version="openai-gpt-5.6-2026-07",
|
||||
),
|
||||
(
|
||||
"openai",
|
||||
|
|
@ -145,7 +145,7 @@ _OFFICIAL_DOCS_PRICING: Dict[tuple[str, str], PricingEntry] = {
|
|||
cache_write_cost_per_million=Decimal("1.25"),
|
||||
source="official_docs_snapshot",
|
||||
source_url="https://openai.com/index/previewing-gpt-5-6-sol/",
|
||||
pricing_version="openai-gpt-5.6-preview-2026-07",
|
||||
pricing_version="openai-gpt-5.6-2026-07",
|
||||
),
|
||||
# ── Anthropic Claude 4.8 ─────────────────────────────────────────────
|
||||
# Same $5/$25 base pricing as 4.6/4.7. Fast-mode variant is a separate
|
||||
|
|
@ -646,7 +646,11 @@ def resolve_billing_route(
|
|||
return BillingRoute(provider="nous", model=model, base_url=base_url or _NOUS_DEFAULT_BASE_URL, billing_mode="official_models_api")
|
||||
if provider_name == "anthropic":
|
||||
return BillingRoute(provider="anthropic", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot")
|
||||
if provider_name == "openai":
|
||||
# "openai-api" is the picker/registry slug for direct api.openai.com; it
|
||||
# bills identically to bare "openai", so normalize it here — otherwise the
|
||||
# ("openai", <model>) _OFFICIAL_DOCS_PRICING keys are unreachable from the
|
||||
# openai-api provider path.
|
||||
if provider_name in {"openai", "openai-api"}:
|
||||
return BillingRoute(provider="openai", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot")
|
||||
if provider_name in {"minimax", "minimax-cn"}:
|
||||
return BillingRoute(provider=provider_name, model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot")
|
||||
|
|
|
|||
|
|
@ -550,7 +550,9 @@ def _model_sort_key(model_id: str, prefix: str) -> tuple:
|
|||
# Lower number = preferred
|
||||
# "sol" is the flagship tier of the GPT-5.6 series (sol > terra > luna);
|
||||
# without it, alias resolution would tiebreak alphabetically and pick
|
||||
# luna (the cheapest) for `/model gpt`.
|
||||
# luna (the cheapest) for `/model gpt`. Unlike pro/max/plus/turbo it is a
|
||||
# series codename, not a generic quality word — revisit if another vendor
|
||||
# ever ships a "-sol" suffix that isn't a flagship.
|
||||
_SUFFIX_RANK = {"pro": 0, "max": 0, "plus": 0, "turbo": 0, "sol": 0}
|
||||
suffix_rank = _SUFFIX_RANK.get(suffix, 1)
|
||||
|
||||
|
|
|
|||
66
tests/hermes_cli/test_gpt56_registration.py
Normal file
66
tests/hermes_cli/test_gpt56_registration.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
"""Behavior contracts for the GPT-5.6 (Sol/Terra/Luna) registration.
|
||||
|
||||
Invariant tests only — no list snapshots (per the no-change-detector-tests
|
||||
policy). These pin the two behaviors that would silently regress:
|
||||
|
||||
1. Version/tier sorting: the flagship Sol must outrank Terra/Luna and the
|
||||
whole 5.6 series must outrank 5.5, so `/model gpt` resolves to the
|
||||
flagship rather than an alphabetical-first cheap tier.
|
||||
2. Pricing reachability: the ("openai", <model>) official-docs pricing keys
|
||||
must be reachable from BOTH the bare "openai" provider and the
|
||||
"openai-api" picker slug (resolve_billing_route normalizes the latter).
|
||||
"""
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
from agent.usage_pricing import (
|
||||
_OFFICIAL_DOCS_PRICING,
|
||||
_lookup_official_docs_pricing,
|
||||
resolve_billing_route,
|
||||
)
|
||||
from hermes_cli.model_switch import _model_sort_key
|
||||
|
||||
|
||||
class TestGpt56SortInvariants:
|
||||
def test_sol_outranks_terra_and_luna(self):
|
||||
models = ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"]
|
||||
models.sort(key=lambda m: _model_sort_key(m, "gpt"))
|
||||
assert models[0] == "gpt-5.6-sol"
|
||||
|
||||
def test_56_series_outranks_55(self):
|
||||
models = ["gpt-5.5", "gpt-5.5-pro", "gpt-5.6-sol"]
|
||||
models.sort(key=lambda m: _model_sort_key(m, "gpt"))
|
||||
assert models[0] == "gpt-5.6-sol"
|
||||
|
||||
def test_aggregator_prefix_form(self):
|
||||
models = ["openai/gpt-5.5-pro", "openai/gpt-5.6-sol"]
|
||||
models.sort(key=lambda m: _model_sort_key(m, "openai/gpt"))
|
||||
assert models[0] == "openai/gpt-5.6-sol"
|
||||
|
||||
|
||||
class TestGpt56PricingRoute:
|
||||
def test_official_pricing_reachable_from_openai(self):
|
||||
route = resolve_billing_route("gpt-5.6-sol", provider="openai")
|
||||
entry = _lookup_official_docs_pricing(route)
|
||||
assert entry is not None
|
||||
assert entry.input_cost_per_million == Decimal("5.00")
|
||||
|
||||
def test_official_pricing_reachable_from_openai_api_slug(self):
|
||||
# "openai-api" is the picker slug for direct api.openai.com and must
|
||||
# normalize to the "openai" pricing key space.
|
||||
route = resolve_billing_route("gpt-5.6-sol", provider="openai-api")
|
||||
assert route.provider == "openai"
|
||||
entry = _lookup_official_docs_pricing(route)
|
||||
assert entry is not None
|
||||
assert entry.input_cost_per_million == Decimal("5.00")
|
||||
|
||||
def test_cache_write_is_1_25x_input_for_56_series(self):
|
||||
for slug in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"):
|
||||
entry = _OFFICIAL_DOCS_PRICING[("openai", slug)]
|
||||
assert entry.input_cost_per_million is not None, slug
|
||||
assert entry.cache_write_cost_per_million == (
|
||||
entry.input_cost_per_million * Decimal("1.25")
|
||||
), slug
|
||||
assert entry.cache_read_cost_per_million == (
|
||||
entry.input_cost_per_million * Decimal("0.10")
|
||||
), slug
|
||||
Loading…
Add table
Add a link
Reference in a new issue