mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-23 16:36:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

fix(context): revalidate Codex OAuth context windows

Browse source
This commit is contained in:
sbe27 2026-07-10 02:43:26 +02:00 • committed by Teknium
parent 279be8211d
commit 8a0701ca48
2 changed files with 102 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

69
agent/model_metadata.py
View file

@ -581,8 +581,13 @@ def _skip_persistent_context_cache(base_url: str, provider: str) -> bool:
LM Studio excludes caching because loaded context is transient — the user
can reload the model with a different context_length at any time.
"""
return provider == "lmstudio"
Codex OAuth excludes caching because its context window is account- and
entitlement-specific metadata supplied by the authenticated /models
endpoint. A fallback value written after a transient probe failure must
not prevent a later live probe from observing an updated allocation.
"""
return (provider or "").strip().lower() in {"lmstudio", "openai-codex"}
def _maybe_cache_local_context_length(
@ -1974,27 +1979,31 @@ def _fetch_codex_oauth_context_lengths(access_token: str) -> Dict[str, int]:
return result
def _resolve_codex_oauth_context_length(
def _resolve_codex_oauth_context_length_with_source(
model: str, access_token: str = ""
) -> Optional[int]:
) -> Tuple[Optional[int], str]:
"""Resolve a Codex OAuth model's real context window.
Prefers a live probe of chatgpt.com/backend-api/codex/models (when we
have a bearer token), then falls back to ``_CODEX_OAUTH_CONTEXT_FALLBACK``.
Returns ``(context_length, source)`` where source is ``"live"`` for a
value returned by the authenticated endpoint or ``"fallback"`` for the
static conservative table. Callers must not persist the latter.
"""
model_bare = _strip_provider_prefix(model).strip()
if not model_bare:
return None
return None, ""
if access_token:
live = _fetch_codex_oauth_context_lengths(access_token)
if model_bare in live:
return live[model_bare]
return live[model_bare], "live"
# Case-insensitive match in case casing drifts
model_lower = model_bare.lower()
for slug, ctx in live.items():
if slug.lower() == model_lower:
return ctx
return ctx, "live"
# Fallback: longest-key-first substring match over hardcoded defaults.
model_lower = model_bare.lower()
@ -2002,9 +2011,19 @@ def _resolve_codex_oauth_context_length(
_CODEX_OAUTH_CONTEXT_FALLBACK.items(), key=lambda x: len(x[0]), reverse=True
):
if slug in model_lower:
return ctx
return ctx, "fallback"
return None
return None, ""
def _resolve_codex_oauth_context_length(
model: str, access_token: str = ""
) -> Optional[int]:
"""Resolve a Codex OAuth model's context length (compatibility wrapper)."""
context_length, _source = _resolve_codex_oauth_context_length_with_source(
model, access_token=access_token,
)
return context_length
def _resolve_nous_context_length(
@ -2094,9 +2113,9 @@ def get_model_context_length(
Resolution order:
0. Explicit config override (model.context_length or custom_providers per-model)
0c. Endpoint-scoped metadata for models validated on one multiplexed endpoint
1. Persistent cache (previously discovered via probing). Nous URLs
bypass the cache here so step 5b can always reconcile against
the authoritative portal /v1/models response.
1. Persistent cache (previously discovered via probing). Nous URLs,
LM Studio, and Codex OAuth bypass the cache here so their provider
metadata can be reconciled against the authoritative live source.
1b. AWS Bedrock static table (must precede custom-endpoint probe)
2. Active endpoint metadata (/models for explicit custom endpoints)
3. Local server query (for local endpoints)
@ -2196,24 +2215,13 @@ def get_model_context_length(
# LM Studio is excluded — its loaded context length is transient (the
# user can reload the model with a different context_length at any time
# via /api/v1/models/load), so a stale cached value would mask reloads.
# Codex OAuth is excluded because the authenticated /models catalogue is
# account-specific and a fallback must never suppress later revalidation.
if base_url and not _skip_persistent_context_cache(base_url, provider):
cached = get_cached_context_length(model, base_url)
if cached is not None:
# Invalidate stale Codex OAuth cache entries: pre-PR #14935 builds
# resolved gpt-5.x to the direct-API value (e.g. 1.05M) via
# models.dev and persisted it. Codex OAuth caps at 272K for every
# slug, so any cached Codex entry at or above 400K is a leftover
# from the old resolution path. Drop it and fall through to the
# live /models probe in step 5 below.
if provider == "openai-codex" and cached >= 400_000:
logger.info(
"Dropping stale Codex cache entry %s@%s -> %s (pre-fix value); "
"re-resolving via live /models probe",
model, base_url, f"{cached:,}",
)
_invalidate_cached_context_length(model, base_url)
# Invalidate stale 32k cache entries for Kimi-family models.
elif cached <= 32768 and _model_name_suggests_kimi(model):
if cached <= 32768 and _model_name_suggests_kimi(model):
logger.info(
"Dropping stale Kimi cache entry %s@%s -> %s (OpenRouter underreport); "
"re-resolving via hardcoded defaults",
@ -2452,9 +2460,14 @@ def get_model_context_length(
# Codex OAuth enforces lower context limits than the direct OpenAI
# API for the same slug (e.g. gpt-5.5 is 1.05M on the API but 272K
# on Codex). Authoritative source is Codex's own /models endpoint.
codex_ctx = _resolve_codex_oauth_context_length(model, access_token=api_key or "")
codex_ctx, codex_source = _resolve_codex_oauth_context_length_with_source(
model, access_token=api_key or "",
)
if codex_ctx:
if base_url:
# Only a successful authenticated catalogue response is safe to
# persist. The static fallback is deliberately runtime-only so a
# transient OAuth/network failure cannot poison future probes.
if base_url and codex_source == "live":
save_context_length(model, base_url, codex_ctx)
return codex_ctx
if effective_provider == "gmi" and base_url:

125
tests/agent/test_model_metadata.py
View file

@ -431,11 +431,10 @@ class TestDefaultContextLengths:
# =========================================================================
class TestCodexOAuthContextLength:
"""ChatGPT Codex OAuth imposes lower context limits than the direct
OpenAI API for the same slugs. Verified Apr 2026 via live probe of
chatgpt.com/backend-api/codex/models: most models return 272k, while
models.dev reports 1.05M for gpt-5.5/gpt-5.4 and 400k for the rest.
(Known exception: gpt-5.3-codex-spark is 128k.)
"""ChatGPT Codex OAuth context windows come from the authenticated
/models catalogue and may differ from the static fallback table or the
direct OpenAI API allocation. The fallback values below are conservative
defaults used only when the live probe is unavailable.
"""
def setup_method(self):
@ -551,97 +550,95 @@ class TestCodexOAuthContextLength:
"leaked outside openai-codex provider"
)
def test_stale_codex_cache_over_400k_is_invalidated(self, tmp_path, monkeypatch):
"""Pre-PR #14935 builds cached gpt-5.5 at 1.05M (from models.dev)
before the Codex-aware branch existed. Upgrading users keep that
stale entry on disk and the cache-first lookup returns it forever.
Codex OAuth caps at 272k for every slug, so any cached Codex
entry >= 400k must be dropped and re-resolved via the live probe.
"""
def test_stale_codex_cache_is_bypassed_and_live_probe_wins(self, tmp_path, monkeypatch):
"""A stale Codex disk entry must not mask the authenticated catalogue."""
from agent import model_metadata as mm
# Isolate the cache file to tmp_path
cache_file = tmp_path / "context_length_cache.yaml"
monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file)
base_url = "https://chatgpt.com/backend-api/codex/"
stale_key = f"gpt-5.5@{base_url}"
base_url = "https://chatgpt.com/backend-api/codex"
stale_key = f"gpt-5.6-terra@{base_url}"
other_key = "other-model@https://api.openai.com/v1/"
import yaml as _yaml
cache_file.write_text(_yaml.dump({"context_lengths": {
stale_key: 1_050_000, # stale pre-fix value
other_key: 128_000, # unrelated, must survive
stale_key: 272_000,
other_key: 128_000,
}}))
fake_response = MagicMock()
fake_response.status_code = 200
fake_response.json.return_value = {
"models": [{"slug": "gpt-5.5", "context_window": 272_000}]
"models": [{"slug": "gpt-5.6-terra", "context_window": 372_000}]
}
with patch("agent.model_metadata.requests.get", return_value=fake_response) as mock_get:
ctx = mm.get_model_context_length(
model="gpt-5.6-terra",
base_url=base_url,
api_key="fake-token",
provider="openai-codex",
)
assert ctx == 372_000
mock_get.assert_called_once()
remaining = _yaml.safe_load(cache_file.read_text()).get("context_lengths", {})
assert remaining.get(stale_key) == 372_000
assert remaining.get(other_key) == 128_000
def test_codex_fallback_is_not_persisted(self, tmp_path, monkeypatch):
"""A failed live probe must not poison the persistent cache."""
from agent import model_metadata as mm
cache_file = tmp_path / "context_length_cache.yaml"
monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file)
base_url = "https://chatgpt.com/backend-api/codex"
fake_response = MagicMock()
fake_response.status_code = 401
fake_response.json.return_value = {}
with patch("agent.model_metadata.requests.get", return_value=fake_response), \
patch("agent.model_metadata.save_context_length") as mock_save:
ctx = mm.get_model_context_length(
model="gpt-5.5",
model="gpt-5.6-terra",
base_url=base_url,
api_key="fake-token",
api_key="expired-token",
provider="openai-codex",
)
assert ctx == 272_000, f"Stale entry should have been re-resolved to 272k, got {ctx}"
# Live save was called with the fresh value
mock_save.assert_called_with("gpt-5.5", base_url, 272_000)
# The stale entry was removed from disk; unrelated entries survived
remaining = _yaml.safe_load(cache_file.read_text()).get("context_lengths", {})
assert stale_key not in remaining, "Stale entry was not invalidated from the cache file"
assert remaining.get(other_key) == 128_000, "Unrelated cache entries must not be touched"
def test_fresh_codex_cache_under_400k_is_respected(self, tmp_path, monkeypatch):
"""Codex entries at the correct 272k must NOT be invalidated —
only stale pre-fix values (>= 400k) get dropped."""
from agent import model_metadata as mm
cache_file = tmp_path / "context_length_cache.yaml"
monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file)
base_url = "https://chatgpt.com/backend-api/codex/"
import yaml as _yaml
cache_file.write_text(_yaml.dump({"context_lengths": {
f"gpt-5.5@{base_url}": 272_000,
}}))
# If the invalidation incorrectly fired, this would be called; assert it isn't.
with patch("agent.model_metadata.requests.get") as mock_get:
ctx = mm.get_model_context_length(
model="gpt-5.5",
base_url=base_url,
api_key="fake-token",
provider="openai-codex",
)
assert ctx == 272_000
mock_get.assert_not_called()
mock_save.assert_not_called()
assert not cache_file.exists()
def test_stale_invalidation_scoped_to_codex_provider(self, tmp_path, monkeypatch):
"""A cached 1M entry for a non-Codex provider (e.g. Anthropic opus on
OpenRouter, legitimately 1M) must NOT be invalidated by this guard."""
def test_codex_cache_is_not_used_when_probe_fails(self, tmp_path, monkeypatch):
"""Even a previously live-looking Codex row must not suppress probing."""
from agent import model_metadata as mm
cache_file = tmp_path / "context_length_cache.yaml"
monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file)
base_url = "https://openrouter.ai/api/v1"
base_url = "https://chatgpt.com/backend-api/codex"
import yaml as _yaml
cache_file.write_text(_yaml.dump({"context_lengths": {
f"anthropic/claude-opus-4.6@{base_url}": 1_000_000,
f"gpt-5.6-terra@{base_url}": 372_000,
}}))
ctx = mm.get_model_context_length(
model="anthropic/claude-opus-4.6",
base_url=base_url,
api_key="fake",
provider="openrouter",
)
assert ctx == 1_000_000, "Non-codex 1M cache entries must be respected"
fake_response = MagicMock()
fake_response.status_code = 401
fake_response.json.return_value = {}
with patch("agent.model_metadata.requests.get", return_value=fake_response) as mock_get:
ctx = mm.get_model_context_length(
model="gpt-5.6-terra",
base_url=base_url,
api_key="expired-token",
provider="openai-codex",
)
assert ctx == 272_000
mock_get.assert_called_once()
remaining = _yaml.safe_load(cache_file.read_text()).get("context_lengths", {})
assert remaining.get(f"gpt-5.6-terra@{base_url}") == 372_000
# =========================================================================