diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 8c56cdf3c6a8..2cbb62de17ae 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -581,8 +581,13 @@ def _skip_persistent_context_cache(base_url: str, provider: str) -> bool: LM Studio excludes caching because loaded context is transient — the user can reload the model with a different context_length at any time. - """ - return provider == "lmstudio" + + Codex OAuth excludes caching because its context window is account- and + entitlement-specific metadata supplied by the authenticated /models + endpoint. A fallback value written after a transient probe failure must + not prevent a later live probe from observing an updated allocation. + """ + return (provider or "").strip().lower() in {"lmstudio", "openai-codex"} def _maybe_cache_local_context_length( @@ -1974,27 +1979,31 @@ def _fetch_codex_oauth_context_lengths(access_token: str) -> Dict[str, int]: return result -def _resolve_codex_oauth_context_length( +def _resolve_codex_oauth_context_length_with_source( model: str, access_token: str = "" -) -> Optional[int]: +) -> Tuple[Optional[int], str]: """Resolve a Codex OAuth model's real context window. Prefers a live probe of chatgpt.com/backend-api/codex/models (when we have a bearer token), then falls back to ``_CODEX_OAUTH_CONTEXT_FALLBACK``. + + Returns ``(context_length, source)`` where source is ``"live"`` for a + value returned by the authenticated endpoint or ``"fallback"`` for the + static conservative table. Callers must not persist the latter. """ model_bare = _strip_provider_prefix(model).strip() if not model_bare: - return None + return None, "" if access_token: live = _fetch_codex_oauth_context_lengths(access_token) if model_bare in live: - return live[model_bare] + return live[model_bare], "live" # Case-insensitive match in case casing drifts model_lower = model_bare.lower() for slug, ctx in live.items(): if slug.lower() == model_lower: - return ctx + return ctx, "live" # Fallback: longest-key-first substring match over hardcoded defaults. model_lower = model_bare.lower() @@ -2002,9 +2011,19 @@ def _resolve_codex_oauth_context_length( _CODEX_OAUTH_CONTEXT_FALLBACK.items(), key=lambda x: len(x[0]), reverse=True ): if slug in model_lower: - return ctx + return ctx, "fallback" - return None + return None, "" + + +def _resolve_codex_oauth_context_length( + model: str, access_token: str = "" +) -> Optional[int]: + """Resolve a Codex OAuth model's context length (compatibility wrapper).""" + context_length, _source = _resolve_codex_oauth_context_length_with_source( + model, access_token=access_token, + ) + return context_length def _resolve_nous_context_length( @@ -2094,9 +2113,9 @@ def get_model_context_length( Resolution order: 0. Explicit config override (model.context_length or custom_providers per-model) 0c. Endpoint-scoped metadata for models validated on one multiplexed endpoint - 1. Persistent cache (previously discovered via probing). Nous URLs - bypass the cache here so step 5b can always reconcile against - the authoritative portal /v1/models response. + 1. Persistent cache (previously discovered via probing). Nous URLs, + LM Studio, and Codex OAuth bypass the cache here so their provider + metadata can be reconciled against the authoritative live source. 1b. AWS Bedrock static table (must precede custom-endpoint probe) 2. Active endpoint metadata (/models for explicit custom endpoints) 3. Local server query (for local endpoints) @@ -2196,24 +2215,13 @@ def get_model_context_length( # LM Studio is excluded — its loaded context length is transient (the # user can reload the model with a different context_length at any time # via /api/v1/models/load), so a stale cached value would mask reloads. + # Codex OAuth is excluded because the authenticated /models catalogue is + # account-specific and a fallback must never suppress later revalidation. if base_url and not _skip_persistent_context_cache(base_url, provider): cached = get_cached_context_length(model, base_url) if cached is not None: - # Invalidate stale Codex OAuth cache entries: pre-PR #14935 builds - # resolved gpt-5.x to the direct-API value (e.g. 1.05M) via - # models.dev and persisted it. Codex OAuth caps at 272K for every - # slug, so any cached Codex entry at or above 400K is a leftover - # from the old resolution path. Drop it and fall through to the - # live /models probe in step 5 below. - if provider == "openai-codex" and cached >= 400_000: - logger.info( - "Dropping stale Codex cache entry %s@%s -> %s (pre-fix value); " - "re-resolving via live /models probe", - model, base_url, f"{cached:,}", - ) - _invalidate_cached_context_length(model, base_url) # Invalidate stale 32k cache entries for Kimi-family models. - elif cached <= 32768 and _model_name_suggests_kimi(model): + if cached <= 32768 and _model_name_suggests_kimi(model): logger.info( "Dropping stale Kimi cache entry %s@%s -> %s (OpenRouter underreport); " "re-resolving via hardcoded defaults", @@ -2452,9 +2460,14 @@ def get_model_context_length( # Codex OAuth enforces lower context limits than the direct OpenAI # API for the same slug (e.g. gpt-5.5 is 1.05M on the API but 272K # on Codex). Authoritative source is Codex's own /models endpoint. - codex_ctx = _resolve_codex_oauth_context_length(model, access_token=api_key or "") + codex_ctx, codex_source = _resolve_codex_oauth_context_length_with_source( + model, access_token=api_key or "", + ) if codex_ctx: - if base_url: + # Only a successful authenticated catalogue response is safe to + # persist. The static fallback is deliberately runtime-only so a + # transient OAuth/network failure cannot poison future probes. + if base_url and codex_source == "live": save_context_length(model, base_url, codex_ctx) return codex_ctx if effective_provider == "gmi" and base_url: diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index 4b0b327682f2..c40e035e93d6 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -431,11 +431,10 @@ class TestDefaultContextLengths: # ========================================================================= class TestCodexOAuthContextLength: - """ChatGPT Codex OAuth imposes lower context limits than the direct - OpenAI API for the same slugs. Verified Apr 2026 via live probe of - chatgpt.com/backend-api/codex/models: most models return 272k, while - models.dev reports 1.05M for gpt-5.5/gpt-5.4 and 400k for the rest. - (Known exception: gpt-5.3-codex-spark is 128k.) + """ChatGPT Codex OAuth context windows come from the authenticated + /models catalogue and may differ from the static fallback table or the + direct OpenAI API allocation. The fallback values below are conservative + defaults used only when the live probe is unavailable. """ def setup_method(self): @@ -551,97 +550,95 @@ class TestCodexOAuthContextLength: "leaked outside openai-codex provider" ) - def test_stale_codex_cache_over_400k_is_invalidated(self, tmp_path, monkeypatch): - """Pre-PR #14935 builds cached gpt-5.5 at 1.05M (from models.dev) - before the Codex-aware branch existed. Upgrading users keep that - stale entry on disk and the cache-first lookup returns it forever. - Codex OAuth caps at 272k for every slug, so any cached Codex - entry >= 400k must be dropped and re-resolved via the live probe. - """ + def test_stale_codex_cache_is_bypassed_and_live_probe_wins(self, tmp_path, monkeypatch): + """A stale Codex disk entry must not mask the authenticated catalogue.""" from agent import model_metadata as mm - # Isolate the cache file to tmp_path cache_file = tmp_path / "context_length_cache.yaml" monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file) - base_url = "https://chatgpt.com/backend-api/codex/" - stale_key = f"gpt-5.5@{base_url}" + base_url = "https://chatgpt.com/backend-api/codex" + stale_key = f"gpt-5.6-terra@{base_url}" other_key = "other-model@https://api.openai.com/v1/" import yaml as _yaml cache_file.write_text(_yaml.dump({"context_lengths": { - stale_key: 1_050_000, # stale pre-fix value - other_key: 128_000, # unrelated, must survive + stale_key: 272_000, + other_key: 128_000, }})) fake_response = MagicMock() fake_response.status_code = 200 fake_response.json.return_value = { - "models": [{"slug": "gpt-5.5", "context_window": 272_000}] + "models": [{"slug": "gpt-5.6-terra", "context_window": 372_000}] } + with patch("agent.model_metadata.requests.get", return_value=fake_response) as mock_get: + ctx = mm.get_model_context_length( + model="gpt-5.6-terra", + base_url=base_url, + api_key="fake-token", + provider="openai-codex", + ) + + assert ctx == 372_000 + mock_get.assert_called_once() + remaining = _yaml.safe_load(cache_file.read_text()).get("context_lengths", {}) + assert remaining.get(stale_key) == 372_000 + assert remaining.get(other_key) == 128_000 + + def test_codex_fallback_is_not_persisted(self, tmp_path, monkeypatch): + """A failed live probe must not poison the persistent cache.""" + from agent import model_metadata as mm + + cache_file = tmp_path / "context_length_cache.yaml" + monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file) + base_url = "https://chatgpt.com/backend-api/codex" + + fake_response = MagicMock() + fake_response.status_code = 401 + fake_response.json.return_value = {} + with patch("agent.model_metadata.requests.get", return_value=fake_response), \ patch("agent.model_metadata.save_context_length") as mock_save: ctx = mm.get_model_context_length( - model="gpt-5.5", + model="gpt-5.6-terra", base_url=base_url, - api_key="fake-token", + api_key="expired-token", provider="openai-codex", ) - assert ctx == 272_000, f"Stale entry should have been re-resolved to 272k, got {ctx}" - # Live save was called with the fresh value - mock_save.assert_called_with("gpt-5.5", base_url, 272_000) - # The stale entry was removed from disk; unrelated entries survived - remaining = _yaml.safe_load(cache_file.read_text()).get("context_lengths", {}) - assert stale_key not in remaining, "Stale entry was not invalidated from the cache file" - assert remaining.get(other_key) == 128_000, "Unrelated cache entries must not be touched" - - def test_fresh_codex_cache_under_400k_is_respected(self, tmp_path, monkeypatch): - """Codex entries at the correct 272k must NOT be invalidated — - only stale pre-fix values (>= 400k) get dropped.""" - from agent import model_metadata as mm - - cache_file = tmp_path / "context_length_cache.yaml" - monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file) - - base_url = "https://chatgpt.com/backend-api/codex/" - import yaml as _yaml - cache_file.write_text(_yaml.dump({"context_lengths": { - f"gpt-5.5@{base_url}": 272_000, - }})) - - # If the invalidation incorrectly fired, this would be called; assert it isn't. - with patch("agent.model_metadata.requests.get") as mock_get: - ctx = mm.get_model_context_length( - model="gpt-5.5", - base_url=base_url, - api_key="fake-token", - provider="openai-codex", - ) assert ctx == 272_000 - mock_get.assert_not_called() + mock_save.assert_not_called() + assert not cache_file.exists() - def test_stale_invalidation_scoped_to_codex_provider(self, tmp_path, monkeypatch): - """A cached 1M entry for a non-Codex provider (e.g. Anthropic opus on - OpenRouter, legitimately 1M) must NOT be invalidated by this guard.""" + def test_codex_cache_is_not_used_when_probe_fails(self, tmp_path, monkeypatch): + """Even a previously live-looking Codex row must not suppress probing.""" from agent import model_metadata as mm cache_file = tmp_path / "context_length_cache.yaml" monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file) - - base_url = "https://openrouter.ai/api/v1" + base_url = "https://chatgpt.com/backend-api/codex" import yaml as _yaml cache_file.write_text(_yaml.dump({"context_lengths": { - f"anthropic/claude-opus-4.6@{base_url}": 1_000_000, + f"gpt-5.6-terra@{base_url}": 372_000, }})) - ctx = mm.get_model_context_length( - model="anthropic/claude-opus-4.6", - base_url=base_url, - api_key="fake", - provider="openrouter", - ) - assert ctx == 1_000_000, "Non-codex 1M cache entries must be respected" + fake_response = MagicMock() + fake_response.status_code = 401 + fake_response.json.return_value = {} + + with patch("agent.model_metadata.requests.get", return_value=fake_response) as mock_get: + ctx = mm.get_model_context_length( + model="gpt-5.6-terra", + base_url=base_url, + api_key="expired-token", + provider="openai-codex", + ) + + assert ctx == 272_000 + mock_get.assert_called_once() + remaining = _yaml.safe_load(cache_file.read_text()).get("context_lengths", {}) + assert remaining.get(f"gpt-5.6-terra@{base_url}") == 372_000 # =========================================================================