test(codex-spark): add live-API regression and make picker test deterministic

Two follow-ups from self-review:

1. Add unit test for _fetch_models_from_api covering the live HTTP path.
   The salvaged PR #19530 dropped the supported_in_api:false filter in
   both _fetch_models_from_api and _read_cache_models, but only the
   cache path had a regression test. This adds the symmetric live-fetch
   test (mocked httpx) so a future drive-by change to the HTTP path
   can't silently re-introduce the filter.

2. Pin test_codex_picker_uses_live_codex_catalog to the cache fallback.
   The test wrote a fake JWT and a CODEX_HOME cache, but provider_model_ids
   ('openai-codex') still issued a real 10s HTTP probe to
   chatgpt.com/backend-api/codex/models before falling back to the cache.
   That made the test slow and non-deterministic in restricted/CI
   networks. Patch _fetch_models_from_api to return [] so we go straight
   to the cache path the test actually means to exercise.
This commit is contained in:
kshitijk4poor 2026-05-10 10:53:49 +05:30 committed by kshitij
parent 9ee9a4297d
commit 826e7171e9
2 changed files with 43 additions and 0 deletions

View file

@ -88,6 +88,13 @@ def test_codex_picker_uses_live_codex_catalog(hermes_auth_only_env, tmp_path, mo
]
}))
monkeypatch.setenv("CODEX_HOME", str(codex_home))
# Force the cache fallback path — without this the test issues a real
# 10s HTTP probe to chatgpt.com/backend-api/codex/models which is both
# slow and non-deterministic in CI/sandboxed environments.
monkeypatch.setattr(
"hermes_cli.codex_models._fetch_models_from_api",
lambda access_token: [],
)
providers = list_authenticated_providers(
current_provider="openai-codex",

View file

@ -74,6 +74,42 @@ def test_get_codex_model_ids_adds_forward_compat_models_from_templates(monkeypat
]
def test_fetch_from_api_keeps_supported_in_api_false_models(monkeypatch):
"""Regression: gpt-5.3-codex-spark is returned by the live Codex backend
with ``supported_in_api: false`` because it isn't in the public OpenAI
API. The Codex CLI / OAuth route still serves it for ChatGPT Pro
accounts, so we must not drop it on that flag. visibility=hidden is
the separate signal that *should* still filter entries out.
"""
import sys
from hermes_cli import codex_models
class _FakeResp:
status_code = 200
def json(self):
return {
"models": [
{"slug": "gpt-5.5", "priority": 0, "supported_in_api": True},
{"slug": "gpt-5.3-codex-spark", "priority": 7, "supported_in_api": False},
{"slug": "gpt-5-internal", "priority": 99, "visibility": "hidden"},
]
}
class _FakeHttpx:
@staticmethod
def get(url, headers=None, timeout=None):
return _FakeResp()
monkeypatch.setitem(sys.modules, "httpx", _FakeHttpx)
models = codex_models._fetch_models_from_api(access_token="tok")
assert "gpt-5.5" in models
assert "gpt-5.3-codex-spark" in models
assert "gpt-5-internal" not in models
def test_model_command_uses_runtime_access_token_for_codex_list(monkeypatch):
from hermes_cli.main import _model_flow_openai_codex