From 826e7171e97a97215785a517491705f7c695e4f2 Mon Sep 17 00:00:00 2001
From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com>
Date: Sun, 10 May 2026 10:53:49 +0530
Subject: [PATCH] test(codex-spark): add live-API regression and make picker
 test deterministic

Two follow-ups from self-review:

1. Add unit test for _fetch_models_from_api covering the live HTTP path.
   The salvaged PR #19530 dropped the supported_in_api:false filter in
   both _fetch_models_from_api and _read_cache_models, but only the
   cache path had a regression test. This adds the symmetric live-fetch
   test (mocked httpx) so a future drive-by change to the HTTP path
   can't silently re-introduce the filter.

2. Pin test_codex_picker_uses_live_codex_catalog to the cache fallback.
   The test wrote a fake JWT and a CODEX_HOME cache, but provider_model_ids
   ('openai-codex') still issued a real 10s HTTP probe to
   chatgpt.com/backend-api/codex/models before falling back to the cache.
   That made the test slow and non-deterministic in restricted/CI
   networks. Patch _fetch_models_from_api to return [] so we go straight
   to the cache path the test actually means to exercise.
---
 .../hermes_cli/test_codex_cli_model_picker.py |  7 ++++
 tests/hermes_cli/test_codex_models.py         | 36 +++++++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/tests/hermes_cli/test_codex_cli_model_picker.py b/tests/hermes_cli/test_codex_cli_model_picker.py
index 0143297b308..4edbef2dea0 100644
--- a/tests/hermes_cli/test_codex_cli_model_picker.py
+++ b/tests/hermes_cli/test_codex_cli_model_picker.py
@@ -88,6 +88,13 @@ def test_codex_picker_uses_live_codex_catalog(hermes_auth_only_env, tmp_path, mo
         ]
     }))
     monkeypatch.setenv("CODEX_HOME", str(codex_home))
+    # Force the cache fallback path — without this the test issues a real
+    # 10s HTTP probe to chatgpt.com/backend-api/codex/models which is both
+    # slow and non-deterministic in CI/sandboxed environments.
+    monkeypatch.setattr(
+        "hermes_cli.codex_models._fetch_models_from_api",
+        lambda access_token: [],
+    )
 
     providers = list_authenticated_providers(
         current_provider="openai-codex",
diff --git a/tests/hermes_cli/test_codex_models.py b/tests/hermes_cli/test_codex_models.py
index 41cbfeb058c..c1e92df755a 100644
--- a/tests/hermes_cli/test_codex_models.py
+++ b/tests/hermes_cli/test_codex_models.py
@@ -74,6 +74,42 @@ def test_get_codex_model_ids_adds_forward_compat_models_from_templates(monkeypat
     ]
 
 
+def test_fetch_from_api_keeps_supported_in_api_false_models(monkeypatch):
+    """Regression: gpt-5.3-codex-spark is returned by the live Codex backend
+    with ``supported_in_api: false`` because it isn't in the public OpenAI
+    API. The Codex CLI / OAuth route still serves it for ChatGPT Pro
+    accounts, so we must not drop it on that flag. visibility=hidden is
+    the separate signal that *should* still filter entries out.
+    """
+    import sys
+    from hermes_cli import codex_models
+
+    class _FakeResp:
+        status_code = 200
+
+        def json(self):
+            return {
+                "models": [
+                    {"slug": "gpt-5.5", "priority": 0, "supported_in_api": True},
+                    {"slug": "gpt-5.3-codex-spark", "priority": 7, "supported_in_api": False},
+                    {"slug": "gpt-5-internal", "priority": 99, "visibility": "hidden"},
+                ]
+            }
+
+    class _FakeHttpx:
+        @staticmethod
+        def get(url, headers=None, timeout=None):
+            return _FakeResp()
+
+    monkeypatch.setitem(sys.modules, "httpx", _FakeHttpx)
+
+    models = codex_models._fetch_models_from_api(access_token="tok")
+
+    assert "gpt-5.5" in models
+    assert "gpt-5.3-codex-spark" in models
+    assert "gpt-5-internal" not in models
+
+
 def test_model_command_uses_runtime_access_token_for_codex_list(monkeypatch):
     from hermes_cli.main import _model_flow_openai_codex