diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 1f406564def4..e46780d23372 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -2700,6 +2700,60 @@ def _is_model_not_found_error(exc: Exception) -> bool: )) +def _is_model_incompatible_error(exc: Exception) -> bool: + """Detect "this route cannot serve this model" 400s (capability mismatch). + + Distinct from :func:`_is_model_not_found_error` (the model does not exist + anywhere): here the model name is valid but the *current provider/account* + is structurally unable to run it. The canonical case is a configured + fallback that cannot run the main model — e.g. an ``openai-codex`` / + ChatGPT-account fallback asked to compress a ``glm-5.2`` conversation:: + + Error code: 400 - {'detail': "The 'glm-5.2' model is not supported + when using Codex with a ChatGPT account."} + + The candidate authenticates fine and builds a client, so the auth and + payment predicates don't fire and the call would otherwise raise and + abort the whole auxiliary task (commonly compression — which then drops + middle turns and churns the session, destroying the prompt cache). + Treating it as a fallback-worthy capability error lets the chain skip the + incapable route and continue to the next candidate, mirroring the + context-window feasibility screen (#52392). + + Billing/quota 400s belong to :func:`_is_payment_error`; "model does not + exist" 400s belong to :func:`_is_model_not_found_error`. This predicate + explicitly excludes both so the three don't overlap. + """ + status = getattr(exc, "status_code", None) + if status not in {400, None}: + return False + err_lower = str(exc).lower() + # Not-found 400s ("invalid model ID", "model does not exist") are owned by + # _is_model_not_found_error. Billing/free-tier 400s are owned by the + # payment path — key on the billing keywords directly here rather than + # calling _is_payment_error(), because that predicate is status-gated + # ({402,403,404,429,None}) and would not recognise a 400-coded billing + # body, letting it leak into this capability bucket. + if _is_model_not_found_error(exc): + return False + if any(kw in err_lower for kw in ( + "credits", "insufficient funds", "billing", "out of funds", + "balance_depleted", "no usable credits", "payment required", + "free tier", "free-tier", "not available on the free tier", + "model_not_supported_on_free_tier", "quota", + )): + return False + return any(kw in err_lower for kw in ( + "is not supported when using", # codex/ChatGPT-account model gating + "model is not supported", + "not supported with this", + "not supported for this account", + "model_not_supported", + "does not support this model", + "unsupported model", + )) + + def _evict_cached_clients(provider: str) -> None: """Drop cached auxiliary clients for a provider so fresh creds are used.""" normalized = _normalize_aux_provider(provider) @@ -5803,6 +5857,7 @@ def call_llm( _is_payment_error(first_err) or _is_connection_error(first_err) or _is_rate_limit_error(first_err) + or _is_model_incompatible_error(first_err) ) # Respect explicit provider choice for transient errors (auth, request # validation, etc.) but allow fallback when the provider clearly cannot @@ -5815,7 +5870,17 @@ def call_llm( # literally cannot serve this request regardless of user intent. # Rate limits are included: after retries are exhausted, a 429 means # the provider cannot serve this request — fall back. See #52228. - is_capacity_error = _is_payment_error(first_err) or _is_connection_error(first_err) or _is_rate_limit_error(first_err) + # Model-incompatibility 400s are also a hard capability mismatch (the + # route cannot run this model at all — e.g. a codex/ChatGPT-account + # fallback asked to compress a glm-5.2 conversation), so they bypass + # the explicit-provider gate and continue to the next candidate + # instead of aborting the auxiliary task and churning the session. + is_capacity_error = ( + _is_payment_error(first_err) + or _is_connection_error(first_err) + or _is_rate_limit_error(first_err) + or _is_model_incompatible_error(first_err) + ) if should_fallback and (is_auto or is_capacity_error): if _is_payment_error(first_err): reason = "payment error" @@ -5828,6 +5893,8 @@ def call_llm( ) elif _is_rate_limit_error(first_err): reason = "rate limit" + elif _is_model_incompatible_error(first_err): + reason = "model incompatible with route" else: reason = "connection error" logger.info("Auxiliary %s: %s on %s (%s), trying fallback", @@ -6266,14 +6333,22 @@ async def async_call_llm( _is_payment_error(first_err) or _is_connection_error(first_err) or _is_rate_limit_error(first_err) + or _is_model_incompatible_error(first_err) ) # Capacity errors (payment/quota/connection/rate-limit) bypass the # explicit-provider gate — the provider cannot serve the request # regardless of user intent. Rate limits are included: after retries # are exhausted, a 429 means the provider is at capacity. See #52228. # See #26803: daily token quota must fall back like a 402 credit error. + # Model-incompatibility 400s (route cannot run this model at all) + # bypass the gate too — see the sync call_llm() path for rationale. is_auto = resolved_provider in {"auto", "", None} - is_capacity_error = _is_payment_error(first_err) or _is_connection_error(first_err) or _is_rate_limit_error(first_err) + is_capacity_error = ( + _is_payment_error(first_err) + or _is_connection_error(first_err) + or _is_rate_limit_error(first_err) + or _is_model_incompatible_error(first_err) + ) if should_fallback and (is_auto or is_capacity_error): if _is_payment_error(first_err): reason = "payment error" @@ -6282,6 +6357,8 @@ async def async_call_llm( ) elif _is_rate_limit_error(first_err): reason = "rate limit" + elif _is_model_incompatible_error(first_err): + reason = "model incompatible with route" else: reason = "connection error" logger.info("Auxiliary %s (async): %s on %s (%s), trying fallback", diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 0c5c43973817..527c3bf0b2c9 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -23,6 +23,7 @@ from agent.auxiliary_client import ( _is_payment_error, _is_rate_limit_error, _is_model_not_found_error, + _is_model_incompatible_error, _refresh_nous_recommended_model, _normalize_aux_provider, _try_payment_fallback, @@ -1542,6 +1543,60 @@ class TestIsModelNotFoundError: assert _is_model_not_found_error(exc) is False +class TestIsModelIncompatibleError: + """_is_model_incompatible_error detects 400s where the route cannot run + the model at all (capability mismatch), distinct from not-found and + payment errors.""" + + def test_codex_chatgpt_account_model_gating(self): + """The exact incident: an openai-codex/ChatGPT-account fallback asked + to compress a glm-5.2 conversation.""" + exc = Exception( + "Error code: 400 - {'detail': \"The 'glm-5.2' model is not " + "supported when using Codex with a ChatGPT account.\"}" + ) + exc.status_code = 400 + assert _is_model_incompatible_error(exc) is True + + def test_model_is_not_supported_phrasing(self): + exc = Exception("This model is not supported for this endpoint") + exc.status_code = 400 + assert _is_model_incompatible_error(exc) is True + + def test_unsupported_model_keyword(self): + exc = Exception("unsupported model for this account tier") + exc.status_code = 400 + assert _is_model_incompatible_error(exc) is True + + def test_not_found_is_not_incompatible(self): + """A model-does-not-exist 400 belongs to _is_model_not_found_error — + the two predicates must not overlap.""" + exc = Exception("openrouter/foo/bar is not a valid model ID") + exc.status_code = 400 + assert _is_model_incompatible_error(exc) is False + assert _is_model_not_found_error(exc) is True + + def test_payment_400_is_not_incompatible(self): + """A billing 400 that also contains capability-ish phrasing must be + rejected here — billing keywords win so the payment path owns it and + the two buckets don't overlap.""" + exc = Exception("insufficient credits: model is not supported on free tier") + exc.status_code = 400 + assert _is_model_incompatible_error(exc) is False + + def test_wrong_status_is_not_incompatible(self): + exc = Exception("model is not supported") # right phrase, wrong status + exc.status_code = 500 + assert _is_model_incompatible_error(exc) is False + + def test_generic_400_is_not_incompatible(self): + """A plain request-validation 400 without capability phrasing must not + trigger fallback (we respect explicit-provider choice for those).""" + exc = Exception("invalid value for parameter temperature") + exc.status_code = 400 + assert _is_model_incompatible_error(exc) is False + + class TestRefreshNousRecommendedModel: """_refresh_nous_recommended_model picks a fresh model after a stale 404."""