mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-18 14:52:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

fix(auxiliary): fall back when a route can't run the model at all (400 capability mismatch)

Browse source 
The salvaged context-window screen (#52392) skips fallback candidates that
are too small, and the rate-limit/403 fixes skip candidates that are at
capacity. A third hard failure remained uncovered: a fallback that builds a
client fine but returns a 400 because it structurally cannot run the model.
The canonical case is a configured openai-codex / ChatGPT-account fallback
asked to compress a glm-5.2 conversation:

    400 - {'detail': "The 'glm-5.2' model is not supported when using
    Codex with a ChatGPT account."}

This is a request-validation error, so should_fallback was False and the
explicit-provider gate blocked it — the auxiliary task (compression) aborted
every turn, dropping middle turns without a summary and churning the session,
which is exactly what destroys the prompt cache.

Adds _is_model_incompatible_error() (400 + capability phrasing, excluding
not-found and billing 400s which the sibling predicates own) and treats it as
a fallback-worthy capacity error in both sync and async call_llm, so the chain
skips the incapable route and continues to the next viable candidate.
This commit is contained in:
teknium1 2026-06-25 12:55:17 -07:00 • committed by Teknium
parent e4d026aa3b
commit 0d777453fa
2 changed files with 134 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

81
agent/auxiliary_client.py
View file

@ -2700,6 +2700,60 @@ def _is_model_not_found_error(exc: Exception) -> bool:
))
def _is_model_incompatible_error(exc: Exception) -> bool:
"""Detect "this route cannot serve this model" 400s (capability mismatch).
Distinct from :func:`_is_model_not_found_error` (the model does not exist
anywhere): here the model name is valid but the *current provider/account*
is structurally unable to run it. The canonical case is a configured
fallback that cannot run the main model — e.g. an ``openai-codex`` /
ChatGPT-account fallback asked to compress a ``glm-5.2`` conversation::
Error code: 400 - {'detail': "The 'glm-5.2' model is not supported
when using Codex with a ChatGPT account."}
The candidate authenticates fine and builds a client, so the auth and
payment predicates don't fire and the call would otherwise raise and
abort the whole auxiliary task (commonly compression — which then drops
middle turns and churns the session, destroying the prompt cache).
Treating it as a fallback-worthy capability error lets the chain skip the
incapable route and continue to the next candidate, mirroring the
context-window feasibility screen (#52392).
Billing/quota 400s belong to :func:`_is_payment_error`; "model does not
exist" 400s belong to :func:`_is_model_not_found_error`. This predicate
explicitly excludes both so the three don't overlap.
"""
status = getattr(exc, "status_code", None)
if status not in {400, None}:
return False
err_lower = str(exc).lower()
# Not-found 400s ("invalid model ID", "model does not exist") are owned by
# _is_model_not_found_error. Billing/free-tier 400s are owned by the
# payment path — key on the billing keywords directly here rather than
# calling _is_payment_error(), because that predicate is status-gated
# ({402,403,404,429,None}) and would not recognise a 400-coded billing
# body, letting it leak into this capability bucket.
if _is_model_not_found_error(exc):
return False
if any(kw in err_lower for kw in (
"credits", "insufficient funds", "billing", "out of funds",
"balance_depleted", "no usable credits", "payment required",
"free tier", "free-tier", "not available on the free tier",
"model_not_supported_on_free_tier", "quota",
)):
return False
return any(kw in err_lower for kw in (
"is not supported when using", # codex/ChatGPT-account model gating
"model is not supported",
"not supported with this",
"not supported for this account",
"model_not_supported",
"does not support this model",
"unsupported model",
))
def _evict_cached_clients(provider: str) -> None:
"""Drop cached auxiliary clients for a provider so fresh creds are used."""
normalized = _normalize_aux_provider(provider)
@ -5803,6 +5857,7 @@ def call_llm(
_is_payment_error(first_err)
or _is_connection_error(first_err)
or _is_rate_limit_error(first_err)
or _is_model_incompatible_error(first_err)
)
# Respect explicit provider choice for transient errors (auth, request
# validation, etc.) but allow fallback when the provider clearly cannot
@ -5815,7 +5870,17 @@ def call_llm(
# literally cannot serve this request regardless of user intent.
# Rate limits are included: after retries are exhausted, a 429 means
# the provider cannot serve this request — fall back. See #52228.
is_capacity_error = _is_payment_error(first_err) or _is_connection_error(first_err) or _is_rate_limit_error(first_err)
# Model-incompatibility 400s are also a hard capability mismatch (the
# route cannot run this model at all — e.g. a codex/ChatGPT-account
# fallback asked to compress a glm-5.2 conversation), so they bypass
# the explicit-provider gate and continue to the next candidate
# instead of aborting the auxiliary task and churning the session.
is_capacity_error = (
_is_payment_error(first_err)
or _is_connection_error(first_err)
or _is_rate_limit_error(first_err)
or _is_model_incompatible_error(first_err)
)
if should_fallback and (is_auto or is_capacity_error):
if _is_payment_error(first_err):
reason = "payment error"
@ -5828,6 +5893,8 @@ def call_llm(
)
elif _is_rate_limit_error(first_err):
reason = "rate limit"
elif _is_model_incompatible_error(first_err):
reason = "model incompatible with route"
else:
reason = "connection error"
logger.info("Auxiliary %s: %s on %s (%s), trying fallback",
@ -6266,14 +6333,22 @@ async def async_call_llm(
_is_payment_error(first_err)
or _is_connection_error(first_err)
or _is_rate_limit_error(first_err)
or _is_model_incompatible_error(first_err)
)
# Capacity errors (payment/quota/connection/rate-limit) bypass the
# explicit-provider gate — the provider cannot serve the request
# regardless of user intent. Rate limits are included: after retries
# are exhausted, a 429 means the provider is at capacity. See #52228.
# See #26803: daily token quota must fall back like a 402 credit error.
# Model-incompatibility 400s (route cannot run this model at all)
# bypass the gate too — see the sync call_llm() path for rationale.
is_auto = resolved_provider in {"auto", "", None}
is_capacity_error = _is_payment_error(first_err) or _is_connection_error(first_err) or _is_rate_limit_error(first_err)
is_capacity_error = (
_is_payment_error(first_err)
or _is_connection_error(first_err)
or _is_rate_limit_error(first_err)
or _is_model_incompatible_error(first_err)
)
if should_fallback and (is_auto or is_capacity_error):
if _is_payment_error(first_err):
reason = "payment error"
@ -6282,6 +6357,8 @@ async def async_call_llm(
)
elif _is_rate_limit_error(first_err):
reason = "rate limit"
elif _is_model_incompatible_error(first_err):
reason = "model incompatible with route"
else:
reason = "connection error"
logger.info("Auxiliary %s (async): %s on %s (%s), trying fallback",

55
tests/agent/test_auxiliary_client.py
View file

@ -23,6 +23,7 @@ from agent.auxiliary_client import (
_is_payment_error,
_is_rate_limit_error,
_is_model_not_found_error,
_is_model_incompatible_error,
_refresh_nous_recommended_model,
_normalize_aux_provider,
_try_payment_fallback,
@ -1542,6 +1543,60 @@ class TestIsModelNotFoundError:
assert _is_model_not_found_error(exc) is False
class TestIsModelIncompatibleError:
"""_is_model_incompatible_error detects 400s where the route cannot run
the model at all (capability mismatch), distinct from not-found and
payment errors."""
def test_codex_chatgpt_account_model_gating(self):
"""The exact incident: an openai-codex/ChatGPT-account fallback asked
to compress a glm-5.2 conversation."""
exc = Exception(
"Error code: 400 - {'detail': \"The 'glm-5.2' model is not "
"supported when using Codex with a ChatGPT account.\"}"
)
exc.status_code = 400
assert _is_model_incompatible_error(exc) is True
def test_model_is_not_supported_phrasing(self):
exc = Exception("This model is not supported for this endpoint")
exc.status_code = 400
assert _is_model_incompatible_error(exc) is True
def test_unsupported_model_keyword(self):
exc = Exception("unsupported model for this account tier")
exc.status_code = 400
assert _is_model_incompatible_error(exc) is True
def test_not_found_is_not_incompatible(self):
"""A model-does-not-exist 400 belongs to _is_model_not_found_error —
the two predicates must not overlap."""
exc = Exception("openrouter/foo/bar is not a valid model ID")
exc.status_code = 400
assert _is_model_incompatible_error(exc) is False
assert _is_model_not_found_error(exc) is True
def test_payment_400_is_not_incompatible(self):
"""A billing 400 that also contains capability-ish phrasing must be
rejected here — billing keywords win so the payment path owns it and
the two buckets don't overlap."""
exc = Exception("insufficient credits: model is not supported on free tier")
exc.status_code = 400
assert _is_model_incompatible_error(exc) is False
def test_wrong_status_is_not_incompatible(self):
exc = Exception("model is not supported") # right phrase, wrong status
exc.status_code = 500
assert _is_model_incompatible_error(exc) is False
def test_generic_400_is_not_incompatible(self):
"""A plain request-validation 400 without capability phrasing must not
trigger fallback (we respect explicit-provider choice for those)."""
exc = Exception("invalid value for parameter temperature")
exc.status_code = 400
assert _is_model_incompatible_error(exc) is False
class TestRefreshNousRecommendedModel:
"""_refresh_nous_recommended_model picks a fresh model after a stale 404."""