mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
Add distinct, greppable log lines at both fix sites so the condition is observable in the field instead of only inferable from the absence of the old 'Cannot compress further' spiral: - chat_completion_helpers: warn when an empty partial-stream stub is replaced with the placeholder (0 chars recovered, no tool call). - error_classifier: warn when a malformed-body 400 is classified as format_error rather than context overflow, with num_messages/approx_tokens. Extend the litellm-shape classifier test to assert the warning is emitted.
2329 lines
101 KiB
Python
2329 lines
101 KiB
Python
"""Tests for agent.error_classifier — structured API error classification."""
|
|
|
|
import pytest
|
|
from agent.error_classifier import (
|
|
ClassifiedError,
|
|
FailoverReason,
|
|
classify_api_error,
|
|
_extract_status_code,
|
|
_extract_error_body,
|
|
_extract_error_code,
|
|
_classify_402,
|
|
)
|
|
|
|
|
|
# ── Helper: mock API errors ────────────────────────────────────────────
|
|
|
|
class MockAPIError(Exception):
|
|
"""Simulates an OpenAI SDK APIStatusError."""
|
|
def __init__(self, message, status_code=None, body=None):
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.body = body or {}
|
|
|
|
|
|
class MockTransportError(Exception):
|
|
"""Simulates a transport-level error with a specific type name."""
|
|
pass
|
|
|
|
|
|
class ReadTimeout(MockTransportError):
|
|
pass
|
|
|
|
|
|
class ConnectError(MockTransportError):
|
|
pass
|
|
|
|
|
|
class RemoteProtocolError(MockTransportError):
|
|
pass
|
|
|
|
|
|
class ServerDisconnectedError(MockTransportError):
|
|
pass
|
|
|
|
|
|
# ── Test: FailoverReason enum ──────────────────────────────────────────
|
|
|
|
class TestFailoverReason:
|
|
def test_all_reasons_have_string_values(self):
|
|
for reason in FailoverReason:
|
|
assert isinstance(reason.value, str)
|
|
|
|
def test_enum_members_exist(self):
|
|
expected = {
|
|
"auth", "auth_permanent", "billing", "rate_limit",
|
|
"upstream_rate_limit",
|
|
"overloaded", "server_error", "timeout",
|
|
"ssl_cert_verification",
|
|
"context_overflow", "payload_too_large", "image_too_large",
|
|
"model_not_found", "format_error",
|
|
"invalid_encrypted_content",
|
|
"multimodal_tool_content_unsupported",
|
|
"provider_policy_blocked",
|
|
"content_policy_blocked",
|
|
"thinking_signature", "long_context_tier",
|
|
"oauth_long_context_beta_forbidden",
|
|
"llama_cpp_grammar_pattern",
|
|
"unknown",
|
|
}
|
|
actual = {r.value for r in FailoverReason}
|
|
assert expected == actual
|
|
|
|
|
|
# ── Test: ClassifiedError ──────────────────────────────────────────────
|
|
|
|
class TestClassifiedError:
|
|
def test_is_auth_property(self):
|
|
e1 = ClassifiedError(reason=FailoverReason.auth)
|
|
assert e1.is_auth is True
|
|
|
|
e2 = ClassifiedError(reason=FailoverReason.auth_permanent)
|
|
assert e2.is_auth is True
|
|
|
|
e3 = ClassifiedError(reason=FailoverReason.billing)
|
|
assert e3.is_auth is False
|
|
|
|
def test_defaults(self):
|
|
e = ClassifiedError(reason=FailoverReason.unknown)
|
|
assert e.retryable is True
|
|
assert e.should_compress is False
|
|
assert e.should_rotate_credential is False
|
|
assert e.should_fallback is False
|
|
assert e.status_code is None
|
|
assert e.message == ""
|
|
|
|
|
|
# ── Test: Status code extraction ───────────────────────────────────────
|
|
|
|
class TestExtractStatusCode:
|
|
def test_from_status_code_attr(self):
|
|
e = MockAPIError("fail", status_code=429)
|
|
assert _extract_status_code(e) == 429
|
|
|
|
def test_from_status_attr(self):
|
|
class ErrWithStatus(Exception):
|
|
status = 503
|
|
assert _extract_status_code(ErrWithStatus()) == 503
|
|
|
|
def test_from_cause_chain(self):
|
|
inner = MockAPIError("inner", status_code=401)
|
|
outer = Exception("outer")
|
|
outer.__cause__ = inner
|
|
assert _extract_status_code(outer) == 401
|
|
|
|
def test_none_when_missing(self):
|
|
assert _extract_status_code(Exception("generic")) is None
|
|
|
|
def test_rejects_non_http_status(self):
|
|
"""Integers outside 100-599 on .status should be ignored."""
|
|
class ErrWeirdStatus(Exception):
|
|
status = 42
|
|
assert _extract_status_code(ErrWeirdStatus()) is None
|
|
|
|
|
|
# ── Test: Error body extraction ────────────────────────────────────────
|
|
|
|
class TestExtractErrorBody:
|
|
def test_from_body_attr(self):
|
|
e = MockAPIError("fail", body={"error": {"message": "bad"}})
|
|
assert _extract_error_body(e) == {"error": {"message": "bad"}}
|
|
|
|
def test_from_cause_chain_body_attr(self):
|
|
inner = MockAPIError(
|
|
"inner",
|
|
status_code=402,
|
|
body={"error": {"message": "Usage limit reached, try again in 5 minutes"}},
|
|
)
|
|
outer = Exception("outer")
|
|
outer.__cause__ = inner
|
|
assert _extract_error_body(outer) == {
|
|
"error": {"message": "Usage limit reached, try again in 5 minutes"},
|
|
}
|
|
|
|
def test_empty_when_no_body(self):
|
|
assert _extract_error_body(Exception("generic")) == {}
|
|
|
|
|
|
# ── Test: Error code extraction ────────────────────────────────────────
|
|
|
|
class TestExtractErrorCode:
|
|
def test_from_nested_error_code(self):
|
|
body = {"error": {"code": "rate_limit_exceeded"}}
|
|
assert _extract_error_code(body) == "rate_limit_exceeded"
|
|
|
|
def test_from_nested_error_type(self):
|
|
body = {"error": {"type": "invalid_request_error"}}
|
|
assert _extract_error_code(body) == "invalid_request_error"
|
|
|
|
def test_from_top_level_code(self):
|
|
body = {"code": "model_not_found"}
|
|
assert _extract_error_code(body) == "model_not_found"
|
|
|
|
def test_from_wrapped_json_message(self):
|
|
body = {
|
|
"error": {
|
|
"message": (
|
|
'{"error":{"message":"The encrypted content for item rs_001 could not be verified. '
|
|
'Reason: Encrypted content could not be decrypted or parsed.",'
|
|
'"type":"invalid_request_error","param":"","code":"invalid_encrypted_content"}}'
|
|
),
|
|
"type": "400",
|
|
}
|
|
}
|
|
assert _extract_error_code(body) == "invalid_encrypted_content"
|
|
|
|
def test_empty_when_no_code(self):
|
|
assert _extract_error_code({}) == ""
|
|
assert _extract_error_code({"error": {"message": "oops"}}) == ""
|
|
|
|
|
|
# ── Test: 402 disambiguation ───────────────────────────────────────────
|
|
|
|
class TestClassify402:
|
|
"""The critical 402 billing vs rate_limit disambiguation."""
|
|
|
|
def test_billing_exhaustion(self):
|
|
"""Plain 402 = billing."""
|
|
result = _classify_402(
|
|
"payment required",
|
|
lambda reason, **kw: ClassifiedError(reason=reason, **kw),
|
|
)
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.should_rotate_credential is True
|
|
|
|
def test_transient_usage_limit(self):
|
|
"""402 with 'usage limit' + 'try again' = rate limit, not billing."""
|
|
result = _classify_402(
|
|
"usage limit exceeded. try again in 5 minutes",
|
|
lambda reason, **kw: ClassifiedError(reason=reason, **kw),
|
|
)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_rotate_credential is True
|
|
|
|
def test_quota_with_retry(self):
|
|
"""402 with 'quota' + 'retry' = rate limit."""
|
|
result = _classify_402(
|
|
"quota exceeded, please retry after the window resets",
|
|
lambda reason, **kw: ClassifiedError(reason=reason, **kw),
|
|
)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_quota_without_retry(self):
|
|
"""402 with just 'quota' but no transient signal = billing."""
|
|
result = _classify_402(
|
|
"quota exceeded",
|
|
lambda reason, **kw: ClassifiedError(reason=reason, **kw),
|
|
)
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
def test_insufficient_credits(self):
|
|
result = _classify_402(
|
|
"insufficient credits to complete request",
|
|
lambda reason, **kw: ClassifiedError(reason=reason, **kw),
|
|
)
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
|
|
# ── Test: Full classification pipeline ─────────────────────────────────
|
|
|
|
class TestClassifyApiError:
|
|
"""End-to-end classification tests."""
|
|
|
|
# ── Auth errors ──
|
|
|
|
def test_401_classified_as_auth(self):
|
|
e = MockAPIError("Unauthorized", status_code=401)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.auth
|
|
assert result.should_rotate_credential is True
|
|
# 401 is non-retryable on its own — credential rotation runs
|
|
# before the retryability check in the agent loop.
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
def test_403_classified_as_auth(self):
|
|
e = MockAPIError("Forbidden", status_code=403)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.auth
|
|
assert result.should_fallback is True
|
|
|
|
def test_403_key_limit_classified_as_billing(self):
|
|
"""OpenRouter 403 'key limit exceeded' is billing, not auth."""
|
|
e = MockAPIError("Key limit exceeded for this key", status_code=403)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.should_rotate_credential is True
|
|
assert result.should_fallback is True
|
|
|
|
def test_403_spending_limit_classified_as_billing(self):
|
|
e = MockAPIError("spending limit reached", status_code=403)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
def test_xai_403_structured_spending_limit_code_classified_as_billing(self):
|
|
"""xAI reports exhausted Grok credits as a provider-specific 403 code."""
|
|
e = MockAPIError(
|
|
"Error code: 403",
|
|
status_code=403,
|
|
body={
|
|
"code": "personal-team-blocked:spending-limit",
|
|
"error": (
|
|
"You have run out of credits or need a Grok subscription. "
|
|
"Add credits at Grok or upgrade at Grok."
|
|
),
|
|
},
|
|
)
|
|
|
|
result = classify_api_error(e, provider="xai-oauth")
|
|
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.retryable is False
|
|
assert result.should_rotate_credential is True
|
|
assert result.should_fallback is True
|
|
|
|
def test_non_xai_403_generic_billing_code_remains_auth(self):
|
|
"""Do not broaden generic providers' historical structured-403 behavior."""
|
|
e = MockAPIError(
|
|
"Error code: 403",
|
|
status_code=403,
|
|
body={"code": "insufficient_quota", "error": "Forbidden"},
|
|
)
|
|
|
|
result = classify_api_error(e, provider="openrouter")
|
|
|
|
assert result.reason == FailoverReason.auth
|
|
assert result.should_rotate_credential is False
|
|
|
|
# ── Billing ──
|
|
|
|
def test_402_plain_billing(self):
|
|
e = MockAPIError("Payment Required", status_code=402)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.retryable is False
|
|
|
|
def test_402_out_of_funds_billing(self):
|
|
e = MockAPIError(
|
|
"Payment Required",
|
|
status_code=402,
|
|
body={
|
|
"status": 402,
|
|
"message": (
|
|
"Your API key has run out of funds. Please go visit the "
|
|
"portal to sort that out: https://portal.nousresearch.com"
|
|
),
|
|
},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.retryable is False
|
|
|
|
def test_402_transient_usage_limit(self):
|
|
e = MockAPIError("usage limit exceeded, try again later", status_code=402)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.retryable is True
|
|
|
|
def test_403_plan_entitlement_billing(self):
|
|
e = MockAPIError("This plan does not include the requested model", status_code=403)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.retryable is False
|
|
|
|
def test_404_free_tier_model_block_is_billing(self):
|
|
e = MockAPIError(
|
|
"Not Found",
|
|
status_code=404,
|
|
body={
|
|
"status": 404,
|
|
"message": (
|
|
"Model 'gpt-5' is not available on the Free Tier. "
|
|
"Upgrade at https://portal.nousresearch.com or pick a free model."
|
|
),
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="nous", model="gpt-5")
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
def test_wrapped_402_uses_nested_body_message(self):
|
|
inner = MockAPIError(
|
|
"inner",
|
|
status_code=402,
|
|
body={"error": {"message": "Usage limit reached, try again in 5 minutes"}},
|
|
)
|
|
outer = Exception("outer")
|
|
outer.__cause__ = inner
|
|
|
|
result = classify_api_error(outer)
|
|
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.retryable is True
|
|
assert result.message == "Usage limit reached, try again in 5 minutes"
|
|
|
|
# ── Rate limit ──
|
|
|
|
def test_429_rate_limit(self):
|
|
e = MockAPIError("Too Many Requests", status_code=429)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_fallback is True
|
|
|
|
def test_alibaba_rate_increased_too_quickly(self):
|
|
"""Alibaba/DashScope returns a unique throttling message.
|
|
|
|
Port from anomalyco/opencode#21355.
|
|
"""
|
|
msg = (
|
|
"Upstream error from Alibaba: Request rate increased too quickly. "
|
|
"To ensure system stability, please adjust your client logic to "
|
|
"scale requests more smoothly over time."
|
|
)
|
|
e = MockAPIError(msg, status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.retryable is True
|
|
assert result.should_rotate_credential is True
|
|
|
|
# ── Server errors ──
|
|
|
|
def test_500_server_error(self):
|
|
e = MockAPIError("Internal Server Error", status_code=500)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.server_error
|
|
assert result.retryable is True
|
|
|
|
def test_502_server_error(self):
|
|
e = MockAPIError("Bad Gateway", status_code=502)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.server_error
|
|
|
|
def test_503_overloaded(self):
|
|
e = MockAPIError("Service Unavailable", status_code=503)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.overloaded
|
|
|
|
def test_529_anthropic_overloaded(self):
|
|
e = MockAPIError("Overloaded", status_code=529)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.overloaded
|
|
|
|
def test_408_request_timeout_is_retryable_timeout(self):
|
|
"""HTTP 408 Request Timeout is a transient timing failure the server
|
|
itself flags as safe to retry (RFC 9110 §15.5.9) — commonly emitted by
|
|
reverse proxies in front of self-hosted backends (llama.cpp / Ollama /
|
|
vLLM) when a long generation outruns the proxy's request-read window.
|
|
It must NOT fall into the generic 4xx bucket as a non-retryable
|
|
format_error, which would abort the turn on a retry-safe error."""
|
|
e = MockAPIError("Request Timeout", status_code=408)
|
|
result = classify_api_error(e, provider="vllm")
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_400_bad_request_still_non_retryable_format_error(self):
|
|
"""Guard the boundary: a genuine 400 Bad Request must remain a
|
|
non-retryable format_error and must not be swept up by the 408 branch."""
|
|
e = MockAPIError("Bad Request", status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
|
|
def test_message_only_overloaded_without_status_is_overloaded(self):
|
|
"""Some Anthropic-compatible proxies surface 'overloaded' in the
|
|
message with no 503/529 status_code. It must classify as overloaded
|
|
(transient backoff+retry), not unknown / credential rotation. (#14261)"""
|
|
e = MockAPIError(
|
|
"Anthropic API error: Overloaded - the service is temporarily overloaded"
|
|
) # no status_code
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.overloaded
|
|
assert result.retryable is True
|
|
assert result.should_rotate_credential is False
|
|
|
|
def test_429_with_overloaded_body_is_overloaded_not_rate_limit(self):
|
|
"""Z.AI / Zhipu reuse HTTP 429 for server-wide overload. The credential
|
|
is valid — the server is just busy — so it must classify as overloaded
|
|
(back off + retry the same key), NOT rate_limit (which would rotate and
|
|
exhaust the pool, doing nothing for a single-key user). (#14038)"""
|
|
e = MockAPIError(
|
|
"The service may be temporarily overloaded, please try again later",
|
|
status_code=429,
|
|
)
|
|
result = classify_api_error(e, provider="zai")
|
|
assert result.reason == FailoverReason.overloaded
|
|
assert result.retryable is True
|
|
assert result.should_rotate_credential is False
|
|
|
|
def test_429_normal_rate_limit_still_rotates(self):
|
|
"""Guard: a genuine 429 rate limit (no overload language) must still
|
|
classify as rate_limit and rotate the credential. (#14038)"""
|
|
e = MockAPIError(
|
|
"Rate limit exceeded: too many requests", status_code=429
|
|
)
|
|
result = classify_api_error(e, provider="zai")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_rotate_credential is True
|
|
|
|
# ── 5xx that are actually request-validation errors ──
|
|
# Some OpenAI-compatible gateways (e.g. codex.nekos.me) return
|
|
# request-validation failures with a 5xx status. These are
|
|
# deterministic, so they must NOT be retried — otherwise the retry
|
|
# loop hammers the identical bad request into a flood.
|
|
|
|
def test_502_with_unknown_parameter_is_non_retryable(self):
|
|
e = MockAPIError(
|
|
"Unknown parameter: 'input[617]._empty_recovery_synthetic'",
|
|
status_code=502,
|
|
body={
|
|
"error": {
|
|
"type": "invalid_request_error",
|
|
"message": (
|
|
"[ObjectParam] [input[617]._empty_recovery_synthetic] "
|
|
"[unknown_parameter] Unknown parameter: "
|
|
"'input[617]._empty_recovery_synthetic'."
|
|
),
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
def test_502_with_unsupported_parameter_is_non_retryable(self):
|
|
e = MockAPIError(
|
|
"Unsupported parameter: logprobs",
|
|
status_code=502,
|
|
body={
|
|
"error": {
|
|
"type": "invalid_request_error",
|
|
"message": "Unsupported parameter: logprobs",
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
|
|
def test_500_with_invalid_request_error_type_is_non_retryable(self):
|
|
e = MockAPIError(
|
|
"bad request",
|
|
status_code=500,
|
|
body={"error": {"type": "invalid_request_error", "message": "bad request"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
|
|
def test_502_plain_bad_gateway_still_retryable(self):
|
|
"""A genuine 502 with no request-validation signal stays retryable."""
|
|
e = MockAPIError("Bad Gateway", status_code=502)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.server_error
|
|
assert result.retryable is True
|
|
|
|
# ── 5xx that are actually context overflow ──
|
|
# Some local inference servers (llama.cpp / llama-server, and vLLM/Ollama
|
|
# behind a Cloudflare/Tailscale hop) report context overflow with a 5xx
|
|
# status instead of the standard 400/413. These must route into the
|
|
# compression-and-retry path, not the blind server_error/overloaded retry
|
|
# that exhausts and drops the turn.
|
|
|
|
@pytest.mark.parametrize("status_code", [500, 502, 503, 529])
|
|
def test_5xx_context_overflow_routes_to_compression(self, status_code):
|
|
"""Explicit context-overflow wording on any of the codes the fix covers
|
|
(500/502/503/529) must route to context_overflow + compression, not a
|
|
blind server_error/overloaded retry. Covers all four branches the code
|
|
touches (the original PR only asserted 500 and 503)."""
|
|
e = MockAPIError(
|
|
"Context size has been exceeded.",
|
|
status_code=status_code,
|
|
body={"error": {"code": status_code, "message": "Context size has been exceeded.", "type": "server_error"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
assert result.retryable is True
|
|
|
|
def test_500_plain_server_error_not_compressed(self):
|
|
"""A genuine 500 crash without overflow wording must NOT be swallowed
|
|
into compression — it stays a retryable server_error."""
|
|
e = MockAPIError("Internal Server Error", status_code=500)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.server_error
|
|
assert result.should_compress is False
|
|
|
|
def test_503_plain_overloaded_not_compressed(self):
|
|
"""A genuine 503 overload without overflow wording stays overloaded."""
|
|
e = MockAPIError("Service Unavailable", status_code=503)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.overloaded
|
|
assert result.should_compress is False
|
|
|
|
# ── Model not found ──
|
|
|
|
def test_404_model_not_found(self):
|
|
e = MockAPIError("model not found", status_code=404)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.model_not_found
|
|
assert result.should_fallback is True
|
|
assert result.retryable is False
|
|
|
|
def test_404_generic(self):
|
|
# Generic 404 with no "model not found" signal — common for local
|
|
# llama.cpp/Ollama/vLLM endpoints with slightly wrong paths. Treat
|
|
# as unknown (retryable) so the real error surfaces, rather than
|
|
# claiming the model is missing and silently falling back.
|
|
e = MockAPIError("Not Found", status_code=404)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.unknown
|
|
assert result.retryable is True
|
|
assert result.should_fallback is False
|
|
|
|
# ── Provider policy-block (OpenRouter privacy/guardrail) ──
|
|
|
|
def test_404_openrouter_policy_blocked(self):
|
|
# Real OpenRouter error when the user's account privacy setting
|
|
# excludes the only endpoint serving a model (e.g. DeepSeek V4 Pro
|
|
# which is hosted only by DeepSeek, and their endpoint may log
|
|
# inputs). Must NOT classify as model_not_found — the model
|
|
# exists, falling back won't help (same account setting applies),
|
|
# and the error body already tells the user where to fix it.
|
|
e = MockAPIError(
|
|
"No endpoints available matching your guardrail restrictions "
|
|
"and data policy. Configure: https://openrouter.ai/settings/privacy",
|
|
status_code=404,
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.provider_policy_blocked
|
|
assert result.retryable is False
|
|
assert result.should_fallback is False
|
|
|
|
def test_400_openrouter_policy_blocked(self):
|
|
# Defense-in-depth: if OpenRouter ever returns this as 400 instead
|
|
# of 404, still classify it distinctly rather than as format_error
|
|
# or model_not_found.
|
|
e = MockAPIError(
|
|
"No endpoints available matching your data policy",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.provider_policy_blocked
|
|
assert result.retryable is False
|
|
assert result.should_fallback is False
|
|
|
|
def test_message_only_openrouter_policy_blocked(self):
|
|
# No status code — classifier should still catch the fingerprint
|
|
# via the message-pattern fallback.
|
|
e = Exception(
|
|
"No endpoints available matching your guardrail restrictions "
|
|
"and data policy"
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.provider_policy_blocked
|
|
|
|
# ── Provider content-policy block (per-prompt safety filter) ──
|
|
#
|
|
# Distinct from ``provider_policy_blocked`` above — these are upstream
|
|
# model-provider safety refusals for THIS prompt, not OpenRouter
|
|
# account-level data policy. Recovery is fallback model, not config fix.
|
|
# See issue #18028 — OpenAI Codex was burning 3 retries on identical
|
|
# refusals before users saw "API failed after 3 retries" on Telegram.
|
|
|
|
def test_message_only_cyber_content_policy_blocked(self):
|
|
# OpenAI Codex returns this without an HTTP status. Retrying the
|
|
# same prompt three times only repeats the same policy decision, so
|
|
# the classifier must jump straight to fallback / abort instead of
|
|
# leaving it in the retryable ``unknown`` bucket.
|
|
e = Exception(
|
|
"This content was flagged for possible cybersecurity risk. If this "
|
|
"seems wrong, try rephrasing your request. To get authorized for "
|
|
"security work, join the Trusted Access for Cyber program."
|
|
)
|
|
result = classify_api_error(e, provider="openai-codex", model="gpt-5.5")
|
|
assert result.reason == FailoverReason.content_policy_blocked
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
assert result.should_compress is False
|
|
|
|
def test_400_cyber_content_policy_blocked(self):
|
|
# When the SDK does attach a status (e.g. 400), the safety pattern
|
|
# must still beat the format_error fallthrough.
|
|
e = MockAPIError(
|
|
"This content was flagged for possible cybersecurity risk",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="openai-codex", model="gpt-5.5")
|
|
assert result.reason == FailoverReason.content_policy_blocked
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
def test_openai_usage_policy_violation_content_policy_blocked(self):
|
|
# OpenAI moderation refusal wording from chat completions / responses.
|
|
e = MockAPIError(
|
|
"Your request was flagged by the moderation system as potentially "
|
|
"violating OpenAI's usage policies.",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="openai", model="gpt-4o")
|
|
assert result.reason == FailoverReason.content_policy_blocked
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
def test_anthropic_safety_system_content_policy_blocked(self):
|
|
# Anthropic safety refusal — distinct phrasing from OpenAI.
|
|
e = Exception(
|
|
"Your prompt was flagged by our safety system. Please rephrase "
|
|
"and try again."
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", model="claude-3-5-sonnet")
|
|
assert result.reason == FailoverReason.content_policy_blocked
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
def test_azure_content_filter_content_policy_blocked(self):
|
|
# Azure OpenAI returns ``content_filter`` finish reason / error code
|
|
# and ``ResponsibleAIPolicyViolation`` in error bodies — both narrow
|
|
# tokens, not the generic English phrase.
|
|
e = MockAPIError(
|
|
"The response was filtered: ResponsibleAIPolicyViolation "
|
|
"(finish_reason=content_filter).",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="azure", model="gpt-4o")
|
|
assert result.reason == FailoverReason.content_policy_blocked
|
|
assert result.retryable is False
|
|
|
|
def test_404_model_not_found_still_works(self):
|
|
# Regression guard: the new policy-block check must not swallow
|
|
# genuine model_not_found 404s.
|
|
e = MockAPIError(
|
|
"openrouter/nonexistent-model is not a valid model ID",
|
|
status_code=404,
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.model_not_found
|
|
assert result.should_fallback is True
|
|
|
|
# ── Payload too large ──
|
|
|
|
def test_413_payload_too_large(self):
|
|
e = MockAPIError("Request Entity Too Large", status_code=413)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.payload_too_large
|
|
assert result.should_compress is True
|
|
|
|
# ── Context overflow ──
|
|
|
|
def test_400_context_length(self):
|
|
e = MockAPIError("context length exceeded: 250000 > 200000", status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_400_too_many_tokens(self):
|
|
e = MockAPIError("This model's maximum context is 128000 tokens, too many tokens", status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_400_prompt_too_long(self):
|
|
e = MockAPIError("prompt is too long: 300000 tokens > 200000 maximum", status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_400_generic_large_session(self):
|
|
"""Generic 400 with large session → context overflow heuristic."""
|
|
e = MockAPIError(
|
|
"Error",
|
|
status_code=400,
|
|
body={"error": {"message": "Error"}},
|
|
)
|
|
result = classify_api_error(e, approx_tokens=100000, context_length=200000)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_400_generic_small_session_is_format_error(self):
|
|
"""Generic 400 with small session → format error, not context overflow."""
|
|
e = MockAPIError(
|
|
"Error",
|
|
status_code=400,
|
|
body={"error": {"message": "Error"}},
|
|
)
|
|
result = classify_api_error(e, approx_tokens=1000, context_length=200000)
|
|
assert result.reason == FailoverReason.format_error
|
|
|
|
def test_400_generic_many_messages_below_large_context_pressure_is_format_error(self):
|
|
"""Large-context sessions should not overflow solely due to message count."""
|
|
e = MockAPIError(
|
|
"Error",
|
|
status_code=400,
|
|
body={"error": {"message": "Error"}},
|
|
)
|
|
result = classify_api_error(
|
|
e,
|
|
provider="openai-codex",
|
|
model="gpt-5.5",
|
|
approx_tokens=74320,
|
|
context_length=1_000_000,
|
|
num_messages=432,
|
|
)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.should_compress is False
|
|
|
|
# ── Server disconnect + large session ──
|
|
|
|
def test_disconnect_large_session_context_overflow(self):
|
|
"""Server disconnect with large session → context overflow."""
|
|
e = Exception("server disconnected without sending complete message")
|
|
result = classify_api_error(e, approx_tokens=150000, context_length=200000)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_disconnect_small_session_timeout(self):
|
|
"""Server disconnect with small session → timeout."""
|
|
e = Exception("server disconnected without sending complete message")
|
|
result = classify_api_error(e, approx_tokens=5000, context_length=200000)
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
def test_disconnect_many_messages_below_large_context_pressure_is_timeout(self):
|
|
"""Large-context disconnects should not overflow solely due to message count."""
|
|
e = Exception("server disconnected without sending complete message")
|
|
result = classify_api_error(
|
|
e,
|
|
provider="openai-codex",
|
|
model="gpt-5.5",
|
|
approx_tokens=74320,
|
|
context_length=1_000_000,
|
|
num_messages=432,
|
|
)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.should_compress is False
|
|
|
|
# ── Provider-specific: Anthropic thinking signature ──
|
|
|
|
def test_anthropic_thinking_signature(self):
|
|
e = MockAPIError(
|
|
"thinking block has invalid signature",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.thinking_signature
|
|
assert result.retryable is True
|
|
|
|
def test_non_anthropic_400_with_signature_not_classified_as_thinking(self):
|
|
"""400 with 'signature' but from non-Anthropic → format error."""
|
|
e = MockAPIError("invalid signature", status_code=400)
|
|
result = classify_api_error(e, provider="openrouter", approx_tokens=0)
|
|
# Without "thinking" in the message, it shouldn't be thinking_signature
|
|
assert result.reason != FailoverReason.thinking_signature
|
|
|
|
def test_anthropic_thinking_blocks_cannot_be_modified(self):
|
|
"""Frozen-block mutation 400 (no 'signature' token) must route to
|
|
thinking_signature recovery, not hard-abort. Regression for the
|
|
real-world error: latest-assistant thinking blocks 'cannot be
|
|
modified' after upstream message mutation."""
|
|
e = MockAPIError(
|
|
"messages.73.content.10: `thinking` or `redacted_thinking` blocks "
|
|
"in the latest assistant message cannot be modified. These blocks "
|
|
"must remain as they were in the original response.",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.thinking_signature
|
|
assert result.retryable is True
|
|
|
|
def test_anthropic_thinking_cannot_be_modified_via_openrouter(self):
|
|
"""Same frozen-block error proxied through OpenRouter must also be
|
|
caught (provider is not gated)."""
|
|
e = MockAPIError(
|
|
"`thinking` or `redacted_thinking` blocks in the latest assistant "
|
|
"message cannot be modified.",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.thinking_signature
|
|
assert result.retryable is True
|
|
|
|
def test_400_cannot_be_modified_without_thinking_not_classified(self):
|
|
"""A 400 'cannot be modified' that has nothing to do with thinking
|
|
blocks must NOT be swept into thinking_signature recovery."""
|
|
e = MockAPIError(
|
|
"this field cannot be modified after creation", status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", approx_tokens=0)
|
|
assert result.reason != FailoverReason.thinking_signature
|
|
|
|
def test_invalid_encrypted_content_classified_as_retryable_replay_failure(self):
|
|
body = {
|
|
"error": {
|
|
"message": (
|
|
'{"error":{"message":"The encrypted content for item rs_001 could not be verified. '
|
|
'Reason: Encrypted content could not be decrypted or parsed.",'
|
|
'"type":"invalid_request_error","param":"","code":"invalid_encrypted_content"}}'
|
|
),
|
|
"type": "400",
|
|
}
|
|
}
|
|
e = MockAPIError(
|
|
"Error code: 400 - invalid_encrypted_content",
|
|
status_code=400,
|
|
body=body,
|
|
)
|
|
result = classify_api_error(e, provider="custom", model="gpt-5.4")
|
|
assert result.reason == FailoverReason.invalid_encrypted_content
|
|
assert result.retryable is True
|
|
assert result.should_fallback is False
|
|
|
|
def test_xai_invalid_encrypted_content_wording_uses_replay_recovery(self):
|
|
e = MockAPIError(
|
|
"Error code: 400 - Could not decrypt the provided encrypted_content. "
|
|
"Ensure the value is the unmodified encrypted_content from a previous response.",
|
|
status_code=400,
|
|
body={
|
|
"code": "Client specified an invalid argument",
|
|
"error": (
|
|
"Could not decrypt the provided encrypted_content. Ensure the value "
|
|
"is the unmodified encrypted_content from a previous response."
|
|
),
|
|
},
|
|
)
|
|
|
|
result = classify_api_error(e, provider="xai-oauth", model="grok-4.3")
|
|
|
|
assert result.reason == FailoverReason.invalid_encrypted_content
|
|
assert result.retryable is True
|
|
assert result.should_fallback is False
|
|
|
|
def test_invalid_encrypted_content_broad_message_match_does_not_catch_generic_parse_error(self):
|
|
message = "Encrypted content could not be decrypted or parsed."
|
|
e = MockAPIError(
|
|
message,
|
|
status_code=400,
|
|
body={"error": {"message": message}},
|
|
)
|
|
result = classify_api_error(e, provider="custom", model="gpt-5.4")
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
@pytest.mark.parametrize("error_code", ["Invalid_Encrypted_Content", "INVALID_ENCRYPTED_CONTENT"])
|
|
def test_invalid_encrypted_content_code_is_case_insensitive_for_400(self, error_code):
|
|
e = MockAPIError(
|
|
"Error code: 400 - bad request",
|
|
status_code=400,
|
|
body={"error": {"code": error_code, "message": "Bad request"}},
|
|
)
|
|
result = classify_api_error(e, provider="custom", model="gpt-5.4")
|
|
assert result.reason == FailoverReason.invalid_encrypted_content
|
|
assert result.retryable is True
|
|
assert result.should_fallback is False
|
|
|
|
# ── Provider-specific: llama.cpp grammar-parse ──
|
|
|
|
def test_llama_cpp_grammar_parse_error(self):
|
|
"""llama.cpp rejects regex escapes in JSON Schema `pattern`."""
|
|
e = MockAPIError(
|
|
"parse: error parsing grammar: unknown escape at \\d",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="openai-compatible")
|
|
assert result.reason == FailoverReason.llama_cpp_grammar_pattern
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
def test_llama_cpp_unable_to_generate_parser(self):
|
|
"""Older llama.cpp builds surface the error as 'unable to generate parser'."""
|
|
e = MockAPIError(
|
|
"Unable to generate parser for this template",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="openai-compatible")
|
|
assert result.reason == FailoverReason.llama_cpp_grammar_pattern
|
|
|
|
def test_llama_cpp_json_schema_to_grammar_phrase(self):
|
|
"""Some builds mention the module name explicitly."""
|
|
e = MockAPIError(
|
|
"json-schema-to-grammar failed to convert schema",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="openai-compatible")
|
|
assert result.reason == FailoverReason.llama_cpp_grammar_pattern
|
|
|
|
def test_llama_cpp_grammar_requires_400(self):
|
|
"""A 500 with the same phrase isn't the llama.cpp grammar case."""
|
|
e = MockAPIError("error parsing grammar", status_code=500)
|
|
result = classify_api_error(e, provider="openai-compatible")
|
|
assert result.reason != FailoverReason.llama_cpp_grammar_pattern
|
|
|
|
# ── Provider-specific: Anthropic long-context tier ──
|
|
|
|
def test_anthropic_long_context_tier(self):
|
|
e = MockAPIError(
|
|
"Extra usage is required for long context requests over 200k tokens",
|
|
status_code=429,
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", model="claude-sonnet-4")
|
|
assert result.reason == FailoverReason.long_context_tier
|
|
assert result.should_compress is True
|
|
|
|
def test_normal_429_not_long_context(self):
|
|
"""Normal 429 without 'extra usage' + 'long context' → rate_limit."""
|
|
e = MockAPIError("Too Many Requests", status_code=429)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
# ── Provider-specific: Anthropic OAuth 1M-context beta forbidden ──
|
|
|
|
def test_anthropic_oauth_1m_beta_forbidden(self):
|
|
"""400 + 'long context beta is not yet available for this subscription'
|
|
→ oauth_long_context_beta_forbidden (retryable, no compression)."""
|
|
e = MockAPIError(
|
|
"The long context beta is not yet available for this subscription.",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", model="claude-sonnet-4.6")
|
|
assert result.reason == FailoverReason.oauth_long_context_beta_forbidden
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
def test_anthropic_oauth_1m_beta_forbidden_does_not_collide_with_tier_gate(self):
|
|
"""The 429 'extra usage' + 'long context' tier gate keeps its own
|
|
classification even though its message mentions 'long context'."""
|
|
e = MockAPIError(
|
|
"Extra usage is required for long context requests over 200k tokens",
|
|
status_code=429,
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", model="claude-sonnet-4.6")
|
|
assert result.reason == FailoverReason.long_context_tier
|
|
|
|
def test_400_without_beta_phrase_is_not_1m_beta_forbidden(self):
|
|
"""A generic 400 that happens to mention 'long context' but not the
|
|
exact beta-availability phrase should not be misclassified."""
|
|
e = MockAPIError(
|
|
"long context window exceeded",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason != FailoverReason.oauth_long_context_beta_forbidden
|
|
|
|
# ── Transport errors ──
|
|
|
|
def test_read_timeout(self):
|
|
e = ReadTimeout("Read timed out")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_connect_error(self):
|
|
e = ConnectError("Connection refused")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
def test_connection_error_builtin(self):
|
|
e = ConnectionError("Connection reset by peer")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
def test_timeout_error_builtin(self):
|
|
e = TimeoutError("timed out")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
def test_runtime_error_cli_turn_timed_out_classifies_as_timeout(self):
|
|
# RuntimeError from a local claude-cli shim that wraps a subprocess
|
|
# timeout must classify as FailoverReason.timeout, not unknown, so
|
|
# the retry loop rebuilds the client instead of treating the turn as
|
|
# an empty model response (#22548).
|
|
e = RuntimeError("claude CLI turn timed out")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_runtime_error_request_timed_out_classifies_as_timeout(self):
|
|
e = RuntimeError("request timed out after 120s")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_runtime_error_deadline_exceeded_classifies_as_timeout(self):
|
|
e = RuntimeError("deadline exceeded")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
# ── Error code classification ──
|
|
|
|
def test_error_code_resource_exhausted(self):
|
|
e = MockAPIError(
|
|
"Resource exhausted",
|
|
body={"error": {"code": "resource_exhausted", "message": "Too many requests"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_error_code_model_not_found(self):
|
|
e = MockAPIError(
|
|
"Model not available",
|
|
body={"error": {"code": "model_not_found"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.model_not_found
|
|
|
|
def test_error_code_context_length_exceeded(self):
|
|
e = MockAPIError(
|
|
"Context too large",
|
|
body={"error": {"code": "context_length_exceeded"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_error_code_model_not_supported_on_free_tier_is_billing(self):
|
|
e = MockAPIError(
|
|
"Model unavailable",
|
|
body={
|
|
"error": {
|
|
"code": "model_not_supported_on_free_tier",
|
|
"message": "Model 'gpt-5' is not available on the Free Tier.",
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="nous", model="gpt-5")
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
# ── Message-only patterns (no status code) ──
|
|
|
|
def test_message_billing_pattern(self):
|
|
e = Exception("insufficient credits to complete this request")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
def test_message_free_tier_model_block_is_billing(self):
|
|
e = Exception("Model 'gpt-5' is not available on the Free Tier.")
|
|
result = classify_api_error(e, provider="nous", model="gpt-5")
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
def test_message_rate_limit_pattern(self):
|
|
e = Exception("rate limit reached for this model")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_message_auth_pattern(self):
|
|
e = Exception("invalid api key provided")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.auth
|
|
|
|
def test_message_model_not_found_pattern(self):
|
|
e = Exception("gpt-99 is not a valid model")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.model_not_found
|
|
|
|
def test_message_context_overflow_pattern(self):
|
|
e = Exception("maximum context length exceeded")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
# ── Message-only usage limit disambiguation (no status code) ──
|
|
|
|
def test_message_usage_limit_transient_is_rate_limit(self):
|
|
"""'usage limit' + 'try again' with no status code → rate_limit, not billing."""
|
|
e = Exception("usage limit exceeded, try again in 5 minutes")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.retryable is True
|
|
assert result.should_rotate_credential is True
|
|
assert result.should_fallback is True
|
|
|
|
def test_message_usage_limit_no_retry_signal_is_billing(self):
|
|
"""'usage limit' with no transient signal and no status code → billing."""
|
|
e = Exception("usage limit reached")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.retryable is False
|
|
assert result.should_rotate_credential is True
|
|
|
|
def test_message_quota_with_reset_window_is_rate_limit(self):
|
|
"""'quota' + 'resets at' with no status code → rate_limit."""
|
|
e = Exception("quota exceeded, resets at midnight UTC")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.retryable is True
|
|
|
|
def test_message_limit_exceeded_with_wait_is_rate_limit(self):
|
|
"""'limit exceeded' + 'wait' with no status code → rate_limit."""
|
|
e = Exception("key limit exceeded, please wait before retrying")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.retryable is True
|
|
|
|
# ── Unknown / fallback ──
|
|
|
|
def test_generic_exception_is_unknown(self):
|
|
e = Exception("something weird happened")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.unknown
|
|
assert result.retryable is True
|
|
|
|
# ── Format error ──
|
|
|
|
def test_400_descriptive_format_error(self):
|
|
"""400 with descriptive message (not context overflow) → format error."""
|
|
e = MockAPIError(
|
|
"Invalid value for parameter 'temperature': must be between 0 and 2",
|
|
status_code=400,
|
|
body={"error": {"message": "Invalid value for parameter 'temperature': must be between 0 and 2"}},
|
|
)
|
|
result = classify_api_error(e, approx_tokens=1000)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
|
|
def test_400_unsupported_max_tokens_param_not_context_overflow(self):
|
|
"""A GPT-5 model rejecting max_tokens must NOT be misclassified as
|
|
context overflow. The OpenAI error string contains the literal
|
|
'max_tokens' (a _CONTEXT_OVERFLOW_PATTERNS entry), so without the
|
|
request-validation guard it was routed into the compression loop,
|
|
re-sent with the same bad param, and ended in "Cannot compress
|
|
further". Regression for gpt-5-context-overflow-misclassification."""
|
|
msg = ("Unsupported parameter: 'max_tokens' is not supported with this "
|
|
"model. Use 'max_completion_tokens' instead.")
|
|
e = MockAPIError(
|
|
msg,
|
|
status_code=400,
|
|
body={"error": {"message": msg, "type": "invalid_request_error",
|
|
"code": "unsupported_parameter"}},
|
|
)
|
|
# Tiny context against a huge window — definitely not a real overflow.
|
|
result = classify_api_error(e, model="gpt-5.4",
|
|
approx_tokens=6962, context_length=1050000)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
assert result.should_compress is False
|
|
|
|
def test_empty_provider_response_advisory_not_context_overflow(self):
|
|
"""nano-gpt / OpenRouter empty-response advisories mention
|
|
'very low max_tokens' as a possible cause. That used to match the
|
|
bare 'max_tokens' overflow pattern and thrash compression until
|
|
'Cannot compress further' on a healthy session."""
|
|
msg = (
|
|
"The model returned an empty response despite retries across "
|
|
"available sources. This is usually a temporary upstream issue "
|
|
"and retrying the request often succeeds. Less commonly it can "
|
|
"be caused by stop sequences matching the output, a very low "
|
|
"max_tokens, or content filtering. No charge was applied."
|
|
)
|
|
result = classify_api_error(
|
|
Exception(msg),
|
|
approx_tokens=143000,
|
|
context_length=1_048_576,
|
|
num_messages=300,
|
|
)
|
|
assert result.reason == FailoverReason.server_error
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
def test_max_tokens_exceeded_still_context_overflow(self):
|
|
"""Specific max_tokens-exceeded phrasing must keep compressing."""
|
|
result = classify_api_error(
|
|
Exception("Request failed: max_tokens exceeded for this model"),
|
|
approx_tokens=200000,
|
|
context_length=128000,
|
|
)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_400_unknown_parameter_not_context_overflow(self):
|
|
"""'Unknown parameter' 400s are deterministic request-validation
|
|
failures, not overflows."""
|
|
e = MockAPIError(
|
|
"Unknown parameter: 'foo'.",
|
|
status_code=400,
|
|
body={"error": {"message": "Unknown parameter: 'foo'.",
|
|
"code": "unknown_parameter"}},
|
|
)
|
|
result = classify_api_error(e, approx_tokens=1000)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.should_compress is False
|
|
|
|
def test_400_real_overflow_with_invalid_request_error_code_still_compresses(self):
|
|
"""Guard the guard: OpenAI stamps genuine context-overflow 400s with
|
|
the generic 'invalid_request_error' code. The request-validation guard
|
|
must NOT key off that code, or real overflows stop compressing."""
|
|
msg = ("This model's maximum context length is 128000 tokens, however "
|
|
"you requested 150000 tokens.")
|
|
e = MockAPIError(
|
|
msg,
|
|
status_code=400,
|
|
body={"error": {"message": msg, "type": "invalid_request_error"}},
|
|
)
|
|
result = classify_api_error(e, model="gpt-5.4",
|
|
approx_tokens=150000, context_length=128000)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_422_format_error(self):
|
|
e = MockAPIError("Unprocessable Entity", status_code=422)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
|
|
def test_400_flat_body_descriptive_not_context_overflow(self):
|
|
"""Responses API flat body with descriptive error + large session → format error.
|
|
|
|
The Codex Responses API returns errors in flat body format:
|
|
{"message": "...", "type": "..."} without an "error" wrapper.
|
|
A descriptive 400 must NOT be misclassified as context overflow
|
|
just because the session is large.
|
|
"""
|
|
e = MockAPIError(
|
|
"Invalid 'input[index].name': string does not match pattern.",
|
|
status_code=400,
|
|
body={"message": "Invalid 'input[index].name': string does not match pattern.",
|
|
"type": "invalid_request_error"},
|
|
)
|
|
result = classify_api_error(e, approx_tokens=200000, context_length=400000, num_messages=500)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
|
|
def test_400_flat_body_generic_large_session_still_context_overflow(self):
|
|
"""Flat body with generic 'Error' message + large session → context overflow.
|
|
|
|
Regression: the flat-body fallback must not break the existing heuristic
|
|
for genuinely generic errors from providers that use flat bodies.
|
|
"""
|
|
e = MockAPIError(
|
|
"Error",
|
|
status_code=400,
|
|
body={"message": "Error"},
|
|
)
|
|
result = classify_api_error(e, approx_tokens=100000, context_length=200000)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_400_empty_content_message_not_context_overflow(self):
|
|
"""Anthropic 'non-empty content' 400 → format_error, NOT compression.
|
|
|
|
Regression for the empty-assistant-stub bug: a stream dies with 0
|
|
recovered chars, an empty assistant message is persisted, and every
|
|
subsequent request 400s with 'all messages must have non-empty
|
|
content'. On a large session the generic '400 + large session'
|
|
heuristic used to mis-route this into the compression loop, ending in
|
|
'Cannot compress further' on every retry (compression can't fix a
|
|
malformed transcript). It must classify as a non-retryable
|
|
format_error so the loop stops looping.
|
|
"""
|
|
msg = ("all messages must have non-empty content except for the "
|
|
"optional final assistant message")
|
|
e = MockAPIError(
|
|
msg,
|
|
status_code=400,
|
|
body={"error": {"message": msg, "type": "invalid_request_error"}},
|
|
)
|
|
# Large session (many messages / tokens) to prove the overflow
|
|
# heuristic does NOT capture it.
|
|
result = classify_api_error(
|
|
e, approx_tokens=66000, context_length=200000, num_messages=219,
|
|
)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
assert result.should_compress is not True
|
|
|
|
def test_400_litellm_invalid_request_body_shape(self, caplog):
|
|
"""litellm/Bedrock proxy shape (errorMessage/errorCode) → format_error.
|
|
|
|
The proxy in front of Anthropic surfaces the empty-content rejection
|
|
as {"errorMessage": "...non-empty content...", "errorCode":
|
|
"INVALID_REQUEST_BODY", "errorArgs": {"reason": "..."}}. Those keys
|
|
are not the standard error.message / message, so err_body_msg used to
|
|
come back empty → is_generic=True → mis-routed into compression on a
|
|
large session. Both the message pattern and the errorCode must be
|
|
recognized, and a distinct warning must be logged so the condition is
|
|
observable in the field.
|
|
"""
|
|
import logging
|
|
proxy_msg = ("The provided request body is invalid: claude "
|
|
"messages.208: all messages must have non-empty content "
|
|
"except for the optional final assistant message")
|
|
e = MockAPIError(
|
|
proxy_msg,
|
|
status_code=400,
|
|
body={
|
|
"errorMessage": proxy_msg,
|
|
"errorCode": "INVALID_REQUEST_BODY",
|
|
"statusCode": 400,
|
|
"errorArgs": {"reason": "claude messages.208: ..."},
|
|
},
|
|
)
|
|
with caplog.at_level(logging.WARNING, logger="agent.error_classifier"):
|
|
result = classify_api_error(
|
|
e, approx_tokens=66000, context_length=200000, num_messages=219,
|
|
)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
assert result.should_compress is not True
|
|
assert any(
|
|
"Malformed message array 400" in r.getMessage()
|
|
for r in caplog.records
|
|
), "Expected a distinct warning identifying the malformed-body 400"
|
|
|
|
def test_400_real_context_overflow_still_compresses(self):
|
|
"""Guard: the new empty-content guard must NOT swallow real overflows.
|
|
|
|
A genuine 'maximum context length' 400 must still route into
|
|
compression — the fix is surgical, not a blanket 400→format_error.
|
|
"""
|
|
msg = ("This model's maximum context length is 200000 tokens. "
|
|
"However, your messages resulted in 250000 tokens.")
|
|
e = MockAPIError(
|
|
msg,
|
|
status_code=400,
|
|
body={"error": {"message": msg, "type": "invalid_request_error"}},
|
|
)
|
|
result = classify_api_error(
|
|
e, approx_tokens=250000, context_length=200000, num_messages=219,
|
|
)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
# ── Peer closed + large session ──
|
|
|
|
def test_peer_closed_large_session(self):
|
|
e = Exception("peer closed connection without sending complete message")
|
|
result = classify_api_error(e, approx_tokens=130000, context_length=200000)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
# ── Chinese error messages ──
|
|
|
|
def test_chinese_context_overflow(self):
|
|
e = MockAPIError("超过最大长度限制", status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
# ── Z.AI / Zhipu GLM error messages ──
|
|
|
|
def test_zai_glm_token_limit_overflow(self):
|
|
"""Z.AI GLM's 'tokens in request more than max tokens allowed'
|
|
(error code 1210) → context_overflow, so the agent compresses
|
|
instead of blindly retrying. Port of anomalyco/opencode#35671."""
|
|
e = MockAPIError(
|
|
'{"error": {"code": "1210", "message": '
|
|
'"tokens in request more than max tokens allowed"}}',
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="zai")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
# ── vLLM / local inference server error messages ──
|
|
|
|
def test_vllm_max_model_len_overflow(self):
|
|
"""vLLM's 'exceeds the max_model_len' error → context_overflow."""
|
|
e = MockAPIError(
|
|
"The engine prompt length 1327246 exceeds the max_model_len 131072. "
|
|
"Please reduce prompt.",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_vllm_prompt_length_exceeds(self):
|
|
"""vLLM prompt length error → context_overflow."""
|
|
e = MockAPIError(
|
|
"prompt length 200000 exceeds maximum model length 131072",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_vllm_input_too_long(self):
|
|
"""vLLM 'input is too long' error → context_overflow."""
|
|
e = MockAPIError("input is too long for model", status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_ollama_context_length_exceeded(self):
|
|
"""Ollama 'context length exceeded' error → context_overflow."""
|
|
e = MockAPIError("context length exceeded", status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_llamacpp_slot_context(self):
|
|
"""llama.cpp / llama-server 'slot context' error → context_overflow."""
|
|
e = MockAPIError(
|
|
"slot context: 4096 tokens, prompt 8192 tokens — not enough space",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
# ── Result metadata ──
|
|
|
|
def test_provider_and_model_in_result(self):
|
|
e = MockAPIError("fail", status_code=500)
|
|
result = classify_api_error(e, provider="openrouter", model="gpt-5")
|
|
assert result.provider == "openrouter"
|
|
assert result.model == "gpt-5"
|
|
assert result.status_code == 500
|
|
|
|
def test_message_extracted(self):
|
|
e = MockAPIError(
|
|
"outer",
|
|
status_code=500,
|
|
body={"error": {"message": "Internal server error occurred"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.message == "Internal server error occurred"
|
|
|
|
|
|
# ── Test: Adversarial / edge cases (from live testing) ─────────────────
|
|
|
|
class TestAdversarialEdgeCases:
|
|
"""Edge cases discovered during live testing with real SDK objects."""
|
|
|
|
def test_empty_exception_message(self):
|
|
result = classify_api_error(Exception(""))
|
|
assert result.reason == FailoverReason.unknown
|
|
assert result.retryable is True
|
|
|
|
def test_500_with_none_body(self):
|
|
e = MockAPIError("fail", status_code=500, body=None)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.server_error
|
|
|
|
def test_non_dict_body(self):
|
|
"""Some providers return strings instead of JSON."""
|
|
class StringBodyError(Exception):
|
|
status_code = 400
|
|
body = "just a string"
|
|
result = classify_api_error(StringBodyError("bad"))
|
|
assert result.reason == FailoverReason.format_error
|
|
|
|
def test_list_body(self):
|
|
class ListBodyError(Exception):
|
|
status_code = 500
|
|
body = [{"error": "something"}]
|
|
result = classify_api_error(ListBodyError("server error"))
|
|
assert result.reason == FailoverReason.server_error
|
|
|
|
def test_circular_cause_chain(self):
|
|
"""Must not infinite-loop on circular __cause__."""
|
|
e = Exception("circular")
|
|
e.__cause__ = e
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.unknown
|
|
|
|
def test_three_level_cause_chain(self):
|
|
inner = MockAPIError("inner", status_code=429)
|
|
middle = Exception("middle")
|
|
middle.__cause__ = inner
|
|
outer = RuntimeError("outer")
|
|
outer.__cause__ = middle
|
|
result = classify_api_error(outer)
|
|
assert result.status_code == 429
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_400_with_rate_limit_text(self):
|
|
"""Some providers send rate limits as 400 instead of 429."""
|
|
e = MockAPIError(
|
|
"rate limit policy",
|
|
status_code=400,
|
|
body={"error": {"message": "rate limit exceeded on this model"}},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_400_with_billing_text(self):
|
|
"""Some providers send billing errors as 400."""
|
|
e = MockAPIError(
|
|
"billing",
|
|
status_code=400,
|
|
body={"error": {"message": "insufficient credits for this request"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
def test_400_anthropic_extra_usage_exhausted(self):
|
|
"""Anthropic returns 400 with 'out of extra usage' when the user's
|
|
extra-usage allowance is depleted. Must classify as billing so the
|
|
fallback chain engages (with credential rotation) instead of the
|
|
generic format_error path, which never rotates. (#11736, #13170)"""
|
|
e = MockAPIError(
|
|
"You're out of extra usage. Add more at claude.ai/settings/usage and keep going.",
|
|
status_code=400,
|
|
body={"error": {
|
|
"type": "invalid_request_error",
|
|
"message": "You're out of extra usage. Add more at claude.ai/settings/usage and keep going.",
|
|
}},
|
|
)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.should_fallback is True
|
|
assert result.retryable is False
|
|
assert result.should_rotate_credential is True
|
|
|
|
def test_200_with_error_body(self):
|
|
"""200 status with error in body — should be unknown, not crash."""
|
|
class WeirdSuccess(Exception):
|
|
status_code = 200
|
|
body = {"error": {"message": "loading"}}
|
|
result = classify_api_error(WeirdSuccess("model loading"))
|
|
assert result.reason == FailoverReason.unknown
|
|
|
|
def test_ollama_context_size_exceeded(self):
|
|
e = MockAPIError(
|
|
"Error",
|
|
status_code=400,
|
|
body={"error": {"message": "context size has been exceeded"}},
|
|
)
|
|
result = classify_api_error(e, provider="ollama")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_connection_refused_error(self):
|
|
e = ConnectionRefusedError("Connection refused: localhost:11434")
|
|
result = classify_api_error(e, provider="ollama")
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
def test_body_message_enrichment(self):
|
|
"""Body message must be included in pattern matching even when
|
|
str(error) doesn't contain it (OpenAI SDK APIStatusError)."""
|
|
e = MockAPIError(
|
|
"Usage limit", # str(e) = "usage limit"
|
|
status_code=402,
|
|
body={"error": {"message": "Usage limit reached, try again in 5 minutes"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
# "try again" is only in body, not in str(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_disconnect_pattern_ordering(self):
|
|
"""Disconnect + large session must beat generic transport catch."""
|
|
class FakeRemoteProtocol(Exception):
|
|
pass
|
|
# Type name isn't in _TRANSPORT_ERROR_TYPES but message has disconnect pattern
|
|
e = Exception("peer closed connection without sending complete message")
|
|
result = classify_api_error(e, approx_tokens=150000, context_length=200000)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_credit_balance_too_low(self):
|
|
e = MockAPIError(
|
|
"Credits low",
|
|
status_code=402,
|
|
body={"error": {"message": "Your credit balance is too low"}},
|
|
)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
def test_deepseek_402_chinese(self):
|
|
"""Chinese billing message should still match billing patterns."""
|
|
# "余额不足" doesn't match English billing patterns, but 402 defaults to billing
|
|
e = MockAPIError("余额不足", status_code=402)
|
|
result = classify_api_error(e, provider="deepseek")
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
def test_openrouter_wrapped_context_overflow_in_metadata_raw(self):
|
|
"""OpenRouter wraps provider errors in metadata.raw JSON string."""
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=400,
|
|
body={
|
|
"error": {
|
|
"message": "Provider returned error",
|
|
"code": 400,
|
|
"metadata": {
|
|
"raw": '{"error":{"message":"context length exceeded: 50000 > 32768"}}'
|
|
}
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", approx_tokens=10000)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_openrouter_wrapped_rate_limit_in_metadata_raw(self):
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=400,
|
|
body={
|
|
"error": {
|
|
"message": "Provider returned error",
|
|
"metadata": {
|
|
"raw": '{"error":{"message":"Rate limit exceeded. Please retry after 30s."}}'
|
|
}
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_thinking_signature_via_openrouter(self):
|
|
"""Thinking signature errors proxied through OpenRouter must be caught."""
|
|
e = MockAPIError(
|
|
"thinking block has invalid signature",
|
|
status_code=400,
|
|
)
|
|
# provider is openrouter, not anthropic — old code missed this
|
|
result = classify_api_error(e, provider="openrouter", model="anthropic/claude-sonnet-4")
|
|
assert result.reason == FailoverReason.thinking_signature
|
|
|
|
def test_generic_400_large_by_message_count(self):
|
|
"""Many small messages (>80) should trigger context overflow heuristic."""
|
|
e = MockAPIError(
|
|
"Error",
|
|
status_code=400,
|
|
body={"error": {"message": "Error"}},
|
|
)
|
|
# Low token count but high message count
|
|
result = classify_api_error(
|
|
e, approx_tokens=5000, context_length=200000, num_messages=100,
|
|
)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_disconnect_large_by_message_count(self):
|
|
"""Server disconnect with 200+ messages should trigger context overflow."""
|
|
e = Exception("server disconnected without sending complete message")
|
|
result = classify_api_error(
|
|
e, approx_tokens=5000, context_length=200000, num_messages=250,
|
|
)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_openrouter_wrapped_model_not_found_in_metadata_raw(self):
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=400,
|
|
body={
|
|
"error": {
|
|
"message": "Provider returned error",
|
|
"metadata": {
|
|
"raw": '{"error":{"message":"The model gpt-99 does not exist"}}'
|
|
}
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.model_not_found
|
|
|
|
# ── Regression: dict-typed message field (Issue #11233) ──
|
|
|
|
def test_pydantic_dict_message_no_crash(self):
|
|
"""Pydantic validation errors return message as dict, not string.
|
|
|
|
Regression: classify_api_error must not crash when body['message']
|
|
is a dict (e.g. {"detail": [...]} from FastAPI/Pydantic). The
|
|
'or ""' fallback only handles None/falsy values — a non-empty
|
|
dict is truthy and passed to .lower(), causing AttributeError.
|
|
"""
|
|
e = MockAPIError(
|
|
"Unprocessable Entity",
|
|
status_code=422,
|
|
body={
|
|
"object": "error",
|
|
"message": {
|
|
"detail": [
|
|
{
|
|
"type": "extra_forbidden",
|
|
"loc": ["body", "think"],
|
|
"msg": "Extra inputs are not permitted",
|
|
}
|
|
]
|
|
},
|
|
},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.status_code == 422
|
|
assert result.retryable is False
|
|
|
|
def test_nested_error_dict_message_no_crash(self):
|
|
"""Nested body['error']['message'] as dict must not crash.
|
|
|
|
Some providers wrap Pydantic errors in an 'error' object.
|
|
"""
|
|
e = MockAPIError(
|
|
"Validation error",
|
|
status_code=400,
|
|
body={
|
|
"error": {
|
|
"message": {
|
|
"detail": [
|
|
{"type": "missing", "loc": ["body", "required"]}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, approx_tokens=1000)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.status_code == 400
|
|
|
|
def test_metadata_raw_dict_message_no_crash(self):
|
|
"""OpenRouter metadata.raw with dict message must not crash."""
|
|
e = MockAPIError(
|
|
"Provider error",
|
|
status_code=400,
|
|
body={
|
|
"error": {
|
|
"message": "Provider error",
|
|
"metadata": {
|
|
"raw": '{"error":{"message":{"detail":[{"type":"invalid"}]}}}'
|
|
}
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.format_error
|
|
|
|
# Broader non-string type guards — defense against other provider quirks.
|
|
|
|
def test_list_message_no_crash(self):
|
|
"""Some providers return message as a list of error entries."""
|
|
e = MockAPIError(
|
|
"validation",
|
|
status_code=400,
|
|
body={"message": [{"msg": "field required"}]},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result is not None
|
|
|
|
def test_int_message_no_crash(self):
|
|
"""Any non-string type must be coerced safely."""
|
|
e = MockAPIError("server error", status_code=500, body={"message": 42})
|
|
result = classify_api_error(e)
|
|
assert result is not None
|
|
|
|
def test_none_message_still_works(self):
|
|
"""Regression: None fallback (the 'or \"\"' path) must still work."""
|
|
e = MockAPIError("server error", status_code=500, body={"message": None})
|
|
result = classify_api_error(e)
|
|
assert result is not None
|
|
|
|
|
|
# ── Test: SSL/TLS transient errors ─────────────────────────────────────
|
|
|
|
class TestSSLTransientPatterns:
|
|
"""SSL/TLS alerts mid-stream should retry as timeout, not unknown, and
|
|
should NOT trigger context compression even on a large session.
|
|
|
|
Motivation: OpenSSL 3.x changed TLS alert error code format
|
|
(`SSLV3_ALERT_BAD_RECORD_MAC` → `SSL/TLS_ALERT_BAD_RECORD_MAC`),
|
|
breaking string-exact matching in downstream retry logic. We match
|
|
stable substrings instead.
|
|
"""
|
|
|
|
def test_bad_record_mac_classifies_as_timeout(self):
|
|
"""OpenSSL 3.x mid-stream bad record mac alert."""
|
|
e = Exception("[SSL: BAD_RECORD_MAC] sslv3 alert bad record mac (_ssl.c:2580)")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
def test_openssl_3x_format_classifies_as_timeout(self):
|
|
"""New format `ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC` still matches
|
|
because we key on both space- and underscore-separated forms of
|
|
the stable `bad_record_mac` token."""
|
|
e = Exception("ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC during streaming")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
def test_tls_alert_internal_error_classifies_as_timeout(self):
|
|
e = Exception("[SSL: TLSV1_ALERT_INTERNAL_ERROR] tlsv1 alert internal error")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
def test_ssl_handshake_failure_classifies_as_timeout(self):
|
|
e = Exception("ssl handshake failure during mid-stream")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_ssl_prefix_classifies_as_timeout(self):
|
|
"""Python's generic '[SSL: XYZ]' prefix from the ssl module."""
|
|
e = Exception("[SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_ssl_alert_on_large_session_does_not_compress(self):
|
|
"""Critical: SSL alerts on big contexts must NOT trigger context
|
|
compression — compression is expensive and won't fix a transport
|
|
hiccup. This is why _SSL_TRANSIENT_PATTERNS is separate from
|
|
_SERVER_DISCONNECT_PATTERNS.
|
|
"""
|
|
e = Exception("[SSL: BAD_RECORD_MAC] sslv3 alert bad record mac")
|
|
result = classify_api_error(
|
|
e,
|
|
approx_tokens=180000, # 90% of a 200k-context window
|
|
context_length=200000,
|
|
num_messages=300,
|
|
)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.should_compress is False
|
|
|
|
def test_plain_disconnect_on_large_session_still_compresses(self):
|
|
"""Regression guard: the context-overflow-via-disconnect path
|
|
(non-SSL disconnects on large sessions) must still trigger
|
|
compression. Only SSL-specific disconnects skip it.
|
|
"""
|
|
e = Exception("Server disconnected without sending a response")
|
|
result = classify_api_error(
|
|
e,
|
|
approx_tokens=180000,
|
|
context_length=200000,
|
|
num_messages=300,
|
|
)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_real_ssl_error_type_classifies_as_timeout(self):
|
|
"""Real ssl.SSLError instance — the type name alone (not message)
|
|
should route to the transport bucket."""
|
|
import ssl
|
|
e = ssl.SSLError("arbitrary ssl error")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
|
|
# ── Test: SSL certificate verification failures (fail fast) ────────────
|
|
|
|
class TestSSLCertVerificationFailFast:
|
|
"""Certificate verification failures are deterministic for the host —
|
|
a TLS-inspecting proxy, missing custom CA, expired or self-signed cert
|
|
fails identically on every retry. They must classify as non-retryable
|
|
``ssl_cert_verification`` so the user sees the fix hint immediately,
|
|
instead of matching the transient "[ssl:" pattern and retrying forever.
|
|
|
|
Inspired by Claude Code v2.1.199 (July 2026).
|
|
"""
|
|
|
|
def test_python_cert_verify_failed_is_non_retryable(self):
|
|
import ssl
|
|
e = ssl.SSLCertVerificationError(
|
|
1,
|
|
"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: "
|
|
"unable to get local issuer certificate (_ssl.c:1006)",
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.ssl_cert_verification
|
|
assert result.retryable is False
|
|
assert result.should_compress is False
|
|
|
|
def test_wrapped_cert_verify_message_is_non_retryable(self):
|
|
"""SDKs often re-raise without chaining — match on message alone."""
|
|
e = Exception(
|
|
"Connection error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate "
|
|
"verify failed: self-signed certificate in certificate chain"
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.ssl_cert_verification
|
|
assert result.retryable is False
|
|
|
|
def test_expired_certificate_is_non_retryable(self):
|
|
e = Exception("certificate verify failed: certificate has expired")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.ssl_cert_verification
|
|
assert result.retryable is False
|
|
|
|
def test_node_undici_phrasing_is_non_retryable(self):
|
|
"""MCP bridges surface Node's phrasing."""
|
|
e = Exception("fetch failed: unable to verify the first certificate")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.ssl_cert_verification
|
|
assert result.retryable is False
|
|
|
|
def test_cert_verify_wins_over_transient_ssl_prefix(self):
|
|
"""The '[SSL:' prefix also appears in cert-verify messages; the
|
|
cert check must run first so this doesn't retry as timeout."""
|
|
e = Exception("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.ssl_cert_verification
|
|
assert result.retryable is False
|
|
|
|
def test_transient_ssl_alert_still_retries(self):
|
|
"""Regression guard: genuine transient alerts keep retrying."""
|
|
e = Exception("[SSL: BAD_RECORD_MAC] sslv3 alert bad record mac")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_cert_verify_on_large_session_does_not_compress(self):
|
|
e = Exception("certificate verify failed: unable to get local issuer certificate")
|
|
result = classify_api_error(
|
|
e, approx_tokens=180000, context_length=200000, num_messages=300,
|
|
)
|
|
assert result.reason == FailoverReason.ssl_cert_verification
|
|
assert result.should_compress is False
|
|
|
|
# ── Test: RateLimitError without status_code (Copilot/GitHub Models) ──────────
|
|
|
|
class TestRateLimitErrorWithoutStatusCode:
|
|
"""Regression tests for the Copilot/GitHub Models edge case where the
|
|
OpenAI SDK raises RateLimitError but does not populate .status_code."""
|
|
|
|
def _make_rate_limit_error(self, status_code=None):
|
|
"""Create an exception whose class name is 'RateLimitError' with
|
|
an optionally missing status_code, mirroring the OpenAI SDK shape."""
|
|
cls = type("RateLimitError", (Exception,), {})
|
|
e = cls("You have exceeded your rate limit.")
|
|
e.status_code = status_code # None simulates the Copilot case
|
|
return e
|
|
|
|
def test_rate_limit_error_without_status_code_classified_as_rate_limit(self):
|
|
"""RateLimitError with status_code=None must classify as rate_limit."""
|
|
e = self._make_rate_limit_error(status_code=None)
|
|
result = classify_api_error(e, provider="copilot", model="gpt-4o")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_rate_limit_error_with_status_code_429_classified_as_rate_limit(self):
|
|
"""RateLimitError that does set status_code=429 still classifies correctly."""
|
|
e = self._make_rate_limit_error(status_code=429)
|
|
result = classify_api_error(e, provider="copilot", model="gpt-4o")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_other_error_without_status_code_not_forced_to_rate_limit(self):
|
|
"""A non-RateLimitError with missing status_code must NOT be forced to 429."""
|
|
cls = type("APIError", (Exception,), {})
|
|
e = cls("something went wrong")
|
|
e.status_code = None
|
|
result = classify_api_error(e, provider="copilot", model="gpt-4o")
|
|
assert result.reason != FailoverReason.rate_limit
|
|
|
|
|
|
|
|
# ── Test: multimodal_tool_content_unsupported pattern ───────────────────
|
|
|
|
class TestMultimodalToolContentUnsupported:
|
|
"""Issue #27344 — providers that reject list-type tool message content
|
|
should be classified as ``multimodal_tool_content_unsupported`` so the
|
|
retry loop can downgrade screenshots to text and try again.
|
|
"""
|
|
|
|
def test_xiaomi_mimo_text_is_not_set_pattern(self):
|
|
"""The actual Xiaomi MiMo 400 wording from the bug report."""
|
|
e = MockAPIError(
|
|
"Error code: 400 - {'error': {'code': '400', 'message': 'Param Incorrect', 'param': 'text is not set', 'type': ''}}",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="xiaomi", model="mimo-v2.5")
|
|
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
|
|
assert result.retryable is True
|
|
|
|
def test_generic_tool_message_must_be_string(self):
|
|
e = MockAPIError(
|
|
"tool message content must be a string",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="custom", model="some-model")
|
|
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
|
|
|
|
def test_expected_string_got_list(self):
|
|
e = MockAPIError(
|
|
"Schema validation failed: expected string, got list",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="custom", model="some-model")
|
|
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
|
|
|
|
def test_multimodal_tool_content_takes_priority_over_context_overflow(self):
|
|
"""Some providers return a 400 whose message contains BOTH
|
|
'text is not set' and a length-shaped phrase; the tool-content
|
|
recovery is cheaper than compression so it must win the priority.
|
|
"""
|
|
e = MockAPIError(
|
|
"text is not set; context length exceeded",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="xiaomi", model="mimo-v2.5")
|
|
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
|
|
|
|
def test_no_status_code_path_also_classifies(self):
|
|
"""When the error reaches us without a status code (transport
|
|
layer ate it) the message-only classifier branch must also
|
|
recognise the pattern.
|
|
"""
|
|
e = MockTransportError("tool_call.content must be string")
|
|
result = classify_api_error(e, provider="alibaba", model="qwen3.5-plus")
|
|
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
|
|
|
|
def test_unrelated_400_is_not_misclassified(self):
|
|
"""Make sure the patterns don't false-positive on normal 400s."""
|
|
e = MockAPIError("bad request: missing field 'model'", status_code=400)
|
|
result = classify_api_error(e, provider="openrouter", model="anthropic/claude-sonnet-4")
|
|
|
|
|
|
class TestOpenRouterUpstreamRateLimit:
|
|
"""Distinguish upstream-provider 429 from account-level 429 on OpenRouter.
|
|
|
|
When an upstream model (DeepSeek, Anthropic, etc.) rate-limits OpenRouter's
|
|
aggregate traffic, OpenRouter returns 429 with the outer message "Provider
|
|
returned error". The user's key is healthy — we must fall back to a
|
|
different model, NOT mark the credential exhausted.
|
|
"""
|
|
|
|
def test_openrouter_upstream_429_classified_as_upstream_rate_limit(self):
|
|
"""OpenRouter 429 with 'Provider returned error' → upstream_rate_limit."""
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=429,
|
|
body={
|
|
"error": {
|
|
"message": "Provider returned error",
|
|
"code": 429,
|
|
"metadata": {
|
|
"provider_name": "DeepSeek",
|
|
"raw": '{"error":{"message":"Rate limit exceeded"}}',
|
|
},
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", model="deepseek/deepseek-v4-flash")
|
|
assert result.reason == FailoverReason.upstream_rate_limit
|
|
assert result.should_rotate_credential is False
|
|
assert result.should_fallback is True
|
|
assert result.error_context.get("upstream_provider") == "DeepSeek"
|
|
|
|
def test_upstream_429_metadata_shape_without_explicit_provider(self):
|
|
"""metadata.raw shape alone (provider != openrouter literal) still detected."""
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=429,
|
|
body={
|
|
"error": {
|
|
"message": "Provider returned error",
|
|
"metadata": {"raw": '{"error":{"code":429}}'},
|
|
}
|
|
},
|
|
)
|
|
# provider passed as the slug-form some callers use
|
|
result = classify_api_error(e, provider="openrouter", model="x")
|
|
assert result.reason == FailoverReason.upstream_rate_limit
|
|
|
|
def test_account_level_429_still_rotates_credential(self):
|
|
"""A real account-level 429 (no upstream wrapper) → rate_limit, rotates."""
|
|
e = MockAPIError(
|
|
"Rate limit exceeded: 200 requests per minute",
|
|
status_code=429,
|
|
body={
|
|
"error": {
|
|
"message": "Rate limit exceeded: 200 requests per minute",
|
|
"code": 429,
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", model="deepseek/deepseek-v4-flash")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_rotate_credential is True
|
|
|
|
def test_upstream_wrapper_without_metadata_on_non_openrouter_not_matched(self):
|
|
"""'Provider returned error' alone on a non-openrouter provider → plain rate_limit."""
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=429,
|
|
body={"error": {"message": "Provider returned error", "code": 429}},
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", model="claude-sonnet-4")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_upstream_provider_name_missing_yields_empty_context(self):
|
|
"""No provider_name in metadata → upstream_rate_limit with empty context."""
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=429,
|
|
body={
|
|
"error": {
|
|
"message": "Provider returned error",
|
|
"metadata": {"raw": '{"error":{"code":429}}'},
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", model="x")
|
|
assert result.reason == FailoverReason.upstream_rate_limit
|
|
assert result.error_context.get("upstream_provider") is None
|
|
|
|
def test_overload_429_takes_precedence_over_upstream(self):
|
|
"""A 429 carrying overload language stays overloaded (retry same key)."""
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=429,
|
|
body={
|
|
"error": {
|
|
"message": "service is temporarily overloaded",
|
|
"metadata": {"provider_name": "DeepSeek"},
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", model="x")
|
|
# Overload disambiguation runs first; the outer message is the overload
|
|
# phrase, so this is an overload, not an upstream rate-limit.
|
|
assert result.reason == FailoverReason.overloaded
|
|
|
|
|
|
# ── HTTP 408 request timeout ────────────────────────────────────────────
|
|
|
|
class Test408RequestTimeout:
|
|
"""HTTP 408 must never fall through to the non-retryable 'other 4xx'
|
|
bucket (that abort persists an empty assistant turn — the "disappeared
|
|
conversation" / blank-bubble symptom). ALL 408s are classified as a transient
|
|
``timeout``: retryable, and explicitly NOT should_compress.
|
|
|
|
Design decision (field 2026-07-02): even the GitHub Copilot
|
|
``user_request_timeout`` / "Timed out reading request body ... use a
|
|
smaller request size" case is a plain retry, NOT auto-compression. Real
|
|
data showed the 408 is probabilistic jitter well below the hard prompt
|
|
ceiling — the same ~785k-token request that 408'd once succeeded on the
|
|
next attempt at ~786k — so retrying the same body usually works, and
|
|
auto-compaction would silently delete conversation history for a merely
|
|
transient timeout. Genuine over-window prompts surface as 413 /
|
|
context_overflow (their own compression path); users compact 408-prone
|
|
long sessions deliberately via ``/compress``.
|
|
"""
|
|
|
|
def test_copilot_oversized_body_408_retries_as_timeout_not_compress(self):
|
|
# The exact shape GitHub Copilot returns on a long session. It must
|
|
# retry (timeout), and must NOT auto-compress.
|
|
e = MockAPIError(
|
|
"Error code: 408 - {'error': {'message': 'Timed out reading "
|
|
"request body. Try again, or use a smaller request size.', "
|
|
"'code': 'user_request_timeout'}}",
|
|
status_code=408,
|
|
body={"error": {"message": "Timed out reading request body. "
|
|
"Try again, or use a smaller request size.",
|
|
"code": "user_request_timeout"}},
|
|
)
|
|
result = classify_api_error(e, provider="copilot", model="claude-opus-4.8")
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
def test_408_never_auto_compresses(self):
|
|
# Hard guard on the user's explicit preference: a 408 must NEVER
|
|
# trigger auto-compaction (which would delete history unprompted).
|
|
# This must FAIL if anyone re-routes 408 to payload_too_large.
|
|
for msg, body in [
|
|
("Timed out reading request body. Use a smaller request size.", {}),
|
|
("Request timed out.", {"error": {"code": "user_request_timeout"}}),
|
|
("Request Timeout", {}),
|
|
]:
|
|
e = MockAPIError(msg, status_code=408, body=body)
|
|
result = classify_api_error(e, provider="copilot", model="claude-opus-4.8")
|
|
assert result.should_compress is False, msg
|
|
assert result.reason != FailoverReason.payload_too_large, msg
|
|
|
|
def test_oversized_body_408_is_not_non_retryable_format_error(self):
|
|
# Falsification guard: if the 408 branch is removed, this 408 would
|
|
# be classified as a non-retryable format_error and the turn would
|
|
# abort into a blank bubble. This assertion must FAIL on buggy code.
|
|
e = MockAPIError(
|
|
"Timed out reading request body. Try again, or use a smaller "
|
|
"request size.",
|
|
status_code=408,
|
|
)
|
|
result = classify_api_error(e, provider="copilot", model="claude-opus-4.8")
|
|
assert result.retryable is True
|
|
assert result.reason != FailoverReason.format_error
|
|
|
|
def test_plain_408_is_transient_timeout(self):
|
|
# A generic gateway/request timeout must retry as a transport timeout.
|
|
e = MockAPIError("Request Timeout", status_code=408)
|
|
result = classify_api_error(e, provider="openai", model="gpt-5.5")
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
def test_stale_breaker_runtime_error_triggers_fallback_not_retry(self):
|
|
# The cross-turn stale-call circuit breaker (_check_stale_giveup in
|
|
# chat_completion_helpers.py) raises a RuntimeError when the provider
|
|
# has been unresponsive for N consecutive stale attempts. This must
|
|
# be classified as non-retryable + should_fallback so the retry loop
|
|
# activates the fallback provider immediately instead of burning all
|
|
# max_retries against the same dead provider (each retry hitting the
|
|
# circuit breaker instantly with zero network overhead).
|
|
e = RuntimeError(
|
|
"Provider has been unresponsive (no response received) for "
|
|
"6 consecutive stale attempts — aborting this call to "
|
|
"avoid an indefinite stall. Switch models or start a new "
|
|
"session, then retry."
|
|
)
|
|
result = classify_api_error(
|
|
e, provider="openrouter", model="anthropic/claude-fable-5",
|
|
approx_tokens=126327, context_length=200000, num_messages=274,
|
|
)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
assert result.should_compress is False
|
|
|
|
|
|
# ── Test: throttle vs overflow disambiguation + new overflow shapes ─────
|
|
# Port of anomalyco/opencode#37848 (expand context overflow patterns +
|
|
# rate-limit exclusion guard).
|
|
|
|
class TestThrottleVsOverflowDisambiguation:
|
|
"""Throttle messages that mention tokens must NOT route to compression."""
|
|
|
|
def test_bedrock_throttling_too_many_tokens_is_rate_limit(self):
|
|
# AWS Bedrock (and some proxies) surface throttling as
|
|
# "Throttling error: Too many tokens, please wait before trying
|
|
# again." — the "too many tokens" fragment sits in
|
|
# _CONTEXT_OVERFLOW_PATTERNS, so before the "throttling" rate-limit
|
|
# pattern this compressed a healthy session on every throttle.
|
|
e = Exception(
|
|
"Throttling error: Too many tokens, please wait before trying again."
|
|
)
|
|
result = classify_api_error(e, provider="bedrock", model="claude")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_compress is False
|
|
|
|
def test_plain_too_many_tokens_still_overflow(self):
|
|
# Without any throttle wording, "Too many tokens" remains a
|
|
# context-overflow signal (Z.AI / GLM family wording).
|
|
e = Exception("Too many tokens")
|
|
result = classify_api_error(e, provider="zai", model="glm-5")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
|
|
class TestExpandedOverflowPatterns:
|
|
"""New provider overflow wordings route into compression recovery."""
|
|
|
|
def test_maximum_allowed_input_length_is_overflow(self):
|
|
# Together/Fireworks-style wording — matched no pattern before.
|
|
e = Exception(
|
|
"Input length 131393 exceeds the maximum allowed input length "
|
|
"of 131040 tokens."
|
|
)
|
|
result = classify_api_error(e, provider="together", model="m")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_request_too_large_message_only_is_payload_too_large(self):
|
|
# Anthropic's structured 413 type re-wrapped by a proxy with no
|
|
# status attribute — was falling through to `unknown`.
|
|
e = Exception(
|
|
'{"error":{"type":"request_too_large",'
|
|
'"message":"Request exceeds the maximum size"}}'
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", model="m")
|
|
assert result.reason == FailoverReason.payload_too_large
|
|
assert result.should_compress is True
|
|
|
|
def test_longer_than_context_length_still_overflow(self):
|
|
# Regression guard for wordings that already matched.
|
|
e = Exception(
|
|
"The input (516368 tokens) is longer than the model's context "
|
|
"length (262144 tokens)."
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", model="m")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
def test_configured_context_size_still_overflow(self):
|
|
e = Exception(
|
|
"Prompt has 5,958,968 tokens, but the configured context size "
|
|
"is 256,000 tokens"
|
|
)
|
|
result = classify_api_error(e, provider="ollama", model="m")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
|