fix(error-classifier): route 5xx context-overflow into compression

Local inference servers (llama.cpp/llama-server, vLLM/Ollama behind a
Cloudflare/Tailscale hop) report context overflow with HTTP 500/502/503/529
instead of 400/413. _classify_by_status returned server_error/overloaded and
retried blindly, then dropped the turn with no compaction. Route explicit
_CONTEXT_OVERFLOW_PATTERNS matches on those 5xx codes to context_overflow
(should_compress=True); plain 500 stays server_error, plain 503 overloaded.
This commit is contained in:
pefontana 2026-07-01 15:46:47 +05:30 committed by kshitij
parent 74809b4e94
commit a04b7024ff
2 changed files with 67 additions and 0 deletions

View file

@ -964,9 +964,31 @@ def _classify_by_status(
retryable=False,
should_fallback=True,
)
# Some local inference servers (notably llama.cpp / llama-server)
# report context overflow with an HTTP 500 instead of the standard
# 400/413. The request-validation guard above already ran, so any
# remaining explicit context-overflow signal routes into the
# compression-and-retry path (mirroring _classify_400) instead of
# blind server_error retries that exhaust and drop the turn.
if any(p in error_msg for p in _CONTEXT_OVERFLOW_PATTERNS):
return result_fn(
FailoverReason.context_overflow,
retryable=True,
should_compress=True,
)
return result_fn(FailoverReason.server_error, retryable=True)
if status_code in {503, 529}:
# Same overflow-as-5xx variant (server busy / model-load OOM, or a
# Cloudflare/Tailscale hop relabeling the status). Route explicit
# overflow bodies into compression; otherwise treat as transient
# overload and retry.
if any(p in error_msg for p in _CONTEXT_OVERFLOW_PATTERNS):
return result_fn(
FailoverReason.context_overflow,
retryable=True,
should_compress=True,
)
return result_fn(FailoverReason.overloaded, retryable=True)
# Other 4xx — non-retryable

View file

@ -469,6 +469,51 @@ class TestClassifyApiError:
assert result.reason == FailoverReason.server_error
assert result.retryable is True
# ── 5xx that are actually context overflow ──
# Some local inference servers (llama.cpp / llama-server, and vLLM/Ollama
# behind a Cloudflare/Tailscale hop) report context overflow with a 5xx
# status instead of the standard 400/413. These must route into the
# compression-and-retry path, not the blind server_error/overloaded retry
# that exhausts and drops the turn.
def test_500_context_overflow_routes_to_compression(self):
"""A llama.cpp 500 'Context size has been exceeded.' must compress."""
e = MockAPIError(
"Context size has been exceeded.",
status_code=500,
body={"error": {"code": 500, "message": "Context size has been exceeded.", "type": "server_error"}},
)
result = classify_api_error(e)
assert result.reason == FailoverReason.context_overflow
assert result.should_compress is True
assert result.retryable is True
def test_503_context_overflow_routes_to_compression(self):
"""An overflow surfaced as 503 (busy / model-load OOM) must compress."""
e = MockAPIError(
"the request exceeds the available context size",
status_code=503,
body={"error": {"message": "the request exceeds the available context size"}},
)
result = classify_api_error(e)
assert result.reason == FailoverReason.context_overflow
assert result.should_compress is True
def test_500_plain_server_error_not_compressed(self):
"""A genuine 500 crash without overflow wording must NOT be swallowed
into compression it stays a retryable server_error."""
e = MockAPIError("Internal Server Error", status_code=500)
result = classify_api_error(e)
assert result.reason == FailoverReason.server_error
assert result.should_compress is False
def test_503_plain_overloaded_not_compressed(self):
"""A genuine 503 overload without overflow wording stays overloaded."""
e = MockAPIError("Service Unavailable", status_code=503)
result = classify_api_error(e)
assert result.reason == FailoverReason.overloaded
assert result.should_compress is False
# ── Model not found ──
def test_404_model_not_found(self):