From 4751af0a0bab56177693f305712d3750e6fe11d9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 5 Jul 2026 02:05:51 -0700 Subject: [PATCH] feat(errors): fail fast on TLS certificate verification failures with fix hints (#57992) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inspired by Claude Code v2.1.199 (July 2, 2026): SSL certificate errors (TLS-inspecting proxies, missing CA bundles, expired certs) no longer burn retries before showing actionable guidance — they fail immediately with the fix hint. - agent/error_classifier.py: new FailoverReason.ssl_cert_verification + _SSL_CERT_VERIFY_PATTERNS, checked BEFORE the transient-SSL patterns (cert-verify messages also contain '[SSL:' and previously retried forever as timeout). Non-retryable, no compression, no fallback churn. - agent/conversation_loop.py: dedicated status line + per-cause fix hints (corporate proxy CA bundle, certifi refresh, self-signed local endpoints) on the non-retryable abort path. - 7 new tests incl. regression guards (transient alerts still retry, large-session cert failure doesn't trigger compression). --- agent/conversation_loop.py | 44 +++++++++++++++++ agent/error_classifier.py | 45 ++++++++++++++++- tests/agent/test_error_classifier.py | 72 ++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 1 deletion(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index dff11c489b5..5a254b89e6b 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3700,6 +3700,8 @@ def run_conversation( if agent._has_pending_fallback(): if classified.reason == FailoverReason.content_policy_blocked: agent._buffer_status("⚠️ Provider safety filter blocked this request — trying fallback...") + elif classified.reason == FailoverReason.ssl_cert_verification: + agent._buffer_status("⚠️ TLS certificate verification failed — trying fallback...") else: agent._buffer_status(f"⚠️ Non-retryable error (HTTP {status_code}) — trying fallback...") if agent._try_activate_fallback(): @@ -3728,6 +3730,11 @@ def run_conversation( f"❌ Provider safety filter blocked this request: " f"{_nonretryable_summary}" ) + elif classified.reason == FailoverReason.ssl_cert_verification: + agent._emit_status( + f"❌ TLS certificate verification failed: " + f"{_nonretryable_summary}" + ) else: agent._emit_status( f"❌ Non-retryable error (HTTP {status_code}): " @@ -3801,6 +3808,43 @@ def run_conversation( f"{agent.log_prefix} hermes fallback add (interactive picker — same as `hermes model`)", force=True, ) + # TLS certificate failures are environment problems, not + # provider/prompt problems — tell the user exactly which + # knobs fix each common cause. Inspired by Claude Code + # v2.1.199's immediate SSL fix hints. + if classified.reason == FailoverReason.ssl_cert_verification: + agent._vprint( + f"{agent.log_prefix} 💡 The TLS certificate chain could not be verified. This fails the same", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} way on every retry — fix the environment, then try again:", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} • Corporate TLS-inspecting proxy? Point Python at its CA bundle:", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} export SSL_CERT_FILE=/path/to/corp-ca.pem (also REQUESTS_CA_BUNDLE)", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} • Missing/stale system CA store? Install/refresh it:", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} pip install --upgrade certifi (macOS: run 'Install Certificates.command')", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} • Self-signed local endpoint (llama.cpp, LM Studio, vLLM)? Use http://", + force=True, + ) + agent._vprint( + f"{agent.log_prefix} for localhost, or add the server's cert to your trust store.", + force=True, + ) logger.error(f"{agent.log_prefix}Non-retryable client error: {api_error}") # Skip session persistence when the error is likely # context-overflow related (status 400 + large session). diff --git a/agent/error_classifier.py b/agent/error_classifier.py index 27311609de6..0cb397f3322 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -41,6 +41,11 @@ class FailoverReason(enum.Enum): # Transport timeout = "timeout" # Connection/read timeout — rebuild client + retry + # TLS certificate verification failure — deterministic for the host + # (TLS-inspecting proxy, missing/expired CA bundle, self-signed cert). + # Retrying reproduces the identical handshake failure, so fail fast + # with actionable guidance instead of burning retries. + ssl_cert_verification = "ssl_cert_verification" # Context / payload context_overflow = "context_overflow" # Context too large — compress, not failover @@ -447,6 +452,29 @@ _SERVER_DISCONNECT_PATTERNS = [ "incomplete chunked read", ] +# SSL certificate verification failures — deterministic, NOT transient. +# +# A failed certificate chain (TLS-inspecting corporate proxy, missing +# custom CA in the trust store, expired certificate, self-signed cert) +# fails identically on every retry. Burning the retry budget before +# surfacing the error hides the actionable fix from the user for minutes. +# Inspired by Claude Code v2.1.199 (July 2026), which made SSL certificate +# errors fail immediately with a fix hint instead of retrying. +# +# Must be checked BEFORE _SSL_TRANSIENT_PATTERNS — "certificate verify +# failed" messages usually also contain "[SSL:" which would otherwise +# match the transient list and retry forever. +_SSL_CERT_VERIFY_PATTERNS = [ + "certificate verify failed", # Python ssl module canonical text + "certificate_verify_failed", # OpenSSL error token + "unable to get local issuer certificate", + "self-signed certificate", + "self signed certificate", + "certificate has expired", + "hostname mismatch, certificate is not valid", + "unable to verify the first certificate", # Node/undici phrasing (MCP bridges) +] + # SSL/TLS transient failure patterns — intentionally distinct from # _SERVER_DISCONNECT_PATTERNS above. # @@ -744,7 +772,22 @@ def classify_api_error( if classified is not None: return classified - # ── 5. SSL/TLS transient errors → retry as timeout (not compression) ── + # ── 5. SSL certificate verification failures → fail fast ──────── + # A broken certificate chain (TLS-inspecting proxy, missing custom CA, + # expired/self-signed cert) is deterministic for the host — every retry + # reproduces the identical handshake failure. Fail immediately with + # actionable guidance instead of burning the retry budget first. + # Checked BEFORE the transient-SSL patterns: cert-verify messages also + # contain "[ssl:" which would otherwise match the transient list. + # Inspired by Claude Code v2.1.199 (July 2026). + if any(p in error_msg for p in _SSL_CERT_VERIFY_PATTERNS): + return _result( + FailoverReason.ssl_cert_verification, + retryable=False, + should_fallback=False, + ) + + # ── 5b. SSL/TLS transient errors → retry as timeout (not compression) ── # SSL alerts mid-stream are transport hiccups, not server-side context # overflow signals. Classify before the disconnect check so a large # session doesn't incorrectly trigger context compression when the real diff --git a/tests/agent/test_error_classifier.py b/tests/agent/test_error_classifier.py index 6c533089986..3aff558062e 100644 --- a/tests/agent/test_error_classifier.py +++ b/tests/agent/test_error_classifier.py @@ -55,6 +55,7 @@ class TestFailoverReason: "auth", "auth_permanent", "billing", "rate_limit", "upstream_rate_limit", "overloaded", "server_error", "timeout", + "ssl_cert_verification", "context_overflow", "payload_too_large", "image_too_large", "model_not_found", "format_error", "invalid_encrypted_content", @@ -1679,6 +1680,77 @@ class TestSSLTransientPatterns: assert result.reason == FailoverReason.timeout assert result.retryable is True + +# ── Test: SSL certificate verification failures (fail fast) ──────────── + +class TestSSLCertVerificationFailFast: + """Certificate verification failures are deterministic for the host — + a TLS-inspecting proxy, missing custom CA, expired or self-signed cert + fails identically on every retry. They must classify as non-retryable + ``ssl_cert_verification`` so the user sees the fix hint immediately, + instead of matching the transient "[ssl:" pattern and retrying forever. + + Inspired by Claude Code v2.1.199 (July 2026). + """ + + def test_python_cert_verify_failed_is_non_retryable(self): + import ssl + e = ssl.SSLCertVerificationError( + 1, + "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: " + "unable to get local issuer certificate (_ssl.c:1006)", + ) + result = classify_api_error(e) + assert result.reason == FailoverReason.ssl_cert_verification + assert result.retryable is False + assert result.should_compress is False + + def test_wrapped_cert_verify_message_is_non_retryable(self): + """SDKs often re-raise without chaining — match on message alone.""" + e = Exception( + "Connection error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate " + "verify failed: self-signed certificate in certificate chain" + ) + result = classify_api_error(e) + assert result.reason == FailoverReason.ssl_cert_verification + assert result.retryable is False + + def test_expired_certificate_is_non_retryable(self): + e = Exception("certificate verify failed: certificate has expired") + result = classify_api_error(e) + assert result.reason == FailoverReason.ssl_cert_verification + assert result.retryable is False + + def test_node_undici_phrasing_is_non_retryable(self): + """MCP bridges surface Node's phrasing.""" + e = Exception("fetch failed: unable to verify the first certificate") + result = classify_api_error(e) + assert result.reason == FailoverReason.ssl_cert_verification + assert result.retryable is False + + def test_cert_verify_wins_over_transient_ssl_prefix(self): + """The '[SSL:' prefix also appears in cert-verify messages; the + cert check must run first so this doesn't retry as timeout.""" + e = Exception("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed") + result = classify_api_error(e) + assert result.reason == FailoverReason.ssl_cert_verification + assert result.retryable is False + + def test_transient_ssl_alert_still_retries(self): + """Regression guard: genuine transient alerts keep retrying.""" + e = Exception("[SSL: BAD_RECORD_MAC] sslv3 alert bad record mac") + result = classify_api_error(e) + assert result.reason == FailoverReason.timeout + assert result.retryable is True + + def test_cert_verify_on_large_session_does_not_compress(self): + e = Exception("certificate verify failed: unable to get local issuer certificate") + result = classify_api_error( + e, approx_tokens=180000, context_length=200000, num_messages=300, + ) + assert result.reason == FailoverReason.ssl_cert_verification + assert result.should_compress is False + # ── Test: RateLimitError without status_code (Copilot/GitHub Models) ────────── class TestRateLimitErrorWithoutStatusCode: