mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-08 13:12:08 +00:00
feat(errors): fail fast on TLS certificate verification failures with fix hints (#57992)
Inspired by Claude Code v2.1.199 (July 2, 2026): SSL certificate errors (TLS-inspecting proxies, missing CA bundles, expired certs) no longer burn retries before showing actionable guidance — they fail immediately with the fix hint. - agent/error_classifier.py: new FailoverReason.ssl_cert_verification + _SSL_CERT_VERIFY_PATTERNS, checked BEFORE the transient-SSL patterns (cert-verify messages also contain '[SSL:' and previously retried forever as timeout). Non-retryable, no compression, no fallback churn. - agent/conversation_loop.py: dedicated status line + per-cause fix hints (corporate proxy CA bundle, certifi refresh, self-signed local endpoints) on the non-retryable abort path. - 7 new tests incl. regression guards (transient alerts still retry, large-session cert failure doesn't trigger compression).
This commit is contained in:
Teknium
GitHub
parent
619db0175d
commit
4751af0a0b
3 changed files with 160 additions and 1 deletions
|
|
@ -3700,6 +3700,8 @@ def run_conversation(
|
|||
if agent._has_pending_fallback():
|
||||
if classified.reason == FailoverReason.content_policy_blocked:
|
||||
agent._buffer_status("⚠️ Provider safety filter blocked this request — trying fallback...")
|
||||
elif classified.reason == FailoverReason.ssl_cert_verification:
|
||||
agent._buffer_status("⚠️ TLS certificate verification failed — trying fallback...")
|
||||
else:
|
||||
agent._buffer_status(f"⚠️ Non-retryable error (HTTP {status_code}) — trying fallback...")
|
||||
if agent._try_activate_fallback():
|
||||
|
|
@ -3728,6 +3730,11 @@ def run_conversation(
|
|||
f"❌ Provider safety filter blocked this request: "
|
||||
f"{_nonretryable_summary}"
|
||||
)
|
||||
elif classified.reason == FailoverReason.ssl_cert_verification:
|
||||
agent._emit_status(
|
||||
f"❌ TLS certificate verification failed: "
|
||||
f"{_nonretryable_summary}"
|
||||
)
|
||||
else:
|
||||
agent._emit_status(
|
||||
f"❌ Non-retryable error (HTTP {status_code}): "
|
||||
|
|
@ -3801,6 +3808,43 @@ def run_conversation(
|
|||
f"{agent.log_prefix} hermes fallback add (interactive picker — same as `hermes model`)",
|
||||
force=True,
|
||||
)
|
||||
# TLS certificate failures are environment problems, not
|
||||
# provider/prompt problems — tell the user exactly which
|
||||
# knobs fix each common cause. Inspired by Claude Code
|
||||
# v2.1.199's immediate SSL fix hints.
|
||||
if classified.reason == FailoverReason.ssl_cert_verification:
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix} 💡 The TLS certificate chain could not be verified. This fails the same",
|
||||
force=True,
|
||||
)
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix} way on every retry — fix the environment, then try again:",
|
||||
force=True,
|
||||
)
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix} • Corporate TLS-inspecting proxy? Point Python at its CA bundle:",
|
||||
force=True,
|
||||
)
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix} export SSL_CERT_FILE=/path/to/corp-ca.pem (also REQUESTS_CA_BUNDLE)",
|
||||
force=True,
|
||||
)
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix} • Missing/stale system CA store? Install/refresh it:",
|
||||
force=True,
|
||||
)
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix} pip install --upgrade certifi (macOS: run 'Install Certificates.command')",
|
||||
force=True,
|
||||
)
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix} • Self-signed local endpoint (llama.cpp, LM Studio, vLLM)? Use http://",
|
||||
force=True,
|
||||
)
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix} for localhost, or add the server's cert to your trust store.",
|
||||
force=True,
|
||||
)
|
||||
logger.error(f"{agent.log_prefix}Non-retryable client error: {api_error}")
|
||||
# Skip session persistence when the error is likely
|
||||
# context-overflow related (status 400 + large session).
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ class FailoverReason(enum.Enum):
|
|||
|
||||
# Transport
|
||||
timeout = "timeout" # Connection/read timeout — rebuild client + retry
|
||||
# TLS certificate verification failure — deterministic for the host
|
||||
# (TLS-inspecting proxy, missing/expired CA bundle, self-signed cert).
|
||||
# Retrying reproduces the identical handshake failure, so fail fast
|
||||
# with actionable guidance instead of burning retries.
|
||||
ssl_cert_verification = "ssl_cert_verification"
|
||||
|
||||
# Context / payload
|
||||
context_overflow = "context_overflow" # Context too large — compress, not failover
|
||||
|
|
@ -447,6 +452,29 @@ _SERVER_DISCONNECT_PATTERNS = [
|
|||
"incomplete chunked read",
|
||||
]
|
||||
|
||||
# SSL certificate verification failures — deterministic, NOT transient.
|
||||
#
|
||||
# A failed certificate chain (TLS-inspecting corporate proxy, missing
|
||||
# custom CA in the trust store, expired certificate, self-signed cert)
|
||||
# fails identically on every retry. Burning the retry budget before
|
||||
# surfacing the error hides the actionable fix from the user for minutes.
|
||||
# Inspired by Claude Code v2.1.199 (July 2026), which made SSL certificate
|
||||
# errors fail immediately with a fix hint instead of retrying.
|
||||
#
|
||||
# Must be checked BEFORE _SSL_TRANSIENT_PATTERNS — "certificate verify
|
||||
# failed" messages usually also contain "[SSL:" which would otherwise
|
||||
# match the transient list and retry forever.
|
||||
_SSL_CERT_VERIFY_PATTERNS = [
|
||||
"certificate verify failed", # Python ssl module canonical text
|
||||
"certificate_verify_failed", # OpenSSL error token
|
||||
"unable to get local issuer certificate",
|
||||
"self-signed certificate",
|
||||
"self signed certificate",
|
||||
"certificate has expired",
|
||||
"hostname mismatch, certificate is not valid",
|
||||
"unable to verify the first certificate", # Node/undici phrasing (MCP bridges)
|
||||
]
|
||||
|
||||
# SSL/TLS transient failure patterns — intentionally distinct from
|
||||
# _SERVER_DISCONNECT_PATTERNS above.
|
||||
#
|
||||
|
|
@ -744,7 +772,22 @@ def classify_api_error(
|
|||
if classified is not None:
|
||||
return classified
|
||||
|
||||
# ── 5. SSL/TLS transient errors → retry as timeout (not compression) ──
|
||||
# ── 5. SSL certificate verification failures → fail fast ────────
|
||||
# A broken certificate chain (TLS-inspecting proxy, missing custom CA,
|
||||
# expired/self-signed cert) is deterministic for the host — every retry
|
||||
# reproduces the identical handshake failure. Fail immediately with
|
||||
# actionable guidance instead of burning the retry budget first.
|
||||
# Checked BEFORE the transient-SSL patterns: cert-verify messages also
|
||||
# contain "[ssl:" which would otherwise match the transient list.
|
||||
# Inspired by Claude Code v2.1.199 (July 2026).
|
||||
if any(p in error_msg for p in _SSL_CERT_VERIFY_PATTERNS):
|
||||
return _result(
|
||||
FailoverReason.ssl_cert_verification,
|
||||
retryable=False,
|
||||
should_fallback=False,
|
||||
)
|
||||
|
||||
# ── 5b. SSL/TLS transient errors → retry as timeout (not compression) ──
|
||||
# SSL alerts mid-stream are transport hiccups, not server-side context
|
||||
# overflow signals. Classify before the disconnect check so a large
|
||||
# session doesn't incorrectly trigger context compression when the real
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ class TestFailoverReason:
|
|||
"auth", "auth_permanent", "billing", "rate_limit",
|
||||
"upstream_rate_limit",
|
||||
"overloaded", "server_error", "timeout",
|
||||
"ssl_cert_verification",
|
||||
"context_overflow", "payload_too_large", "image_too_large",
|
||||
"model_not_found", "format_error",
|
||||
"invalid_encrypted_content",
|
||||
|
|
@ -1679,6 +1680,77 @@ class TestSSLTransientPatterns:
|
|||
assert result.reason == FailoverReason.timeout
|
||||
assert result.retryable is True
|
||||
|
||||
|
||||
# ── Test: SSL certificate verification failures (fail fast) ────────────
|
||||
|
||||
class TestSSLCertVerificationFailFast:
|
||||
"""Certificate verification failures are deterministic for the host —
|
||||
a TLS-inspecting proxy, missing custom CA, expired or self-signed cert
|
||||
fails identically on every retry. They must classify as non-retryable
|
||||
``ssl_cert_verification`` so the user sees the fix hint immediately,
|
||||
instead of matching the transient "[ssl:" pattern and retrying forever.
|
||||
|
||||
Inspired by Claude Code v2.1.199 (July 2026).
|
||||
"""
|
||||
|
||||
def test_python_cert_verify_failed_is_non_retryable(self):
|
||||
import ssl
|
||||
e = ssl.SSLCertVerificationError(
|
||||
1,
|
||||
"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: "
|
||||
"unable to get local issuer certificate (_ssl.c:1006)",
|
||||
)
|
||||
result = classify_api_error(e)
|
||||
assert result.reason == FailoverReason.ssl_cert_verification
|
||||
assert result.retryable is False
|
||||
assert result.should_compress is False
|
||||
|
||||
def test_wrapped_cert_verify_message_is_non_retryable(self):
|
||||
"""SDKs often re-raise without chaining — match on message alone."""
|
||||
e = Exception(
|
||||
"Connection error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate "
|
||||
"verify failed: self-signed certificate in certificate chain"
|
||||
)
|
||||
result = classify_api_error(e)
|
||||
assert result.reason == FailoverReason.ssl_cert_verification
|
||||
assert result.retryable is False
|
||||
|
||||
def test_expired_certificate_is_non_retryable(self):
|
||||
e = Exception("certificate verify failed: certificate has expired")
|
||||
result = classify_api_error(e)
|
||||
assert result.reason == FailoverReason.ssl_cert_verification
|
||||
assert result.retryable is False
|
||||
|
||||
def test_node_undici_phrasing_is_non_retryable(self):
|
||||
"""MCP bridges surface Node's phrasing."""
|
||||
e = Exception("fetch failed: unable to verify the first certificate")
|
||||
result = classify_api_error(e)
|
||||
assert result.reason == FailoverReason.ssl_cert_verification
|
||||
assert result.retryable is False
|
||||
|
||||
def test_cert_verify_wins_over_transient_ssl_prefix(self):
|
||||
"""The '[SSL:' prefix also appears in cert-verify messages; the
|
||||
cert check must run first so this doesn't retry as timeout."""
|
||||
e = Exception("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed")
|
||||
result = classify_api_error(e)
|
||||
assert result.reason == FailoverReason.ssl_cert_verification
|
||||
assert result.retryable is False
|
||||
|
||||
def test_transient_ssl_alert_still_retries(self):
|
||||
"""Regression guard: genuine transient alerts keep retrying."""
|
||||
e = Exception("[SSL: BAD_RECORD_MAC] sslv3 alert bad record mac")
|
||||
result = classify_api_error(e)
|
||||
assert result.reason == FailoverReason.timeout
|
||||
assert result.retryable is True
|
||||
|
||||
def test_cert_verify_on_large_session_does_not_compress(self):
|
||||
e = Exception("certificate verify failed: unable to get local issuer certificate")
|
||||
result = classify_api_error(
|
||||
e, approx_tokens=180000, context_length=200000, num_messages=300,
|
||||
)
|
||||
assert result.reason == FailoverReason.ssl_cert_verification
|
||||
assert result.should_compress is False
|
||||
|
||||
# ── Test: RateLimitError without status_code (Copilot/GitHub Models) ──────────
|
||||
|
||||
class TestRateLimitErrorWithoutStatusCode:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue