mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
28524adb0e
- Remove tests/-shadowing sys.path.insert(dirname/'..') from 11 test files: it prepended the tests/ dir itself to sys.path, so 'import agent' / 'import hermes_cli' resolved to the test packages and collection died with ModuleNotFoundError depending on import order (2 files failed in every full-suite run; 9 more were latent). - Patch call_llm in 5 context-compressor tests that called compress() unmocked: each burned ~50s attempting live LLM traffic through the relay before falling back (572s file — the slowest in the suite, and flaky under the 300s per-file timeout). File now runs in ~5s. - agent/redact.py: fix two catastrophically-backtracking regexes hit by the compressor's redaction pass on large payloads — _STRICT_URL_USERINFO_RE anchors on the mandatory '//' (optional-scheme prefix backtracked O(n^2): ~55s on a 320KB payload, now sub-ms; output-equivalence fuzz-verified on 20k random strings), and the _CFG_DOTTED_RE/_CFG_ANCHORED_RE subs gain an exact linear keyword pre-gate so secret-free text skips the quadratic pattern entirely. - tests/gateway/test_feishu.py: version-guard the extra_ua_tags SDK signature check; the repo pins lark-oapi==1.6.8 but stale local installs (1.5.3) fail the assertion — skip below the pin. - tests/tools/test_managed_browserbase_and_modal.py: stub agent.redact + agent.credential_persistence in the fake agent package (empty __path__ blocks all real agent.* imports added since the fake was written). - tests/gateway/test_startup_restart_race.py: raise wait_for timeouts 2s -> 30s; 2s wall-clock on a loaded 40-worker box flaked in the baseline run (passes instantly when the box is quiet).
286 lines
11 KiB
Python
286 lines
11 KiB
Python
"""Tests for the context-halving bugfix.
|
|
|
|
Background
|
|
----------
|
|
When the API returns "max_tokens too large given prompt" (input is fine,
|
|
but input_tokens + requested max_tokens > context_window), the old code
|
|
incorrectly halved context_length via get_next_probe_tier().
|
|
|
|
The fix introduces:
|
|
* parse_available_output_tokens_from_error() — detects this specific
|
|
error class and returns the available output token budget.
|
|
* _ephemeral_max_output_tokens on AIAgent — a one-shot override that
|
|
caps the output for one retry without touching context_length.
|
|
* get_context_length_from_provider_error() — accepts only concrete
|
|
provider-reported lower context limits and refuses guessed probe-tier
|
|
step-downs when the provider gives no maximum.
|
|
|
|
Naming note
|
|
-----------
|
|
max_tokens = OUTPUT token cap (a single response).
|
|
context_length = TOTAL context window (input + output combined).
|
|
These are different and the old code conflated them; the fix keeps them
|
|
separate.
|
|
"""
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# parse_available_output_tokens_from_error — unit tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestParseAvailableOutputTokens:
|
|
"""Pure-function tests; no I/O required."""
|
|
|
|
def _parse(self, msg):
|
|
from agent.model_metadata import parse_available_output_tokens_from_error
|
|
return parse_available_output_tokens_from_error(msg)
|
|
|
|
# ── Should detect and extract ────────────────────────────────────────
|
|
|
|
def test_anthropic_canonical_format(self):
|
|
"""Canonical Anthropic error: max_tokens: X > context_window: Y - input_tokens: Z = available_tokens: W"""
|
|
msg = (
|
|
"max_tokens: 32768 > context_window: 200000 "
|
|
"- input_tokens: 190000 = available_tokens: 10000"
|
|
)
|
|
assert self._parse(msg) == 10000
|
|
|
|
|
|
|
|
def test_available_tokens_natural_language(self):
|
|
"""'available tokens: N' wording (no underscore)."""
|
|
msg = "max_tokens must be at most 10000 given your prompt (available tokens: 10000)"
|
|
assert self._parse(msg) == 10000
|
|
|
|
|
|
# ── Should NOT detect (returns None) ─────────────────────────────────
|
|
|
|
def test_prompt_too_long_is_not_output_cap_error(self):
|
|
"""'prompt is too long' errors must NOT be caught — they need context-overflow recovery."""
|
|
msg = "prompt is too long: 205000 tokens > 200000 maximum"
|
|
assert self._parse(msg) is None
|
|
|
|
|
|
|
|
def test_no_max_tokens_keyword(self):
|
|
"""Error not related to max_tokens at all."""
|
|
msg = "invalid_api_key: the API key is invalid"
|
|
assert self._parse(msg) is None
|
|
|
|
|
|
def test_rate_limit_error(self):
|
|
msg = "rate_limit_error: too many requests per minute"
|
|
assert self._parse(msg) is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Context-overflow recovery — only trust provider-reported limits
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestContextOverflowLimitSelection:
|
|
"""Context-overflow recovery must not invent a lower window size.
|
|
|
|
Some providers only say "input exceeds the context window" without telling
|
|
Hermes what the actual maximum is. In that case we may compress the
|
|
conversation, but must not silently probe-step from a user-configured 1M
|
|
window down to 256K/128K/64K/etc.
|
|
"""
|
|
|
|
def test_generic_overflow_without_provider_limit_keeps_context_length(self):
|
|
from agent.model_metadata import get_context_length_from_provider_error
|
|
from agent.model_metadata import get_next_probe_tier
|
|
from agent.model_metadata import parse_context_limit_from_error
|
|
|
|
old_ctx = 1_000_000
|
|
error_msg = (
|
|
"Your input exceeds the context window of this model. "
|
|
"Please adjust your input and try again."
|
|
)
|
|
|
|
assert parse_context_limit_from_error(error_msg) is None
|
|
assert get_next_probe_tier(old_ctx) == 256_000
|
|
assert get_context_length_from_provider_error(error_msg, old_ctx) is None
|
|
|
|
def test_explicit_provider_limit_still_selects_that_limit(self):
|
|
from agent.model_metadata import get_context_length_from_provider_error
|
|
|
|
error_msg = "prompt is too long: 300000 tokens > 272000 maximum"
|
|
|
|
assert get_context_length_from_provider_error(error_msg, 1_000_000) == 272_000
|
|
|
|
def test_reported_limit_not_lower_than_current_is_ignored(self):
|
|
from agent.model_metadata import get_context_length_from_provider_error
|
|
|
|
error_msg = "maximum context length is 1000000 tokens"
|
|
|
|
assert get_context_length_from_provider_error(error_msg, 272_000) is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# build_anthropic_kwargs — output cap clamping
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestBuildAnthropicKwargsClamping:
|
|
"""The context_length clamp only fires when output ceiling > window.
|
|
For standard Anthropic models (output ceiling < window) it must not fire.
|
|
"""
|
|
|
|
def _build(self, model, max_tokens=None, context_length=None):
|
|
from agent.anthropic_adapter import build_anthropic_kwargs
|
|
return build_anthropic_kwargs(
|
|
model=model,
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
tools=None,
|
|
max_tokens=max_tokens,
|
|
reasoning_config=None,
|
|
context_length=context_length,
|
|
)
|
|
|
|
def test_no_clamping_when_output_ceiling_fits_in_window(self):
|
|
"""Opus 4.6 native output (128K) < context window (200K) — no clamping."""
|
|
kwargs = self._build("claude-opus-4-6", context_length=200_000)
|
|
assert kwargs["max_tokens"] == 128_000
|
|
|
|
|
|
|
|
def test_explicit_max_tokens_clamped_when_exceeds_window(self):
|
|
"""Explicit max_tokens larger than a small window is clamped."""
|
|
kwargs = self._build("claude-opus-4-6", max_tokens=32_768, context_length=16_000)
|
|
assert kwargs["max_tokens"] == 15_999
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Ephemeral max_tokens mechanism — _build_api_kwargs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestEphemeralMaxOutputTokens:
|
|
"""_build_api_kwargs consumes _ephemeral_max_output_tokens exactly once
|
|
and falls back to self.max_tokens on subsequent calls.
|
|
"""
|
|
|
|
def _make_agent(self):
|
|
"""Return a minimal AIAgent with api_mode='anthropic_messages' and
|
|
a stubbed context_compressor, bypassing full __init__ cost."""
|
|
from run_agent import AIAgent
|
|
agent = object.__new__(AIAgent)
|
|
# Minimal attributes used by _build_api_kwargs
|
|
agent.api_mode = "anthropic_messages"
|
|
agent.model = "claude-opus-4-6"
|
|
agent.tools = []
|
|
agent.max_tokens = None
|
|
agent.reasoning_config = None
|
|
agent._is_anthropic_oauth = False
|
|
agent._ephemeral_max_output_tokens = None
|
|
|
|
compressor = MagicMock()
|
|
compressor.context_length = 200_000
|
|
agent.context_compressor = compressor
|
|
|
|
# Stub out the internal message-preparation helper
|
|
agent._prepare_anthropic_messages_for_api = MagicMock(
|
|
return_value=[{"role": "user", "content": "hi"}]
|
|
)
|
|
agent._anthropic_preserve_dots = MagicMock(return_value=False)
|
|
agent.request_overrides = {}
|
|
return agent
|
|
|
|
def test_ephemeral_override_is_used_on_first_call(self):
|
|
"""When _ephemeral_max_output_tokens is set, it overrides self.max_tokens."""
|
|
agent = self._make_agent()
|
|
agent._ephemeral_max_output_tokens = 5_000
|
|
|
|
kwargs = agent._build_api_kwargs([{"role": "user", "content": "hi"}])
|
|
assert kwargs["max_tokens"] == 5_000
|
|
|
|
def test_ephemeral_override_is_consumed_after_one_call(self):
|
|
"""After one call the ephemeral override is cleared to None."""
|
|
agent = self._make_agent()
|
|
agent._ephemeral_max_output_tokens = 5_000
|
|
|
|
agent._build_api_kwargs([{"role": "user", "content": "hi"}])
|
|
assert agent._ephemeral_max_output_tokens is None
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Integration: error handler does NOT halve context_length for output-cap errors
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestContextNotHalvedOnOutputCapError:
|
|
"""When the API returns 'max_tokens too large given prompt', the handler
|
|
must set _ephemeral_max_output_tokens and NOT modify context_length.
|
|
"""
|
|
|
|
def _make_agent_with_compressor(self, context_length=200_000):
|
|
from run_agent import AIAgent
|
|
from agent.context_compressor import ContextCompressor
|
|
|
|
agent = object.__new__(AIAgent)
|
|
agent.api_mode = "anthropic_messages"
|
|
agent.model = "claude-opus-4-6"
|
|
agent.base_url = "https://api.anthropic.com"
|
|
agent.tools = []
|
|
agent.max_tokens = None
|
|
agent.reasoning_config = None
|
|
agent._is_anthropic_oauth = False
|
|
agent._ephemeral_max_output_tokens = None
|
|
agent.log_prefix = ""
|
|
agent.quiet_mode = True
|
|
agent.verbose_logging = False
|
|
|
|
compressor = MagicMock(spec=ContextCompressor)
|
|
compressor.context_length = context_length
|
|
compressor.threshold_percent = 0.75
|
|
agent.context_compressor = compressor
|
|
|
|
agent._prepare_anthropic_messages_for_api = MagicMock(
|
|
return_value=[{"role": "user", "content": "hi"}]
|
|
)
|
|
agent._anthropic_preserve_dots = MagicMock(return_value=False)
|
|
agent._vprint = MagicMock()
|
|
agent.request_overrides = {}
|
|
return agent
|
|
|
|
def test_output_cap_error_sets_ephemeral_not_context_length(self):
|
|
"""On 'max_tokens too large' error, _ephemeral_max_output_tokens is set
|
|
and compressor.context_length is left unchanged."""
|
|
from agent.model_metadata import parse_available_output_tokens_from_error
|
|
|
|
error_msg = (
|
|
"max_tokens: 128000 > context_window: 200000 "
|
|
"- input_tokens: 180000 = available_tokens: 20000"
|
|
)
|
|
|
|
# Simulate the handler logic from run_agent.py
|
|
agent = self._make_agent_with_compressor(context_length=200_000)
|
|
old_ctx = agent.context_compressor.context_length
|
|
|
|
available_out = parse_available_output_tokens_from_error(error_msg)
|
|
assert available_out == 20_000, "parser must detect the error"
|
|
|
|
# The fix: set ephemeral, skip context_length modification
|
|
agent._ephemeral_max_output_tokens = max(1, available_out - 64)
|
|
|
|
# context_length must be untouched
|
|
assert agent.context_compressor.context_length == old_ctx
|
|
assert agent._ephemeral_max_output_tokens == 19_936
|
|
|
|
|
|
def test_output_cap_error_safety_margin(self):
|
|
"""The ephemeral value includes a 64-token safety margin below available_out."""
|
|
from agent.model_metadata import parse_available_output_tokens_from_error
|
|
|
|
error_msg = (
|
|
"max_tokens: 32768 > context_window: 200000 "
|
|
"- input_tokens: 190000 = available_tokens: 10000"
|
|
)
|
|
available_out = parse_available_output_tokens_from_error(error_msg)
|
|
safe_out = max(1, available_out - 64)
|
|
assert safe_out == 9_936
|
|
|