From 320c587256aa5a0a73822abb2a9733ece8c5794c Mon Sep 17 00:00:00 2001 From: Tyler Merritt Date: Wed, 10 Jun 2026 08:47:01 -0500 Subject: [PATCH] fix(context): parse vLLM's token-based output-cap error format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vLLM (and other OpenAI-compatible servers) report context overflow with both the window and the prompt in tokens: "This model's maximum context length is 131072 tokens. However, you requested 65536 output tokens and your prompt contains at least 65537 input tokens, for a total of at least 131073 tokens." parse_available_output_tokens_from_error() already classified this as an output-cap error (the "requested N output tokens" gate), but none of the extraction patterns matched the "prompt contains [at least] N input tokens" phrasing, so it returned None. The recovery path then misclassified the failure as prompt-too-long and looped through compression — which frees little while each retry keeps requesting the same oversized max_tokens — terminating in "cannot compress further" even though simply lowering the output cap would have succeeded. Add an extraction branch for the token-based phrasing: available output = window - reported input. When the input alone is at or over the window it still returns None, so the caller correctly falls through to compression. Relates to #43547. Co-Authored-By: Claude Opus 4.8 --- agent/model_metadata.py | 17 ++++++++++++ tests/test_output_cap_parsing.py | 46 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index e286485a4f3..547fe2f5791 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -1145,6 +1145,23 @@ def parse_available_output_tokens_from_error(error_msg: str) -> Optional[int]: if _available >= 1: return _available + # vLLM style: both the window and the prompt are reported in TOKENS, e.g. + # "This model's maximum context length is 131072 tokens. However, you + # requested 65536 output tokens and your prompt contains at least 65537 + # input tokens, for a total of at least 131073 tokens. Please reduce + # the length of the input prompt or the number of requested output + # tokens." + # Available output = window - input. When the input alone is at or over + # the window this stays None, so the caller correctly falls through to + # compression instead of futilely shrinking the output cap. + _m_vllm_input = re.search( + r'prompt contains (?:at least )?(\d+)\s*input tokens', error_lower + ) + if _m_ctx_tok and _m_vllm_input: + _available = int(_m_ctx_tok.group(1)) - int(_m_vllm_input.group(1)) + if _available >= 1: + return _available + return None diff --git a/tests/test_output_cap_parsing.py b/tests/test_output_cap_parsing.py index ddeee6045dd..f915102b844 100644 --- a/tests/test_output_cap_parsing.py +++ b/tests/test_output_cap_parsing.py @@ -120,3 +120,49 @@ class TestIsOutputCapError: def test_unrelated_error_is_not_output_cap(self): assert is_output_cap_error("some unrelated 400 error") is False + + +class TestParseVllmTokenBasedOutputCap: + """vLLM reports both the window and the prompt in TOKENS. + + Until this format was parsed, the recovery path misclassified it as + prompt-too-long and looped through compression (which frees little) while + retrying with the same oversized max_tokens — terminating in "cannot + compress further" even though simply lowering the output cap would have + succeeded. + """ + + # Verbatim vLLM 0.22 / OpenAI-compatible server response (max_tokens set). + _VLLM_MSG = ( + "This model's maximum context length is 131072 tokens. However, you " + "requested 65536 output tokens and your prompt contains at least " + "65537 input tokens, for a total of at least 131073 tokens. Please " + "reduce the length of the input prompt or the number of requested " + "output tokens." + ) + + def test_vllm_token_based_format(self): + # available output = 131072 - 65537 = 65535 + assert parse_available_output_tokens_from_error(self._VLLM_MSG) == 65535 + + def test_vllm_without_at_least_qualifier(self): + # Some versions omit the "at least" hedge. + msg = ("This model's maximum context length is 131072 tokens. However, " + "you requested 4096 output tokens and your prompt contains " + "100000 input tokens, for a total of 104096 tokens.") + assert parse_available_output_tokens_from_error(msg) == 31072 + + def test_vllm_retry_fits_inside_window(self): + # The retried cap plus the reported input must fit in the window. + available = parse_available_output_tokens_from_error(self._VLLM_MSG) + assert available is not None + assert available + 65537 <= 131072 + + def test_vllm_input_alone_exceeds_window_returns_none(self): + # Input >= window -> lowering the output cap cannot help; the caller + # must fall through to the compression path. + msg = ("This model's maximum context length is 131072 tokens. However, " + "you requested 1024 output tokens and your prompt contains at " + "least 140000 input tokens, for a total of at least 141024 " + "tokens.") + assert parse_available_output_tokens_from_error(msg) is None