From c1c4e56e7ee0dbd5932b7d58531cf28a9ba97ad8 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 19 Jul 2026 19:48:12 -0500 Subject: [PATCH] perf(agent): drop per-call base64 re-serialization from request-size estimate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every API iteration computed `total_chars = sum(len(str(msg)) ...)`, which str()-serializes the ENTIRE history — including base64 images and large tool results — just to take its length, then called estimate_request_tokens_rough, which walked the messages a SECOND time (it re-runs estimate_messages_tokens_rough internally, already computed one line above). Now derive both from one image-stripped message estimate: approx_tokens = estimate_messages_tokens_rough(api_messages) # once request_pressure_tokens = approx_tokens + tools_tokens # == old value total_chars = approx_tokens * 4 # log/metric only request_pressure_tokens is byte-identical to the old estimate_request_tokens_rough(api_messages, tools=agent.tools or None) (no system_prompt arg → messages + tools). total_chars only feeds a verbose log and the pre-api-request hook's request_char_count, so a rough proxy is fine and it no longer balloons on image turns. On the TTFT critical path for every call. tests/agent/test_model_metadata.py + test_compressor_image_tokens.py green. --- agent/conversation_loop.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 83c534087773..28cff8d44457 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -52,6 +52,7 @@ from agent.message_sanitization import ( ) from agent.model_metadata import ( MINIMUM_CONTEXT_LENGTH, + _estimate_tools_tokens_rough, estimate_messages_tokens_rough, estimate_request_tokens_rough, get_context_length_from_provider_error, @@ -1072,17 +1073,19 @@ def run_conversation( # the OpenAI SDK. Sanitizing here prevents the 3-retry cycle. _sanitize_messages_surrogates(api_messages) - # Calculate approximate request size for logging and pressure checks. - # estimate_messages_tokens_rough(api_messages) includes the system - # prompt copy but not the tool schema payload, which is sent as a - # separate field. Add tools back for compression decisions so long - # tool-heavy turns do not creep up to the context ceiling and leave - # no room for the model's final answer. - total_chars = sum(len(str(msg)) for msg in api_messages) + # Approximate request size for logging + pressure checks, from ONE + # message-token estimate. Tool schemas ship as a separate field, so add + # them back for compression decisions (50+ tools = 20-30K tokens). + # (estimate_messages_tokens_rough already image-strips base64.) This + # replaces a `str(msg)` char walk that re-serialized the whole history + # incl. base64 every call, plus estimate_request_tokens_rough re-walking + # the messages a second time. total_chars stays a rough (~) proxy — it + # only feeds a verbose log + the pre-api-request hook metric. approx_tokens = estimate_messages_tokens_rough(api_messages) - request_pressure_tokens = estimate_request_tokens_rough( - api_messages, tools=agent.tools or None + request_pressure_tokens = approx_tokens + ( + _estimate_tools_tokens_rough(agent.tools) if agent.tools else 0 ) + total_chars = approx_tokens * 4 _runtime_context_error = _ollama_context_limit_error( agent, request_pressure_tokens