mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
0034da2975
The streaming hot loop computed len(repr(chunk)) on EVERY chunk to feed the retry-diagnostic byte counter — a full recursive pydantic repr at 5.5-8.8 us per chunk (measured), ~20-30 ms of pure CPU per 3,000-chunk response, paid on every streaming response on every platform. New _estimate_chunk_bytes() sizes the chunk from its delta payload strings (content / reasoning / tool-call arguments) plus a 40-byte framing floor: 2.1-2.4 us per chunk (~3x cheaper), independent of pydantic field count, never raises on unknown shapes (Anthropic events, stub providers fall back to the floor). Both call sites switched (chat-completions loop + anthropic event loop). The counter feeds only the stream-retry diagnostic log line (agent/stream_diag.py) — an estimate proportional to traffic preserves its purpose (distinguishing 'died at 0 bytes' from 'died mid-stream'). 6 new contract tests; 64 targeted stream tests green.
88 lines
2.3 KiB
Python
88 lines
2.3 KiB
Python
"""Tests for _estimate_chunk_bytes — the cheap per-chunk stream size proxy.
|
|
|
|
Replaced ``len(repr(chunk))`` in the streaming hot loop. Contract:
|
|
- proportional to delta payload length (content, reasoning, tool args);
|
|
- never raises on unknown shapes (returns the framing floor);
|
|
- monotonic accumulation stays positive.
|
|
"""
|
|
|
|
from openai.types.chat import ChatCompletionChunk
|
|
from openai.types.chat.chat_completion_chunk import (
|
|
Choice,
|
|
ChoiceDelta,
|
|
ChoiceDeltaToolCall,
|
|
ChoiceDeltaToolCallFunction,
|
|
)
|
|
|
|
from agent.chat_completion_helpers import _estimate_chunk_bytes
|
|
|
|
|
|
def _chunk(delta):
|
|
return ChatCompletionChunk(
|
|
id="chatcmpl-1",
|
|
created=1730000000,
|
|
model="m",
|
|
object="chat.completion.chunk",
|
|
choices=[Choice(index=0, delta=delta, finish_reason=None)],
|
|
)
|
|
|
|
|
|
def test_content_chunk_scales_with_payload():
|
|
small = _estimate_chunk_bytes(_chunk(ChoiceDelta(content="hi")))
|
|
large = _estimate_chunk_bytes(_chunk(ChoiceDelta(content="x" * 500)))
|
|
assert large - small == 498
|
|
assert small > 0
|
|
|
|
|
|
def test_reasoning_content_counted():
|
|
base = _estimate_chunk_bytes(_chunk(ChoiceDelta()))
|
|
|
|
class Delta:
|
|
content = None
|
|
reasoning_content = "thinking..."
|
|
reasoning = None
|
|
tool_calls = None
|
|
|
|
class Choice0:
|
|
delta = Delta()
|
|
|
|
class Chunk:
|
|
choices = [Choice0()]
|
|
|
|
assert _estimate_chunk_bytes(Chunk()) == base + len("thinking...")
|
|
|
|
|
|
def test_tool_call_arguments_counted():
|
|
delta = ChoiceDelta(
|
|
tool_calls=[
|
|
ChoiceDeltaToolCall(
|
|
index=0,
|
|
function=ChoiceDeltaToolCallFunction(
|
|
arguments='{"path": "/tmp/file"}', name="read_file"
|
|
),
|
|
)
|
|
]
|
|
)
|
|
est = _estimate_chunk_bytes(_chunk(delta))
|
|
assert est >= 40 + len('{"path": "/tmp/file"}') + len("read_file")
|
|
|
|
|
|
def test_unknown_shape_returns_floor_never_raises():
|
|
class Weird:
|
|
pass
|
|
|
|
assert _estimate_chunk_bytes(Weird()) == 40
|
|
assert _estimate_chunk_bytes(None) == 40
|
|
assert _estimate_chunk_bytes(object()) == 40
|
|
|
|
|
|
def test_anthropic_style_event_text_counted():
|
|
class Delta:
|
|
text = "anthropic text delta"
|
|
partial_json = None
|
|
|
|
class Event:
|
|
choices = None
|
|
delta = Delta()
|
|
|
|
assert _estimate_chunk_bytes(Event()) == 40 + len("anthropic text delta")
|