mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
6b81590c55
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
90 lines
3.2 KiB
Python
90 lines
3.2 KiB
Python
"""Tests for image-token accounting in the context compressor.
|
||
|
||
Covers the native-image-routing PR's companion change: the compressor's
|
||
multimodal message length counter now charges ~1600 tokens per attached
|
||
image part instead of 0, so tail-cut / prune decisions are accurate for
|
||
creative workflows that iterate on images across many turns.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
|
||
from agent.context_compressor import (
|
||
_CHARS_PER_TOKEN,
|
||
_IMAGE_CHAR_EQUIVALENT,
|
||
_IMAGE_TOKEN_ESTIMATE,
|
||
_content_length_for_budget,
|
||
)
|
||
|
||
|
||
class TestContentLengthForBudget:
|
||
def test_plain_string(self):
|
||
assert _content_length_for_budget("hello world") == 11
|
||
|
||
|
||
|
||
def test_text_only_list(self):
|
||
content = [
|
||
{"type": "text", "text": "first"},
|
||
{"type": "text", "text": "second"},
|
||
]
|
||
assert _content_length_for_budget(content) == 5 + 6
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def test_image_estimate_constant_is_reasonable(self):
|
||
"""Sanity-check the estimate aligns with real provider billing.
|
||
|
||
Anthropic ≈ width*height/750 → ~1600 for 1000×1200.
|
||
OpenAI GPT-4o high-detail 2048×2048 ≈ 1445.
|
||
Gemini 258/tile × 6 tiles for a 2048×2048 ≈ 1548.
|
||
Anything in the 800-2000 range is defensible. Enforce bounds so an
|
||
accidental edit doesn't drop it to e.g. 16.
|
||
"""
|
||
assert 800 <= _IMAGE_TOKEN_ESTIMATE <= 2500
|
||
assert _IMAGE_CHAR_EQUIVALENT == _IMAGE_TOKEN_ESTIMATE * _CHARS_PER_TOKEN
|
||
|
||
|
||
class TestTokenBudgetWithImages:
|
||
"""Integration: the compressor's tail-cut decision now respects image cost."""
|
||
|
||
def test_image_heavy_turns_count_toward_budget(self):
|
||
"""A tail with 5 image-bearing turns should blow past a 5K token budget."""
|
||
from agent.context_compressor import ContextCompressor
|
||
|
||
# Minimal compressor fixture — just enough to call _find_tail_cut_by_tokens
|
||
cc = object.__new__(ContextCompressor)
|
||
cc.tail_token_budget = 5000
|
||
|
||
# Build 10 messages: 5 with images, 5 with short text. Without the
|
||
# image-tokens fix, the compressor would think all 10 fit in 5K and
|
||
# protect them all. With the fix, images alone cost 5 × 1600 = 8K,
|
||
# so the tail should be trimmed.
|
||
messages = [{"role": "system", "content": "sys"}]
|
||
for i in range(5):
|
||
messages.append({
|
||
"role": "user",
|
||
"content": [
|
||
{"type": "text", "text": f"turn {i}"},
|
||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}},
|
||
],
|
||
})
|
||
messages.append({
|
||
"role": "assistant",
|
||
"content": f"response {i}",
|
||
})
|
||
|
||
cut = cc._find_tail_cut_by_tokens(messages, head_end=0, token_budget=5000)
|
||
|
||
# Budget is 5K, soft ceiling 7.5K. 5 images alone = 8000 image-tokens.
|
||
# Walking backward, the compressor should stop before including all 5.
|
||
# Exact cut depends on text lengths and min_tail, but it MUST be > 1
|
||
# (at least some head-side messages should be compressible).
|
||
assert cut > 1, (
|
||
f"Expected image-heavy tail to be trimmed; compressor placed cut at "
|
||
f"{cut} out of {len(messages)} (image tokens were likely ignored)."
|
||
)
|