mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-17 14:42:06 +00:00
251 lines
11 KiB
Python
251 lines
11 KiB
Python
"""The anti-thrashing guard must actually fire.
|
|
|
|
Two defects made it dead code:
|
|
|
|
1. Effectiveness was scored as ``(current_tokens - estimate(compressed)) / current_tokens``.
|
|
``current_tokens`` is the provider's FULL prompt (system prompt + tool schemas +
|
|
messages); ``estimate(compressed)`` covers messages only. The mixed basis reported
|
|
~96% savings on every pass, so ``_ineffective_compression_count`` reset each time.
|
|
|
|
2. Even scored correctly, message shrinkage is the wrong yardstick. ``should_compress()``
|
|
trips on the full prompt, but compaction can only shrink messages. When the
|
|
incompressible floor alone meets the threshold, each pass can shrink messages by a
|
|
healthy margin, reset the counter, and still leave the prompt over the line -- so the
|
|
next turn compacts again, forever.
|
|
|
|
Together these made a mis-sized context window present as a hung CLI rather than a
|
|
warning. Effectiveness is now scored against the goal -- did the prompt get under the
|
|
threshold? -- judged in ``update_from_response()``, the one place that sees the
|
|
provider's real prompt count for the just-compacted conversation.
|
|
|
|
Two subtleties this pins:
|
|
|
|
* The verdict must NOT live in ``should_compress()``. That runs twice per turn with
|
|
two different measures (a rough preflight estimate and the real post-response count,
|
|
#36718); the rough one can dip below the threshold and reset the strike every turn,
|
|
re-opening the loop. ``test_rough_preflight_reading_does_not_reopen_the_loop``.
|
|
|
|
* It must not be computed analytically as ``floor = current_tokens - rough_estimate``.
|
|
That subtracts an estimate from a real count, absorbs the tokenizer skew into "floor",
|
|
and disables compaction on a healthy session.
|
|
``test_no_false_positive_under_tokenizer_skew``.
|
|
"""
|
|
import pytest
|
|
|
|
from agent.context_compressor import ContextCompressor
|
|
|
|
|
|
def _compressor(threshold_tokens: int) -> ContextCompressor:
|
|
cc = ContextCompressor(
|
|
model="test-model",
|
|
threshold_percent=0.75,
|
|
protect_first_n=5,
|
|
protect_last_n=20,
|
|
quiet_mode=True,
|
|
config_context_length=40960,
|
|
provider="test",
|
|
)
|
|
cc.threshold_tokens = threshold_tokens # pin; don't couple to window math
|
|
cc._generate_summary = lambda *a, **k: "Summary of earlier turns."
|
|
return cc
|
|
|
|
|
|
def _messages(n: int, size: int = 1500) -> list:
|
|
msgs = [{"role": "system", "content": "sys"}]
|
|
for i in range(n):
|
|
role = "user" if i % 2 == 0 else "assistant"
|
|
msgs.append({"role": role, "content": f"m{i} " + "z" * size})
|
|
return msgs
|
|
|
|
|
|
def _turn(cc, msgs, real_prompt_tokens):
|
|
"""One agent turn as conversation_loop drives it.
|
|
|
|
should_compress() gates on the real prompt count; if it fires, compress()
|
|
runs and the provider then reports real usage for the shorter conversation.
|
|
The anti-thrashing verdict is judged in update_from_response() against that
|
|
real count -- not inside should_compress(), which is called twice per turn
|
|
with two different measures. Returns (msgs, did_compact).
|
|
"""
|
|
if not cc.should_compress(real_prompt_tokens):
|
|
return msgs, False
|
|
msgs = cc.compress(msgs, current_tokens=real_prompt_tokens)
|
|
cc._verify_compaction_cleared_threshold = True
|
|
cc.update_from_response({"prompt_tokens": real_prompt_tokens})
|
|
return msgs, True
|
|
|
|
|
|
class TestSavingsBasis:
|
|
def test_savings_does_not_depend_on_current_tokens(self):
|
|
"""Savings is a property of the messages, not of the caller's token count.
|
|
|
|
Before the fix, passing the full-prompt count made an identical compaction
|
|
report a wildly higher savings percentage.
|
|
"""
|
|
msgs = _messages(14)
|
|
|
|
a = _compressor(threshold_tokens=24_576)
|
|
a.compress([m.copy() for m in msgs], current_tokens=100_000)
|
|
with_full_prompt = a._last_compression_savings_pct
|
|
|
|
b = _compressor(threshold_tokens=24_576)
|
|
b.compress([m.copy() for m in msgs], current_tokens=None)
|
|
messages_only = b._last_compression_savings_pct
|
|
|
|
assert with_full_prompt == pytest.approx(messages_only, abs=0.01), (
|
|
"savings must be scored on a messages-vs-messages basis; got "
|
|
f"{with_full_prompt:.1f}% with a full-prompt count vs "
|
|
f"{messages_only:.1f}% without"
|
|
)
|
|
|
|
def test_no_op_compaction_is_not_reported_as_huge_savings(self):
|
|
cc = _compressor(threshold_tokens=24_576)
|
|
msgs = _messages(4, size=10) # below the minimum compressible size
|
|
cc.compress(msgs, current_tokens=100_000)
|
|
assert cc._last_compression_savings_pct < 50
|
|
|
|
|
|
class TestFutilityGuard:
|
|
def test_stops_when_floor_alone_meets_threshold(self):
|
|
"""Incompressible floor >= threshold -> shrinking messages cannot help."""
|
|
cc = _compressor(threshold_tokens=24_576)
|
|
msgs = _messages(14)
|
|
# Full prompt is far above threshold and dominated by system + tools,
|
|
# so the real count never drops no matter how the message list shrinks.
|
|
real_prompt_tokens = 33_564
|
|
|
|
fired = 0
|
|
for _ in range(6):
|
|
msgs, did = _turn(cc, msgs, real_prompt_tokens)
|
|
if did:
|
|
fired += 1
|
|
msgs.append({"role": "user", "content": "next " + "w" * 4000})
|
|
|
|
assert cc._ineffective_compression_count >= 2
|
|
assert not cc.should_compress(real_prompt_tokens), (
|
|
"compaction that cannot clear the threshold must stop"
|
|
)
|
|
assert fired <= 3, f"expected the loop to break early, compacted {fired}x"
|
|
|
|
def test_rough_preflight_reading_does_not_reopen_the_loop(self):
|
|
"""should_compress() runs twice per turn with two different measures.
|
|
|
|
The pre-API gate uses a rough estimate that can dip below the threshold;
|
|
the post-response gate uses the real count that does not. If the verdict
|
|
lived in should_compress(), the rough reading would reset the strike
|
|
every turn and the loop would never stop. Judging it in
|
|
update_from_response() (real-vs-real) closes that hole.
|
|
"""
|
|
cc = _compressor(threshold_tokens=24_576)
|
|
msgs = _messages(13)
|
|
rough, real = 20_000, 33_564 # rough dips under; real never does
|
|
|
|
fired = 0
|
|
for _ in range(8):
|
|
cc.should_compress(rough) # pre-API gate (rough)
|
|
msgs, did = _turn(cc, msgs, real) # post-response gate (real) + usage
|
|
if did:
|
|
fired += 1
|
|
msgs.append({"role": "user", "content": "more " + "w" * 3000})
|
|
|
|
assert fired <= 2, (
|
|
f"a sub-threshold rough reading must not re-open the loop; "
|
|
f"compacted {fired}x"
|
|
)
|
|
|
|
def test_effective_compaction_still_resets_the_counter(self):
|
|
"""A compaction that gets the prompt under the threshold is not thrashing."""
|
|
cc = _compressor(threshold_tokens=750_000)
|
|
msgs = _messages(120, size=2000)
|
|
before = len(msgs)
|
|
out = cc.compress(msgs, current_tokens=None)
|
|
|
|
assert len(out) < before, "a long transcript must still compact"
|
|
assert cc._last_compression_savings_pct > 10
|
|
assert cc._last_compression_made_progress is True
|
|
assert cc._ineffective_compression_count == 0
|
|
|
|
def test_no_false_positive_under_tokenizer_skew(self):
|
|
"""The provider's real count exceeds the rough estimate; that is not a floor.
|
|
|
|
An analytic `floor = real_prompt - rough_estimate` misreads tokenizer skew
|
|
as incompressible overhead and disables compaction on a healthy session.
|
|
The check must compare the caller's own measure on both sides.
|
|
"""
|
|
from agent.context_compressor import estimate_messages_tokens_rough
|
|
|
|
skew, floor = 1.6, 30_000
|
|
cc = _compressor(threshold_tokens=150_000)
|
|
msgs = _messages(160, size=2500)
|
|
|
|
real_prompt = floor + int(skew * estimate_messages_tokens_rough(msgs))
|
|
assert cc.should_compress(real_prompt)
|
|
msgs = cc.compress(msgs, current_tokens=real_prompt)
|
|
real_after = floor + int(skew * estimate_messages_tokens_rough(msgs))
|
|
cc._verify_compaction_cleared_threshold = True
|
|
cc.update_from_response({"prompt_tokens": real_after}) # provider's real count
|
|
|
|
assert real_after < cc.threshold_tokens, "compaction really did clear it"
|
|
assert cc._ineffective_compression_count == 0, (
|
|
"tokenizer skew must not be mistaken for an incompressible floor"
|
|
)
|
|
|
|
def test_latched_counter_resets_after_any_real_prompt_fits(self):
|
|
cc = _compressor(threshold_tokens=24_576)
|
|
cc._ineffective_compression_count = 2
|
|
|
|
cc.update_from_response({"prompt_tokens": 20_000})
|
|
|
|
assert cc._ineffective_compression_count == 0
|
|
assert cc.should_compress(33_564)
|
|
|
|
def test_usage_less_response_consumes_pending_verdict(self):
|
|
cc = _compressor(threshold_tokens=24_576)
|
|
cc._verify_compaction_cleared_threshold = True
|
|
cc.awaiting_real_usage_after_compression = True
|
|
|
|
cc.update_from_response({})
|
|
|
|
assert cc._verify_compaction_cleared_threshold is False
|
|
assert cc.awaiting_real_usage_after_compression is False
|
|
assert cc._ineffective_compression_count == 0
|
|
|
|
def test_a_failed_pass_records_exactly_one_strike(self):
|
|
"""A compaction that leaves the real prompt over the threshold: one strike.
|
|
|
|
The verdict is judged once, when the provider reports real usage — not on
|
|
every should_compress() reading.
|
|
"""
|
|
cc = _compressor(threshold_tokens=24_576)
|
|
msgs = _messages(14)
|
|
|
|
assert cc.should_compress(33_564)
|
|
cc.compress(msgs, current_tokens=33_564)
|
|
cc._verify_compaction_cleared_threshold = True
|
|
assert cc._ineffective_compression_count == 0, "no verdict before real usage"
|
|
|
|
cc.update_from_response({"prompt_tokens": 33_564}) # still over
|
|
assert cc._ineffective_compression_count == 1
|
|
|
|
# A later reading, rough or real, must not add phantom strikes.
|
|
cc.should_compress(33_564)
|
|
cc.should_compress(20_000)
|
|
assert cc._ineffective_compression_count == 1
|
|
|
|
|
|
class TestMinimumMessagesBranch:
|
|
def test_too_few_messages_records_an_ineffective_pass(self):
|
|
"""Returning the transcript unchanged must move the anti-thrash state.
|
|
|
|
Otherwise should_compress() keeps saying True about a transcript that can
|
|
never shrink, and every turn re-enters a no-op compaction.
|
|
"""
|
|
cc = _compressor(threshold_tokens=1)
|
|
msgs = _messages(3, size=10)
|
|
before = cc._ineffective_compression_count
|
|
|
|
out = cc.compress(msgs, current_tokens=100_000)
|
|
|
|
assert len(out) == len(msgs), "nothing should have been compressed"
|
|
assert cc._last_compression_made_progress is False
|
|
assert cc._ineffective_compression_count == before + 1
|