mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-31 19:16:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat(agent): report context occupancy, not just tokens saved

Browse source 
Tokens saved is the wrong headline for this feature. Micro-compaction is
not an efficiency optimisation — the same summarization work happens either
way. What it buys is (a) that work amortized across turns instead of one
stall, and (b) a window kept low enough that a session runs much further
before needing a hard compaction at all.

Neither shows up in "net tokens saved". A session can save nothing on paper
and still be a clear win on both counts.

So the telemetry now carries occupancy: tokens_after as a share of the
compaction threshold, plus the threshold and resolved window it was
computed from. That is the number that says whether a session has headroom
left. The report leads with it, and cross-references the batch
`compression_attempt` lines already in the log so it can show how often the
long pause actually fired — ideally never.

Occupancy is read from the cached threshold only. The public
`threshold_tokens` property resolves lazily and can issue a synchronous
/models probe (#32221); telemetry must never be the thing that blocks a
turn, so an unresolved window reports null. In practice a pass has already
resolved it via the tail calculation, so the field is populated. A test
pins the no-forcing behaviour directly against the emitter.

The report is pure ASCII: `scripts/check_subprocess_stdin.py` currently
dies on a cp1252 console before printing its results, and a diagnostic tool
that crashes on the platform it is diagnosing is worse than no tool.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Michael Jordan 2026-07-29 16:15:35 -04:00 • committed by kshitij
parent cac9526d2a
commit ac48add3a7
4 changed files with 201 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

153
scripts/micro_compaction_report.py
View file

@ -1,9 +1,9 @@
#!/usr/bin/env python3
"""Summarize micro-compaction telemetry from Hermes logs.
Reads the content-free ``micro compaction telemetry:`` JSON lines emitted by
Reads the content-free JSON lines emitted by
``ContextCompressor._emit_micro_compaction_telemetry`` and reports what the
feature actually did, per session and overall.
feature actually bought you.
Usage:
python scripts/micro_compaction_report.py [LOGFILE ...]
@ -11,11 +11,23 @@ Usage:
With no LOGFILE, reads ``$HERMES_HOME/logs/agent.log`` (default ~/.hermes).
Note on reading the numbers: the first pass in a session inserts the summary
marker, which costs a fixed ~400 tokens of scaffolding. That overhead is paid
once; from the second pass on the marker is replaced rather than added, so
each absorbed exchange is pure saving. A session with a single pass can
therefore show a net loss and still be working correctly.
What to look at
---------------
The point of micro-compaction is not saving tokens or time. It is:
(a) amortizing the one long batch-compaction pause across many turns, and
(b) keeping the context window low enough that a session runs much further
before it needs a hard compaction at all.
So the headline numbers here are OCCUPANCY (how full the window is kept, as a
percentage of the compaction threshold) and BATCH COMPACTIONS (how often the
long pause actually fired). Net tokens saved is reported too, but it is the
least interesting figure -- a session can save nothing on paper and still be a
clear win because the stalls disappeared and the window never filled.
Caveat: running the test suite writes telemetry into the same log. Test lines
cluster inside a sub-second window and carry an empty session_id (they group
as "(unknown)"). Use --per-session to spot them.
"""
from __future__ import annotations
@ -27,7 +39,8 @@ import sys
from collections import defaultdict
from pathlib import Path
MARKER = "micro compaction telemetry: "
MICRO_MARKER = "micro compaction telemetry: "
BATCH_MARKER = "context compression attempt telemetry: "
def default_log() -> Path:
@ -35,8 +48,9 @@ def default_log() -> Path:
return Path(home) / "logs" / "agent.log"
def load(paths: list[Path]) -> list[dict]:
events: list[dict] = []
def load(paths: list[Path]) -> tuple[list[dict], list[dict]]:
micro: list[dict] = []
batch: list[dict] = []
for path in paths:
try:
text = path.read_text(encoding="utf-8", errors="replace")
@ -44,88 +58,113 @@ def load(paths: list[Path]) -> list[dict]:
print(f"warning: cannot read {path}: {exc}", file=sys.stderr)
continue
for line in text.splitlines():
idx = line.find(MARKER)
if idx == -1:
continue
try:
events.append(json.loads(line[idx + len(MARKER):]))
except ValueError:
continue
return events
for marker, sink in ((MICRO_MARKER, micro), (BATCH_MARKER, batch)):
idx = line.find(marker)
if idx == -1:
continue
try:
sink.append(json.loads(line[idx + len(marker):]))
except ValueError:
pass
break
return micro, batch
def fmt(n: int | None) -> str:
def pct(values: list[float]) -> tuple[float, float, float] | None:
if not values:
return None
ordered = sorted(values)
return ordered[0], ordered[len(ordered) // 2], ordered[-1]
def fmt(n) -> str:
return "-" if n is None else f"{n:,}"
def report(events: list[dict], per_session: bool) -> int:
if not events:
def report(micro: list[dict], batch: list[dict], per_session: bool) -> int:
if not micro:
print("No micro-compaction telemetry found.")
print("Micro-compaction may be disabled (compression.micro_compact),")
print("or no session has run long enough to trigger a pass yet.")
print("It may be disabled (compression.micro_compact), or no session")
print("has run long enough to trigger a pass yet.")
return 1
by_session: dict[str, list[dict]] = defaultdict(list)
for e in events:
for e in micro:
by_session[e.get("session_id") or "(unknown)"].append(e)
outcomes: dict[str, int] = defaultdict(int)
for e in events:
for e in micro:
outcomes[e.get("outcome", "?")] += 1
saved = sum(-(e.get("tokens_delta") or 0) for e in events)
absorbed = [e for e in events if e.get("outcome") == "absorbed"]
exchange_tokens = [e.get("exchange_tokens") or 0 for e in absorbed]
durations = [e.get("duration_ms") or 0 for e in events]
occupancies = [e["occupancy_pct"] for e in micro if e.get("occupancy_pct") is not None]
saved = sum(-(e.get("tokens_delta") or 0) for e in micro)
absorbed = [e for e in micro if e.get("outcome") == "absorbed"]
durations = [e.get("duration_ms") or 0 for e in micro]
if per_session:
print(f"{'session':<38} {'passes':>7} {'saved':>10} {'first':>8} {'last':>8}")
print("-" * 76)
print(f"{'session':<26} {'passes':>6} {'occupancy%':>18} {'batch':>6} {'saved':>10}")
print("-" * 72)
batch_by_session: dict[str, int] = defaultdict(int)
for b in batch:
batch_by_session[b.get("session_id") or "(unknown)"] += 1
for sid, evs in sorted(by_session.items(), key=lambda kv: -len(kv[1])):
occ = [e["occupancy_pct"] for e in evs if e.get("occupancy_pct") is not None]
spread = pct(occ)
occ_s = f"{spread[0]:.0f}-{spread[2]:.0f} (med {spread[1]:.0f})" if spread else "-"
s = sum(-(e.get("tokens_delta") or 0) for e in evs)
first = evs[0].get("tokens_before")
last = evs[-1].get("tokens_after")
print(f"{sid[:38]:<38} {len(evs):>7} {s:>+10,} {fmt(first):>8} {fmt(last):>8}")
print(f"{sid[:26]:<26} {len(evs):>6} {occ_s:>18} "
f"{batch_by_session.get(sid, 0):>6} {s:>+10,}")
print()
print("-- headroom ----------------------------------")
spread = pct(occupancies)
if spread:
print(f"context occupancy min {spread[0]:.0f}% median {spread[1]:.0f}% max {spread[2]:.0f}%")
print(" (% of the batch-compaction threshold)")
else:
print("context occupancy unavailable (window not resolved when logged)")
print(f"batch compactions {len(batch):,}")
if batch:
print(f" micro passes each {len(micro) / len(batch):.1f}")
else:
print(" none fired -- the long pause never happened in this log")
print()
print("-- activity ----------------------------------")
print(f"sessions {len(by_session):,}")
print(f"passes {len(events):,}")
print(f"passes {len(micro):,}")
for name, count in sorted(outcomes.items(), key=lambda kv: -kv[1]):
print(f" {name:<20} {count:,}")
print(f"net tokens saved {saved:+,}")
if absorbed:
print(f"exchanges absorbed {len(absorbed):,}")
print(f" mean exchange size {sum(exchange_tokens) // len(absorbed):,} tokens")
print(f" mean saving/pass {saved // len(events):+,} tokens")
if durations:
ordered = sorted(durations)
print(f"pass duration median {ordered[len(ordered) // 2]:,} ms")
print(f" max {ordered[-1]:,} ms")
print(f"pass duration median {ordered[len(ordered) // 2]:,} ms "
f"max {ordered[-1]:,} ms")
multi = {s: e for s, e in by_session.items() if len(e) > 1}
if multi:
net = sum(
(evs[-1].get("tokens_after") or 0) - (evs[0].get("tokens_before") or 0)
for evs in multi.values()
)
print()
print(f"sessions with >1 pass {len(multi):,}")
print(f" net context change {net:+,} tokens (negative is shrinkage)")
print()
print("-- tokens (least interesting) ----------------")
print(f"net tokens saved {saved:+,}")
if absorbed:
sizes = [e.get("exchange_tokens") or 0 for e in absorbed]
print(f"exchanges absorbed {len(absorbed):,} "
f"(mean {sum(sizes) // len(absorbed):,} tokens each)")
print("note: the first pass in a session costs ~400 tokens of marker")
print("scaffolding; it pays back from the second pass on.")
return 0
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("logs", nargs="*", type=Path, help="log files (default: agent.log)")
ap.add_argument("--per-session", action="store_true", help="break down by session")
args = ap.parse_args()
paths = args.logs or [default_log()]
missing = [p for p in paths if not p.exists()]
for p in missing:
print(f"warning: {p} does not exist", file=sys.stderr)
return report(load([p for p in paths if p.exists()]), args.per_session)
for p in paths:
if not p.exists():
print(f"warning: {p} does not exist", file=sys.stderr)
micro, batch = load([p for p in paths if p.exists()])
return report(micro, batch, args.per_session)
if __name__ == "__main__":