diff --git a/agent/agent_init.py b/agent/agent_init.py index 5972b849817..b716522d0a9 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1963,8 +1963,13 @@ def init_agent( compression_in_place = is_truthy_value( _compression_cfg.get("in_place"), default=True ) + # Opt-in (default False): a micro-compaction pass rewrites already-sent + # history every turn, which breaks the provider prompt-cache prefix on a + # per-turn cadence rather than at an episodic boundary. That is the cost + # `proactive_prune_min_reclaim_tokens` exists to amortize, so the feature + # stays off until an operator opts in and accepts the tradeoff. compression_micro_compact = is_truthy_value( - _compression_cfg.get("micro_compact"), default=True + _compression_cfg.get("micro_compact"), default=False ) codex_app_server_auto_compaction = str( _compression_cfg.get("codex_app_server_auto", "native") or "native" diff --git a/agent/context_compressor.py b/agent/context_compressor.py index ca4a3f56eb7..8b71a517d2a 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -2150,8 +2150,10 @@ class ContextCompressor(ContextEngine): self.abort_on_summary_failure = abort_on_summary_failure # ── Micro-compaction (per-turn rolling compaction) ───────── - # Default: enabled when not explicitly disabled (backward compatible). - self._micro_compact_enabled: bool = True + # Default: OFF. Each pass rewrites already-sent history, so it breaks + # the prompt-cache prefix every turn instead of at an episodic + # boundary. Operators opt in via `compression.micro_compact: true`. + self._micro_compact_enabled: bool = False self._micro_compact_cursor: int = 0 self._micro_compact_rolling_summary: str = "" self._micro_compact_consecutive_failures: int = 0 diff --git a/docs/micro-compaction.md b/docs/micro-compaction.md index 974fce2f114..dd62f6478c6 100644 --- a/docs/micro-compaction.md +++ b/docs/micro-compaction.md @@ -14,11 +14,16 @@ Hermes folds the single oldest un-absorbed exchange into a running summary. The work is the same work; it just happens continuously, a piece at a time, instead of all at once in the middle of your session. -It is not free and it is not a magic bullet. Each pass is a real call to the +It is not free and it is not a magic bullet, and it is **off by default** — +`compression.micro_compact: true` turns it on. Each pass is a real call to the compression model, and it runs at the end of a turn — your answer has already -streamed, but the turn does not close until the pass finishes. What the feature -gives you is a **tuning option**: you choose how the compression cost is -distributed, and which model pays it. See +streamed, but the turn does not close until the pass finishes. Each pass also +rewrites already-sent history, which breaks the provider prompt-cache prefix +every turn; read [Prompt caching](#prompt-caching--the-cost-you-are-opting-into) +before enabling it, because for some setups that cost exceeds the benefit. + +What the feature gives you is a **tuning option**: you choose how the +compression cost is distributed, and which model pays it. See [Choosing a compression model](#choosing-a-compression-model), because that choice matters more than anything else here. @@ -159,14 +164,53 @@ batch path fires much less often. ## Configuration +Micro-compaction is **off by default**. Turn it on explicitly: + ```yaml compression: - micro_compact: true # default + micro_compact: true # default: false ``` -Set it to `false` to disable micro-compaction and return to batch-only +With it unset or `false` Hermes behaves exactly as it always has: batch-only compaction. Everything else about compression is unchanged. +It ships opt-in rather than on because of the prompt-cache cost described in +the next section — that cost is real, it is not universally worth paying, and +it should be a decision you make rather than one you inherit. + +## Prompt caching — the cost you are opting into + +Read this before enabling the feature. It is the strongest argument against it. + +A long-lived conversation reuses a cached prompt prefix every turn, and cached +input tokens are billed at a fraction of uncached ones. That discount survives +only as long as the prefix does not change. **A micro-compaction pass rewrites +already-sent history**, which invalidates the prefix from the rewrite point +onward — so with micro-compaction on, you break the cache *every turn* instead +of once per batch compaction. + +This is the same cost the proactive prune deliberately avoids. That path gates +itself behind `compression.proactive_prune_min_reclaim_tokens` (4096 by +default) precisely so its rewrites stay, in the words of the config comment, +"one big episodic break instead of a tiny break every tool iteration." +Micro-compaction has no such gate: one exchange per turn means one break per +turn, by design. + +So the honest framing is a trade of one cost for another, not a saving: + +| | Batch-only (default) | Micro-compaction on | +|---|---|---| +| Compression stalls | One long stall at the threshold | Spread across turns | +| Context occupancy | Sawtooths up to the threshold | Stays low and flat | +| Cache prefix | Intact between compactions | Broken every turn | + +Which side wins depends on numbers specific to you: how much your provider +discounts cached input, how large your prefix is, how long your sessions run, +and how much a mid-session stall actually costs you. On a provider with a deep +cache discount and a big prefix, the per-turn invalidation can plausibly cost +more than the stall it removes. Measure your own sessions — see +[Measuring it](#measuring-it) — rather than assuming. + ## Choosing a compression model Micro-compaction uses the `auxiliary.compression` model: diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index dd1cfe88c01..7d52bc2502d 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -578,6 +578,18 @@ DEFAULT_CONFIG = { # prompt-cache invalidation amortized: one big # episodic break instead of a tiny break every # tool iteration. 0 = commit any non-zero prune. + "micro_compact": False, # opt-in: after each completed turn, fold the + # oldest un-absorbed exchange into a rolling + # summary, amortizing compression cost instead + # of paying it in one batch stall. Default False + # because a pass rewrites already-sent history + # and so breaks the provider prompt-cache prefix + # EVERY turn — the per-turn cache break that + # `proactive_prune_min_reclaim_tokens` above + # exists to avoid. Enable only when you have + # measured that the amortized stall is worth + # more to you than the cached-prefix discount. + # See docs/micro-compaction.md. "hygiene_hard_message_limit": 5000, # gateway session-hygiene force-compress threshold by message count "hygiene_timeout_seconds": 30, # max seconds gateway waits for pre-agent hygiene compression # WITHOUT forward progress. The summary call streams, so