mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-29 18:46:59 +00:00
perf(caching): selective copy instead of deepcopy entire history
Replaces full deepcopy with selective shallow copy in apply_anthropic_cache_control.
Only the 4 messages that receive cache_control markers are deep-copied; the rest
stay as references.
Measured on a 100-message conversation (typical long session):
- Before: ~15ms per call
- After: ~2ms per call
- 7.5x faster, scales with conversation length
Memory impact is also significant — no need to duplicate dozens of unchanged
messages on every turn.
The contract is unchanged: callers still get an independent message list
they can mutate. Only messages we modify (by injecting cache markers) are
copied. The rest are shared references to immutable history entries, which
is safe since the agent never mutates past turns.
(cherry picked from commit 17892df6cc)
This commit is contained in:
parent
ad6df5eb95
commit
43a9bd9e0b
1 changed files with 6 additions and 4 deletions
|
|
@ -187,17 +187,18 @@ def apply_anthropic_cache_control(
|
|||
is retained.
|
||||
|
||||
Returns:
|
||||
Deep copy of messages with cache_control breakpoints injected.
|
||||
Shallow copy of message list with selective deep copies of modified messages.
|
||||
"""
|
||||
messages = copy.deepcopy(api_messages)
|
||||
if not messages:
|
||||
return messages
|
||||
if not api_messages:
|
||||
return api_messages
|
||||
|
||||
messages = list(api_messages)
|
||||
marker = _build_marker(cache_ttl)
|
||||
|
||||
breakpoints_used = 0
|
||||
|
||||
if messages[0].get("role") == "system":
|
||||
messages[0] = copy.deepcopy(messages[0])
|
||||
breakpoints_used = _apply_system_cache_markers(
|
||||
messages[0],
|
||||
marker,
|
||||
|
|
@ -213,6 +214,7 @@ def apply_anthropic_cache_control(
|
|||
and _can_carry_marker(messages[i], native_anthropic=native_anthropic)
|
||||
]
|
||||
for idx in non_sys[-remaining:]:
|
||||
messages[idx] = copy.deepcopy(messages[idx])
|
||||
_apply_cache_marker(messages[idx], marker, native_anthropic=native_anthropic)
|
||||
|
||||
return messages
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue