mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-15 14:22:43 +00:00
fix(model_metadata): bound the tools-token estimate cache
Follow-up to the salvaged str(tools) fix. The id()-keyed _TOOLS_TOKENS_CACHE had no eviction, so a long-lived gateway/desktop backend could accumulate an unbounded number of stale entries as it builds transient tool lists. Cap it at 256 with oldest-first eviction (insertion-ordered dict) and add a regression test asserting the cache never exceeds the cap.
This commit is contained in:
parent
f4d5cfd0fd
commit
55dbc3ffb5
2 changed files with 37 additions and 0 deletions
|
|
@ -142,6 +142,32 @@ class TestEstimateRequestTokensRough:
|
|||
estimate_request_tokens_rough(messages, system_prompt="x" * 8, tools=tools)
|
||||
assert dumps.call_count == 1
|
||||
|
||||
def test_tools_cache_is_bounded(self):
|
||||
# A long-lived process builds many transient tool lists; the cache must
|
||||
# not grow without bound. Feed more distinct lists than the cap and
|
||||
# confirm the cache never exceeds it.
|
||||
import agent.model_metadata as mm
|
||||
|
||||
mm._TOOLS_TOKENS_CACHE.clear()
|
||||
cap = mm._TOOLS_TOKENS_CACHE_MAX
|
||||
# Keep references so ids are not recycled mid-loop, forcing distinct keys.
|
||||
held = []
|
||||
for i in range(cap + 50):
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": f"tool_{i}",
|
||||
"description": "d",
|
||||
"parameters": {"type": "object"},
|
||||
},
|
||||
}
|
||||
]
|
||||
held.append(tools)
|
||||
mm._estimate_tools_tokens_rough(tools)
|
||||
assert len(mm._TOOLS_TOKENS_CACHE) <= cap
|
||||
assert len(mm._TOOLS_TOKENS_CACHE) == cap
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Default context lengths
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue