fix(model_metadata): bound the tools-token estimate cache

Follow-up to the salvaged str(tools) fix. The id()-keyed
_TOOLS_TOKENS_CACHE had no eviction, so a long-lived gateway/desktop
backend could accumulate an unbounded number of stale entries as it
builds transient tool lists. Cap it at 256 with oldest-first eviction
(insertion-ordered dict) and add a regression test asserting the cache
never exceeds the cap.
This commit is contained in:
kshitijk4poor 2026-07-09 12:21:42 +05:30 committed by kshitij
parent f4d5cfd0fd
commit 55dbc3ffb5
2 changed files with 37 additions and 0 deletions

View file

@ -142,6 +142,32 @@ class TestEstimateRequestTokensRough:
estimate_request_tokens_rough(messages, system_prompt="x" * 8, tools=tools)
assert dumps.call_count == 1
def test_tools_cache_is_bounded(self):
# A long-lived process builds many transient tool lists; the cache must
# not grow without bound. Feed more distinct lists than the cap and
# confirm the cache never exceeds it.
import agent.model_metadata as mm
mm._TOOLS_TOKENS_CACHE.clear()
cap = mm._TOOLS_TOKENS_CACHE_MAX
# Keep references so ids are not recycled mid-loop, forcing distinct keys.
held = []
for i in range(cap + 50):
tools = [
{
"type": "function",
"function": {
"name": f"tool_{i}",
"description": "d",
"parameters": {"type": "object"},
},
}
]
held.append(tools)
mm._estimate_tools_tokens_rough(tools)
assert len(mm._TOOLS_TOKENS_CACHE) <= cap
assert len(mm._TOOLS_TOKENS_CACHE) == cap
# =========================================================================
# Default context lengths