mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-15 14:22:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

fix(model_metadata): bound the tools-token estimate cache

Browse source 
Follow-up to the salvaged str(tools) fix. The id()-keyed
_TOOLS_TOKENS_CACHE had no eviction, so a long-lived gateway/desktop
backend could accumulate an unbounded number of stale entries as it
builds transient tool lists. Cap it at 256 with oldest-first eviction
(insertion-ordered dict) and add a regression test asserting the cache
never exceeds the cap.
This commit is contained in:
kshitijk4poor 2026-07-09 12:21:42 +05:30 • committed by kshitij
parent f4d5cfd0fd
commit 55dbc3ffb5
2 changed files with 37 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

11
agent/model_metadata.py
View file

@ -2445,7 +2445,13 @@ def estimate_request_tokens_rough(
# NOTE: tool schemas can be large. Avoid repeated `str(tools)` conversions,
# which are CPU-heavy and can stall GUI event loops under GIL pressure.
#
# Keyed by ``id(tools)``. A long-lived gateway/desktop backend builds many
# transient tool lists over its lifetime, so the cache is bounded and evicts
# oldest-first (insertion-ordered dict) once it exceeds the cap. The cap is
# generous relative to how rarely toolsets are rebuilt within a process.
_TOOLS_TOKENS_CACHE: dict[int, Tuple[int, str, str, int]] = {}
_TOOLS_TOKENS_CACHE_MAX = 256
def _tool_name_for_cache(tool: Any) -> str:
@ -2505,5 +2511,10 @@ def _estimate_tools_tokens_rough(tools: List[Dict[str, Any]]) -> int:
total_chars += len(str(params))
tokens = (total_chars + 3) // 4
# Bound the cache: drop the oldest entry when the cap is exceeded so a
# long-running process can't accumulate an unbounded number of stale
# ``id(tools)`` entries (id values are recycled after GC anyway).
if len(_TOOLS_TOKENS_CACHE) >= _TOOLS_TOKENS_CACHE_MAX:
_TOOLS_TOKENS_CACHE.pop(next(iter(_TOOLS_TOKENS_CACHE)), None)
_TOOLS_TOKENS_CACHE[key] = (n, first, last, tokens)
return tokens