diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 751f98472f60..5b9acef131ce 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2947,16 +2947,18 @@ DEFAULT_CONFIG = { # Tier 0 — no MCP/plugin tools: everything stays eager. # Tier 1 — catalog listing fits the budget: bridge + skills-style # name+description manifest (degrades to names-only). - # Tier 2 — listing over budget even names-only (e.g. Cloudflare's - # ~3,300-tool flat API surface): bare bridge, discovery through - # tool_search only. + # Tier 2 — per-tool listing over budget even names-only (e.g. + # Cloudflare's ~3,300-tool flat API surface): bare bridge + + # a one-line-per-server summary (name + tool count) so the + # model knows which domains are reachable; individual tools + # discoverable through tool_search only. # "auto"/"on" — activate when at least one deferrable tool exists. # "off" — disable entirely. Tools-array assembly is a pass-through. "enabled": "auto", # Listing budget as a percentage of the active model's context # length. Effective budget = min(this % of context, - # listing_max_tokens). 10 matches the Claude Code default. Range 0..100. - "threshold_pct": 10, + # listing_max_tokens). Range 0..100. + "threshold_pct": 5, # When the model calls tool_search without a ``limit`` argument, # how many hits to return. Range 1..max_search_limit. "search_default_limit": 5, diff --git a/model_tools.py b/model_tools.py index 60fb8fd82f3b..109a9662da2a 100644 --- a/model_tools.py +++ b/model_tools.py @@ -557,6 +557,8 @@ def _compute_tool_definitions( if assembly.activated and not quiet_mode: _forms = {"full": "catalog listing embedded", "names": "names-only listing embedded", + "mixed": "listing embedded (oversized servers summarized)", + "groups": "server summary embedded (search-only discovery)", "none": "no listing (search-only)"} print( f"🔎 Tool Search (tier {assembly.tier}): {assembly.deferred_count} " diff --git a/tests/tools/test_tool_search.py b/tests/tools/test_tool_search.py index a0ad19c0527f..117047ecfeb7 100644 --- a/tests/tools/test_tool_search.py +++ b/tests/tools/test_tool_search.py @@ -44,7 +44,7 @@ class TestConfigParsing: from tools.tool_search import ToolSearchConfig cfg = ToolSearchConfig.from_raw(None) assert cfg.enabled == "auto" - assert cfg.threshold_pct == 10.0 + assert cfg.threshold_pct == 5.0 def test_bool_true_maps_to_auto(self): from tools.tool_search import ToolSearchConfig @@ -162,14 +162,16 @@ class TestThresholdGate: def test_listing_budget_min_of_pct_and_cap(self): from tools.tool_search import ToolSearchConfig, listing_token_budget cfg = ToolSearchConfig.from_raw( - {"threshold_pct": 10, "listing_max_tokens": 8000}) - # 10% of 200K = 20K > cap 8K → cap wins + {"threshold_pct": 5, "listing_max_tokens": 8000}) + # 5% of 200K = 10K > cap 8K → cap wins assert listing_token_budget(cfg, 200_000) == 8000 - # 10% of 50K = 5K < cap 8K → pct leg wins - assert listing_token_budget(cfg, 50_000) == 5000 - # unknown context → 20K fallback for the pct leg, still capped + # 5% of 50K = 2.5K < cap 8K → pct leg wins + assert listing_token_budget(cfg, 50_000) == 2500 + # unknown context → 10K fallback for the pct leg, still capped assert listing_token_budget(cfg, 0) == 8000 assert listing_token_budget(cfg, None) == 8000 + # default threshold is 5% + assert ToolSearchConfig.from_raw(None).threshold_pct == 5.0 def test_token_estimate_proportional_to_schema_size(self): from tools.tool_search import estimate_tokens_from_schemas @@ -291,9 +293,9 @@ class TestAssembly: if t["function"]["name"] == "tool_search") assert "tier_small_a" in search["function"]["description"] - def test_oversized_catalog_degrades_to_bare_bridge_tier2(self): + def test_oversized_catalog_degrades_to_server_summary_tier2(self): """When even the names-only listing exceeds the budget, tier 2: - bare bridge, no listing.""" + bare bridge + one-line-per-server summary (no per-tool names).""" from tools.tool_search import assemble_tool_defs, ToolSearchConfig names = [f"tier2_very_long_tool_name_number_{i:04d}_extra" for i in range(400)] for n in names: @@ -308,10 +310,52 @@ class TestAssembly: ) assert result.activated assert result.tier == 2 - assert result.listing_form == "none" + assert result.listing_form == "groups" search = next(t for t in result.tool_defs if t["function"]["name"] == "tool_search") - assert "tier2_very_long_tool_name_number_0000" not in search["function"]["description"] + desc = search["function"]["description"] + # No individual tool names... + assert "tier2_very_long_tool_name_number_0000" not in desc + # ...but the server (toolset) is named with its tool count, and the + # model is told to search rather than substitute/deny. + assert "tiertest" in desc + assert "(400 tools" in desc + assert "search here FIRST" in desc + + def test_mixed_catalog_small_server_keeps_listing(self): + """Per-server degradation: an oversized server collapses to a + summary line while a small co-attached server keeps per-tool names + (the Cloudflare+Linear shape).""" + from tools.tool_search import build_catalog_listing_with_form + from tools.registry import registry + import json as _json + + def _h(args, task_id=None, **kw): + return _json.dumps({"ok": True}) + + big = [f"bigsrv_tool_{i:04d}_with_a_long_name" for i in range(300)] + small = ["smallsrv_create_item", "smallsrv_list_items"] + for n in big: + registry.register(name=n, handler=_h, + schema=_td(n, "Big server tool.")["function"], + toolset="mcp-bigsrv") + for n in small: + registry.register(name=n, handler=_h, + schema=_td(n, "Small server tool.")["function"], + toolset="mcp-smallsrv") + defs = ([_td(n, "Big server tool.") for n in big] + + [_td(n, "Small server tool.") for n in small]) + # Budget fits the small server's lines + big server's summary, + # but not the big server's 300 names. + text, form = build_catalog_listing_with_form(defs, max_tokens=300) + assert form == "mixed" + assert text is not None + assert "smallsrv_create_item" in text # small server listed + assert "bigsrv_tool_0000" not in text # big server names dropped + assert "bigsrv (300 tools" in text # ...but summarized + # deterministic (cache safety) + text2, _ = build_catalog_listing_with_form(list(reversed(defs)), max_tokens=300) + assert text == text2 def test_idempotent_when_bridge_already_present(self): from tools.tool_search import assemble_tool_defs, ToolSearchConfig, BRIDGE_TOOL_NAMES @@ -675,7 +719,7 @@ class TestCatalogListing: for i in range(30) ] result = assemble_tool_defs( - defs, context_length=1000, + defs, context_length=200_000, config=ToolSearchConfig.from_raw({"enabled": "on"}), ) assert result.activated diff --git a/tools/tool_search.py b/tools/tool_search.py index cdab8279ba93..ccfddcc46e45 100644 --- a/tools/tool_search.py +++ b/tools/tool_search.py @@ -14,12 +14,14 @@ for the full rationale): size is the *listing*, not the activation decision: - Tier 0 — no MCP/plugin tools: pure passthrough, everything eager. - Tier 1 — deferred tools whose catalog listing fits the listing budget - (``min(threshold_pct`` of context, ``listing_max_tokens)``): bridge + - skills-style listing (name + short description per tool), degrading to - a names-only listing when the full form is over budget. - - Tier 2 — listing over budget even names-only (e.g. Cloudflare's flat - API surface, ~3,300 tools whose names alone are ~32K tokens): bare - bridge; tools are discoverable only through ``tool_search``. + (``min(threshold_pct`` of context — default 5% — ``, listing_max_tokens)``): + bridge + skills-style listing (name + short description per tool), + degrading to a names-only listing when the full form is over budget. + - Tier 2 — per-tool listing over budget even names-only (e.g. + Cloudflare's flat API surface, ~3,300 tools whose names alone are + ~32K tokens): bare bridge + a one-line-per-server summary (server + name + tool count) so the model still knows WHICH domains are + reachable; individual tools are discoverable only via ``tool_search``. * The catalog is stateless across turns and tools-array assemblies. It is rebuilt from the current tool-defs list every time. This is the lesson from OpenClaw's cron regression (openclaw/openclaw#84141): a session-keyed @@ -104,13 +106,13 @@ class ToolSearchConfig: break the agent. """ if raw is True: - return cls(enabled="auto", threshold_pct=10.0, + return cls(enabled="auto", threshold_pct=5.0, search_default_limit=5, max_search_limit=20) if raw is False: - return cls(enabled="off", threshold_pct=10.0, + return cls(enabled="off", threshold_pct=5.0, search_default_limit=5, max_search_limit=20) if not isinstance(raw, dict): - return cls(enabled="auto", threshold_pct=10.0, + return cls(enabled="auto", threshold_pct=5.0, search_default_limit=5, max_search_limit=20) enabled_raw = str(raw.get("enabled", "auto")).strip().lower() @@ -123,7 +125,7 @@ class ToolSearchConfig: else: enabled = "auto" - threshold_pct = _safe_float(raw.get("threshold_pct"), 10.0) + threshold_pct = _safe_float(raw.get("threshold_pct"), 5.0) threshold_pct = max(0.0, min(100.0, threshold_pct)) max_search_limit = max(1, min(50, _safe_int(raw.get("max_search_limit"), 20))) @@ -298,13 +300,13 @@ def listing_token_budget( """Effective token budget for the embedded catalog listing. ``min(listing_max_tokens, threshold_pct% of context)``. Without a known - context size, the percentage leg falls back to the fixed 20K cutoff the - activation gate historically used (10% of a typical 200K window). + context size, the percentage leg falls back to a fixed 10K cutoff + (5% of a typical 200K window). """ if context_length and context_length > 0: pct_leg = int(context_length * (config.threshold_pct / 100.0)) else: - pct_leg = 20_000 + pct_leg = 10_000 return max(0, min(config.listing_max_tokens, pct_leg)) @@ -529,7 +531,10 @@ def build_catalog_listing( activation gate): 1. full listing (names + short descriptions) 2. names-only listing, still grouped - 3. ``None`` — caller falls back to the legacy bare-count description + 3. server-level summary — one line per MCP server / plugin toolset + (name + tool count), so the model always knows WHICH domains are + reachable through the bridge even when per-tool names don't fit + 4. ``None`` — only when the summary itself exceeds the budget """ text, _form = build_catalog_listing_with_form(deferrable, max_tokens=max_tokens) return text @@ -543,8 +548,15 @@ def build_catalog_listing_with_form( """Like :func:`build_catalog_listing` but also reports the form used. Returns ``(text, form)`` where ``form`` is ``"full"`` (names + short - descriptions), ``"names"`` (names-only fallback), or ``"none"`` (over - budget in every form — tier-2 bare bridge). + descriptions), ``"names"`` (names-only fallback), ``"mixed"`` (per-server + degradation: small servers keep per-tool lines, oversized servers + collapse to a name + tool-count summary line), ``"groups"`` (every + server summarized), or ``"none"`` (over budget in every form). + + Degradation is PER SERVER, not global: one huge server (Cloudflare's + 3,320 flat tools) must not cost a small co-attached server (Linear's 24) + its listing. Greedy fit, smallest rendered group first, is deterministic + for a given catalog — byte-stable across assemblies, cache-safe. """ if not deferrable: return None, "none" @@ -562,29 +574,59 @@ def build_catalog_listing_with_form( if not groups: return None, "none" - def render(with_descriptions: bool) -> str: - lines: List[str] = ["Deferred tool catalog (call schemas via " - f"`{TOOL_DESCRIBE_NAME}`, invoke via `{TOOL_CALL_NAME}`):"] - for label in sorted(groups): - tools = sorted(groups[label]) - lines.append(f"{label} tools ({len(tools)}):") - if with_descriptions: - for name, desc in tools: - lines.append(f"- {name}: {desc}" if desc else f"- {name}") - else: - lines.append(", ".join(name for name, _ in tools)) + def render_group(label: str, mode: str) -> str: + """Render one server's block. mode: 'full' | 'names' | 'summary'.""" + tools = sorted(groups[label]) + if mode == "summary": + return (f"{label} ({len(tools)} tools — names not listed; " + f"discover via `{TOOL_SEARCH_NAME}`)") + lines = [f"{label} tools ({len(tools)}):"] + if mode == "full": + for name, desc in tools: + lines.append(f"- {name}: {desc}" if desc else f"- {name}") + else: + lines.append(", ".join(name for name, _ in tools)) return "\n".join(lines) - for with_desc, form in ((True, "full"), (False, "names")): - text = render(with_desc) - if math.ceil(len(text) / CHARS_PER_TOKEN) <= max_tokens: - return text, form + header = ("Deferred tool catalog (call schemas via " + f"`{TOOL_DESCRIBE_NAME}`, invoke via `{TOOL_CALL_NAME}`):") + + def assemble(modes: Dict[str, str]) -> str: + return "\n".join([header] + [render_group(lbl, modes[lbl]) + for lbl in sorted(groups)]) + + def fits(text: str) -> bool: + return math.ceil(len(text) / CHARS_PER_TOKEN) <= max_tokens + + # 1. Everything full. + modes = {lbl: "full" for lbl in groups} + if fits(assemble(modes)): + return assemble(modes), "full" + + # 2. Everything names-only. + modes = {lbl: "names" for lbl in groups} + if fits(assemble(modes)): + return assemble(modes), "names" + + # 3. Per-server degradation: collapse the LARGEST rendered groups to + # summary lines first, keeping per-tool names for small servers. + # Deterministic: size then label. One oversized server (Cloudflare) + # must not cost a small co-attached server (Linear) its listing. + by_size = sorted(groups, key=lambda lbl: (-len(render_group(lbl, "names")), lbl)) + for lbl in by_size: + modes[lbl] = "summary" + if fits(assemble(modes)): + form = "groups" if all(m == "summary" for m in modes.values()) else "mixed" + return assemble(modes), form + + # 4. Even the all-summary form is over budget. return None, "none" def bridge_tool_schemas( deferred_count: int, listing: Optional[str] = None, + listing_form: str = "", ) -> List[Dict[str, Any]]: """Build the bridge tool schemas to inject in place of deferred tools. @@ -596,7 +638,10 @@ def bridge_tool_schemas( embedded in the ``tool_search`` description so every deferred capability stays *visible* by name — the skills-listing pattern — closing the "model doesn't know what it doesn't know" gap while full parameter - schemas remain deferred. + schemas remain deferred. ``listing_form`` selects the framing: per-tool + forms ("full"/"names") tell the model it may skip the search when it + sees the exact name; the server-summary form ("groups") tells it which + DOMAINS are reachable and that search is mandatory for tool discovery. """ desc_search = ( f"Search {deferred_count} additional tools that are loaded on demand. " @@ -605,13 +650,28 @@ def bridge_tool_schemas( f"then `{TOOL_CALL_NAME}` to invoke it. Tools listed at the top of this " "system prompt are already available and do not need to be searched." ) - if listing: + if listing and listing_form == "groups": desc_search += ( - "\n\nEvery deferred tool is listed below. If the capability you " - "need appears here, do NOT claim it is unavailable — load it with " - f"`{TOOL_DESCRIBE_NAME}` (skip `{TOOL_SEARCH_NAME}` when you " - "already see the exact name).\n\n" + listing + "\n\nThe servers below are connected and their tools ARE available " + "through this bridge. For any request in these domains, search " + "here FIRST — do not claim the capability is unavailable and do " + "not substitute a generic tool (terminal/browser) without " + "searching.\n\n" + listing ) + elif listing: + desc_search += ( + "\n\nEvery deferred capability is listed below. If a tool name " + "appears here, do NOT claim it is unavailable — load it with " + f"`{TOOL_DESCRIBE_NAME}` (skip `{TOOL_SEARCH_NAME}` when you " + "already see the exact name)." + ) + if listing_form == "mixed": + desc_search += ( + " For servers marked 'names not listed', the tools exist " + f"too — find them with `{TOOL_SEARCH_NAME}` before " + "concluding anything is missing." + ) + desc_search += "\n\n" + listing desc_describe = ( f"Load the full JSON schema for one tool returned by `{TOOL_SEARCH_NAME}`. " f"Required before `{TOOL_CALL_NAME}` if the tool's parameters are unknown." @@ -753,9 +813,14 @@ def assemble_tool_defs( if config.listing != "off": listing, listing_form = build_catalog_listing_with_form( deferrable, max_tokens=listing_budget) - bridge = bridge_tool_schemas(len(deferrable), listing=listing) + bridge = bridge_tool_schemas(len(deferrable), listing=listing, + listing_form=listing_form) result = visible + bridge - tier = 1 if listing else 2 + # Tier 1 = per-tool listing for at least part of the catalog (full, + # names, or mixed). Tier 2 = search-only discovery; the server-level + # "groups" summary keeps domains visible but individual tools are only + # reachable via tool_search. + tier = 1 if listing_form in ("full", "names", "mixed") else 2 logger.info( "tool_search activated (tier %d): %d core/visible tools kept, %d deferred " diff --git a/website/docs/user-guide/features/tool-search.md b/website/docs/user-guide/features/tool-search.md index 6aff6075cfcb..ebb9b9d87603 100644 --- a/website/docs/user-guide/features/tool-search.md +++ b/website/docs/user-guide/features/tool-search.md @@ -62,8 +62,8 @@ how much of the catalog stays visible, not whether schemas defer. | Tier | Condition | What the model sees | | --- | --- | --- | | **0** | No MCP/plugin tools | Every tool eager, no bridge. Pass-through. | -| **1** | Deferred catalog's listing fits the budget | Bridge + a skills-style manifest of every deferred tool (name + short description, degrading to names-only when over budget). | -| **2** | Listing exceeds the budget even names-only (e.g. Cloudflare's flat API surface: ~3,300 tools whose names alone are ~32K tokens) | Bare bridge — tools are discoverable only through `tool_search`. | +| **1** | Deferred catalog's listing fits the budget | Bridge + a skills-style manifest of every deferred tool (name + short description, degrading to names-only when over budget). Degradation is **per server**: when one oversized server (Cloudflare) is attached alongside small ones (Linear), the small servers keep their per-tool listings and only the oversized server collapses to a summary line. | +| **2** | Per-tool listing exceeds the budget even names-only for every server (e.g. Cloudflare's flat API surface alone: ~3,300 tools whose names are ~32K tokens) | Bare bridge + a one-line-per-server summary (server name + tool count), so the model knows which domains are reachable; individual tools are discoverable only through `tool_search`. | The listing budget is `min(threshold_pct% of context, listing_max_tokens)`. The decision is re-evaluated every time the tools array is built, so @@ -76,7 +76,7 @@ tiers on the next assembly. tools: tool_search: enabled: auto # auto (default), on, or off - threshold_pct: 10 # listing budget as a percentage of context + threshold_pct: 5 # listing budget as a percentage of context search_default_limit: 5 max_search_limit: 20 listing: auto # embed a grouped name+description catalog manifest @@ -86,10 +86,10 @@ tools: | Key | Default | Meaning | | --- | --- | --- | | `enabled` | `auto` | `auto`/`on` activate whenever at least one deferrable tool exists; `off` disables entirely (everything stays eager). | -| `threshold_pct` | `10` | Listing budget as a percentage of the active model's context length. Range 0–100. | +| `threshold_pct` | `5` | Listing budget as a percentage of the active model's context length. Range 0–100. | | `search_default_limit` | `5` | Hits returned when the model calls `tool_search` without a `limit`. | | `max_search_limit` | `20` | Hard upper bound the model can request via `limit`. Range 1–50. | -| `listing` | `auto` | Embed a skills-style manifest of every deferred tool (name + first sentence of its description, ≤60 chars, grouped by MCP server) in the `tool_search` bridge description. `auto` includes it when it fits the budget (falling back to names-only, then to the bare tier-2 bridge); `on`/`off` force either way. | +| `listing` | `auto` | Embed a skills-style manifest of every deferred tool (name + first sentence of its description, ≤60 chars, grouped by MCP server) in the `tool_search` bridge description. `auto` includes it when it fits the budget (falling back to names-only, then to the tier-2 server summary); `on`/`off` force either way. | | `listing_max_tokens` | `20000` | Absolute cap on the embedded listing, regardless of context size. Range 200–60000. | ### Why the listing exists