From 3ee3394907ef50970e58d2e9f24ba22cb9d8ef1d Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 18 Jul 2026 10:06:14 -0700 Subject: [PATCH] feat(tools): skills-style catalog listing for tool_search progressive disclosure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deferred MCP/plugin tools become invisible once the tool_search bridge activates — live benchmarking (48 runs, Claude Haiku 4.5) showed models substituting visible core tools (terminal/web_search/browser) for deferred capabilities or declaring them nonexistent instead of searching: 16/24 task success vs 24/24 with eager loading. Skills never had this failure mode because every skill keeps a ~21-token name+description listing line in the system prompt. This ports that exact pattern to the tool bridge: when tool_search activates, a grouped manifest of every deferred tool (name + first sentence of description, clipped to 60 chars, grouped per MCP server / toolset) is embedded in the tool_search bridge description. - tools/tool_search.py: build_catalog_listing() with deterministic ordering (byte-stable across assemblies -> prompt prefix stays cacheable); token-budget fallbacks full -> names-only -> legacy bare count; bridge_tool_schemas(listing=...) embeds it and instructs the model to skip tool_search when the exact name is visible (one fewer round-trip per use) - config: tools.tool_search.listing auto|on|off (default auto), listing_max_tokens (default 4000, clamped 200..20000); legacy bool shapes keep working - tests: 8 new tests (config parsing/clamps, short-desc clipping, deterministic rendering, budget fallbacks, bridge embedding, assembly on/off paths); full file green (47 passed) - docs: tool-search.md config table + rationale - scripts/tool_search_livetest2.py: benchmark harness v2 with real per-call token accounting (normalize_usage spy) and a third 'listing' mode for A/B/C comparison --- hermes_cli/config.py | 12 + scripts/tool_search_livetest.py | 3 +- scripts/tool_search_livetest2.py | 218 ++++++++++++++++++ tests/tools/test_tool_search.py | 111 +++++++++ tools/tool_search.py | 142 +++++++++++- .../docs/user-guide/features/tool-search.md | 15 ++ 6 files changed, 497 insertions(+), 4 deletions(-) create mode 100644 scripts/tool_search_livetest2.py diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 7a527b75f53c..debbb58ac2b1 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2958,6 +2958,18 @@ DEFAULT_CONFIG = { "search_default_limit": 5, # Hard upper bound the model can request via ``limit``. Range 1..50. "max_search_limit": 20, + # Skills-style catalog listing embedded in the tool_search bridge + # description: every deferred tool's name + first sentence of its + # description (≤60 chars), grouped by MCP server / toolset. Keeps + # capabilities discoverable while schemas stay deferred. + # "auto" (default) — include when the listing fits listing_max_tokens + # (falls back to names-only, then to the bare count). + # "on" — same rendering, but explicit intent to always list. + # "off" — legacy bare-count bridge description. + "listing": "auto", + # Token budget for the embedded listing (chars/4 estimate). + # Range 200..20000. + "listing_max_tokens": 4000, }, }, diff --git a/scripts/tool_search_livetest.py b/scripts/tool_search_livetest.py index 86aca38ccab2..d6a9bbeb3a64 100644 --- a/scripts/tool_search_livetest.py +++ b/scripts/tool_search_livetest.py @@ -248,7 +248,7 @@ SCENARIOS: List[Dict[str, Any]] = [ # --------------------------------------------------------------------------- -def setup_isolated_home(enabled: bool) -> Path: +def setup_isolated_home(enabled: bool, listing: str = "off") -> Path: """Create a fresh ~/.hermes/ for one test, copying minimal credentials. Also reads OPENROUTER_API_KEY from the user's real ``~/.hermes/.env`` so @@ -286,6 +286,7 @@ def setup_isolated_home(enabled: bool) -> Path: "threshold_pct": 10, "search_default_limit": 5, "max_search_limit": 20, + "listing": listing, }, }, "logging": {"level": "WARNING"}, diff --git a/scripts/tool_search_livetest2.py b/scripts/tool_search_livetest2.py new file mode 100644 index 000000000000..d444e545cf16 --- /dev/null +++ b/scripts/tool_search_livetest2.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Tool Search live benchmark v2 — real token accounting + more scenarios + reps. + +Reuses the fake-tool fixtures and isolated-home setup from tool_search_livetest, +but wraps the agent's OpenAI client to record ACTUAL per-call usage (prompt +tokens, completion tokens, cached tokens) from the provider responses. + +Runs each scenario N_REPS times in each mode (on/off). Output: + scripts/out2/____rep.json + scripts/out2/_bench_summary.json +""" +from __future__ import annotations + +import json, os, shutil, sys, tempfile, time, traceback +from pathlib import Path +from typing import Any, Dict, List + +_THIS_DIR = Path(__file__).resolve().parent +_WORKTREE_ROOT = _THIS_DIR.parent +sys.path.insert(0, str(_WORKTREE_ROOT)) +sys.path.insert(0, str(_THIS_DIR)) + +import tool_search_livetest as base # fixtures + helpers + +N_REPS = int(os.environ.get("TS_BENCH_REPS", "3")) + +SCENARIOS: List[Dict[str, Any]] = base.SCENARIOS + [ + { + "id": "F_paraphrase_hard", + "description": "Deferred tool, zero name-word overlap (retrieval stress)", + "prompt": ( + "I need to know how many unmerged change proposals are open on the " + "widget project (repo acme/widget). Just the count. Then you're done." + ), + "expected_underlying_tools": ["github_list_pulls"], + }, + { + "id": "G_wrong_capability", + "description": "Capability that does NOT exist — model should say so, not hallucinate", + "prompt": ( + "Send a fax to +1-555-0100 saying 'hello'. If you truly can't, say " + "'CANNOT: ' plus a one-line reason." + ), + "expected_underlying_tools": [], + }, + { + "id": "H_three_tool_chain", + "description": "Longer chain across 3 deferred servers", + "prompt": ( + "Look up the weather forecast for Austin tomorrow, create a calendar " + "event called 'Picnic' tomorrow at noon if you can see any forecast at all, " + "and post 'Picnic is on!' to the #random Slack channel. Then say done." + ), + "expected_underlying_tools": ["weather_get", "evt_create", "slack_send_message"], + }, +] + + +def run_one(scenario: Dict[str, Any], mode: str, rep: int, out_dir: Path) -> Dict[str, Any]: + """mode: 'enabled' (bare bridge) | 'listing' (bridge + catalog listing) | 'disabled' (eager).""" + enabled = mode in ("enabled", "listing") + hermes_home = base.setup_isolated_home(enabled, listing=("auto" if mode == "listing" else "off")) + os.environ["HERMES_HOME"] = str(hermes_home) + base.reset_module_state() + n_registered = base.register_fake_tools() + + Path("/tmp/livetest").mkdir(exist_ok=True) + (Path("/tmp/livetest/notes.txt")).write_text("Hello from the test fixture.\n") + + from tools.registry import registry + original_dispatch = registry.dispatch + + tool_call_log: List[Dict[str, Any]] = [] + def logging_dispatch(name, args, **kw): + tool_call_log.append({"name": name}) + return original_dispatch(name, args, **kw) + registry.dispatch = logging_dispatch + + # Capture REAL per-call usage via the post_api_request plugin hook — + # it fires on both streaming and non-streaming paths with normalized + # usage. NOTE: registered AFTER AIAgent construction because plugin + # discovery during init calls _hooks.clear(). + usage_log: List[Dict[str, Any]] = [] + def usage_hook(**kw): + u = kw.get("usage") or {} + if u: + usage_log.append({ + "prompt_tokens": u.get("prompt_tokens"), + "completion_tokens": u.get("completion_tokens"), + "cached_tokens": u.get("cached_tokens") or u.get("cache_read_input_tokens") or 0, + }) + + started = time.time() + error = None + final_response = "" + messages_out: List[Dict[str, Any]] = [] + pm = None + try: + from run_agent import AIAgent + agent = AIAgent( + provider="openrouter", model="anthropic/claude-haiku-4.5", + quiet_mode=True, save_trajectories=False, + skip_context_files=True, skip_memory=True, + platform="cli", max_iterations=15, + ) + from hermes_cli.plugins import get_plugin_manager, discover_plugins + discover_plugins() # idempotent; ensures no later clear wipes our hook + pm = get_plugin_manager() + pm._hooks.setdefault("post_api_request", []).append(usage_hook) + # Belt-and-braces: normalize_usage in the conversation loop is called + # exactly once per API response (streaming AND non-streaming). Wrap it + # to capture canonical usage the hook path may miss. + import agent.conversation_loop as _cl + _orig_norm = _cl.normalize_usage + def _norm_spy(raw, **kw): + cu = _orig_norm(raw, **kw) + try: + usage_log.append({ + "prompt_tokens": cu.prompt_tokens, + "completion_tokens": getattr(cu, "output_tokens", 0) or 0, + "cached_tokens": getattr(cu, "cache_read_tokens", 0) or 0, + "src": "norm", + }) + except Exception: + pass + return cu + _cl.normalize_usage = _norm_spy + result = agent.run_conversation( + user_message=scenario["prompt"], + system_message=("You are a test agent. Complete the user's task using available " + "tools. Be concise; don't add commentary beyond what's needed."), + ) + if isinstance(result, dict): + final_response = result.get("final_response") or "" + messages_out = result.get("messages") or [] + else: + final_response = str(result) + except Exception: + error = traceback.format_exc() + finally: + registry.dispatch = original_dispatch + try: + import agent.conversation_loop as _cl2 + if "_orig_norm" in dir() or True: + try: + _cl2.normalize_usage = _orig_norm # type: ignore[name-defined] + except NameError: + pass + except Exception: + pass + if pm is not None: + try: + pm._hooks.get("post_api_request", []).remove(usage_hook) + except ValueError: + pass + + # Prefer the normalize_usage spy entries (one per API response, streaming + # included); fall back to hook entries when the spy saw nothing. + norm_entries = [u for u in usage_log if u.get("src") == "norm"] + if norm_entries: + usage_log = norm_entries + + elapsed = time.time() - started + bridge_call_log = base._extract_bridge_calls(messages_out) + + expected = scenario.get("expected_underlying_tools", []) + called_names = [c.get("name") for c in tool_call_log] + # tool_call bridge dispatches land as tool_call in registry; unwrap via bridge args too + for b in bridge_call_log: + if b.get("name") == "tool_call": + inner = (b.get("args") or {}).get("name") + if inner: + called_names.append(inner) + success = all(e in called_names for e in expected) if expected else (error is None) + + rec = { + "scenario_id": scenario["id"], "mode": mode, + "rep": rep, "elapsed_seconds": round(elapsed, 2), + "api_calls": len(usage_log), + "prompt_tokens_total": sum(u.get("prompt_tokens") or 0 for u in usage_log), + "completion_tokens_total": sum(u.get("completion_tokens") or 0 for u in usage_log), + "cached_tokens_total": sum(u.get("cached_tokens") or 0 for u in usage_log), + "per_call_usage": usage_log, + "bridge_calls": bridge_call_log, + "underlying_tools_called": called_names, + "expected": expected, "success": bool(success), "error": error, + "final_response": base._redact_secrets(final_response)[:500], + } + out_path = out_dir / f"{scenario['id']}__{'enabled' if enabled else 'disabled'}__rep{rep}.json" + out_path.write_text(json.dumps(rec, indent=1)) + shutil.rmtree(Path(os.environ["HERMES_HOME"]).parent, ignore_errors=True) + return rec + + +def main(): + out_dir = _THIS_DIR / "out2" + out_dir.mkdir(exist_ok=True) + modes = [m for m in os.environ.get("TS_BENCH_MODES", "enabled,listing,disabled").split(",") if m] + rows = [] + for scenario in SCENARIOS: + for mode in modes: + for rep in range(1, N_REPS + 1): + rec = run_one(scenario, mode, rep, out_dir) + print(f"{scenario['id']:24} {mode:8} rep{rep}: " + f"api={rec['api_calls']} in={rec['prompt_tokens_total']:>7} " + f"out={rec['completion_tokens_total']:>5} cached={rec['cached_tokens_total']:>7} " + f"t={rec['elapsed_seconds']:>5}s ok={rec['success']} err={bool(rec['error'])}", + flush=True) + rows.append(rec) + summary_name = os.environ.get("TS_BENCH_SUMMARY", "_bench_summary.json") + (out_dir / summary_name).write_text(json.dumps( + [{k: v for k, v in r.items() if k not in ("per_call_usage", "bridge_calls", "final_response")} for r in rows], + indent=1)) + print("done ->", out_dir / summary_name) + + +if __name__ == "__main__": + main() diff --git a/tests/tools/test_tool_search.py b/tests/tools/test_tool_search.py index 9c8c8a33c178..f4dee25b7c48 100644 --- a/tests/tools/test_tool_search.py +++ b/tests/tools/test_tool_search.py @@ -536,3 +536,114 @@ class TestRegression_ToolsetScoping: # core tools are never deferrable assert "terminal" not in names + + +# --------------------------------------------------------------------------- +# Catalog listing (skills-style progressive disclosure) +# --------------------------------------------------------------------------- + + +class TestCatalogListing: + def test_config_defaults(self): + from tools.tool_search import ToolSearchConfig + cfg = ToolSearchConfig.from_raw(None) + assert cfg.listing == "auto" + assert cfg.listing_max_tokens == 4000 + # legacy bool shapes keep defaults too + assert ToolSearchConfig.from_raw(True).listing == "auto" + + def test_config_listing_off_and_clamp(self): + from tools.tool_search import ToolSearchConfig + cfg = ToolSearchConfig.from_raw({"listing": "off", "listing_max_tokens": 999999}) + assert cfg.listing == "off" + assert cfg.listing_max_tokens == 20000 + cfg2 = ToolSearchConfig.from_raw({"listing": "garbage", "listing_max_tokens": -5}) + assert cfg2.listing == "auto" + assert cfg2.listing_max_tokens == 200 + + def test_short_desc_first_sentence_and_clip(self): + from tools.tool_search import _short_desc + assert _short_desc("Open an issue. Second sentence dropped.") == "Open an issue." + long = "word " * 40 + s = _short_desc(long) + assert len(s) <= 61 # 60 + ellipsis char + assert s.endswith("…") + assert _short_desc("") == "" + + def test_listing_grouped_and_deterministic(self): + from tools.tool_search import build_catalog_listing + defs = [ + _td("zeta_tool", "Does zeta."), + _td("alpha_tool", "Does alpha."), + ] + a = build_catalog_listing(defs) + b = build_catalog_listing(list(reversed(defs))) + assert a == b # byte-stable regardless of input order (cache safety) + assert a.index("alpha_tool") < a.index("zeta_tool") + + def test_listing_budget_falls_back_to_names_then_none(self): + from tools.tool_search import build_catalog_listing + defs = [_td(f"tool_{i:03d}", "A tool that does something moderately verbose.") + for i in range(50)] + full = build_catalog_listing(defs, max_tokens=20000) + assert full is not None and "- tool_000:" in full + names_only = build_catalog_listing(defs, max_tokens=300) + assert names_only is not None + assert "- tool_000:" not in names_only # descriptions dropped + assert "tool_000" in names_only + assert build_catalog_listing(defs, max_tokens=200) is None or "tool_000" in build_catalog_listing(defs, max_tokens=200) + + def test_bridge_embeds_listing(self): + from tools.tool_search import bridge_tool_schemas + bridges = bridge_tool_schemas(5, listing="github tools (2):\n- a: x\n- b: y") + search = next(b for b in bridges if b["function"]["name"] == "tool_search") + assert "github tools (2)" in search["function"]["description"] + assert "do NOT claim it is unavailable" in search["function"]["description"] + # other bridges unchanged + bare = bridge_tool_schemas(5) + assert bare[1] == bridges[1] and bare[2] == bridges[2] + + @staticmethod + def _register(name): + from tools.registry import registry + + def _handler(args, task_id=None, **kw): + return json.dumps({"ok": True}) + + registry.register( + name=name, + handler=_handler, + schema=_td(name, "Deferred capability description.")["function"], + toolset="mcp-listingtest", + ) + + def test_assembly_embeds_listing_when_active(self): + from tools.tool_search import assemble_tool_defs, ToolSearchConfig + for i in range(30): + self._register(f"mcp_x_{i}") + defs = [_td("terminal", "Run shell")] + [ + _td(f"mcp_x_{i}", "Deferred capability description.", + {"a": {"type": "string", "description": "x" * 200}}) + for i in range(30) + ] + result = assemble_tool_defs( + defs, context_length=1000, + config=ToolSearchConfig.from_raw({"enabled": "on"}), + ) + assert result.activated + search = next(t for t in result.tool_defs if t["function"]["name"] == "tool_search") + assert "mcp_x_0" in search["function"]["description"] + assert "listingtest tools (30):" in search["function"]["description"] + + def test_assembly_listing_off_keeps_legacy_description(self): + from tools.tool_search import assemble_tool_defs, ToolSearchConfig + for i in range(30): + self._register(f"mcp_x_{i}") + defs = [_td(f"mcp_x_{i}", "Deferred.") for i in range(30)] + result = assemble_tool_defs( + defs, context_length=1000, + config=ToolSearchConfig.from_raw({"enabled": "on", "listing": "off"}), + ) + assert result.activated + search = next(t for t in result.tool_defs if t["function"]["name"] == "tool_search") + assert "mcp_x_0" not in search["function"]["description"] diff --git a/tools/tool_search.py b/tools/tool_search.py index e885a5d7b88c..463571851547 100644 --- a/tools/tool_search.py +++ b/tools/tool_search.py @@ -68,6 +68,14 @@ class ToolSearchConfig: threshold_pct: float # 0..100 — only used when enabled == "auto" search_default_limit: int max_search_limit: int + # Catalog listing ("skills-style" progressive disclosure): when active, + # a grouped name + short-description manifest of every deferred tool is + # embedded in the tool_search bridge description, so capabilities stay + # DISCOVERABLE (like the skills listing in the system prompt) while full + # schemas stay deferred. "auto" = include when it fits listing_max_tokens; + # "on" = always include; "off" = legacy bare-count behavior. + listing: str = "auto" # "auto" | "on" | "off" + listing_max_tokens: int = 4000 @classmethod def from_raw(cls, raw: Any) -> "ToolSearchConfig": @@ -106,11 +114,24 @@ class ToolSearchConfig: search_default_limit = max(1, min(max_search_limit, _safe_int(raw.get("search_default_limit"), 5))) + listing_raw = str(raw.get("listing", "auto")).strip().lower() + if listing_raw in ("true", "1", "yes"): + listing = "on" + elif listing_raw in ("false", "0", "no"): + listing = "off" + elif listing_raw in ("auto", "on", "off"): + listing = listing_raw + else: + listing = "auto" + listing_max_tokens = max(200, min(20000, _safe_int(raw.get("listing_max_tokens"), 4000))) + return cls( enabled=enabled, threshold_pct=threshold_pct, search_default_limit=search_default_limit, max_search_limit=max_search_limit, + listing=listing, + listing_max_tokens=listing_max_tokens, ) @@ -423,12 +444,115 @@ def search_catalog(catalog: List[CatalogEntry], query: str, limit: int = 5) -> L # --------------------------------------------------------------------------- -def bridge_tool_schemas(deferred_count: int) -> List[Dict[str, Any]]: +_SENTENCE_END_RE = re.compile(r"[.!?\n]") + + +def _short_desc(description: str, max_chars: int = 60) -> str: + """First sentence of a tool description, clipped to ``max_chars``. + + Mirrors the skills-listing convention: one terse line per capability. + Whitespace is collapsed; a hard clip never cuts mid-word unless the + first word itself exceeds the budget. + """ + text = " ".join((description or "").split()) + if not text: + return "" + m = _SENTENCE_END_RE.search(text) + if m: + text = text[:m.start() + (1 if text[m.start()] == "." else 0)] + if len(text) <= max_chars: + return text + clipped = text[:max_chars] + if " " in clipped: + clipped = clipped.rsplit(" ", 1)[0] + return clipped.rstrip(",;: ") + "…" + + +def _listing_group_label(source_name: str) -> str: + """Human-facing group heading for a toolset, e.g. ``mcp-github`` -> ``github``.""" + label = source_name or "other" + if label.startswith("mcp-"): + label = label[4:] + return label + + +def build_catalog_listing( + deferrable: List[Dict[str, Any]], + *, + max_tokens: int = 4000, +) -> Optional[str]: + """Render a skills-style manifest of the deferred catalog. + + One line per tool — ``name: short description`` — grouped under a + heading per source (MCP server / plugin toolset), exactly like the + bundled-skills listing in the system prompt: + + github tools: (44) + - create_issue: Open a new issue in a GitHub repository. + - merge_pull_request: Merge an open pull request. + ... + + Ordering is deterministic (groups and tools sorted by name) so the + rendered block is byte-stable across assemblies of the same catalog — + this keeps the request prefix cacheable across turns. + + Token-budget fallbacks (cheap chars/4 estimate, same rule as the + activation gate): + 1. full listing (names + short descriptions) + 2. names-only listing, still grouped + 3. ``None`` — caller falls back to the legacy bare-count description + """ + if not deferrable: + return None + + groups: Dict[str, List[Tuple[str, str]]] = {} + for td in deferrable: + fn = td.get("function") or {} + name = fn.get("name", "") + if not name: + continue + source, source_name = _classify_source(name) + label = _listing_group_label(source_name if source != "other" else "other") + groups.setdefault(label, []).append((name, _short_desc(fn.get("description", "")))) + + if not groups: + return None + + def render(with_descriptions: bool) -> str: + lines: List[str] = ["Deferred tool catalog (call schemas via " + f"`{TOOL_DESCRIBE_NAME}`, invoke via `{TOOL_CALL_NAME}`):"] + for label in sorted(groups): + tools = sorted(groups[label]) + lines.append(f"{label} tools ({len(tools)}):") + if with_descriptions: + for name, desc in tools: + lines.append(f"- {name}: {desc}" if desc else f"- {name}") + else: + lines.append(", ".join(name for name, _ in tools)) + return "\n".join(lines) + + for with_desc in (True, False): + text = render(with_desc) + if math.ceil(len(text) / CHARS_PER_TOKEN) <= max_tokens: + return text + return None + + +def bridge_tool_schemas( + deferred_count: int, + listing: Optional[str] = None, +) -> List[Dict[str, Any]]: """Build the bridge tool schemas to inject in place of deferred tools. The schemas are intentionally short — every byte added here is a byte the user pays on every turn. Descriptions are tuned to be unambiguous about the call sequence the model should follow. + + When ``listing`` is provided (see :func:`build_catalog_listing`), it is + embedded in the ``tool_search`` description so every deferred capability + stays *visible* by name — the skills-listing pattern — closing the + "model doesn't know what it doesn't know" gap while full parameter + schemas remain deferred. """ desc_search = ( f"Search {deferred_count} additional tools that are loaded on demand. " @@ -437,6 +561,13 @@ def bridge_tool_schemas(deferred_count: int) -> List[Dict[str, Any]]: f"then `{TOOL_CALL_NAME}` to invoke it. Tools listed at the top of this " "system prompt are already available and do not need to be searched." ) + if listing: + desc_search += ( + "\n\nEvery deferred tool is listed below. If the capability you " + "need appears here, do NOT claim it is unavailable — load it with " + f"`{TOOL_DESCRIBE_NAME}` (skip `{TOOL_SEARCH_NAME}` when you " + "already see the exact name).\n\n" + listing + ) desc_describe = ( f"Load the full JSON schema for one tool returned by `{TOOL_SEARCH_NAME}`. " f"Required before `{TOOL_CALL_NAME}` if the tool's parameters are unknown." @@ -565,13 +696,17 @@ def assemble_tool_defs( threshold_tokens=int((context_length or 0) * (config.threshold_pct / 100.0)), ) - bridge = bridge_tool_schemas(len(deferrable)) + listing = None + if config.listing != "off": + listing = build_catalog_listing(deferrable, max_tokens=config.listing_max_tokens) + bridge = bridge_tool_schemas(len(deferrable), listing=listing) result = visible + bridge threshold_tokens = int((context_length or 0) * (config.threshold_pct / 100.0)) logger.info( - "tool_search activated: %d core/visible tools kept, %d deferred (~%d tokens, threshold ~%d)", + "tool_search activated: %d core/visible tools kept, %d deferred (~%d tokens, threshold ~%d), catalog listing %s", len(visible), len(deferrable), deferrable_tokens, threshold_tokens, + "embedded" if listing else "omitted", ) return AssemblyResult( @@ -724,6 +859,7 @@ __all__ = [ "estimate_tokens_from_schemas", "should_activate", "build_catalog", + "build_catalog_listing", "search_catalog", "bridge_tool_schemas", "assemble_tool_defs", diff --git a/website/docs/user-guide/features/tool-search.md b/website/docs/user-guide/features/tool-search.md index 726cbbc17285..1e74d51a2985 100644 --- a/website/docs/user-guide/features/tool-search.md +++ b/website/docs/user-guide/features/tool-search.md @@ -78,6 +78,8 @@ tools: threshold_pct: 10 # percentage of context — only used in auto mode search_default_limit: 5 max_search_limit: 20 + listing: auto # embed a grouped name+description catalog manifest + listing_max_tokens: 4000 ``` | Key | Default | Meaning | @@ -86,6 +88,19 @@ tools: | `threshold_pct` | `10` | Percentage of context length at which `auto` mode kicks in. Range 0–100. | | `search_default_limit` | `5` | Hits returned when the model calls `tool_search` without a `limit`. | | `max_search_limit` | `20` | Hard upper bound the model can request via `limit`. Range 1–50. | +| `listing` | `auto` | Embed a skills-style manifest of every deferred tool (name + first sentence of its description, ≤60 chars, grouped by MCP server) in the `tool_search` bridge description. `auto` includes it when it fits the budget (falling back to names-only, then to a bare count); `on`/`off` force either way. | +| `listing_max_tokens` | `4000` | Token budget for the embedded listing. Range 200–20000. | + +### Why the listing exists + +Without it, deferred capabilities are *invisible* — live benchmarking showed +models substituting visible core tools (running `gh` in the terminal instead +of searching for the deferred GitHub tool) or declaring a capability +nonexistent instead of calling `tool_search`. The listing applies the skills +pattern to tools: every capability stays discoverable by name at all times, +while full parameter schemas remain deferred. If the model sees the exact +tool name in the listing, it can skip `tool_search` and go straight to +`tool_describe`, saving a round trip. You can also flip the legacy boolean shape: