mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-23 16:36:23 +00:00
feat(tools): skills-style catalog listing for tool_search progressive disclosure
Deferred MCP/plugin tools become invisible once the tool_search bridge activates — live benchmarking (48 runs, Claude Haiku 4.5) showed models substituting visible core tools (terminal/web_search/browser) for deferred capabilities or declaring them nonexistent instead of searching: 16/24 task success vs 24/24 with eager loading. Skills never had this failure mode because every skill keeps a ~21-token name+description listing line in the system prompt. This ports that exact pattern to the tool bridge: when tool_search activates, a grouped manifest of every deferred tool (name + first sentence of description, clipped to 60 chars, grouped per MCP server / toolset) is embedded in the tool_search bridge description. - tools/tool_search.py: build_catalog_listing() with deterministic ordering (byte-stable across assemblies -> prompt prefix stays cacheable); token-budget fallbacks full -> names-only -> legacy bare count; bridge_tool_schemas(listing=...) embeds it and instructs the model to skip tool_search when the exact name is visible (one fewer round-trip per use) - config: tools.tool_search.listing auto|on|off (default auto), listing_max_tokens (default 4000, clamped 200..20000); legacy bool shapes keep working - tests: 8 new tests (config parsing/clamps, short-desc clipping, deterministic rendering, budget fallbacks, bridge embedding, assembly on/off paths); full file green (47 passed) - docs: tool-search.md config table + rationale - scripts/tool_search_livetest2.py: benchmark harness v2 with real per-call token accounting (normalize_usage spy) and a third 'listing' mode for A/B/C comparison
This commit is contained in:
teknium1
Teknium
parent
9acc4b47f5
commit
3ee3394907
6 changed files with 497 additions and 4 deletions
|
|
@ -2958,6 +2958,18 @@ DEFAULT_CONFIG = {
|
|||
"search_default_limit": 5,
|
||||
# Hard upper bound the model can request via ``limit``. Range 1..50.
|
||||
"max_search_limit": 20,
|
||||
# Skills-style catalog listing embedded in the tool_search bridge
|
||||
# description: every deferred tool's name + first sentence of its
|
||||
# description (≤60 chars), grouped by MCP server / toolset. Keeps
|
||||
# capabilities discoverable while schemas stay deferred.
|
||||
# "auto" (default) — include when the listing fits listing_max_tokens
|
||||
# (falls back to names-only, then to the bare count).
|
||||
# "on" — same rendering, but explicit intent to always list.
|
||||
# "off" — legacy bare-count bridge description.
|
||||
"listing": "auto",
|
||||
# Token budget for the embedded listing (chars/4 estimate).
|
||||
# Range 200..20000.
|
||||
"listing_max_tokens": 4000,
|
||||
},
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -248,7 +248,7 @@ SCENARIOS: List[Dict[str, Any]] = [
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def setup_isolated_home(enabled: bool) -> Path:
|
||||
def setup_isolated_home(enabled: bool, listing: str = "off") -> Path:
|
||||
"""Create a fresh ~/.hermes/ for one test, copying minimal credentials.
|
||||
|
||||
Also reads OPENROUTER_API_KEY from the user's real ``~/.hermes/.env`` so
|
||||
|
|
@ -286,6 +286,7 @@ def setup_isolated_home(enabled: bool) -> Path:
|
|||
"threshold_pct": 10,
|
||||
"search_default_limit": 5,
|
||||
"max_search_limit": 20,
|
||||
"listing": listing,
|
||||
},
|
||||
},
|
||||
"logging": {"level": "WARNING"},
|
||||
|
|
|
|||
218
scripts/tool_search_livetest2.py
Normal file
218
scripts/tool_search_livetest2.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Tool Search live benchmark v2 — real token accounting + more scenarios + reps.
|
||||
|
||||
Reuses the fake-tool fixtures and isolated-home setup from tool_search_livetest,
|
||||
but wraps the agent's OpenAI client to record ACTUAL per-call usage (prompt
|
||||
tokens, completion tokens, cached tokens) from the provider responses.
|
||||
|
||||
Runs each scenario N_REPS times in each mode (on/off). Output:
|
||||
scripts/out2/<scenario>__<mode>__rep<k>.json
|
||||
scripts/out2/_bench_summary.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json, os, shutil, sys, tempfile, time, traceback
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
_THIS_DIR = Path(__file__).resolve().parent
|
||||
_WORKTREE_ROOT = _THIS_DIR.parent
|
||||
sys.path.insert(0, str(_WORKTREE_ROOT))
|
||||
sys.path.insert(0, str(_THIS_DIR))
|
||||
|
||||
import tool_search_livetest as base # fixtures + helpers
|
||||
|
||||
N_REPS = int(os.environ.get("TS_BENCH_REPS", "3"))
|
||||
|
||||
SCENARIOS: List[Dict[str, Any]] = base.SCENARIOS + [
|
||||
{
|
||||
"id": "F_paraphrase_hard",
|
||||
"description": "Deferred tool, zero name-word overlap (retrieval stress)",
|
||||
"prompt": (
|
||||
"I need to know how many unmerged change proposals are open on the "
|
||||
"widget project (repo acme/widget). Just the count. Then you're done."
|
||||
),
|
||||
"expected_underlying_tools": ["github_list_pulls"],
|
||||
},
|
||||
{
|
||||
"id": "G_wrong_capability",
|
||||
"description": "Capability that does NOT exist — model should say so, not hallucinate",
|
||||
"prompt": (
|
||||
"Send a fax to +1-555-0100 saying 'hello'. If you truly can't, say "
|
||||
"'CANNOT: ' plus a one-line reason."
|
||||
),
|
||||
"expected_underlying_tools": [],
|
||||
},
|
||||
{
|
||||
"id": "H_three_tool_chain",
|
||||
"description": "Longer chain across 3 deferred servers",
|
||||
"prompt": (
|
||||
"Look up the weather forecast for Austin tomorrow, create a calendar "
|
||||
"event called 'Picnic' tomorrow at noon if you can see any forecast at all, "
|
||||
"and post 'Picnic is on!' to the #random Slack channel. Then say done."
|
||||
),
|
||||
"expected_underlying_tools": ["weather_get", "evt_create", "slack_send_message"],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def run_one(scenario: Dict[str, Any], mode: str, rep: int, out_dir: Path) -> Dict[str, Any]:
|
||||
"""mode: 'enabled' (bare bridge) | 'listing' (bridge + catalog listing) | 'disabled' (eager)."""
|
||||
enabled = mode in ("enabled", "listing")
|
||||
hermes_home = base.setup_isolated_home(enabled, listing=("auto" if mode == "listing" else "off"))
|
||||
os.environ["HERMES_HOME"] = str(hermes_home)
|
||||
base.reset_module_state()
|
||||
n_registered = base.register_fake_tools()
|
||||
|
||||
Path("/tmp/livetest").mkdir(exist_ok=True)
|
||||
(Path("/tmp/livetest/notes.txt")).write_text("Hello from the test fixture.\n")
|
||||
|
||||
from tools.registry import registry
|
||||
original_dispatch = registry.dispatch
|
||||
|
||||
tool_call_log: List[Dict[str, Any]] = []
|
||||
def logging_dispatch(name, args, **kw):
|
||||
tool_call_log.append({"name": name})
|
||||
return original_dispatch(name, args, **kw)
|
||||
registry.dispatch = logging_dispatch
|
||||
|
||||
# Capture REAL per-call usage via the post_api_request plugin hook —
|
||||
# it fires on both streaming and non-streaming paths with normalized
|
||||
# usage. NOTE: registered AFTER AIAgent construction because plugin
|
||||
# discovery during init calls _hooks.clear().
|
||||
usage_log: List[Dict[str, Any]] = []
|
||||
def usage_hook(**kw):
|
||||
u = kw.get("usage") or {}
|
||||
if u:
|
||||
usage_log.append({
|
||||
"prompt_tokens": u.get("prompt_tokens"),
|
||||
"completion_tokens": u.get("completion_tokens"),
|
||||
"cached_tokens": u.get("cached_tokens") or u.get("cache_read_input_tokens") or 0,
|
||||
})
|
||||
|
||||
started = time.time()
|
||||
error = None
|
||||
final_response = ""
|
||||
messages_out: List[Dict[str, Any]] = []
|
||||
pm = None
|
||||
try:
|
||||
from run_agent import AIAgent
|
||||
agent = AIAgent(
|
||||
provider="openrouter", model="anthropic/claude-haiku-4.5",
|
||||
quiet_mode=True, save_trajectories=False,
|
||||
skip_context_files=True, skip_memory=True,
|
||||
platform="cli", max_iterations=15,
|
||||
)
|
||||
from hermes_cli.plugins import get_plugin_manager, discover_plugins
|
||||
discover_plugins() # idempotent; ensures no later clear wipes our hook
|
||||
pm = get_plugin_manager()
|
||||
pm._hooks.setdefault("post_api_request", []).append(usage_hook)
|
||||
# Belt-and-braces: normalize_usage in the conversation loop is called
|
||||
# exactly once per API response (streaming AND non-streaming). Wrap it
|
||||
# to capture canonical usage the hook path may miss.
|
||||
import agent.conversation_loop as _cl
|
||||
_orig_norm = _cl.normalize_usage
|
||||
def _norm_spy(raw, **kw):
|
||||
cu = _orig_norm(raw, **kw)
|
||||
try:
|
||||
usage_log.append({
|
||||
"prompt_tokens": cu.prompt_tokens,
|
||||
"completion_tokens": getattr(cu, "output_tokens", 0) or 0,
|
||||
"cached_tokens": getattr(cu, "cache_read_tokens", 0) or 0,
|
||||
"src": "norm",
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return cu
|
||||
_cl.normalize_usage = _norm_spy
|
||||
result = agent.run_conversation(
|
||||
user_message=scenario["prompt"],
|
||||
system_message=("You are a test agent. Complete the user's task using available "
|
||||
"tools. Be concise; don't add commentary beyond what's needed."),
|
||||
)
|
||||
if isinstance(result, dict):
|
||||
final_response = result.get("final_response") or ""
|
||||
messages_out = result.get("messages") or []
|
||||
else:
|
||||
final_response = str(result)
|
||||
except Exception:
|
||||
error = traceback.format_exc()
|
||||
finally:
|
||||
registry.dispatch = original_dispatch
|
||||
try:
|
||||
import agent.conversation_loop as _cl2
|
||||
if "_orig_norm" in dir() or True:
|
||||
try:
|
||||
_cl2.normalize_usage = _orig_norm # type: ignore[name-defined]
|
||||
except NameError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
if pm is not None:
|
||||
try:
|
||||
pm._hooks.get("post_api_request", []).remove(usage_hook)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Prefer the normalize_usage spy entries (one per API response, streaming
|
||||
# included); fall back to hook entries when the spy saw nothing.
|
||||
norm_entries = [u for u in usage_log if u.get("src") == "norm"]
|
||||
if norm_entries:
|
||||
usage_log = norm_entries
|
||||
|
||||
elapsed = time.time() - started
|
||||
bridge_call_log = base._extract_bridge_calls(messages_out)
|
||||
|
||||
expected = scenario.get("expected_underlying_tools", [])
|
||||
called_names = [c.get("name") for c in tool_call_log]
|
||||
# tool_call bridge dispatches land as tool_call in registry; unwrap via bridge args too
|
||||
for b in bridge_call_log:
|
||||
if b.get("name") == "tool_call":
|
||||
inner = (b.get("args") or {}).get("name")
|
||||
if inner:
|
||||
called_names.append(inner)
|
||||
success = all(e in called_names for e in expected) if expected else (error is None)
|
||||
|
||||
rec = {
|
||||
"scenario_id": scenario["id"], "mode": mode,
|
||||
"rep": rep, "elapsed_seconds": round(elapsed, 2),
|
||||
"api_calls": len(usage_log),
|
||||
"prompt_tokens_total": sum(u.get("prompt_tokens") or 0 for u in usage_log),
|
||||
"completion_tokens_total": sum(u.get("completion_tokens") or 0 for u in usage_log),
|
||||
"cached_tokens_total": sum(u.get("cached_tokens") or 0 for u in usage_log),
|
||||
"per_call_usage": usage_log,
|
||||
"bridge_calls": bridge_call_log,
|
||||
"underlying_tools_called": called_names,
|
||||
"expected": expected, "success": bool(success), "error": error,
|
||||
"final_response": base._redact_secrets(final_response)[:500],
|
||||
}
|
||||
out_path = out_dir / f"{scenario['id']}__{'enabled' if enabled else 'disabled'}__rep{rep}.json"
|
||||
out_path.write_text(json.dumps(rec, indent=1))
|
||||
shutil.rmtree(Path(os.environ["HERMES_HOME"]).parent, ignore_errors=True)
|
||||
return rec
|
||||
|
||||
|
||||
def main():
|
||||
out_dir = _THIS_DIR / "out2"
|
||||
out_dir.mkdir(exist_ok=True)
|
||||
modes = [m for m in os.environ.get("TS_BENCH_MODES", "enabled,listing,disabled").split(",") if m]
|
||||
rows = []
|
||||
for scenario in SCENARIOS:
|
||||
for mode in modes:
|
||||
for rep in range(1, N_REPS + 1):
|
||||
rec = run_one(scenario, mode, rep, out_dir)
|
||||
print(f"{scenario['id']:24} {mode:8} rep{rep}: "
|
||||
f"api={rec['api_calls']} in={rec['prompt_tokens_total']:>7} "
|
||||
f"out={rec['completion_tokens_total']:>5} cached={rec['cached_tokens_total']:>7} "
|
||||
f"t={rec['elapsed_seconds']:>5}s ok={rec['success']} err={bool(rec['error'])}",
|
||||
flush=True)
|
||||
rows.append(rec)
|
||||
summary_name = os.environ.get("TS_BENCH_SUMMARY", "_bench_summary.json")
|
||||
(out_dir / summary_name).write_text(json.dumps(
|
||||
[{k: v for k, v in r.items() if k not in ("per_call_usage", "bridge_calls", "final_response")} for r in rows],
|
||||
indent=1))
|
||||
print("done ->", out_dir / summary_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -536,3 +536,114 @@ class TestRegression_ToolsetScoping:
|
|||
# core tools are never deferrable
|
||||
assert "terminal" not in names
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Catalog listing (skills-style progressive disclosure)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCatalogListing:
|
||||
def test_config_defaults(self):
|
||||
from tools.tool_search import ToolSearchConfig
|
||||
cfg = ToolSearchConfig.from_raw(None)
|
||||
assert cfg.listing == "auto"
|
||||
assert cfg.listing_max_tokens == 4000
|
||||
# legacy bool shapes keep defaults too
|
||||
assert ToolSearchConfig.from_raw(True).listing == "auto"
|
||||
|
||||
def test_config_listing_off_and_clamp(self):
|
||||
from tools.tool_search import ToolSearchConfig
|
||||
cfg = ToolSearchConfig.from_raw({"listing": "off", "listing_max_tokens": 999999})
|
||||
assert cfg.listing == "off"
|
||||
assert cfg.listing_max_tokens == 20000
|
||||
cfg2 = ToolSearchConfig.from_raw({"listing": "garbage", "listing_max_tokens": -5})
|
||||
assert cfg2.listing == "auto"
|
||||
assert cfg2.listing_max_tokens == 200
|
||||
|
||||
def test_short_desc_first_sentence_and_clip(self):
|
||||
from tools.tool_search import _short_desc
|
||||
assert _short_desc("Open an issue. Second sentence dropped.") == "Open an issue."
|
||||
long = "word " * 40
|
||||
s = _short_desc(long)
|
||||
assert len(s) <= 61 # 60 + ellipsis char
|
||||
assert s.endswith("…")
|
||||
assert _short_desc("") == ""
|
||||
|
||||
def test_listing_grouped_and_deterministic(self):
|
||||
from tools.tool_search import build_catalog_listing
|
||||
defs = [
|
||||
_td("zeta_tool", "Does zeta."),
|
||||
_td("alpha_tool", "Does alpha."),
|
||||
]
|
||||
a = build_catalog_listing(defs)
|
||||
b = build_catalog_listing(list(reversed(defs)))
|
||||
assert a == b # byte-stable regardless of input order (cache safety)
|
||||
assert a.index("alpha_tool") < a.index("zeta_tool")
|
||||
|
||||
def test_listing_budget_falls_back_to_names_then_none(self):
|
||||
from tools.tool_search import build_catalog_listing
|
||||
defs = [_td(f"tool_{i:03d}", "A tool that does something moderately verbose.")
|
||||
for i in range(50)]
|
||||
full = build_catalog_listing(defs, max_tokens=20000)
|
||||
assert full is not None and "- tool_000:" in full
|
||||
names_only = build_catalog_listing(defs, max_tokens=300)
|
||||
assert names_only is not None
|
||||
assert "- tool_000:" not in names_only # descriptions dropped
|
||||
assert "tool_000" in names_only
|
||||
assert build_catalog_listing(defs, max_tokens=200) is None or "tool_000" in build_catalog_listing(defs, max_tokens=200)
|
||||
|
||||
def test_bridge_embeds_listing(self):
|
||||
from tools.tool_search import bridge_tool_schemas
|
||||
bridges = bridge_tool_schemas(5, listing="github tools (2):\n- a: x\n- b: y")
|
||||
search = next(b for b in bridges if b["function"]["name"] == "tool_search")
|
||||
assert "github tools (2)" in search["function"]["description"]
|
||||
assert "do NOT claim it is unavailable" in search["function"]["description"]
|
||||
# other bridges unchanged
|
||||
bare = bridge_tool_schemas(5)
|
||||
assert bare[1] == bridges[1] and bare[2] == bridges[2]
|
||||
|
||||
@staticmethod
|
||||
def _register(name):
|
||||
from tools.registry import registry
|
||||
|
||||
def _handler(args, task_id=None, **kw):
|
||||
return json.dumps({"ok": True})
|
||||
|
||||
registry.register(
|
||||
name=name,
|
||||
handler=_handler,
|
||||
schema=_td(name, "Deferred capability description.")["function"],
|
||||
toolset="mcp-listingtest",
|
||||
)
|
||||
|
||||
def test_assembly_embeds_listing_when_active(self):
|
||||
from tools.tool_search import assemble_tool_defs, ToolSearchConfig
|
||||
for i in range(30):
|
||||
self._register(f"mcp_x_{i}")
|
||||
defs = [_td("terminal", "Run shell")] + [
|
||||
_td(f"mcp_x_{i}", "Deferred capability description.",
|
||||
{"a": {"type": "string", "description": "x" * 200}})
|
||||
for i in range(30)
|
||||
]
|
||||
result = assemble_tool_defs(
|
||||
defs, context_length=1000,
|
||||
config=ToolSearchConfig.from_raw({"enabled": "on"}),
|
||||
)
|
||||
assert result.activated
|
||||
search = next(t for t in result.tool_defs if t["function"]["name"] == "tool_search")
|
||||
assert "mcp_x_0" in search["function"]["description"]
|
||||
assert "listingtest tools (30):" in search["function"]["description"]
|
||||
|
||||
def test_assembly_listing_off_keeps_legacy_description(self):
|
||||
from tools.tool_search import assemble_tool_defs, ToolSearchConfig
|
||||
for i in range(30):
|
||||
self._register(f"mcp_x_{i}")
|
||||
defs = [_td(f"mcp_x_{i}", "Deferred.") for i in range(30)]
|
||||
result = assemble_tool_defs(
|
||||
defs, context_length=1000,
|
||||
config=ToolSearchConfig.from_raw({"enabled": "on", "listing": "off"}),
|
||||
)
|
||||
assert result.activated
|
||||
search = next(t for t in result.tool_defs if t["function"]["name"] == "tool_search")
|
||||
assert "mcp_x_0" not in search["function"]["description"]
|
||||
|
|
|
|||
|
|
@ -68,6 +68,14 @@ class ToolSearchConfig:
|
|||
threshold_pct: float # 0..100 — only used when enabled == "auto"
|
||||
search_default_limit: int
|
||||
max_search_limit: int
|
||||
# Catalog listing ("skills-style" progressive disclosure): when active,
|
||||
# a grouped name + short-description manifest of every deferred tool is
|
||||
# embedded in the tool_search bridge description, so capabilities stay
|
||||
# DISCOVERABLE (like the skills listing in the system prompt) while full
|
||||
# schemas stay deferred. "auto" = include when it fits listing_max_tokens;
|
||||
# "on" = always include; "off" = legacy bare-count behavior.
|
||||
listing: str = "auto" # "auto" | "on" | "off"
|
||||
listing_max_tokens: int = 4000
|
||||
|
||||
@classmethod
|
||||
def from_raw(cls, raw: Any) -> "ToolSearchConfig":
|
||||
|
|
@ -106,11 +114,24 @@ class ToolSearchConfig:
|
|||
search_default_limit = max(1, min(max_search_limit,
|
||||
_safe_int(raw.get("search_default_limit"), 5)))
|
||||
|
||||
listing_raw = str(raw.get("listing", "auto")).strip().lower()
|
||||
if listing_raw in ("true", "1", "yes"):
|
||||
listing = "on"
|
||||
elif listing_raw in ("false", "0", "no"):
|
||||
listing = "off"
|
||||
elif listing_raw in ("auto", "on", "off"):
|
||||
listing = listing_raw
|
||||
else:
|
||||
listing = "auto"
|
||||
listing_max_tokens = max(200, min(20000, _safe_int(raw.get("listing_max_tokens"), 4000)))
|
||||
|
||||
return cls(
|
||||
enabled=enabled,
|
||||
threshold_pct=threshold_pct,
|
||||
search_default_limit=search_default_limit,
|
||||
max_search_limit=max_search_limit,
|
||||
listing=listing,
|
||||
listing_max_tokens=listing_max_tokens,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -423,12 +444,115 @@ def search_catalog(catalog: List[CatalogEntry], query: str, limit: int = 5) -> L
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def bridge_tool_schemas(deferred_count: int) -> List[Dict[str, Any]]:
|
||||
_SENTENCE_END_RE = re.compile(r"[.!?\n]")
|
||||
|
||||
|
||||
def _short_desc(description: str, max_chars: int = 60) -> str:
|
||||
"""First sentence of a tool description, clipped to ``max_chars``.
|
||||
|
||||
Mirrors the skills-listing convention: one terse line per capability.
|
||||
Whitespace is collapsed; a hard clip never cuts mid-word unless the
|
||||
first word itself exceeds the budget.
|
||||
"""
|
||||
text = " ".join((description or "").split())
|
||||
if not text:
|
||||
return ""
|
||||
m = _SENTENCE_END_RE.search(text)
|
||||
if m:
|
||||
text = text[:m.start() + (1 if text[m.start()] == "." else 0)]
|
||||
if len(text) <= max_chars:
|
||||
return text
|
||||
clipped = text[:max_chars]
|
||||
if " " in clipped:
|
||||
clipped = clipped.rsplit(" ", 1)[0]
|
||||
return clipped.rstrip(",;: ") + "…"
|
||||
|
||||
|
||||
def _listing_group_label(source_name: str) -> str:
|
||||
"""Human-facing group heading for a toolset, e.g. ``mcp-github`` -> ``github``."""
|
||||
label = source_name or "other"
|
||||
if label.startswith("mcp-"):
|
||||
label = label[4:]
|
||||
return label
|
||||
|
||||
|
||||
def build_catalog_listing(
|
||||
deferrable: List[Dict[str, Any]],
|
||||
*,
|
||||
max_tokens: int = 4000,
|
||||
) -> Optional[str]:
|
||||
"""Render a skills-style manifest of the deferred catalog.
|
||||
|
||||
One line per tool — ``name: short description`` — grouped under a
|
||||
heading per source (MCP server / plugin toolset), exactly like the
|
||||
bundled-skills listing in the system prompt:
|
||||
|
||||
github tools: (44)
|
||||
- create_issue: Open a new issue in a GitHub repository.
|
||||
- merge_pull_request: Merge an open pull request.
|
||||
...
|
||||
|
||||
Ordering is deterministic (groups and tools sorted by name) so the
|
||||
rendered block is byte-stable across assemblies of the same catalog —
|
||||
this keeps the request prefix cacheable across turns.
|
||||
|
||||
Token-budget fallbacks (cheap chars/4 estimate, same rule as the
|
||||
activation gate):
|
||||
1. full listing (names + short descriptions)
|
||||
2. names-only listing, still grouped
|
||||
3. ``None`` — caller falls back to the legacy bare-count description
|
||||
"""
|
||||
if not deferrable:
|
||||
return None
|
||||
|
||||
groups: Dict[str, List[Tuple[str, str]]] = {}
|
||||
for td in deferrable:
|
||||
fn = td.get("function") or {}
|
||||
name = fn.get("name", "")
|
||||
if not name:
|
||||
continue
|
||||
source, source_name = _classify_source(name)
|
||||
label = _listing_group_label(source_name if source != "other" else "other")
|
||||
groups.setdefault(label, []).append((name, _short_desc(fn.get("description", ""))))
|
||||
|
||||
if not groups:
|
||||
return None
|
||||
|
||||
def render(with_descriptions: bool) -> str:
|
||||
lines: List[str] = ["Deferred tool catalog (call schemas via "
|
||||
f"`{TOOL_DESCRIBE_NAME}`, invoke via `{TOOL_CALL_NAME}`):"]
|
||||
for label in sorted(groups):
|
||||
tools = sorted(groups[label])
|
||||
lines.append(f"{label} tools ({len(tools)}):")
|
||||
if with_descriptions:
|
||||
for name, desc in tools:
|
||||
lines.append(f"- {name}: {desc}" if desc else f"- {name}")
|
||||
else:
|
||||
lines.append(", ".join(name for name, _ in tools))
|
||||
return "\n".join(lines)
|
||||
|
||||
for with_desc in (True, False):
|
||||
text = render(with_desc)
|
||||
if math.ceil(len(text) / CHARS_PER_TOKEN) <= max_tokens:
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def bridge_tool_schemas(
|
||||
deferred_count: int,
|
||||
listing: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Build the bridge tool schemas to inject in place of deferred tools.
|
||||
|
||||
The schemas are intentionally short — every byte added here is a byte
|
||||
the user pays on every turn. Descriptions are tuned to be unambiguous
|
||||
about the call sequence the model should follow.
|
||||
|
||||
When ``listing`` is provided (see :func:`build_catalog_listing`), it is
|
||||
embedded in the ``tool_search`` description so every deferred capability
|
||||
stays *visible* by name — the skills-listing pattern — closing the
|
||||
"model doesn't know what it doesn't know" gap while full parameter
|
||||
schemas remain deferred.
|
||||
"""
|
||||
desc_search = (
|
||||
f"Search {deferred_count} additional tools that are loaded on demand. "
|
||||
|
|
@ -437,6 +561,13 @@ def bridge_tool_schemas(deferred_count: int) -> List[Dict[str, Any]]:
|
|||
f"then `{TOOL_CALL_NAME}` to invoke it. Tools listed at the top of this "
|
||||
"system prompt are already available and do not need to be searched."
|
||||
)
|
||||
if listing:
|
||||
desc_search += (
|
||||
"\n\nEvery deferred tool is listed below. If the capability you "
|
||||
"need appears here, do NOT claim it is unavailable — load it with "
|
||||
f"`{TOOL_DESCRIBE_NAME}` (skip `{TOOL_SEARCH_NAME}` when you "
|
||||
"already see the exact name).\n\n" + listing
|
||||
)
|
||||
desc_describe = (
|
||||
f"Load the full JSON schema for one tool returned by `{TOOL_SEARCH_NAME}`. "
|
||||
f"Required before `{TOOL_CALL_NAME}` if the tool's parameters are unknown."
|
||||
|
|
@ -565,13 +696,17 @@ def assemble_tool_defs(
|
|||
threshold_tokens=int((context_length or 0) * (config.threshold_pct / 100.0)),
|
||||
)
|
||||
|
||||
bridge = bridge_tool_schemas(len(deferrable))
|
||||
listing = None
|
||||
if config.listing != "off":
|
||||
listing = build_catalog_listing(deferrable, max_tokens=config.listing_max_tokens)
|
||||
bridge = bridge_tool_schemas(len(deferrable), listing=listing)
|
||||
result = visible + bridge
|
||||
threshold_tokens = int((context_length or 0) * (config.threshold_pct / 100.0))
|
||||
|
||||
logger.info(
|
||||
"tool_search activated: %d core/visible tools kept, %d deferred (~%d tokens, threshold ~%d)",
|
||||
"tool_search activated: %d core/visible tools kept, %d deferred (~%d tokens, threshold ~%d), catalog listing %s",
|
||||
len(visible), len(deferrable), deferrable_tokens, threshold_tokens,
|
||||
"embedded" if listing else "omitted",
|
||||
)
|
||||
|
||||
return AssemblyResult(
|
||||
|
|
@ -724,6 +859,7 @@ __all__ = [
|
|||
"estimate_tokens_from_schemas",
|
||||
"should_activate",
|
||||
"build_catalog",
|
||||
"build_catalog_listing",
|
||||
"search_catalog",
|
||||
"bridge_tool_schemas",
|
||||
"assemble_tool_defs",
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ tools:
|
|||
threshold_pct: 10 # percentage of context — only used in auto mode
|
||||
search_default_limit: 5
|
||||
max_search_limit: 20
|
||||
listing: auto # embed a grouped name+description catalog manifest
|
||||
listing_max_tokens: 4000
|
||||
```
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|
|
@ -86,6 +88,19 @@ tools:
|
|||
| `threshold_pct` | `10` | Percentage of context length at which `auto` mode kicks in. Range 0–100. |
|
||||
| `search_default_limit` | `5` | Hits returned when the model calls `tool_search` without a `limit`. |
|
||||
| `max_search_limit` | `20` | Hard upper bound the model can request via `limit`. Range 1–50. |
|
||||
| `listing` | `auto` | Embed a skills-style manifest of every deferred tool (name + first sentence of its description, ≤60 chars, grouped by MCP server) in the `tool_search` bridge description. `auto` includes it when it fits the budget (falling back to names-only, then to a bare count); `on`/`off` force either way. |
|
||||
| `listing_max_tokens` | `4000` | Token budget for the embedded listing. Range 200–20000. |
|
||||
|
||||
### Why the listing exists
|
||||
|
||||
Without it, deferred capabilities are *invisible* — live benchmarking showed
|
||||
models substituting visible core tools (running `gh` in the terminal instead
|
||||
of searching for the deferred GitHub tool) or declaring a capability
|
||||
nonexistent instead of calling `tool_search`. The listing applies the skills
|
||||
pattern to tools: every capability stays discoverable by name at all times,
|
||||
while full parameter schemas remain deferred. If the model sees the exact
|
||||
tool name in the listing, it can skip `tool_search` and go straight to
|
||||
`tool_describe`, saving a round trip.
|
||||
|
||||
You can also flip the legacy boolean shape:
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue