mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
feat(tool_search): probe-validate blind tool_call args against the deferred schema
Port from nearai/ironclaw#5149 (the describe-first live-hardening fix in their progressive tool disclosure work): when a model invokes a deferred tool through the tool_call bridge without the schema-required arguments, return the tool's parameter schema instead of dispatching blind. Pre-fix, a blind call produced an opaque downstream failure ("[TOOL_ERROR] Tool execution failed: KeyError: 'document_id'") that teaches the model nothing about what the tool expects — IronClaw observed cheap models looping ~30 identical invalid calls until the iteration budget died. Post-fix, the model repairs the call in one round-trip. - tools/tool_search.py: new validate_deferred_call_args() — key-absence check of schema 'required' fields only; no type checking (coerce_tool_args already repairs types downstream); fails open on any validator error so it can never block a legitimate dispatch. - model_tools.py: probe after the scope gate in the bridge dispatch. - agent/tool_executor.py: probe in both unwrap sites (concurrent + sequential) before the underlying tool replaces the bridge; sequential path flattens the payload to match its {"error": str} wrapping. - tests: TestDeferredCallSchemaProbe — blind call returns schema (not KeyError), valid/optional calls dispatch, unvalidatable tools fail open, out-of-scope rejection unchanged.
This commit is contained in:
parent
9b97dea1e6
commit
8fbe2e388f
4 changed files with 199 additions and 4 deletions
|
|
@ -739,3 +739,112 @@ class TestCatalogListing:
|
|||
assert result.activated
|
||||
search = next(t for t in result.tool_defs if t["function"]["name"] == "tool_search")
|
||||
assert "mcp_x_0" not in search["function"]["description"]
|
||||
|
||||
|
||||
class TestDeferredCallSchemaProbe:
|
||||
"""Blind tool_call invocations missing required arguments must return
|
||||
the tool's parameter schema instead of dispatching into an opaque
|
||||
downstream failure (port of nearai/ironclaw#5149's describe-first fix).
|
||||
|
||||
A deferred tool's schema is invisible until tool_describe is called, so
|
||||
models routinely invoke deferred tools by name alone. Pre-fix, that
|
||||
produced ``KeyError: 'document_id'``-style errors that teach the model
|
||||
nothing; post-fix, the probe returns the schema so the model repairs
|
||||
the call in one round-trip. Valid calls dispatch untouched.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _register(name, toolset, required=("document_id",)):
|
||||
from tools.registry import registry
|
||||
|
||||
def _handler(args, task_id=None, **kw):
|
||||
# Simulates a tool that crashes opaquely on a missing required arg.
|
||||
return json.dumps({"ok": True, "doc": args["document_id"]})
|
||||
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"document_id": {"type": "string", "description": "Doc id"},
|
||||
"format": {"type": "string"},
|
||||
},
|
||||
"required": list(required),
|
||||
}
|
||||
registry.register(
|
||||
name=name,
|
||||
handler=_handler,
|
||||
schema={"type": "function",
|
||||
"function": {"name": name, "description": f"desc {name}",
|
||||
"parameters": params}},
|
||||
toolset=toolset,
|
||||
)
|
||||
|
||||
def test_validator_returns_schema_for_missing_required(self):
|
||||
from tools.tool_search import validate_deferred_call_args
|
||||
|
||||
self._register("mcp_probe_docs_get", "mcp-probe")
|
||||
err = validate_deferred_call_args("mcp_probe_docs_get", {})
|
||||
assert err is not None
|
||||
parsed = json.loads(err)
|
||||
assert "document_id" in parsed["error"]
|
||||
assert "NOT invoked" in parsed["error"]
|
||||
assert parsed["parameters"]["required"] == ["document_id"]
|
||||
assert "document_id" in parsed["parameters"]["properties"]
|
||||
|
||||
def test_validator_passes_valid_and_optional_only_calls(self):
|
||||
from tools.tool_search import validate_deferred_call_args
|
||||
|
||||
self._register("mcp_probe_docs_get2", "mcp-probe")
|
||||
# All required present → dispatch.
|
||||
assert validate_deferred_call_args(
|
||||
"mcp_probe_docs_get2", {"document_id": "abc"}) is None
|
||||
# Extra optional args don't matter.
|
||||
assert validate_deferred_call_args(
|
||||
"mcp_probe_docs_get2", {"document_id": "abc", "format": "md"}) is None
|
||||
|
||||
def test_validator_never_blocks_unvalidatable_tools(self):
|
||||
from tools.tool_search import validate_deferred_call_args
|
||||
|
||||
# Unknown tool → no schema → dispatch (downstream scope gate handles it).
|
||||
assert validate_deferred_call_args("mcp_no_such_tool_xyz", {}) is None
|
||||
|
||||
def test_validator_no_required_list_dispatches(self):
|
||||
from tools.tool_search import validate_deferred_call_args
|
||||
from tools.registry import registry
|
||||
|
||||
registry.register(
|
||||
name="mcp_probe_norequired",
|
||||
handler=lambda args, task_id=None, **kw: json.dumps({"ok": True}),
|
||||
schema={"type": "function",
|
||||
"function": {"name": "mcp_probe_norequired",
|
||||
"description": "d",
|
||||
"parameters": {"type": "object", "properties": {}}}},
|
||||
toolset="mcp-probe",
|
||||
)
|
||||
assert validate_deferred_call_args("mcp_probe_norequired", {}) is None
|
||||
|
||||
def test_blind_tool_call_returns_schema_not_keyerror(self):
|
||||
import model_tools
|
||||
|
||||
self._register("mcp_probe_blind_op", "mcp-probe-blind")
|
||||
result = json.loads(model_tools.handle_function_call(
|
||||
function_name="tool_call",
|
||||
function_args={"name": "mcp_probe_blind_op", "arguments": {}},
|
||||
enabled_toolsets=["mcp-probe-blind"],
|
||||
))
|
||||
assert "error" in result
|
||||
assert "KeyError" not in result["error"]
|
||||
assert "missing required argument" in result["error"]
|
||||
assert result["parameters"]["required"] == ["document_id"]
|
||||
|
||||
def test_valid_tool_call_still_dispatches(self):
|
||||
import model_tools
|
||||
|
||||
self._register("mcp_probe_valid_op", "mcp-probe-valid")
|
||||
result = json.loads(model_tools.handle_function_call(
|
||||
function_name="tool_call",
|
||||
function_args={"name": "mcp_probe_valid_op",
|
||||
"arguments": {"document_id": "abc"}},
|
||||
enabled_toolsets=["mcp-probe-valid"],
|
||||
))
|
||||
assert result.get("ok") is True
|
||||
assert result.get("doc") == "abc"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue