mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-05-25 05:52:34 +00:00
fix(tools): refuse skill_view name collisions instead of guessing
skill_view ran the direct-path strategy across every skill dir before the recursive strategy, so a top-level skill in an external dir could silently shadow a same-named nested local skill. /skills correctly listed the local version (deduped local-first by _find_all_skills) but skill_view loaded the external one — confusing, and a real bug class for users with skills.external_dirs registered alongside categorized local skills. Pick a louder fix than @polkn's PR #6136 proposed: collect every match across all dirs (direct path, recursive by parent dir name, legacy flat <name>.md), and if there's more than one, refuse with an error that surfaces every matching path plus a hint to load by the categorized form. Local-first precedence would have replaced silent external-shadowing with silent same-name collisions between two externals, or made an externally-shadowed-by-local skill unreachable by bare name with no signal. Refusing forces the user to disambiguate once and never wonder which skill ran. Recovery: pass the full categorized path ("foundations/runtime/explore-codebase" instead of "explore-codebase"), or rename one of the colliding skills. Co-authored-by: pol <pol.kuijken@gmail.com>
This commit is contained in:
parent
256bedb632
commit
59da8ec4ec
2 changed files with 231 additions and 32 deletions
|
|
@ -1076,3 +1076,168 @@ Do the legacy thing.
|
||||||
assert result["setup_needed"] is False
|
assert result["setup_needed"] is False
|
||||||
assert result["missing_required_environment_variables"] == []
|
assert result["missing_required_environment_variables"] == []
|
||||||
assert result["readiness_status"] == "available"
|
assert result["readiness_status"] == "available"
|
||||||
|
|
||||||
|
|
||||||
|
class TestSkillViewCollisionDetection:
|
||||||
|
"""Regression tests for skill_view name collision handling.
|
||||||
|
|
||||||
|
When a skill name resolves to multiple paths across the local skills
|
||||||
|
dir and external_dirs, skill_view must refuse to guess. Silent
|
||||||
|
shadowing — where ``/skills`` shows the local version but
|
||||||
|
``skill_view`` loads the external one — is the bug class this guards
|
||||||
|
against. Reproduces with `skills.external_dirs` registered in
|
||||||
|
config.yaml and a same-name skill nested under a category locally.
|
||||||
|
|
||||||
|
Adapted from a regression suite originally proposed by @polkn in PR
|
||||||
|
#6136 (which used local-first precedence). The collision-refusal
|
||||||
|
behavior preserves the same protection without silently picking a
|
||||||
|
side, and gives the user an actionable hint (use the categorized
|
||||||
|
path) to recover.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _patch_dirs(self, local_dir, external_dirs):
|
||||||
|
"""Patch SKILLS_DIR (module-level) and get_external_skills_dirs at source."""
|
||||||
|
return (
|
||||||
|
patch("tools.skills_tool.SKILLS_DIR", local_dir),
|
||||||
|
patch(
|
||||||
|
"agent.skill_utils.get_external_skills_dirs",
|
||||||
|
return_value=list(external_dirs),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_nested_local_collides_with_top_level_external(self, tmp_path):
|
||||||
|
"""The original bug scenario: nested local + top-level external,
|
||||||
|
same name. Now refuses with both paths surfaced."""
|
||||||
|
local_dir = tmp_path / "local"
|
||||||
|
external_dir = tmp_path / "external"
|
||||||
|
local_dir.mkdir()
|
||||||
|
external_dir.mkdir()
|
||||||
|
|
||||||
|
_make_skill(
|
||||||
|
local_dir,
|
||||||
|
"explore-codebase",
|
||||||
|
category="foundations/runtime",
|
||||||
|
body="LOCAL VERSION",
|
||||||
|
)
|
||||||
|
_make_skill(external_dir, "explore-codebase", body="EXTERNAL VERSION")
|
||||||
|
|
||||||
|
p1, p2 = self._patch_dirs(local_dir, [external_dir])
|
||||||
|
with p1, p2:
|
||||||
|
raw = skill_view("explore-codebase")
|
||||||
|
|
||||||
|
result = json.loads(raw)
|
||||||
|
assert result["success"] is False
|
||||||
|
assert "Ambiguous skill name 'explore-codebase'" in result["error"]
|
||||||
|
assert "matches" in result
|
||||||
|
assert len(result["matches"]) == 2
|
||||||
|
# Both paths surfaced
|
||||||
|
assert any("foundations/runtime" in p for p in result["matches"])
|
||||||
|
assert any("external" in p for p in result["matches"])
|
||||||
|
assert "hint" in result
|
||||||
|
|
||||||
|
def test_top_level_local_collides_with_external(self, tmp_path):
|
||||||
|
"""Top-level local + top-level external with the same name also
|
||||||
|
refuses — same-name shadowing is ambiguous regardless of nesting."""
|
||||||
|
local_dir = tmp_path / "local"
|
||||||
|
external_dir = tmp_path / "external"
|
||||||
|
local_dir.mkdir()
|
||||||
|
external_dir.mkdir()
|
||||||
|
|
||||||
|
_make_skill(local_dir, "shared-name", body="LOCAL VERSION")
|
||||||
|
_make_skill(external_dir, "shared-name", body="EXTERNAL VERSION")
|
||||||
|
|
||||||
|
p1, p2 = self._patch_dirs(local_dir, [external_dir])
|
||||||
|
with p1, p2:
|
||||||
|
raw = skill_view("shared-name")
|
||||||
|
|
||||||
|
result = json.loads(raw)
|
||||||
|
assert result["success"] is False
|
||||||
|
assert "Ambiguous" in result["error"]
|
||||||
|
assert len(result["matches"]) == 2
|
||||||
|
|
||||||
|
def test_collision_resolvable_via_categorized_path(self, tmp_path):
|
||||||
|
"""User can recover from a collision by passing the full
|
||||||
|
categorized path — the bare name is ambiguous, the path is not."""
|
||||||
|
local_dir = tmp_path / "local"
|
||||||
|
external_dir = tmp_path / "external"
|
||||||
|
local_dir.mkdir()
|
||||||
|
external_dir.mkdir()
|
||||||
|
|
||||||
|
_make_skill(
|
||||||
|
local_dir,
|
||||||
|
"explore-codebase",
|
||||||
|
category="foundations/runtime",
|
||||||
|
body="LOCAL VERSION",
|
||||||
|
)
|
||||||
|
_make_skill(external_dir, "explore-codebase", body="EXTERNAL VERSION")
|
||||||
|
|
||||||
|
p1, p2 = self._patch_dirs(local_dir, [external_dir])
|
||||||
|
with p1, p2:
|
||||||
|
raw = skill_view("foundations/runtime/explore-codebase")
|
||||||
|
|
||||||
|
result = json.loads(raw)
|
||||||
|
assert result["success"] is True
|
||||||
|
assert "LOCAL VERSION" in result["content"]
|
||||||
|
|
||||||
|
def test_external_skill_resolves_when_no_collision(self, tmp_path):
|
||||||
|
"""External-only skills still resolve normally when there's no
|
||||||
|
local skill of the same name."""
|
||||||
|
local_dir = tmp_path / "local"
|
||||||
|
external_dir = tmp_path / "external"
|
||||||
|
local_dir.mkdir()
|
||||||
|
external_dir.mkdir()
|
||||||
|
|
||||||
|
_make_skill(external_dir, "external-only", body="EXTERNAL BODY")
|
||||||
|
|
||||||
|
p1, p2 = self._patch_dirs(local_dir, [external_dir])
|
||||||
|
with p1, p2:
|
||||||
|
raw = skill_view("external-only")
|
||||||
|
|
||||||
|
result = json.loads(raw)
|
||||||
|
assert result["success"] is True
|
||||||
|
assert "EXTERNAL BODY" in result["content"]
|
||||||
|
|
||||||
|
def test_two_externals_same_name_also_refuse(self, tmp_path):
|
||||||
|
"""Collision detection is symmetric — two external dirs with
|
||||||
|
same-name skills also trigger the refusal."""
|
||||||
|
local_dir = tmp_path / "local"
|
||||||
|
ext_a = tmp_path / "ext_a"
|
||||||
|
ext_b = tmp_path / "ext_b"
|
||||||
|
local_dir.mkdir()
|
||||||
|
ext_a.mkdir()
|
||||||
|
ext_b.mkdir()
|
||||||
|
|
||||||
|
_make_skill(ext_a, "pr", body="EXT_A VERSION")
|
||||||
|
_make_skill(ext_b, "pr", body="EXT_B VERSION")
|
||||||
|
|
||||||
|
p1, p2 = self._patch_dirs(local_dir, [ext_a, ext_b])
|
||||||
|
with p1, p2:
|
||||||
|
raw = skill_view("pr")
|
||||||
|
|
||||||
|
result = json.loads(raw)
|
||||||
|
assert result["success"] is False
|
||||||
|
assert "Ambiguous" in result["error"]
|
||||||
|
assert len(result["matches"]) == 2
|
||||||
|
|
||||||
|
def test_local_only_skill_loads_normally(self, tmp_path):
|
||||||
|
"""Sanity: a single local skill (no external collision) loads
|
||||||
|
without any error."""
|
||||||
|
local_dir = tmp_path / "local"
|
||||||
|
external_dir = tmp_path / "external"
|
||||||
|
local_dir.mkdir()
|
||||||
|
external_dir.mkdir()
|
||||||
|
|
||||||
|
_make_skill(
|
||||||
|
local_dir,
|
||||||
|
"my-skill",
|
||||||
|
category="foundations/runtime",
|
||||||
|
body="LOCAL BODY",
|
||||||
|
)
|
||||||
|
|
||||||
|
p1, p2 = self._patch_dirs(local_dir, [external_dir])
|
||||||
|
with p1, p2:
|
||||||
|
raw = skill_view("my-skill")
|
||||||
|
|
||||||
|
result = json.loads(raw)
|
||||||
|
assert result["success"] is True
|
||||||
|
assert "LOCAL BODY" in result["content"]
|
||||||
|
|
|
||||||
|
|
@ -956,49 +956,83 @@ def skill_view(
|
||||||
skill_dir = None
|
skill_dir = None
|
||||||
skill_md = None
|
skill_md = None
|
||||||
|
|
||||||
# Search all dirs: local first, then external (first match wins)
|
# Collision detection: collect ALL candidates across every dir using
|
||||||
|
# every lookup strategy (direct path, recursive by parent dir name,
|
||||||
|
# legacy flat <name>.md). If more than one matches, refuse and tell
|
||||||
|
# the caller — silent shadowing of a local skill by a same-named
|
||||||
|
# external skill is a real bug class (`/skills` shows one, agent
|
||||||
|
# loaded the other) so we surface it loudly instead of guessing.
|
||||||
|
from agent.skill_utils import iter_skill_index_files
|
||||||
|
|
||||||
|
candidates: List[Tuple[Optional[Path], Path]] = [] # (skill_dir, skill_md)
|
||||||
|
seen_md: set = set()
|
||||||
|
|
||||||
|
def _record(sd: Optional[Path], smd: Path) -> None:
|
||||||
|
try:
|
||||||
|
key = smd.resolve()
|
||||||
|
except Exception:
|
||||||
|
key = smd
|
||||||
|
if key in seen_md:
|
||||||
|
return
|
||||||
|
seen_md.add(key)
|
||||||
|
candidates.append((sd, smd))
|
||||||
|
|
||||||
for search_dir in all_dirs:
|
for search_dir in all_dirs:
|
||||||
# Try direct path first (e.g., "mlops/axolotl")
|
# Strategy 1: direct path (e.g., "mlops/axolotl" or bare "axolotl"
|
||||||
|
# at the top of the dir).
|
||||||
direct_path = search_dir / name
|
direct_path = search_dir / name
|
||||||
if direct_path.is_dir() and (direct_path / "SKILL.md").exists():
|
if direct_path.is_dir() and (direct_path / "SKILL.md").exists():
|
||||||
skill_dir = direct_path
|
_record(direct_path, direct_path / "SKILL.md")
|
||||||
skill_md = direct_path / "SKILL.md"
|
|
||||||
break
|
|
||||||
elif direct_path.with_suffix(".md").exists():
|
elif direct_path.with_suffix(".md").exists():
|
||||||
skill_md = direct_path.with_suffix(".md")
|
_record(None, direct_path.with_suffix(".md"))
|
||||||
break
|
|
||||||
|
# Strategy 1b: categorized form for plugin namespace fall-through
|
||||||
|
# (e.g., a "myplugin:explore" name with no plugin registered also
|
||||||
|
# tries the on-disk path "myplugin/explore").
|
||||||
if local_category_name:
|
if local_category_name:
|
||||||
categorized_path = search_dir / local_category_name
|
categorized_path = search_dir / local_category_name
|
||||||
if categorized_path.is_dir() and (categorized_path / "SKILL.md").exists():
|
if categorized_path.is_dir() and (categorized_path / "SKILL.md").exists():
|
||||||
skill_dir = categorized_path
|
_record(categorized_path, categorized_path / "SKILL.md")
|
||||||
skill_md = categorized_path / "SKILL.md"
|
|
||||||
break
|
|
||||||
elif categorized_path.with_suffix(".md").exists():
|
elif categorized_path.with_suffix(".md").exists():
|
||||||
skill_md = categorized_path.with_suffix(".md")
|
_record(None, categorized_path.with_suffix(".md"))
|
||||||
break
|
|
||||||
|
|
||||||
# Search by directory name across all dirs
|
# Strategy 2: recursive by directory name (catches nested skills
|
||||||
if not skill_md:
|
# like "foundations/runtime/explore-codebase" called by bare name).
|
||||||
for search_dir in all_dirs:
|
for found_skill_md in iter_skill_index_files(search_dir, "SKILL.md"):
|
||||||
from agent.skill_utils import iter_skill_index_files
|
if found_skill_md.parent.name == name:
|
||||||
|
_record(found_skill_md.parent, found_skill_md)
|
||||||
|
|
||||||
for found_skill_md in iter_skill_index_files(search_dir, "SKILL.md"):
|
# Strategy 3: legacy flat <name>.md files anywhere under the dir.
|
||||||
if found_skill_md.parent.name == name:
|
for found_md in search_dir.rglob(f"{name}.md"):
|
||||||
skill_dir = found_skill_md.parent
|
if found_md.name != "SKILL.md":
|
||||||
skill_md = found_skill_md
|
_record(None, found_md)
|
||||||
break
|
|
||||||
if skill_md:
|
|
||||||
break
|
|
||||||
|
|
||||||
# Legacy: flat .md files
|
if len(candidates) > 1:
|
||||||
if not skill_md:
|
paths = [str(smd) for _, smd in candidates]
|
||||||
for search_dir in all_dirs:
|
logging.getLogger(__name__).warning(
|
||||||
for found_md in search_dir.rglob(f"{name}.md"):
|
"Skill name collision for '%s': %d candidates — %s",
|
||||||
if found_md.name != "SKILL.md":
|
name, len(candidates), "; ".join(paths),
|
||||||
skill_md = found_md
|
)
|
||||||
break
|
return json.dumps(
|
||||||
if skill_md:
|
{
|
||||||
break
|
"success": False,
|
||||||
|
"error": (
|
||||||
|
f"Ambiguous skill name '{name}': {len(candidates)} skills "
|
||||||
|
"match across your local skills dir and external_dirs. "
|
||||||
|
"Refusing to guess — load one explicitly by its categorized path."
|
||||||
|
),
|
||||||
|
"matches": paths,
|
||||||
|
"hint": (
|
||||||
|
"Pass the full relative path instead of the bare name "
|
||||||
|
"(e.g., 'category/skill-name'), or rename one of the "
|
||||||
|
"colliding skills so each name is unique."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
if candidates:
|
||||||
|
skill_dir, skill_md = candidates[0]
|
||||||
|
|
||||||
if not skill_md or not skill_md.exists():
|
if not skill_md or not skill_md.exists():
|
||||||
available = [s["name"] for s in _sort_skills(_find_all_skills())[:20]]
|
available = [s["name"] for s in _sort_skills(_find_all_skills())[:20]]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue