mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
6b81590c55
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
522 lines
17 KiB
Python
522 lines
17 KiB
Python
"""Tests for the curator consolidated-vs-pruned classifier.
|
|
|
|
The classifier splits skills that disappeared between the before/after
|
|
snapshots into two buckets:
|
|
|
|
- "consolidated" — absorbed into an umbrella; content still lives
|
|
under another skill's files
|
|
- "pruned" — archived for staleness; content not preserved elsewhere
|
|
|
|
Without the split the report lumped everything under "Skills archived",
|
|
which misled users into thinking consolidated skills had been pruned.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def curator_env(tmp_path, monkeypatch):
|
|
home = tmp_path / ".hermes"
|
|
home.mkdir()
|
|
(home / "skills").mkdir()
|
|
(home / "logs").mkdir()
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
|
|
|
import importlib
|
|
import hermes_constants
|
|
importlib.reload(hermes_constants)
|
|
from agent import curator
|
|
importlib.reload(curator)
|
|
yield curator
|
|
|
|
|
|
def test_classify_consolidated_via_write_file_evidence(curator_env):
|
|
"""skill_manage write_file on umbrella references/<removed>.md = consolidated."""
|
|
result = curator_env._classify_removed_skills(
|
|
removed=["axolotl-training"],
|
|
added=[],
|
|
after_names={"training-platforms", "keeper"},
|
|
tool_calls=[
|
|
{
|
|
"name": "skill_manage",
|
|
"arguments": json.dumps({
|
|
"action": "write_file",
|
|
"name": "training-platforms",
|
|
"file_path": "references/axolotl-training.md",
|
|
"file_content": "# Axolotl\n...",
|
|
}),
|
|
},
|
|
],
|
|
)
|
|
assert len(result["consolidated"]) == 1
|
|
assert result["consolidated"][0]["name"] == "axolotl-training"
|
|
assert result["consolidated"][0]["into"] == "training-platforms"
|
|
assert result["pruned"] == []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_report_md_splits_consolidated_and_pruned_sections(curator_env):
|
|
"""End-to-end: REPORT.md shows both sections distinctly."""
|
|
curator = curator_env
|
|
start = datetime.now(timezone.utc)
|
|
|
|
before = [
|
|
{"name": "absorbed-skill", "state": "active", "pinned": False},
|
|
{"name": "dead-skill", "state": "stale", "pinned": False},
|
|
{"name": "keeper", "state": "active", "pinned": False},
|
|
]
|
|
after = [
|
|
{"name": "keeper", "state": "active", "pinned": False},
|
|
{"name": "umbrella", "state": "active", "pinned": False},
|
|
]
|
|
|
|
run_dir = curator._write_run_report(
|
|
started_at=start,
|
|
elapsed_seconds=60.0,
|
|
auto_counts={"checked": 3, "marked_stale": 0, "archived": 0, "reactivated": 0},
|
|
auto_summary="no auto changes",
|
|
before_report=before,
|
|
before_names={r["name"] for r in before},
|
|
after_report=after,
|
|
llm_meta={
|
|
"final": "Consolidated absorbed-skill into umbrella. Pruned dead-skill.",
|
|
"summary": "1 consolidated, 1 pruned",
|
|
"model": "m",
|
|
"provider": "p",
|
|
"error": None,
|
|
"tool_calls": [
|
|
{
|
|
"name": "skill_manage",
|
|
"arguments": json.dumps({
|
|
"action": "create",
|
|
"name": "umbrella",
|
|
"content": "# umbrella\n\nAbsorbed absorbed-skill.",
|
|
}),
|
|
},
|
|
],
|
|
},
|
|
)
|
|
|
|
payload = json.loads((run_dir / "run.json").read_text())
|
|
# Both lists exist and are disjoint
|
|
consolidated_names = {e["name"] for e in payload["consolidated"]}
|
|
assert consolidated_names == {"absorbed-skill"}
|
|
# `pruned` holds full dicts {name, source, reason}; `pruned_names` is the
|
|
# flat list for quick scans / legacy compat.
|
|
pruned_names = payload["pruned_names"]
|
|
assert pruned_names == ["dead-skill"]
|
|
assert all(isinstance(e, dict) and "name" in e for e in payload["pruned"])
|
|
# The union still matches the legacy "archived" field for backward compat
|
|
assert set(payload["archived"]) == consolidated_names | set(pruned_names)
|
|
# counts exposed
|
|
assert payload["counts"]["consolidated_this_run"] == 1
|
|
assert payload["counts"]["pruned_this_run"] == 1
|
|
|
|
md = (run_dir / "REPORT.md").read_text()
|
|
# Two separate sections, not a single "Skills archived" lump
|
|
assert "Consolidated into umbrella skills" in md
|
|
assert "Pruned — archived for staleness" in md
|
|
assert "`absorbed-skill` → merged into `umbrella`" in md
|
|
assert "`dead-skill`" in md
|
|
# The old single-lump section should not appear
|
|
assert "### Skills archived" not in md
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _parse_structured_summary — extracting the model's required YAML block
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_parse_structured_summary_happy_path(curator_env):
|
|
text = (
|
|
"Long human summary here. I processed clusters X, Y, Z.\n\n"
|
|
"## Structured summary (required)\n"
|
|
"```yaml\n"
|
|
"consolidations:\n"
|
|
" - from: anthropic-api\n"
|
|
" into: llm-providers\n"
|
|
" reason: duplicate of the generic llm-providers skill\n"
|
|
" - from: openai-api\n"
|
|
" into: llm-providers\n"
|
|
" reason: same — merged with sibling\n"
|
|
"prunings:\n"
|
|
" - name: random-old-notes\n"
|
|
" reason: pre-curator garbage, no overlap\n"
|
|
"```\n"
|
|
)
|
|
out = curator_env._parse_structured_summary(text)
|
|
assert len(out["consolidations"]) == 2
|
|
assert out["consolidations"][0] == {
|
|
"from": "anthropic-api",
|
|
"into": "llm-providers",
|
|
"reason": "duplicate of the generic llm-providers skill",
|
|
}
|
|
assert len(out["prunings"]) == 1
|
|
assert out["prunings"][0]["reason"] == "pre-curator garbage, no overlap"
|
|
|
|
|
|
def test_parse_structured_summary_missing_block(curator_env):
|
|
out = curator_env._parse_structured_summary("No block in this text.")
|
|
assert out == {"consolidations": [], "prunings": []}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _reconcile_classification — merging model block with heuristic
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_model_block_visible_in_full_report(curator_env):
|
|
"""End-to-end: LLM final response with the YAML block → reasons in REPORT.md."""
|
|
import json as _json
|
|
from datetime import datetime as _dt, timezone as _tz
|
|
|
|
start = _dt.now(_tz.utc)
|
|
before = [
|
|
{"name": "anthropic-api", "state": "active", "pinned": False},
|
|
{"name": "stale-thing", "state": "stale", "pinned": False},
|
|
]
|
|
after = [{"name": "llm-providers", "state": "active", "pinned": False}]
|
|
|
|
llm_final_text = (
|
|
"Processed 3 clusters. Absorbed anthropic-api into llm-providers.\n\n"
|
|
"## Structured summary (required)\n"
|
|
"```yaml\n"
|
|
"consolidations:\n"
|
|
" - from: anthropic-api\n"
|
|
" into: llm-providers\n"
|
|
" reason: duplicate content, now a subsection\n"
|
|
"prunings:\n"
|
|
" - name: stale-thing\n"
|
|
" reason: pre-curator junk, no overlap with anything\n"
|
|
"```\n"
|
|
)
|
|
|
|
run_dir = curator_env._write_run_report(
|
|
started_at=start,
|
|
elapsed_seconds=30.0,
|
|
auto_counts={"checked": 2, "marked_stale": 0, "archived": 0, "reactivated": 0},
|
|
auto_summary="none",
|
|
before_report=before,
|
|
before_names={r["name"] for r in before},
|
|
after_report=after,
|
|
llm_meta={
|
|
"final": llm_final_text,
|
|
"summary": "1 consolidated, 1 pruned",
|
|
"model": "m",
|
|
"provider": "p",
|
|
"error": None,
|
|
"tool_calls": [
|
|
{"name": "skill_manage", "arguments": _json.dumps({
|
|
"action": "create",
|
|
"name": "llm-providers",
|
|
"content": "# llm-providers\nIncludes anthropic-api",
|
|
})},
|
|
],
|
|
},
|
|
)
|
|
|
|
payload = _json.loads((run_dir / "run.json").read_text())
|
|
cons = payload["consolidated"][0]
|
|
assert cons["name"] == "anthropic-api"
|
|
assert cons["into"] == "llm-providers"
|
|
assert cons["reason"] == "duplicate content, now a subsection"
|
|
assert cons["source"] == "model+audit" # model AND heuristic both had it
|
|
|
|
pruned = payload["pruned"][0]
|
|
assert pruned["name"] == "stale-thing"
|
|
assert pruned["reason"] == "pre-curator junk, no overlap with anything"
|
|
|
|
md = (run_dir / "REPORT.md").read_text()
|
|
assert "duplicate content, now a subsection" in md
|
|
assert "pre-curator junk" in md
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _extract_absorbed_into_declarations — authoritative signal from delete calls
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_extract_absorbed_into_picks_up_consolidation(curator_env):
|
|
"""Delete call with absorbed_into=<umbrella> yields a declaration."""
|
|
declarations = curator_env._extract_absorbed_into_declarations([
|
|
{
|
|
"name": "skill_manage",
|
|
"arguments": json.dumps({
|
|
"action": "delete",
|
|
"name": "narrow-skill",
|
|
"absorbed_into": "umbrella",
|
|
}),
|
|
},
|
|
])
|
|
assert declarations == {
|
|
"narrow-skill": {"into": "umbrella", "declared": True},
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_extract_absorbed_into_ignores_non_delete_actions(curator_env):
|
|
"""Patch, create, write_file etc. must not leak into declarations."""
|
|
declarations = curator_env._extract_absorbed_into_declarations([
|
|
{
|
|
"name": "skill_manage",
|
|
"arguments": json.dumps({
|
|
"action": "patch",
|
|
"name": "umbrella",
|
|
"old_string": "...",
|
|
"new_string": "...",
|
|
"absorbed_into": "something", # bogus on non-delete, must be ignored
|
|
}),
|
|
},
|
|
])
|
|
assert declarations == {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _reconcile_classification with absorbed_into declarations (authoritative)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_reconcile_absorbed_into_beats_everything_else(curator_env):
|
|
"""Model declared absorbed_into at delete; YAML/heuristic disagree — declaration wins.
|
|
|
|
This is the exact #18671 regression: the model forgets to emit the YAML
|
|
summary block, the heuristic's substring match misses because the
|
|
umbrella's patch content doesn't literally contain the old skill's
|
|
slug. Previously this fell through to 'no-evidence fallback' prune,
|
|
which dropped the cron ref instead of rewriting. With absorbed_into
|
|
declared, the model tells us directly.
|
|
"""
|
|
out = curator_env._reconcile_classification(
|
|
removed=["pr-review-format"],
|
|
heuristic={"consolidated": [], "pruned": [{"name": "pr-review-format"}]},
|
|
model_block={"consolidations": [], "prunings": []}, # model forgot YAML block
|
|
destinations={"hermes-agent-dev"},
|
|
absorbed_declarations={
|
|
"pr-review-format": {"into": "hermes-agent-dev", "declared": True},
|
|
},
|
|
)
|
|
assert len(out["consolidated"]) == 1
|
|
assert out["pruned"] == []
|
|
e = out["consolidated"][0]
|
|
assert e["name"] == "pr-review-format"
|
|
assert e["into"] == "hermes-agent-dev"
|
|
assert "absorbed_into" in e["source"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_reconcile_mixed_declarations_and_legacy_calls(curator_env):
|
|
"""Real-world run: some deletes declared absorbed_into, some didn't.
|
|
Declared ones use the authoritative path; others fall through to YAML/heuristic.
|
|
"""
|
|
out = curator_env._reconcile_classification(
|
|
removed=["declared-cons", "declared-prune", "legacy-cons", "legacy-prune"],
|
|
heuristic={
|
|
"consolidated": [
|
|
{"name": "legacy-cons", "into": "umbrella-a", "evidence": "..."},
|
|
],
|
|
"pruned": [{"name": "legacy-prune"}],
|
|
},
|
|
model_block={"consolidations": [], "prunings": []},
|
|
destinations={"umbrella-a", "umbrella-b"},
|
|
absorbed_declarations={
|
|
"declared-cons": {"into": "umbrella-b", "declared": True},
|
|
"declared-prune": {"into": "", "declared": True},
|
|
},
|
|
)
|
|
cons_by_name = {e["name"]: e for e in out["consolidated"]}
|
|
pruned_by_name = {e["name"]: e for e in out["pruned"]}
|
|
|
|
assert "declared-cons" in cons_by_name
|
|
assert cons_by_name["declared-cons"]["into"] == "umbrella-b"
|
|
assert "absorbed_into" in cons_by_name["declared-cons"]["source"]
|
|
|
|
assert "legacy-cons" in cons_by_name
|
|
assert cons_by_name["legacy-cons"]["into"] == "umbrella-a"
|
|
assert "tool-call audit" in cons_by_name["legacy-cons"]["source"]
|
|
|
|
assert "declared-prune" in pruned_by_name
|
|
assert "model-declared prune" in pruned_by_name["declared-prune"]["source"]
|
|
|
|
assert "legacy-prune" in pruned_by_name
|
|
assert "no-evidence fallback" in pruned_by_name["legacy-prune"]["source"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _build_rename_summary — surfaces the "where did my skills go?" map to the
|
|
# user-visible curator summary (gateway 💾 line, CLI Rich panel,
|
|
# `hermes curator status`). The full data has always been in REPORT.md on
|
|
# disk; this helper makes it visible without digging.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_rename_summary_empty_when_nothing_archived(curator_env):
|
|
"""No removals = empty string (no log noise on no-op ticks)."""
|
|
result = curator_env._build_rename_summary(
|
|
before_names={"alpha", "beta"},
|
|
after_report=[
|
|
{"name": "alpha", "state": "active"},
|
|
{"name": "beta", "state": "active"},
|
|
],
|
|
tool_calls=[],
|
|
model_final="",
|
|
)
|
|
assert result == ""
|
|
|
|
|
|
|
|
|
|
def test_rename_summary_pruned_marked_explicitly(curator_env):
|
|
"""Pruned skills (no umbrella) say `pruned (stale)` so users don't think they were merged."""
|
|
result = curator_env._build_rename_summary(
|
|
before_names={"old-flaky-thing", "keeper"},
|
|
after_report=[{"name": "keeper", "state": "active"}],
|
|
tool_calls=[
|
|
{
|
|
"name": "skill_manage",
|
|
"arguments": json.dumps({
|
|
"action": "delete",
|
|
"name": "old-flaky-thing",
|
|
"absorbed_into": "",
|
|
}),
|
|
},
|
|
],
|
|
model_final="",
|
|
)
|
|
assert "old-flaky-thing — pruned (stale)" in result
|
|
assert "→" not in result.split("old-flaky-thing")[1].splitlines()[0]
|
|
|
|
|
|
def test_rename_summary_caps_at_ten_with_more_indicator(curator_env):
|
|
"""Large consolidations don't blow up the log line — cap + `… and N more`."""
|
|
removed = [f"skill-{i}" for i in range(15)]
|
|
tool_calls = [
|
|
{
|
|
"name": "skill_manage",
|
|
"arguments": json.dumps({
|
|
"action": "delete",
|
|
"name": name,
|
|
"absorbed_into": "umbrella",
|
|
}),
|
|
}
|
|
for name in removed
|
|
]
|
|
result = curator_env._build_rename_summary(
|
|
before_names=set(removed) | {"umbrella"},
|
|
after_report=[{"name": "umbrella", "state": "active"}],
|
|
tool_calls=tool_calls,
|
|
model_final="",
|
|
)
|
|
assert "archived 15 skill(s):" in result
|
|
assert "… and 5 more" in result
|
|
# Exactly 10 bullets shown
|
|
bullet_count = sum(1 for ln in result.splitlines() if ln.startswith(" • "))
|
|
assert bullet_count == 10
|
|
|
|
|
|
def test_rename_summary_mixed_consolidation_and_pruning(curator_env):
|
|
"""Consolidated entries come first, pruned entries follow — matches REPORT.md ordering."""
|
|
result = curator_env._build_rename_summary(
|
|
before_names={"merge-me", "drop-me", "umbrella"},
|
|
after_report=[{"name": "umbrella", "state": "active"}],
|
|
tool_calls=[
|
|
{
|
|
"name": "skill_manage",
|
|
"arguments": json.dumps({
|
|
"action": "delete",
|
|
"name": "merge-me",
|
|
"absorbed_into": "umbrella",
|
|
}),
|
|
},
|
|
{
|
|
"name": "skill_manage",
|
|
"arguments": json.dumps({
|
|
"action": "delete",
|
|
"name": "drop-me",
|
|
"absorbed_into": "",
|
|
}),
|
|
},
|
|
],
|
|
model_final="",
|
|
)
|
|
lines = result.splitlines()
|
|
merge_idx = next(i for i, ln in enumerate(lines) if "merge-me" in ln)
|
|
drop_idx = next(i for i, ln in enumerate(lines) if "drop-me" in ln)
|
|
assert merge_idx < drop_idx, "consolidated should render before pruned"
|
|
assert "merge-me → umbrella" in lines[merge_idx]
|
|
assert "drop-me — pruned (stale)" in lines[drop_idx]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pin hint — surfaces `hermes curator pin <umbrella>` in the rename block so
|
|
# users learn the command exists at the moment they care (a consolidation
|
|
# just landed against their library). The hint is gated on having at least
|
|
# one umbrella destination — pruned-only runs skip it.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|