hermes-agent/tests/scripts/test_validate_plugin_catalog.py

"""Behavior tests for scripts/validate_plugin_catalog.py.

The script is the no-install structural validator used by the plugin-catalog
admission CI: it must run with only stdlib + pyyaml, take file paths or a
directory, exit 0/1, and support --json machine output. These tests exercise
the CLI contract via subprocess (the same way CI invokes it).
"""

import json
import subprocess
import sys
from pathlib import Path

import yaml

REPO_ROOT = Path(__file__).resolve().parents[2]
SCRIPT = REPO_ROOT / "scripts" / "validate_plugin_catalog.py"

VALID_ENTRY = {
    "name": "example-plugin",
    "repo": "https://github.com/NousResearch/hermes-example-plugins",
    "sha": "38fe0fb53eff98d477f807432e965429e665ca33",
    "subdir": "",
    "description": "One-line description.",
    "maintainer": "NousResearch",
    "tier": "official",
    "requires_hermes": ">=0.19",
    "docs_url": "",
    "platforms": [],
    "capabilities": {
        "provides_tools": ["example_tool"],
        "provides_hooks": [],
        "provides_middleware": [],
        "requires_env": [],
    },
}


def write_entry(tmp_path: Path, data: dict, filename: str | None = None) -> Path:
    name = filename or f"{data.get('name', 'entry')}.yaml"
    path = tmp_path / name
    path.write_text(yaml.safe_dump(data), encoding="utf-8")
    return path


def run_validator(*args: str) -> subprocess.CompletedProcess:
    return subprocess.run(
        [sys.executable, str(SCRIPT), *args],
        capture_output=True,
        text=True,
    )


# ── valid input ────────────────────────────────────────────────────────


def test_valid_entry_passes(tmp_path):
    path = write_entry(tmp_path, VALID_ENTRY)
    result = run_validator(str(path))
    assert result.returncode == 0, result.stdout + result.stderr


def test_valid_entry_without_optional_fields_passes(tmp_path):
    entry = {
        "name": "minimal-plugin",
        "repo": "https://github.com/example/minimal",
        "sha": "a" * 40,
        "description": "Minimal.",
        "maintainer": "someone",
    }
    path = write_entry(tmp_path, entry)
    result = run_validator(str(path))
    assert result.returncode == 0, result.stdout + result.stderr


# ── each malformed field fails with a pointed error ────────────────────


def _expect_error(tmp_path, mutation: dict, expected_substring: str, drop: str = ""):
    entry = {**VALID_ENTRY, **mutation}
    if drop:
        entry.pop(drop, None)
    path = write_entry(tmp_path, entry, filename="entry.yaml")
    result = run_validator(str(path))
    combined = result.stdout + result.stderr
    assert result.returncode == 1, combined
    assert expected_substring in combined, combined
    assert "entry.yaml" in combined, combined


def test_bad_name_fails(tmp_path):
    _expect_error(tmp_path, {"name": "Bad Name!"}, "name")


def test_name_too_long_fails(tmp_path):
    _expect_error(tmp_path, {"name": "x" * 65}, "name")


def test_non_https_repo_fails(tmp_path):
    _expect_error(tmp_path, {"repo": "git@github.com:evil/x.git"}, "repo")


def test_short_sha_fails(tmp_path):
    _expect_error(tmp_path, {"sha": "abc123"}, "sha")


def test_non_hex_sha_fails(tmp_path):
    _expect_error(tmp_path, {"sha": "z" * 40}, "sha")


def test_bad_tier_fails(tmp_path):
    _expect_error(tmp_path, {"tier": "platinum"}, "tier")


def test_empty_description_fails(tmp_path):
    _expect_error(tmp_path, {"description": ""}, "description")


def test_empty_maintainer_fails(tmp_path):
    _expect_error(tmp_path, {"maintainer": ""}, "maintainer")


def test_missing_required_field_fails(tmp_path):
    _expect_error(tmp_path, {}, "sha", drop="sha")


def test_capabilities_value_not_a_list_fails(tmp_path):
    _expect_error(
        tmp_path,
        {"capabilities": {"provides_tools": "not-a-list"}},
        "provides_tools",
    )


def test_capabilities_list_of_non_strings_fails(tmp_path):
    _expect_error(
        tmp_path,
        {"capabilities": {"requires_env": [1, 2]}},
        "requires_env",
    )


def test_bad_requires_hermes_spec_fails(tmp_path):
    _expect_error(tmp_path, {"requires_hermes": "banana"}, "requires_hermes")


def test_comma_separated_requires_hermes_passes(tmp_path):
    entry = {**VALID_ENTRY, "requires_hermes": ">=0.19, <2.0"}
    path = write_entry(tmp_path, entry)
    result = run_validator(str(path))
    assert result.returncode == 0, result.stdout + result.stderr


def test_unknown_platform_fails(tmp_path):
    _expect_error(tmp_path, {"platforms": ["linux", "amiga"]}, "platforms")


def test_entry_not_a_mapping_fails(tmp_path):
    path = tmp_path / "entry.yaml"
    path.write_text("- just\n- a\n- list\n", encoding="utf-8")
    result = run_validator(str(path))
    assert result.returncode == 1
    assert "mapping" in (result.stdout + result.stderr)


# ── unknown top-level keys warn but do not fail ────────────────────────


def test_unknown_key_warns_but_passes(tmp_path):
    entry = {**VALID_ENTRY, "future_field": "hello"}
    path = write_entry(tmp_path, entry)
    result = run_validator(str(path))
    combined = result.stdout + result.stderr
    assert result.returncode == 0, combined
    assert "future_field" in combined
    assert "warning" in combined.lower()


# ── removed.yaml shape ─────────────────────────────────────────────────


def test_valid_removed_yaml_passes(tmp_path):
    path = tmp_path / "removed.yaml"
    path.write_text(
        yaml.safe_dump(
            {
                "removed": [
                    {
                        "name": "some-plugin",
                        "repo": "https://github.com/evil/some-plugin",
                        "reason": "Exfiltrated env vars",
                        "date": "2026-07-02",
                    }
                ]
            }
        ),
        encoding="utf-8",
    )
    result = run_validator(str(path))
    assert result.returncode == 0, result.stdout + result.stderr


def test_removed_yaml_not_a_list_fails(tmp_path):
    path = tmp_path / "removed.yaml"
    path.write_text(yaml.safe_dump({"removed": "nope"}), encoding="utf-8")
    result = run_validator(str(path))
    assert result.returncode == 1
    assert "removed" in (result.stdout + result.stderr)


def test_removed_item_missing_name_fails(tmp_path):
    path = tmp_path / "removed.yaml"
    path.write_text(
        yaml.safe_dump({"removed": [{"reason": "bad", "date": "2026-01-01"}]}),
        encoding="utf-8",
    )
    result = run_validator(str(path))
    assert result.returncode == 1
    assert "name" in (result.stdout + result.stderr)


# ── --json machine output ──────────────────────────────────────────────


def test_json_output_shape_on_failure(tmp_path):
    bad = write_entry(tmp_path, {**VALID_ENTRY, "sha": "short"}, filename="bad.yaml")
    result = run_validator("--json", str(bad))
    assert result.returncode == 1
    payload = json.loads(result.stdout)
    assert payload["ok"] is False
    assert isinstance(payload["files"], list)
    entry = next(f for f in payload["files"] if f["path"].endswith("bad.yaml"))
    assert entry["ok"] is False
    assert any("sha" in e for e in entry["errors"])


def test_json_output_shape_on_success_with_warning(tmp_path):
    good = write_entry(tmp_path, {**VALID_ENTRY, "future_field": 1})
    result = run_validator("--json", str(good))
    assert result.returncode == 0
    payload = json.loads(result.stdout)
    assert payload["ok"] is True
    (entry,) = payload["files"]
    assert entry["ok"] is True
    assert entry["errors"] == []
    assert any("future_field" in w for w in entry["warnings"])


# ── directory mode ─────────────────────────────────────────────────────


def test_directory_mode_validates_all_entries_and_removed(tmp_path):
    write_entry(tmp_path, VALID_ENTRY)
    write_entry(tmp_path, {**VALID_ENTRY, "name": "bad-one", "sha": "nope"})
    (tmp_path / "removed.yaml").write_text(
        yaml.safe_dump({"removed": [{"name": "gone", "reason": "test"}]}),
        encoding="utf-8",
    )
    result = run_validator(str(tmp_path))
    combined = result.stdout + result.stderr
    assert result.returncode == 1
    assert "bad-one.yaml" in combined
    # the valid entry and removed.yaml must not produce errors
    assert combined.count("ERROR") == combined.count("bad-one.yaml: ERROR")


def test_directory_mode_all_valid_exits_zero(tmp_path):
    write_entry(tmp_path, VALID_ENTRY)
    (tmp_path / "removed.yaml").write_text(
        yaml.safe_dump({"removed": []}), encoding="utf-8"
    )
    result = run_validator(str(tmp_path))
    assert result.returncode == 0, result.stdout + result.stderr