diff --git a/.github/actions/plugin-validate/action.yml b/.github/actions/plugin-validate/action.yml new file mode 100644 index 000000000000..5484aaf56c25 --- /dev/null +++ b/.github/actions/plugin-validate/action.yml @@ -0,0 +1,56 @@ +name: Hermes Plugin Validate +description: >- + Validate a Hermes Agent plugin (plugin.yaml manifest schema AND + declared-vs-actually-registered capabilities) using + `hermes plugins validate`. Drop this into your plugin repo's CI: + + - uses: actions/checkout@ + - uses: NousResearch/hermes-agent/.github/actions/plugin-validate@main + with: + path: . + + The caller's job owns checkout; this action installs Python + hermes-agent + (git install — a supported CI-context install route) and runs the + validator against your plugin directory. + +inputs: + path: + description: Path to the plugin directory (containing plugin.yaml). + default: "." + hermes-ref: + description: hermes-agent git ref (branch/tag/sha) to install and validate with. + default: "main" + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + + - name: Install hermes-agent + shell: bash + env: + _HERMES_REF: ${{ inputs.hermes-ref }} + run: | + set -euo pipefail + # CI-context install from git; the ref lets plugin authors validate + # against a pinned hermes release instead of main. + pip install "git+https://github.com/NousResearch/hermes-agent@${_HERMES_REF}" + + - name: Validate plugin + shell: bash + env: + _PLUGIN_PATH: ${{ inputs.path }} + run: | + set -uo pipefail + # `hermes plugins validate` checks the plugin.yaml manifest schema + # and loads the plugin in a scratch subprocess to verify that the + # capabilities it DECLARES match what it actually registers. + if hermes plugins validate "$_PLUGIN_PATH"; then + echo "✅ PASS: plugin at '$_PLUGIN_PATH' validated cleanly" + else + echo "❌ FAIL: plugin at '$_PLUGIN_PATH' failed validation (see output above)" + exit 1 + fi diff --git a/.github/workflows/plugin-catalog-ci.yml b/.github/workflows/plugin-catalog-ci.yml new file mode 100644 index 000000000000..32fb597c1681 --- /dev/null +++ b/.github/workflows/plugin-catalog-ci.yml @@ -0,0 +1,140 @@ +name: Plugin Catalog CI + +# Admission gate for plugin-catalog entries. Fires ONLY on PRs touching +# plugin-catalog/** so it can never go red on unrelated PRs. +# +# Two gates: +# structural — cheap schema check, no hermes install needed +# pinned-source-validate — supply-chain gate: the pinned sha MUST be +# reachable in the entry's repo, and the plugin +# at that exact commit must pass +# `hermes plugins validate`. + +on: + pull_request: + paths: + - "plugin-catalog/**" + +permissions: + contents: read + +jobs: + structural: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + + - name: Install PyYAML + uses: ./.github/actions/retry + with: + command: pip install pyyaml==6.0.2 + + - name: Validate catalog files (structural) + run: | + set -euo pipefail + # Validating the whole directory is simpler than diffing and keeps + # the invariant that EVERYTHING in plugin-catalog/ stays valid. + python3 scripts/validate_plugin_catalog.py plugin-catalog/ + + pinned-source-validate: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 # need the merge-base to diff changed catalog files + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + + - name: Find changed catalog entries + id: changed + run: | + set -euo pipefail + MERGE_BASE=$(git merge-base "origin/${{ github.base_ref }}" HEAD) + # Added + modified entry files only; deletions and removed.yaml + # have nothing to clone. + CHANGED=$(git diff --name-only --diff-filter=AM "$MERGE_BASE"...HEAD \ + -- 'plugin-catalog/*.yaml' 'plugin-catalog/*.yml' \ + | grep -v '/removed\.yaml$' || true) + echo "Changed catalog entries:" + echo "${CHANGED:-}" + { + echo 'files<<__EOF__' + echo "$CHANGED" + echo '__EOF__' + } >> "$GITHUB_OUTPUT" + + - name: Install hermes-agent from the PR's own checkout + if: steps.changed.outputs.files != '' + uses: ./.github/actions/retry + with: + command: pip install -e . + + - name: Clone each entry at its pinned sha and validate + if: steps.changed.outputs.files != '' + env: + CHANGED_FILES: ${{ steps.changed.outputs.files }} + run: | + set -euo pipefail + FAILED=0 + while IFS= read -r entry; do + [ -z "$entry" ] && continue + echo "::group::validate $entry" + + # Parse repo / sha / subdir from the entry yaml. + eval "$(python3 - "$entry" <<'PYEOF' + import shlex + import sys + + import yaml + + with open(sys.argv[1], encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + print(f"REPO={shlex.quote(str(data.get('repo', '')))}") + print(f"SHA={shlex.quote(str(data.get('sha', '')))}") + print(f"SUBDIR={shlex.quote(str(data.get('subdir', '') or ''))}") + PYEOF + )" + echo "repo=$REPO sha=$SHA subdir=$SUBDIR" + + CLONE_DIR=$(mktemp -d) + # Full clone (no --depth 1): the pinned sha may not be the branch tip. + if ! git clone "$REPO" "$CLONE_DIR"; then + echo "::error file=$entry::clone failed for $REPO" + FAILED=1; echo "::endgroup::"; continue + fi + + # SUPPLY-CHAIN GATE: the pinned sha must be reachable in the repo. + if ! git -C "$CLONE_DIR" checkout --detach "$SHA"; then + echo "::error file=$entry::pinned sha $SHA is not reachable in $REPO" + FAILED=1; echo "::endgroup::"; continue + fi + + PLUGIN_DIR="$CLONE_DIR${SUBDIR:+/$SUBDIR}" + if [ ! -f "$PLUGIN_DIR/plugin.yaml" ]; then + echo "::error file=$entry::no plugin.yaml at subdir '$SUBDIR' of $REPO@$SHA" + FAILED=1; echo "::endgroup::"; continue + fi + + # Manifest schema + declared-vs-registered capability check. + if hermes plugins validate "$PLUGIN_DIR"; then + echo "✅ PASS: $entry" + else + echo "::error file=$entry::hermes plugins validate failed" + FAILED=1 + fi + echo "::endgroup::" + done <<< "$CHANGED_FILES" + + if [ "$FAILED" -ne 0 ]; then + echo "❌ FAIL: one or more catalog entries failed pinned-source validation" + exit 1 + fi + echo "✅ PASS: all changed catalog entries validated at their pinned shas" diff --git a/scripts/validate_plugin_catalog.py b/scripts/validate_plugin_catalog.py new file mode 100644 index 000000000000..eb0078c7c0fe --- /dev/null +++ b/scripts/validate_plugin_catalog.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Standalone structural validator for plugin-catalog entry files. + +Validates ``plugin-catalog/*.yaml`` catalog entries and +``plugin-catalog/removed.yaml`` against the catalog contract schema, using +only stdlib + PyYAML so the admission CI (and third-party repos) can run it +WITHOUT installing hermes-agent. + +NOTE: this script intentionally duplicates the schema rules instead of +importing ``hermes_cli`` — the whole point is the no-install requirement for +cheap cross-repo CI use. The runtime twin of this schema lives in +``hermes_cli/plugin_catalog.py``; if the contract changes there, update the +rules here in lockstep. + +Usage: + python3 scripts/validate_plugin_catalog.py plugin-catalog/ + python3 scripts/validate_plugin_catalog.py entry.yaml removed.yaml + python3 scripts/validate_plugin_catalog.py --json plugin-catalog/ + +Exit codes: 0 = all files valid (warnings allowed), 1 = at least one error. +Human output is one ``: ERROR: ...`` / ``: warning: ...`` line +per finding; ``--json`` emits a machine-readable report on stdout instead. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover - dependency guidance only + print( + "ERROR: PyYAML is required (pip install pyyaml)", + file=sys.stderr, + ) + sys.exit(2) + +NAME_RE = re.compile(r"^[a-z0-9_-]{1,64}$") +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +TIERS = ("official", "community") +PLATFORMS = ("linux", "macos", "windows") +CAPABILITY_KEYS = ( + "provides_tools", + "provides_hooks", + "provides_middleware", + "requires_env", +) +# Top-level keys the contract knows about. Unknown keys WARN (forward +# compatibility: newer catalogs must stay valid under older validators). +KNOWN_KEYS = { + "name", + "repo", + "sha", + "subdir", + "description", + "maintainer", + "tier", + "requires_hermes", + "docs_url", + "platforms", + "capabilities", +} +REQUIRED_KEYS = ("name", "repo", "sha", "description", "maintainer") + +# One comparator clause of a requires_hermes spec, e.g. ">=0.19" or "!=1.2.3". +_COMPARATOR_RE = re.compile(r"^(>=|<=|==|!=|>|<)\s*\d+(\.\d+)*$") + + +def _is_nonempty_str(value: object) -> bool: + return isinstance(value, str) and value.strip() != "" + + +def _check_requires_hermes(spec: object, errors: list[str]) -> None: + if not isinstance(spec, str): + errors.append(f"requires_hermes must be a string, got {type(spec).__name__}") + return + if spec.strip() == "": + return # empty = no constraint + for clause in spec.split(","): + if not _COMPARATOR_RE.match(clause.strip()): + errors.append( + f"requires_hermes clause {clause.strip()!r} is not a valid " + "comparator spec (expected e.g. '>=0.19')" + ) + + +def validate_entry(data: object) -> tuple[list[str], list[str]]: + """Validate one catalog entry document. Returns (errors, warnings).""" + errors: list[str] = [] + warnings: list[str] = [] + + if not isinstance(data, dict): + return ["top-level document must be a YAML mapping"], warnings + + for key in REQUIRED_KEYS: + if key not in data: + errors.append(f"missing required key: {key}") + + for key in sorted(set(data) - KNOWN_KEYS): + warnings.append(f"unknown top-level key {key!r} (ignored by this validator)") + + name = data.get("name") + if "name" in data and (not isinstance(name, str) or not NAME_RE.match(name)): + errors.append(f"name {name!r} must match [a-z0-9_-]{{1,64}}") + + repo = data.get("repo") + if "repo" in data and ( + not isinstance(repo, str) or not repo.startswith("https://") + ): + errors.append(f"repo {repo!r} must be an https:// URL") + + sha = data.get("sha") + if "sha" in data and (not isinstance(sha, str) or not SHA_RE.match(sha)): + errors.append(f"sha {sha!r} must be exactly 40 lowercase hex characters") + + for key in ("description", "maintainer"): + if key in data and not _is_nonempty_str(data[key]): + errors.append(f"{key} must be a non-empty string") + + tier = data.get("tier", "community") + if tier not in TIERS: + errors.append(f"tier {tier!r} must be one of {list(TIERS)}") + + if "requires_hermes" in data: + _check_requires_hermes(data["requires_hermes"], errors) + + platforms = data.get("platforms", []) + if platforms is None: + platforms = [] + if not isinstance(platforms, list): + errors.append("platforms must be a list") + else: + bad = [p for p in platforms if p not in PLATFORMS] + if bad: + errors.append(f"platforms {bad!r} not in allowed set {list(PLATFORMS)}") + + caps = data.get("capabilities", {}) + if caps is None: + caps = {} + if not isinstance(caps, dict): + errors.append("capabilities must be a mapping") + else: + for key in sorted(set(caps) - set(CAPABILITY_KEYS)): + warnings.append(f"unknown capabilities key {key!r}") + for key in CAPABILITY_KEYS: + if key not in caps: + continue + value = caps[key] + if not isinstance(value, list) or not all( + isinstance(item, str) for item in value + ): + errors.append(f"capabilities.{key} must be a list of strings") + + return errors, warnings + + +def validate_removed(data: object) -> tuple[list[str], list[str]]: + """Validate the removed.yaml document. Returns (errors, warnings).""" + errors: list[str] = [] + warnings: list[str] = [] + + if not isinstance(data, dict): + return ["top-level document must be a YAML mapping"], warnings + + removed = data.get("removed") + if removed is None: + errors.append("missing required key: removed") + return errors, warnings + if not isinstance(removed, list): + errors.append("removed must be a list") + return errors, warnings + + for i, item in enumerate(removed): + if not isinstance(item, dict): + errors.append(f"removed[{i}] must be a mapping") + continue + if not _is_nonempty_str(item.get("name")): + errors.append(f"removed[{i}] missing non-empty 'name'") + for key in ("repo", "reason", "date"): + if key in item and not isinstance(item[key], str): + errors.append(f"removed[{i}].{key} must be a string") + + return errors, warnings + + +def validate_file(path: Path) -> tuple[list[str], list[str]]: + """Validate one YAML file (dispatching on filename). Returns (errors, warnings).""" + try: + with open(path, encoding="utf-8") as fh: + data = yaml.safe_load(fh) + except OSError as exc: + return [f"cannot read file: {exc}"], [] + except yaml.YAMLError as exc: + return [f"invalid YAML: {exc}"], [] + + if path.name == "removed.yaml": + return validate_removed(data) + return validate_entry(data) + + +def collect_paths(args: list[str]) -> list[Path]: + paths: list[Path] = [] + for arg in args: + p = Path(arg) + if p.is_dir(): + paths.extend(sorted(p.glob("*.yaml"))) + paths.extend(sorted(p.glob("*.yml"))) + else: + paths.append(p) + return paths + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Standalone structural validator for plugin-catalog entry files." + ) + parser.add_argument( + "paths", + nargs="+", + help="catalog entry files, removed.yaml, or a directory of them", + ) + parser.add_argument( + "--json", + action="store_true", + help="emit a machine-readable JSON report on stdout", + ) + opts = parser.parse_args(argv) + + files = collect_paths(opts.paths) + if not files: + print("ERROR: no YAML files found", file=sys.stderr) + return 1 + + report = [] + any_errors = False + for path in files: + errors, warnings = validate_file(path) + any_errors = any_errors or bool(errors) + report.append( + { + "path": str(path), + "ok": not errors, + "errors": errors, + "warnings": warnings, + } + ) + + if opts.json: + print(json.dumps({"ok": not any_errors, "files": report}, indent=2)) + else: + for entry in report: + for err in entry["errors"]: + print(f"{entry['path']}: ERROR: {err}") + for warn in entry["warnings"]: + print(f"{entry['path']}: warning: {warn}") + checked = len(report) + bad = sum(1 for e in report if not e["ok"]) + if any_errors: + print(f"FAIL: {bad}/{checked} file(s) invalid") + else: + print(f"OK: {checked} file(s) valid") + + return 1 if any_errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/scripts/test_validate_plugin_catalog.py b/tests/scripts/test_validate_plugin_catalog.py new file mode 100644 index 000000000000..7b369c96ec4e --- /dev/null +++ b/tests/scripts/test_validate_plugin_catalog.py @@ -0,0 +1,273 @@ +"""Behavior tests for scripts/validate_plugin_catalog.py. + +The script is the no-install structural validator used by the plugin-catalog +admission CI: it must run with only stdlib + pyyaml, take file paths or a +directory, exit 0/1, and support --json machine output. These tests exercise +the CLI contract via subprocess (the same way CI invokes it). +""" + +import json +import subprocess +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "scripts" / "validate_plugin_catalog.py" + +VALID_ENTRY = { + "name": "example-plugin", + "repo": "https://github.com/NousResearch/hermes-example-plugins", + "sha": "38fe0fb53eff98d477f807432e965429e665ca33", + "subdir": "", + "description": "One-line description.", + "maintainer": "NousResearch", + "tier": "official", + "requires_hermes": ">=0.19", + "docs_url": "", + "platforms": [], + "capabilities": { + "provides_tools": ["example_tool"], + "provides_hooks": [], + "provides_middleware": [], + "requires_env": [], + }, +} + + +def write_entry(tmp_path: Path, data: dict, filename: str | None = None) -> Path: + name = filename or f"{data.get('name', 'entry')}.yaml" + path = tmp_path / name + path.write_text(yaml.safe_dump(data), encoding="utf-8") + return path + + +def run_validator(*args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + ) + + +# ── valid input ──────────────────────────────────────────────────────── + + +def test_valid_entry_passes(tmp_path): + path = write_entry(tmp_path, VALID_ENTRY) + result = run_validator(str(path)) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_valid_entry_without_optional_fields_passes(tmp_path): + entry = { + "name": "minimal-plugin", + "repo": "https://github.com/example/minimal", + "sha": "a" * 40, + "description": "Minimal.", + "maintainer": "someone", + } + path = write_entry(tmp_path, entry) + result = run_validator(str(path)) + assert result.returncode == 0, result.stdout + result.stderr + + +# ── each malformed field fails with a pointed error ──────────────────── + + +def _expect_error(tmp_path, mutation: dict, expected_substring: str, drop: str = ""): + entry = {**VALID_ENTRY, **mutation} + if drop: + entry.pop(drop, None) + path = write_entry(tmp_path, entry, filename="entry.yaml") + result = run_validator(str(path)) + combined = result.stdout + result.stderr + assert result.returncode == 1, combined + assert expected_substring in combined, combined + assert "entry.yaml" in combined, combined + + +def test_bad_name_fails(tmp_path): + _expect_error(tmp_path, {"name": "Bad Name!"}, "name") + + +def test_name_too_long_fails(tmp_path): + _expect_error(tmp_path, {"name": "x" * 65}, "name") + + +def test_non_https_repo_fails(tmp_path): + _expect_error(tmp_path, {"repo": "git@github.com:evil/x.git"}, "repo") + + +def test_short_sha_fails(tmp_path): + _expect_error(tmp_path, {"sha": "abc123"}, "sha") + + +def test_non_hex_sha_fails(tmp_path): + _expect_error(tmp_path, {"sha": "z" * 40}, "sha") + + +def test_bad_tier_fails(tmp_path): + _expect_error(tmp_path, {"tier": "platinum"}, "tier") + + +def test_empty_description_fails(tmp_path): + _expect_error(tmp_path, {"description": ""}, "description") + + +def test_empty_maintainer_fails(tmp_path): + _expect_error(tmp_path, {"maintainer": ""}, "maintainer") + + +def test_missing_required_field_fails(tmp_path): + _expect_error(tmp_path, {}, "sha", drop="sha") + + +def test_capabilities_value_not_a_list_fails(tmp_path): + _expect_error( + tmp_path, + {"capabilities": {"provides_tools": "not-a-list"}}, + "provides_tools", + ) + + +def test_capabilities_list_of_non_strings_fails(tmp_path): + _expect_error( + tmp_path, + {"capabilities": {"requires_env": [1, 2]}}, + "requires_env", + ) + + +def test_bad_requires_hermes_spec_fails(tmp_path): + _expect_error(tmp_path, {"requires_hermes": "banana"}, "requires_hermes") + + +def test_comma_separated_requires_hermes_passes(tmp_path): + entry = {**VALID_ENTRY, "requires_hermes": ">=0.19, <2.0"} + path = write_entry(tmp_path, entry) + result = run_validator(str(path)) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_unknown_platform_fails(tmp_path): + _expect_error(tmp_path, {"platforms": ["linux", "amiga"]}, "platforms") + + +def test_entry_not_a_mapping_fails(tmp_path): + path = tmp_path / "entry.yaml" + path.write_text("- just\n- a\n- list\n", encoding="utf-8") + result = run_validator(str(path)) + assert result.returncode == 1 + assert "mapping" in (result.stdout + result.stderr) + + +# ── unknown top-level keys warn but do not fail ──────────────────────── + + +def test_unknown_key_warns_but_passes(tmp_path): + entry = {**VALID_ENTRY, "future_field": "hello"} + path = write_entry(tmp_path, entry) + result = run_validator(str(path)) + combined = result.stdout + result.stderr + assert result.returncode == 0, combined + assert "future_field" in combined + assert "warning" in combined.lower() + + +# ── removed.yaml shape ───────────────────────────────────────────────── + + +def test_valid_removed_yaml_passes(tmp_path): + path = tmp_path / "removed.yaml" + path.write_text( + yaml.safe_dump( + { + "removed": [ + { + "name": "some-plugin", + "repo": "https://github.com/evil/some-plugin", + "reason": "Exfiltrated env vars", + "date": "2026-07-02", + } + ] + } + ), + encoding="utf-8", + ) + result = run_validator(str(path)) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_removed_yaml_not_a_list_fails(tmp_path): + path = tmp_path / "removed.yaml" + path.write_text(yaml.safe_dump({"removed": "nope"}), encoding="utf-8") + result = run_validator(str(path)) + assert result.returncode == 1 + assert "removed" in (result.stdout + result.stderr) + + +def test_removed_item_missing_name_fails(tmp_path): + path = tmp_path / "removed.yaml" + path.write_text( + yaml.safe_dump({"removed": [{"reason": "bad", "date": "2026-01-01"}]}), + encoding="utf-8", + ) + result = run_validator(str(path)) + assert result.returncode == 1 + assert "name" in (result.stdout + result.stderr) + + +# ── --json machine output ────────────────────────────────────────────── + + +def test_json_output_shape_on_failure(tmp_path): + bad = write_entry(tmp_path, {**VALID_ENTRY, "sha": "short"}, filename="bad.yaml") + result = run_validator("--json", str(bad)) + assert result.returncode == 1 + payload = json.loads(result.stdout) + assert payload["ok"] is False + assert isinstance(payload["files"], list) + entry = next(f for f in payload["files"] if f["path"].endswith("bad.yaml")) + assert entry["ok"] is False + assert any("sha" in e for e in entry["errors"]) + + +def test_json_output_shape_on_success_with_warning(tmp_path): + good = write_entry(tmp_path, {**VALID_ENTRY, "future_field": 1}) + result = run_validator("--json", str(good)) + assert result.returncode == 0 + payload = json.loads(result.stdout) + assert payload["ok"] is True + (entry,) = payload["files"] + assert entry["ok"] is True + assert entry["errors"] == [] + assert any("future_field" in w for w in entry["warnings"]) + + +# ── directory mode ───────────────────────────────────────────────────── + + +def test_directory_mode_validates_all_entries_and_removed(tmp_path): + write_entry(tmp_path, VALID_ENTRY) + write_entry(tmp_path, {**VALID_ENTRY, "name": "bad-one", "sha": "nope"}) + (tmp_path / "removed.yaml").write_text( + yaml.safe_dump({"removed": [{"name": "gone", "reason": "test"}]}), + encoding="utf-8", + ) + result = run_validator(str(tmp_path)) + combined = result.stdout + result.stderr + assert result.returncode == 1 + assert "bad-one.yaml" in combined + # the valid entry and removed.yaml must not produce errors + assert combined.count("ERROR") == combined.count("bad-one.yaml: ERROR") + + +def test_directory_mode_all_valid_exits_zero(tmp_path): + write_entry(tmp_path, VALID_ENTRY) + (tmp_path / "removed.yaml").write_text( + yaml.safe_dump({"removed": []}), encoding="utf-8" + ) + result = run_validator(str(tmp_path)) + assert result.returncode == 0, result.stdout + result.stderr