hermes-agent/tests/hermes_cli/test_plugin_scanner_recursion.py
Teknium 39975613b1
test: prune wave 2 + speed fixes — 28,106 → 19,757 test functions, suite wall 315s → 294s
Second, deeper pass over tools/gateway/hermes_cli plus first pass over
the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker,
dashboard, conformance, monitoring, secret_sources, hermes_state,
providers). Same rubric as wave 1 (AGENTS.md test policy); security,
alternation/caching invariants, issue-number regressions, and E2E kept.

Real test-quality fixes found and rooted out along the way:
- tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls
  (DEFAULT_CONFIG smart-approval leaked in) — pinned approval
  mode=manual via autouse fixture: 17.4s → 0.4s.
- test_model_switch_custom_providers.py / test_user_providers_model_switch.py
  silently probed live provider catalogs (~2s/test) — stubbed
  cached_provider_model_ids/provider_model_ids/fetch_api_models.
- test_telegram_noise_filter.py: 15-platform copy-paste matrix over
  shared gateway.run logic → 3 representative platforms (55s → 3.9s).
- test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on
  MagicMock agents — interrupt.side_effect now clears _running_agents
  (22s → 1.0s).
- test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x
  (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps
  patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait
  5s → 0.5s.
- test_telegram_init_deadline.py: loop-block margin restored to 1.0s
  with rationale comment — the watchdog-dump assertion needs the loop
  blocked well past deadline+grace under parallel load (flaked once in
  the 40-worker verification run at a 0.2s margin).

Verification: full hermetic suite via scripts/run_tests.sh —
2,438 files, 21,718 tests passed, 0 failed, 293.9s wall.
Suite totals vs original baseline: 46,820 → 19,757 test functions
(−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
2026-07-29 13:39:40 -07:00

302 lines
11 KiB
Python

"""Tests for PR1 pluggable image gen: scanner recursion, kinds, path keys.
Covers ``_scan_directory`` recursion into category namespaces
(``plugins/image_gen/openai/``), ``kind`` parsing, path-derived registry
keys, and the new gate logic (bundled backends auto-load; user backends
still opt-in; exclusive kind skipped; unknown kinds → standalone warning).
"""
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict
import pytest
import yaml
from hermes_cli.plugins import PluginManager
# ── Helpers ────────────────────────────────────────────────────────────────
def _write_plugin(
root: Path,
segments: list[str],
*,
manifest_extra: Dict[str, Any] | None = None,
register_body: str = "pass",
) -> Path:
"""Create a plugin dir at ``root/<segments...>/`` with plugin.yaml + __init__.py.
``segments`` lets tests build both flat (``["my-plugin"]``) and
category-namespaced (``["image_gen", "openai"]``) layouts.
"""
plugin_dir = root
for seg in segments:
plugin_dir = plugin_dir / seg
plugin_dir.mkdir(parents=True, exist_ok=True)
manifest = {
"name": segments[-1],
"version": "0.1.0",
"description": f"Test plugin {'/'.join(segments)}",
}
if manifest_extra:
manifest.update(manifest_extra)
(plugin_dir / "plugin.yaml").write_text(yaml.dump(manifest))
(plugin_dir / "__init__.py").write_text(
f"def register(ctx):\n {register_body}\n"
)
return plugin_dir
def _enable(hermes_home: Path, name: str) -> None:
"""Append ``name`` to ``plugins.enabled`` in ``<hermes_home>/config.yaml``."""
cfg_path = hermes_home / "config.yaml"
cfg: dict = {}
if cfg_path.exists():
try:
cfg = yaml.safe_load(cfg_path.read_text()) or {}
except Exception:
cfg = {}
plugins_cfg = cfg.setdefault("plugins", {})
enabled = plugins_cfg.setdefault("enabled", [])
if isinstance(enabled, list) and name not in enabled:
enabled.append(name)
cfg_path.write_text(yaml.safe_dump(cfg))
# ── Scanner recursion ──────────────────────────────────────────────────────
class TestCategoryNamespaceRecursion:
def test_category_namespace_discovered(self, tmp_path, monkeypatch):
"""``<root>/image_gen/openai/plugin.yaml`` is discovered with key
``image_gen/openai`` when the ``image_gen`` parent has no manifest."""
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
user_plugins = hermes_home / "plugins"
_write_plugin(user_plugins, ["image_gen", "openai"])
_enable(hermes_home, "image_gen/openai")
mgr = PluginManager()
mgr.discover_and_load()
assert "image_gen/openai" in mgr._plugins
loaded = mgr._plugins["image_gen/openai"]
assert loaded.manifest.key == "image_gen/openai"
assert loaded.manifest.name == "openai"
assert loaded.enabled is True
def test_depth_cap_two(self, tmp_path, monkeypatch):
"""Plugins nested three levels deep are not discovered.
``<root>/a/b/c/plugin.yaml`` should NOT be picked up — cap is
two segments.
"""
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
user_plugins = hermes_home / "plugins"
_write_plugin(user_plugins, ["a", "b", "c"])
mgr = PluginManager()
mgr.discover_and_load()
non_bundled = [
k for k, p in mgr._plugins.items()
if p.manifest.source != "bundled"
]
assert non_bundled == []
# ── Kind parsing ───────────────────────────────────────────────────────────
class TestKindField:
def test_default_kind_is_standalone(self, tmp_path, monkeypatch):
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
_write_plugin(hermes_home / "plugins", ["p1"])
_enable(hermes_home, "p1")
mgr = PluginManager()
mgr.discover_and_load()
assert mgr._plugins["p1"].manifest.kind == "standalone"
@pytest.mark.parametrize("kind", ["backend", "exclusive", "standalone"])
def test_valid_kinds_parsed(self, kind, tmp_path, monkeypatch):
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
_write_plugin(
hermes_home / "plugins",
["p1"],
manifest_extra={"kind": kind},
)
# Not all kinds auto-load, but manifest should parse.
_enable(hermes_home, "p1")
mgr = PluginManager()
mgr.discover_and_load()
assert "p1" in mgr._plugins
assert mgr._plugins["p1"].manifest.kind == kind
def test_unknown_kind_falls_back_to_standalone(self, tmp_path, monkeypatch, caplog):
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
_write_plugin(
hermes_home / "plugins",
["p1"],
manifest_extra={"kind": "bogus"},
)
_enable(hermes_home, "p1")
with caplog.at_level("WARNING"):
mgr = PluginManager()
mgr.discover_and_load()
assert mgr._plugins["p1"].manifest.kind == "standalone"
assert any(
"unknown kind" in rec.getMessage() for rec in caplog.records
)
# ── Gate logic ─────────────────────────────────────────────────────────────
class TestBackendGate:
def test_user_backend_still_gated_by_enabled(self, tmp_path, monkeypatch):
"""User-installed ``kind: backend`` plugins still require opt-in —
they're not trusted by default."""
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
user_plugins = hermes_home / "plugins"
_write_plugin(
user_plugins,
["image_gen", "fancy"],
manifest_extra={"kind": "backend"},
)
# Do NOT opt in.
mgr = PluginManager()
mgr.discover_and_load()
loaded = mgr._plugins["image_gen/fancy"]
assert loaded.enabled is False
assert "not enabled" in (loaded.error or "")
def test_exclusive_kind_skipped(self, tmp_path, monkeypatch):
"""``kind: exclusive`` plugins are recorded but not loaded — the
category's own discovery system handles them (memory today)."""
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
_write_plugin(
hermes_home / "plugins",
["some-backend"],
manifest_extra={"kind": "exclusive"},
)
_enable(hermes_home, "some-backend")
mgr = PluginManager()
mgr.discover_and_load()
loaded = mgr._plugins["some-backend"]
assert loaded.enabled is False
assert "exclusive" in (loaded.error or "")
# ── Bundled backend auto-load (integration with real bundled plugin) ────────
class TestBundledBackendAutoLoad:
def test_bundled_image_gen_openai_autoloads(self, tmp_path, monkeypatch):
"""The bundled ``plugins/image_gen/openai/`` plugin loads without
any opt-in — it's ``kind: backend`` and shipped in-repo."""
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
mgr = PluginManager()
mgr.discover_and_load()
assert "image_gen/openai" in mgr._plugins
loaded = mgr._plugins["image_gen/openai"]
assert loaded.manifest.source == "bundled"
assert loaded.manifest.kind == "backend"
assert loaded.enabled is True, f"error: {loaded.error}"
# ── PluginContext.register_image_gen_provider ───────────────────────────────
class TestRegisterImageGenProvider:
def test_accepts_valid_provider(self, tmp_path, monkeypatch):
from agent import image_gen_registry
from agent.image_gen_provider import ImageGenProvider
image_gen_registry._reset_for_tests()
class FakeProvider(ImageGenProvider):
@property
def name(self) -> str:
return "fake-test"
def generate(self, prompt, aspect_ratio="landscape", **kw):
return {"success": True, "image": "test://fake"}
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
plugin_dir = _write_plugin(
hermes_home / "plugins",
["my-img-plugin"],
register_body=(
"from agent.image_gen_provider import ImageGenProvider\n"
" class P(ImageGenProvider):\n"
" @property\n"
" def name(self): return 'fake-ctx'\n"
" def generate(self, prompt, aspect_ratio='landscape', **kw):\n"
" return {'success': True, 'image': 'x://y'}\n"
" ctx.register_image_gen_provider(P())"
),
)
_enable(hermes_home, "my-img-plugin")
mgr = PluginManager()
mgr.discover_and_load()
assert mgr._plugins["my-img-plugin"].enabled is True
assert image_gen_registry.get_provider("fake-ctx") is not None
image_gen_registry._reset_for_tests()
def test_rejects_non_provider(self, tmp_path, monkeypatch, caplog):
from agent import image_gen_registry
image_gen_registry._reset_for_tests()
import os
hermes_home = Path(os.environ["HERMES_HOME"]) # set by hermetic conftest fixture
_write_plugin(
hermes_home / "plugins",
["bad-img-plugin"],
register_body="ctx.register_image_gen_provider('not a provider')",
)
_enable(hermes_home, "bad-img-plugin")
with caplog.at_level("WARNING"):
mgr = PluginManager()
mgr.discover_and_load()
# Plugin loaded (register returned normally) but nothing was
# registered in the provider registry.
assert mgr._plugins["bad-img-plugin"].enabled is True
assert image_gen_registry.get_provider("not a provider") is None
image_gen_registry._reset_for_tests()