mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-06-01 07:01:41 +00:00
4254f7dd17
Trim ~600 LOC off the original contribution while keeping the same operator-facing surface and detection coverage. - Collapse three entry points (file / dir / bundle) into one ast_scan_path(path) that handles both files and directories. - Drop AstFinding dataclass + severity field — replaced with plain (file, line, pattern_id, description) tuples. Severity ordering was display-only for a diagnostic that explicitly disclaims security verdicts, so the field added bookkeeping without earning its place. - Replace Rich-markup formatter with plain text grouped by file. - Drop the 'inspect --ast-deep' surface — same scanner, same output as 'audit --deep', single CLI entry is enough. Operators audit after install; pre-install inspection signal isn't worth the second surface. - Trim test file to the cases that earn their place: bypass payload, syntax error survival, RecursionError survival, false-positive guard (importer lookalike), literal-arg false-positive guard, non-.py ignored, directory recursion + cache-dir skipping, missing-path, getattr/__dict__ detection, formatter empty + populated. Net: tools/skills_ast_audit.py 353 -> 133 LOC, tests/tools/test_skills_ast_audit.py 299 -> 103 LOC, full diff +704/-12 -> +264/-6. No change to tools/skills_guard.py — Skills Guard verdicts remain untouched per SECURITY.md §2.4.
103 lines
3.2 KiB
Python
103 lines
3.2 KiB
Python
"""Tests for tools.skills_ast_audit — opt-in AST diagnostic scanner."""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from tools.skills_ast_audit import ast_scan_path, format_ast_report
|
|
|
|
|
|
def _pids(findings):
|
|
return [pid for (_f, _l, pid, _d) in findings]
|
|
|
|
|
|
def test_bypass_payload_detected(tmp_path):
|
|
"""The exact bypass shape from #7072 is caught."""
|
|
f = tmp_path / "exfil.py"
|
|
f.write_text(
|
|
"import importlib\n"
|
|
"parts = ['o', 's']\n"
|
|
"m = importlib.import_module(''.join(parts))\n"
|
|
"e = m.__dict__[''.join(['e','n','v'])]\n"
|
|
)
|
|
pids = _pids(ast_scan_path(f))
|
|
assert "dynamic_import" in pids
|
|
assert "importlib_import" in pids
|
|
assert "dict_access" in pids
|
|
|
|
|
|
def test_syntax_error_does_not_crash(tmp_path):
|
|
f = tmp_path / "bad.py"
|
|
f.write_text("def broken(\n")
|
|
assert ast_scan_path(f) == []
|
|
|
|
|
|
def test_recursion_error_does_not_crash(tmp_path):
|
|
f = tmp_path / "deep.py"
|
|
f.write_text("a" + ".x" * 5000 + "\n")
|
|
orig = sys.getrecursionlimit()
|
|
sys.setrecursionlimit(200)
|
|
try:
|
|
result = ast_scan_path(f)
|
|
finally:
|
|
sys.setrecursionlimit(orig)
|
|
assert isinstance(result, list)
|
|
|
|
|
|
def test_importer_lookalike_not_flagged(tmp_path):
|
|
"""`import importer` must NOT match — dot-bounded prefix."""
|
|
f = tmp_path / "ok.py"
|
|
f.write_text("import importer\nfrom importer import x\n")
|
|
assert _pids(ast_scan_path(f)) == []
|
|
|
|
|
|
def test_literal_dunder_import_not_flagged(tmp_path):
|
|
"""__import__('os') with a literal is not flagged (regex catches those)."""
|
|
f = tmp_path / "ok.py"
|
|
f.write_text("m = __import__('os')\n")
|
|
assert "dynamic_import_computed" not in _pids(ast_scan_path(f))
|
|
|
|
|
|
def test_non_python_file_returns_empty(tmp_path):
|
|
f = tmp_path / "script.sh"
|
|
f.write_text("import importlib\n")
|
|
assert ast_scan_path(f) == []
|
|
|
|
|
|
def test_directory_scans_recursively_and_skips_cache_dirs(tmp_path):
|
|
skill = tmp_path / "s"
|
|
skill.mkdir()
|
|
(skill / "main.py").write_text("import importlib\n")
|
|
(skill / "sub").mkdir()
|
|
(skill / "sub" / "u.py").write_text("from importlib.util import find_spec\n")
|
|
for d in ("__pycache__", ".venv", "venv", "node_modules"):
|
|
ignored = skill / d
|
|
ignored.mkdir()
|
|
(ignored / "junk.py").write_text("import importlib\n")
|
|
pids = _pids(ast_scan_path(skill))
|
|
assert pids.count("importlib_import") == 2
|
|
|
|
|
|
def test_missing_path_returns_empty(tmp_path):
|
|
assert ast_scan_path(tmp_path / "does_not_exist") == []
|
|
|
|
|
|
def test_dynamic_getattr_and_dict_access_detected(tmp_path):
|
|
f = tmp_path / "g.py"
|
|
f.write_text("name = 'x'\nv = getattr(o, name)\nv = o.__dict__[name]\n")
|
|
pids = _pids(ast_scan_path(f))
|
|
assert "dynamic_getattr" in pids
|
|
assert "dict_access" in pids
|
|
|
|
|
|
def test_format_report_empty():
|
|
assert "No dynamic" in format_ast_report([])
|
|
|
|
|
|
def test_format_report_with_findings():
|
|
findings = [
|
|
("a.py", 1, "importlib_import", "import importlib — ..."),
|
|
("a.py", 3, "dynamic_import", "importlib.import_module() — ..."),
|
|
]
|
|
out = format_ast_report(findings, skill_name="test")
|
|
assert "test" in out and "a.py" in out and "L1" in out and "L3" in out
|
|
assert "diagnostic hints" in out
|