mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
173 lines
5.9 KiB
Python
173 lines
5.9 KiB
Python
"""Invariant tests for the bundled office/document skills.
|
||
|
||
Covers skills/productivity/{docx,xlsx,pdf,powerpoint} — the office
|
||
document creation/editing suite. Tests assert contracts (frontmatter
|
||
shape, referenced scripts exist, cross-links resolve), not snapshots
|
||
of skill content.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
import yaml
|
||
|
||
REPO = Path(__file__).resolve().parent.parent.parent
|
||
SKILLS = REPO / "skills"
|
||
OPTIONAL_SKILLS = REPO / "optional-skills"
|
||
|
||
OFFICE_SKILLS = ["docx", "xlsx", "pdf", "powerpoint"]
|
||
|
||
|
||
def _skill_dir(name: str) -> Path:
|
||
return SKILLS / "productivity" / name
|
||
|
||
|
||
def _frontmatter(skill_md: Path) -> dict:
|
||
text = skill_md.read_text(encoding="utf-8")
|
||
match = re.match(r"^---\n(.*?)\n---\n", text, re.DOTALL)
|
||
assert match, f"{skill_md} has no YAML frontmatter"
|
||
return yaml.safe_load(match.group(1))
|
||
|
||
|
||
@pytest.mark.parametrize("name", OFFICE_SKILLS)
|
||
def test_skill_exists_with_frontmatter(name):
|
||
skill_md = _skill_dir(name) / "SKILL.md"
|
||
assert skill_md.exists(), f"missing {skill_md}"
|
||
fm = _frontmatter(skill_md)
|
||
assert fm["name"] == name
|
||
assert fm["description"].strip()
|
||
assert len(fm["description"]) <= 60, (
|
||
f"{name}: description is {len(fm['description'])} chars (max 60)"
|
||
)
|
||
assert fm["description"].rstrip('"').endswith(".")
|
||
platforms = fm.get("platforms")
|
||
assert platforms, f"{name}: missing platforms gating"
|
||
assert set(platforms) <= {"linux", "macos", "windows"}
|
||
|
||
|
||
@pytest.mark.parametrize("name", OFFICE_SKILLS)
|
||
def test_referenced_scripts_exist(name):
|
||
"""Every scripts/... path mentioned in SKILL.md must exist on disk."""
|
||
skill_dir = _skill_dir(name)
|
||
body = (skill_dir / "SKILL.md").read_text(encoding="utf-8")
|
||
refs = set(re.findall(r"scripts/[\w./-]+\.py", body))
|
||
assert refs, f"{name}: SKILL.md references no helper scripts"
|
||
for ref in refs:
|
||
assert (skill_dir / ref).exists(), f"{name}: SKILL.md references missing {ref}"
|
||
|
||
|
||
|
||
|
||
@pytest.mark.parametrize("name", OFFICE_SKILLS)
|
||
def test_license_file_present(name):
|
||
"""Adapted Anthropic skills must carry their LICENSE.txt."""
|
||
fm = _frontmatter(_skill_dir(name) / "SKILL.md")
|
||
if "LICENSE.txt" in str(fm.get("license", "")):
|
||
assert (_skill_dir(name) / "LICENSE.txt").exists(), (
|
||
f"{name}: license points to LICENSE.txt but the file is missing"
|
||
)
|
||
|
||
|
||
|
||
|
||
def test_docx_validator_schema_paths_exist():
|
||
"""base.py maps XML parts to XSD files — every mapped schema must ship."""
|
||
for skill in ("docx", "powerpoint"):
|
||
base = _skill_dir(skill) / "scripts" / "office" / "validators" / "base.py"
|
||
schemas = _skill_dir(skill) / "scripts" / "office" / "schemas"
|
||
text = base.read_text(encoding="utf-8")
|
||
refs = set(re.findall(r'"((?:ecma|ISO|mce|microsoft)[\w./-]+\.xsd)"', text))
|
||
assert refs, f"{skill}: no schema references found in validators/base.py"
|
||
for ref in refs:
|
||
assert (schemas / ref).exists(), f"{skill}: validator references missing schema {ref}"
|
||
|
||
|
||
|
||
|
||
def test_docs_pages_generated():
|
||
"""Each bundled office skill has a generated docs-site page."""
|
||
docs_dir = REPO / "website" / "docs" / "user-guide" / "skills" / "bundled" / "productivity"
|
||
for name in OFFICE_SKILLS:
|
||
assert (docs_dir / f"productivity-{name}.md").exists(), (
|
||
f"missing generated docs page for {name}; run website/scripts/generate-skill-docs.py"
|
||
)
|
||
|
||
|
||
# Document/payload readers must not depend on the host locale. OOXML part
|
||
# files are UTF-8 (declared in the XML prolog) and the form-fields JSON the
|
||
# agent authors is UTF-8, but these sites used locale-default text mode: on
|
||
# Windows (cp1251/GBK/cp932) the validators parsed silently mojibake'd
|
||
# document text and the pdf fill scripts wrote mojibake'd values into the
|
||
# user's form — or crashed outright where the bytes don't decode.
|
||
_ENCODING_SENSITIVE_READS = [
|
||
(
|
||
"docx/scripts/office/validators/base.py",
|
||
'with open(xml_file, "rb") as f:',
|
||
),
|
||
(
|
||
"powerpoint/scripts/office/validators/base.py",
|
||
'with open(xml_file, "rb") as f:',
|
||
),
|
||
(
|
||
"pdf/scripts/check_bounding_boxes.py",
|
||
'with open(sys.argv[1], encoding="utf-8") as f:',
|
||
),
|
||
(
|
||
"pdf/scripts/create_validation_image.py",
|
||
"with open(fields_json_path, 'r', encoding='utf-8') as f:",
|
||
),
|
||
(
|
||
"pdf/scripts/fill_fillable_fields.py",
|
||
'with open(fields_json_path, encoding="utf-8") as f:',
|
||
),
|
||
(
|
||
"pdf/scripts/fill_pdf_form_with_annotations.py",
|
||
'with open(fields_json_path, "r", encoding="utf-8") as f:',
|
||
),
|
||
]
|
||
|
||
|
||
|
||
|
||
def test_check_bounding_boxes_reads_utf8_fields_json(tmp_path):
|
||
"""Run the real script on a UTF-8 fields.json with non-ASCII field
|
||
descriptions under a forced non-UTF-8 locale. Without the explicit
|
||
encoding the json.load crashes (C locale on POSIX; cp1251 chokes on
|
||
the 0x98 byte of U+2018 on Windows)."""
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
|
||
script = _skill_dir("pdf") / "scripts" / "check_bounding_boxes.py"
|
||
fields = {
|
||
"form_fields": [
|
||
{
|
||
"description": "Фамилия (‘label’)",
|
||
"page_number": 1,
|
||
"label_bounding_box": [0, 0, 10, 10],
|
||
"entry_bounding_box": [20, 20, 30, 40],
|
||
}
|
||
]
|
||
}
|
||
fields_json = tmp_path / "fields.json"
|
||
fields_json.write_bytes(
|
||
json.dumps(fields, ensure_ascii=False, indent=2).encode("utf-8")
|
||
)
|
||
|
||
env = dict(os.environ)
|
||
env.update({"LC_ALL": "C", "LANG": "C", "PYTHONUTF8": "0", "PYTHONIOENCODING": "utf-8"})
|
||
result = subprocess.run(
|
||
[sys.executable, str(script), str(fields_json)],
|
||
capture_output=True,
|
||
text=True,
|
||
env=env,
|
||
timeout=60,
|
||
)
|
||
assert result.returncode == 0, (
|
||
f"script failed under non-UTF-8 locale:\n{result.stderr}"
|
||
)
|
||
assert "SUCCESS" in result.stdout
|