mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
6b81590c55
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
125 lines
4.2 KiB
Python
125 lines
4.2 KiB
Python
"""Tests for scripts/ci/timings_report.py — generate_review_status().
|
|
|
|
The review status is a JSON array in the unified nested format consumed
|
|
by the review comment assembler. It classifies the CI timings result as
|
|
info/warning (never error — timings is an observability job, not a gate)
|
|
and provides a one-line summary plus optional per-job delta detail.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "timings_report.py"
|
|
_spec = importlib.util.spec_from_file_location("timings_report", _PATH)
|
|
if _spec is None or _spec.loader is None:
|
|
raise ImportError("Failed to load timings_report.py")
|
|
_mod = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(_mod)
|
|
|
|
_T0 = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def _ts(seconds: float) -> str:
|
|
"""ISO timestamp `seconds` after T0."""
|
|
dt = _T0.timestamp() + seconds
|
|
return datetime.fromtimestamp(dt, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _job(name: str, dur_s: float, start_s: float = 0.0, conclusion: str = "success") -> dict:
|
|
"""Build a normalized job dict with realistic timestamps for wall-time math."""
|
|
return {
|
|
"name": name,
|
|
"duration_s": dur_s,
|
|
"conclusion": conclusion,
|
|
"started_at": _ts(start_s),
|
|
"completed_at": _ts(start_s + dur_s),
|
|
"wait_s": 0.0,
|
|
}
|
|
|
|
|
|
def _timings(jobs: list[dict]) -> dict:
|
|
return {"run_id": "123", "head_sha": "abc", "created_at": "", "jobs": jobs}
|
|
|
|
|
|
def _result(statuses: list[dict]) -> dict:
|
|
"""Extract the single result dict from the nested format."""
|
|
assert len(statuses) == 1
|
|
assert statuses[0]["source"] == "ci timing"
|
|
results = statuses[0]["results"]
|
|
assert len(results) == 1
|
|
return results[0]
|
|
|
|
|
|
def test_no_baseline_is_debug():
|
|
t = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(t, None))
|
|
assert result["kind"] == "debug"
|
|
assert "no baseline" in result["summary"].lower()
|
|
assert "link" not in result # no report_url → no link field
|
|
|
|
|
|
def test_no_regression_is_debug():
|
|
cur = _timings([_job("tests", 60.0)])
|
|
bl = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
assert result["kind"] == "debug"
|
|
assert "+0.0%" in result["summary"]
|
|
|
|
|
|
def test_small_regression_is_debug():
|
|
cur = _timings([_job("tests", 65.0)])
|
|
bl = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
# +8.3% — well under the 25% warning threshold
|
|
assert result["kind"] == "debug"
|
|
|
|
|
|
def test_large_regression_is_warning():
|
|
cur = _timings([_job("tests", 80.0)])
|
|
bl = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
# +33% — above the 25% threshold
|
|
assert result["kind"] == "warning"
|
|
assert "+33" in result["summary"]
|
|
|
|
|
|
|
|
|
|
def test_detail_shows_top_deltas():
|
|
cur = _timings([_job("slow-job", 120.0), _job("fast-job", 30.0, start_s=120.0)])
|
|
bl = _timings([_job("slow-job", 60.0), _job("fast-job", 60.0, start_s=60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
assert "slow-job" in result["detail"]
|
|
assert "fast-job" in result["detail"]
|
|
# Sorted by abs delta — slow-job (+60) before fast-job (-30)
|
|
assert result["detail"].index("slow-job") < result["detail"].index("fast-job")
|
|
|
|
|
|
|
|
|
|
def test_report_url_passed_through():
|
|
t = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(t, None, report_url="https://artifact/123"))
|
|
assert result["link"] == "https://artifact/123"
|
|
assert result["link_label"] == "View report"
|
|
|
|
|
|
|
|
|
|
def test_nested_format_structure():
|
|
"""The return value is a list with one {source, results: [...]} entry."""
|
|
t = _timings([_job("tests", 60.0)])
|
|
statuses = _mod.generate_review_status(t, None)
|
|
assert isinstance(statuses, list)
|
|
assert len(statuses) == 1
|
|
assert statuses[0]["source"] == "ci timing"
|
|
assert isinstance(statuses[0]["results"], list)
|
|
assert len(statuses[0]["results"]) == 1
|
|
r = statuses[0]["results"][0]
|
|
assert r["kind"] == "debug"
|
|
assert r["title"] == "CI timings"
|
|
assert "summary" in r
|
|
assert "detail" in r
|