mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-23 16:36:23 +00:00
b9f82ed39f
Replace the static comment-pending + comment-results two-job pattern
with a live-updating comment system that polls the GitHub Actions API
every 15s, re-assembles the review comment from whatever results are
available, and upserts it via the <!-- hermes-ci-review-bot --> marker.
The comment updates in real time as each job finishes — no waiting for
the full pipeline.
Every CI job that wants to appear in the review comment emits a
review_status output — a JSON array of objects, each with a source
and a results array:
[
{
"source": "review-label-gate",
"results": [
{"kind": "action_required", "title": "...", "summary": "...",
"how_to_fix": "..."},
{"kind": "info", "title": "...", "summary": "..."}
]
},
{
"source": "ci timing",
"results": [
{"kind": "warning", "title": "CI timings", "summary": "...",
"detail": "...", "link": "..."}
]
}
]
One job can emit multiple results of different kinds. The source field
is used to exclude the corresponding job from the synthesized error
list (case-insensitive, hyphen-normalized matching against GitHub
Actions job display names).
| job | source | kind (on failure) | section |
|----------------------------|--------------------------|---------------------------|----------------------|
| review-labels | review label gate | action_required / info | Action required |
| lockfile-diff | lockfile-diff | action_required | Action required |
| ci-timings | ci timing | warning / info | Warnings |
| supply-chain scan | supply chain | error / (none) | Job failures |
| supply-chain dep-bounds | supply chain | action_required / (none) | Action required |
| osv-scanner | osv scan | warning / (none) | Warnings |
| uv-lockfile-check | uv.lock check | action_required / (none) | Action required |
| history-check | unrelated histories | action_required | Action required |
| contributor-check | contributor attribution | action_required | Action required |
Jobs that find nothing emit [] (empty array) — no noise info items.
A single comment-live job polls the GitHub Actions API every 15s,
classifies jobs into (completed, pending), assembles the comment, and
upserts it. Merges review_status outputs from all needs jobs via
toJSON(needs.*.outputs.review_status), and downloads the ci-timings
artifact when it becomes available. Shows commit SHA + message below
the header.
The assembler has ZERO job-specific knowledge. It just:
1. collect_from_statuses() — flattens all nested status objects into ReviewItems
2. collect_failed_jobs() — synthesizes errors for failed jobs with no declared status
3. _attach_job_urls() — fills in per-job log links for ALL items
4. render_comment() — groups by severity, renders with group headers
Each item shows links inline next to the title: View report (job-emitted
URL) and View job (auto-attached logs link). Each info item is its own
collapsible <details> block.
# ૮ >ﻌ< ა ci review
running on abc1234 — commit message first line
## ❌ Job failures
### {title} · [View job](url)
{summary}
## ⚠️ Action required
### {title} · [View job](url)
{summary}
**How to fix:**
{how_to_fix}
## ⚠️ Warnings
### {title} · [View report](url) · [View job](url)
{summary}
{detail}
<details><summary>{title}</summary>
{content}
</details>
Still running 3 jobs: ci-timings, docker
- test_assemble_review_comment.py (48 tests): collect_from_statuses,
collect_failed_jobs with exclude_sources, _attach_job_urls,
render_comment (group headers, inline links, commit info, per-item
details, pending footer), assemble integration
- test_live_comment.py (16 tests): classify_jobs pure function
- test_timings_report.py (10 tests): generate_review_status nested format
- test_lockfile_diff.py (6 tests)
- test_classify_changes.py (32 tests, pre-existing)
143 lines
5.1 KiB
Python
143 lines
5.1 KiB
Python
"""Tests for scripts/ci/timings_report.py — generate_review_status().
|
|
|
|
The review status is a JSON array in the unified nested format consumed
|
|
by the review comment assembler. It classifies the CI timings result as
|
|
info/warning (never error — timings is an observability job, not a gate)
|
|
and provides a one-line summary plus optional per-job delta detail.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "timings_report.py"
|
|
_spec = importlib.util.spec_from_file_location("timings_report", _PATH)
|
|
if _spec is None or _spec.loader is None:
|
|
raise ImportError("Failed to load timings_report.py")
|
|
_mod = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(_mod)
|
|
|
|
_T0 = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def _ts(seconds: float) -> str:
|
|
"""ISO timestamp `seconds` after T0."""
|
|
dt = _T0.timestamp() + seconds
|
|
return datetime.fromtimestamp(dt, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _job(name: str, dur_s: float, start_s: float = 0.0, conclusion: str = "success") -> dict:
|
|
"""Build a normalized job dict with realistic timestamps for wall-time math."""
|
|
return {
|
|
"name": name,
|
|
"duration_s": dur_s,
|
|
"conclusion": conclusion,
|
|
"started_at": _ts(start_s),
|
|
"completed_at": _ts(start_s + dur_s),
|
|
"wait_s": 0.0,
|
|
}
|
|
|
|
|
|
def _timings(jobs: list[dict]) -> dict:
|
|
return {"run_id": "123", "head_sha": "abc", "created_at": "", "jobs": jobs}
|
|
|
|
|
|
def _result(statuses: list[dict]) -> dict:
|
|
"""Extract the single result dict from the nested format."""
|
|
assert len(statuses) == 1
|
|
assert statuses[0]["source"] == "ci timing"
|
|
results = statuses[0]["results"]
|
|
assert len(results) == 1
|
|
return results[0]
|
|
|
|
|
|
def test_no_baseline_is_info():
|
|
t = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(t, None))
|
|
assert result["kind"] == "info"
|
|
assert "no baseline" in result["summary"].lower()
|
|
assert "link" not in result # no report_url → no link field
|
|
|
|
|
|
def test_no_regression_is_info():
|
|
cur = _timings([_job("tests", 60.0)])
|
|
bl = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
assert result["kind"] == "info"
|
|
assert "+0.0%" in result["summary"]
|
|
|
|
|
|
def test_small_regression_is_info():
|
|
cur = _timings([_job("tests", 65.0)])
|
|
bl = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
# +8.3% — well under the 25% warning threshold
|
|
assert result["kind"] == "info"
|
|
|
|
|
|
def test_large_regression_is_warning():
|
|
cur = _timings([_job("tests", 80.0)])
|
|
bl = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
# +33% — above the 25% threshold
|
|
assert result["kind"] == "warning"
|
|
assert "+33" in result["summary"]
|
|
|
|
|
|
def test_improvement_is_info():
|
|
cur = _timings([_job("tests", 40.0)])
|
|
bl = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
assert result["kind"] == "info"
|
|
assert "-33" in result["summary"]
|
|
|
|
|
|
def test_detail_shows_top_deltas():
|
|
cur = _timings([_job("slow-job", 120.0), _job("fast-job", 30.0, start_s=120.0)])
|
|
bl = _timings([_job("slow-job", 60.0), _job("fast-job", 60.0, start_s=60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
assert "slow-job" in result["detail"]
|
|
assert "fast-job" in result["detail"]
|
|
# Sorted by abs delta — slow-job (+60) before fast-job (-30)
|
|
assert result["detail"].index("slow-job") < result["detail"].index("fast-job")
|
|
|
|
|
|
def test_skipped_jobs_excluded_from_detail():
|
|
cur = _timings([_job("skipped-job", 0.0, conclusion="skipped"), _job("tests", 60.0)])
|
|
bl = _timings([_job("skipped-job", 0.0, conclusion="skipped"), _job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
assert "skipped-job" not in result["detail"]
|
|
|
|
|
|
def test_report_url_passed_through():
|
|
t = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(t, None, report_url="https://artifact/123"))
|
|
assert result["link"] == "https://artifact/123"
|
|
assert result["link_label"] == "View report"
|
|
|
|
|
|
def test_never_error_severity():
|
|
"""Timings is observability — even huge regressions are warnings, not errors."""
|
|
cur = _timings([_job("tests", 600.0)])
|
|
bl = _timings([_job("tests", 60.0)])
|
|
result = _result(_mod.generate_review_status(cur, bl))
|
|
assert result["kind"] == "warning"
|
|
assert result["kind"] != "error"
|
|
|
|
|
|
def test_nested_format_structure():
|
|
"""The return value is a list with one {source, results: [...]} entry."""
|
|
t = _timings([_job("tests", 60.0)])
|
|
statuses = _mod.generate_review_status(t, None)
|
|
assert isinstance(statuses, list)
|
|
assert len(statuses) == 1
|
|
assert statuses[0]["source"] == "ci timing"
|
|
assert isinstance(statuses[0]["results"], list)
|
|
assert len(statuses[0]["results"]) == 1
|
|
r = statuses[0]["results"][0]
|
|
assert r["kind"] == "info"
|
|
assert r["title"] == "CI timings"
|
|
assert "summary" in r
|
|
assert "detail" in r
|