mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-24 16:54:43 +00:00
Replace the static comment-pending + comment-results two-job pattern
with a live-updating comment system that polls the GitHub Actions API
every 15s, re-assembles the review comment from whatever results are
available, and upserts it via the <!-- hermes-ci-review-bot --> marker.
The comment updates in real time as each job finishes — no waiting for
the full pipeline.
Every CI job that wants to appear in the review comment emits a
review_status output — a JSON array of objects, each with a source
and a results array:
[
{
"source": "review-label-gate",
"results": [
{"kind": "action_required", "title": "...", "summary": "...",
"how_to_fix": "..."},
{"kind": "info", "title": "...", "summary": "..."}
]
},
{
"source": "ci timing",
"results": [
{"kind": "warning", "title": "CI timings", "summary": "...",
"detail": "...", "link": "..."}
]
}
]
One job can emit multiple results of different kinds. The source field
is used to exclude the corresponding job from the synthesized error
list (case-insensitive, hyphen-normalized matching against GitHub
Actions job display names).
| job | source | kind (on failure) | section |
|----------------------------|--------------------------|---------------------------|----------------------|
| review-labels | review label gate | action_required / info | Action required |
| lockfile-diff | lockfile-diff | action_required | Action required |
| ci-timings | ci timing | warning / info | Warnings |
| supply-chain scan | supply chain | error / (none) | Job failures |
| supply-chain dep-bounds | supply chain | action_required / (none) | Action required |
| osv-scanner | osv scan | warning / (none) | Warnings |
| uv-lockfile-check | uv.lock check | action_required / (none) | Action required |
| history-check | unrelated histories | action_required | Action required |
| contributor-check | contributor attribution | action_required | Action required |
Jobs that find nothing emit [] (empty array) — no noise info items.
A single comment-live job polls the GitHub Actions API every 15s,
classifies jobs into (completed, pending), assembles the comment, and
upserts it. Merges review_status outputs from all needs jobs via
toJSON(needs.*.outputs.review_status), and downloads the ci-timings
artifact when it becomes available. Shows commit SHA + message below
the header.
The assembler has ZERO job-specific knowledge. It just:
1. collect_from_statuses() — flattens all nested status objects into ReviewItems
2. collect_failed_jobs() — synthesizes errors for failed jobs with no declared status
3. _attach_job_urls() — fills in per-job log links for ALL items
4. render_comment() — groups by severity, renders with group headers
Each item shows links inline next to the title: View report (job-emitted
URL) and View job (auto-attached logs link). Each info item is its own
collapsible <details> block.
# ૮ >ﻌ< ა ci review
running on abc1234 — commit message first line
## ❌ Job failures
### {title} · [View job](url)
{summary}
## ⚠️ Action required
### {title} · [View job](url)
{summary}
**How to fix:**
{how_to_fix}
## ⚠️ Warnings
### {title} · [View report](url) · [View job](url)
{summary}
{detail}
<details><summary>{title}</summary>
{content}
</details>
Still running 3 jobs: ci-timings, docker
- test_assemble_review_comment.py (48 tests): collect_from_statuses,
collect_failed_jobs with exclude_sources, _attach_job_urls,
render_comment (group headers, inline links, commit info, per-item
details, pending footer), assemble integration
- test_live_comment.py (16 tests): classify_jobs pure function
- test_timings_report.py (10 tests): generate_review_status nested format
- test_lockfile_diff.py (6 tests)
- test_classify_changes.py (32 tests, pre-existing)
143 lines
5.3 KiB
Python
143 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Emit review_status JSON for the review-labels workflow.
|
|
|
|
Builds a JSON array with one entry::
|
|
|
|
[
|
|
{
|
|
"source": "review-label-gate",
|
|
"results": [
|
|
{"kind": "action_required", "title": "...", "summary": "...",
|
|
"how_to_fix": "..."},
|
|
{"kind": "info", "title": "...", "summary": "..."}
|
|
]
|
|
}
|
|
]
|
|
|
|
The ``source`` field is the workflow name that declared the status; the
|
|
assembler uses it to exclude the corresponding job from the synthesized
|
|
❌ Error list (the job already has its own status section).
|
|
|
|
The array can contain 0, 1, or 2 results — one per lane that ran
|
|
(``ci_review``, ``mcp_catalog``). When the ``ci-reviewed`` label is
|
|
present, the kind is ``info``; when missing, it's ``action_required``
|
|
with the verification checklist.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
|
|
# The source identifier used for error-synthesis exclusion. This must
|
|
# match (as a normalized substring) the job name as it appears in the
|
|
# GitHub Actions API. The ci.yml job key is ``review-labels`` with
|
|
# ``name: Review label gate``, and the reusable workflow's job is also
|
|
# ``name: Review label gate``, so the API shows the job as
|
|
# "Review label gate / Review label gate". Normalizing "review-label-gate"
|
|
# (lowercase, hyphens→spaces) gives "review label gate", which is a
|
|
# substring of "review label gate / review label gate".
|
|
SOURCE = "review-label-gate"
|
|
|
|
|
|
def build_results(
|
|
ci_review: bool,
|
|
mcp_catalog: bool,
|
|
label_present: bool,
|
|
) -> list[dict]:
|
|
"""Build the list of result objects for this source."""
|
|
results: list[dict] = []
|
|
|
|
if ci_review:
|
|
if label_present:
|
|
results.append({
|
|
"kind": "info",
|
|
"title": "CI-sensitive file review",
|
|
"summary": "`ci-reviewed` label is present.",
|
|
})
|
|
else:
|
|
results.append({
|
|
"kind": "action_required",
|
|
"title": "CI-sensitive file review",
|
|
"summary": (
|
|
"This PR changes CI-sensitive files (eslint config, "
|
|
"workflow YAMLs, or composite actions). These influence "
|
|
"what the js-autofix job executes and pushes to main."
|
|
),
|
|
"how_to_fix": (
|
|
"Add the `ci-reviewed` label after verifying:\n"
|
|
"- no new eslint rules with custom `fix` functions that write outside linted paths,\n"
|
|
"- no workflow changes that widen permissions or remove guards,\n"
|
|
"- no composite action changes that alter what gets executed."
|
|
),
|
|
})
|
|
|
|
if mcp_catalog:
|
|
if label_present:
|
|
results.append({
|
|
"kind": "info",
|
|
"title": "MCP catalog security review",
|
|
"summary": "`ci-reviewed` label is present.",
|
|
})
|
|
else:
|
|
results.append({
|
|
"kind": "action_required",
|
|
"title": "MCP catalog security review",
|
|
"summary": (
|
|
"This PR changes the bundled MCP catalog or MCP catalog "
|
|
"installer code. MCP entries can define local commands "
|
|
"that users later install into `mcp_servers`, so this "
|
|
"needs explicit maintainer review before merge."
|
|
),
|
|
"how_to_fix": (
|
|
"Add the `ci-reviewed` label after verifying:\n"
|
|
"- any new/changed `optional-mcps/**/manifest.yaml` command and args are expected,\n"
|
|
"- stdio transports do not use shell+egress/exfiltration payloads,\n"
|
|
"- git install refs are pinned and bootstrap commands are minimal,\n"
|
|
"- requested env vars/secrets match the upstream MCP's documented needs."
|
|
),
|
|
})
|
|
|
|
return results
|
|
|
|
|
|
def build_statuses(
|
|
ci_review: bool,
|
|
mcp_catalog: bool,
|
|
label_present: bool,
|
|
) -> list[dict]:
|
|
"""Build the full review_status array (one entry with a results list)."""
|
|
results = build_results(ci_review, mcp_catalog, label_present)
|
|
if not results:
|
|
return []
|
|
return [{"source": SOURCE, "results": results}]
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--ci-review", action="store_true",
|
|
help="Whether CI-sensitive files changed.")
|
|
parser.add_argument("--mcp-catalog", action="store_true",
|
|
help="Whether the MCP catalog / installer changed.")
|
|
parser.add_argument("--label-present", action="store_true",
|
|
help="Whether the ci-reviewed label is present.")
|
|
parser.add_argument("--output", default="-",
|
|
help="Output file ('-' for stdout, or a GITHUB_OUTPUT path).")
|
|
args = parser.parse_args()
|
|
|
|
statuses = build_statuses(args.ci_review, args.mcp_catalog, args.label_present)
|
|
json_str = json.dumps(statuses)
|
|
|
|
if args.output == "-":
|
|
print(json_str)
|
|
else:
|
|
# GITHUB_OUTPUT format: key=value\n
|
|
with open(args.output, "a", encoding="utf-8") as f:
|
|
f.write(f"review_status={json_str}\n")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|