mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-25 17:18:11 +00:00
de5ece9944
Print gh-image stdout and stderr on attachment failures, then replace the pending inline-evidence marker in the PR review comment with an escaped failure notice before preserving the failing workflow result.
219 lines
7.5 KiB
Python
219 lines
7.5 KiB
Python
"""Tests for scripts/ci/publish_e2e_evidence.py."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "publish_e2e_evidence.py"
|
|
_spec = importlib.util.spec_from_file_location("publish_e2e_evidence", _PATH)
|
|
if _spec is None or _spec.loader is None:
|
|
raise ImportError("Failed to load publish_e2e_evidence.py")
|
|
_mod = importlib.util.module_from_spec(_spec)
|
|
sys.modules["publish_e2e_evidence"] = _mod
|
|
_spec.loader.exec_module(_mod)
|
|
|
|
|
|
def _png(width: int = 4, height: int = 3) -> bytes:
|
|
return _mod.PNG_SIGNATURE + b"\x00\x00\x00\rIHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big")
|
|
|
|
|
|
def test_load_evidence_validates_manifest_and_pngs(tmp_path):
|
|
(tmp_path / "shot.png").write_bytes(_png())
|
|
(tmp_path / "diff.png").write_bytes(_png())
|
|
(tmp_path / "actual.png").write_bytes(_png())
|
|
(tmp_path / "expected.png").write_bytes(_png())
|
|
(tmp_path / "e2e-evidence.json").write_text(
|
|
"""{
|
|
"version": 1,
|
|
"screenshots": [{"name": "main-view.png", "file": "shot.png"}],
|
|
"diffs": [{"name": "main-view", "diff": "diff.png", "actual": "actual.png", "expected": "expected.png"}]
|
|
}""",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
files, payloads = _mod.load_evidence(tmp_path)
|
|
|
|
assert [item.label for item in files] == [
|
|
"new screenshot: main-view.png",
|
|
"visual diff: main-view",
|
|
"visual actual: main-view",
|
|
"visual expected: main-view",
|
|
]
|
|
assert set(payloads) == {"shot.png", "diff.png", "actual.png", "expected.png"}
|
|
|
|
|
|
def test_load_evidence_rejects_path_escape_and_non_png(tmp_path):
|
|
(tmp_path / "e2e-evidence.json").write_text(
|
|
'{"version":1,"screenshots":[{"name":"bad","file":"../secret.png"}],"diffs":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="unsafe filename"):
|
|
_mod.load_evidence(tmp_path)
|
|
|
|
(tmp_path / "e2e-evidence.json").write_text(
|
|
'{"version":1,"screenshots":[{"name":"bad","file":"not-png.png"}],"diffs":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "not-png.png").write_bytes(b"not a png")
|
|
|
|
with pytest.raises(ValueError, match="not a PNG"):
|
|
_mod.load_evidence(tmp_path)
|
|
|
|
|
|
def test_render_and_replace_evidence_uses_validated_attachment_urls():
|
|
evidence = _mod.render_evidence(
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
{"shot.png": "https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc"},
|
|
)
|
|
body = "before\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->\nafter"
|
|
|
|
result = _mod.replace_evidence_marker(body, evidence)
|
|
|
|
assert "pending" not in result
|
|
assert "https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc" in result
|
|
assert result.startswith("before\n")
|
|
assert result.endswith("\nafter")
|
|
|
|
|
|
def test_upload_evidence_accepts_only_attachment_urls(tmp_path, monkeypatch):
|
|
shot = tmp_path / "shot.png"
|
|
shot.write_bytes(_png())
|
|
calls = []
|
|
|
|
def fake_run(args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return _mod.subprocess.CompletedProcess(
|
|
args,
|
|
0,
|
|
stdout="\n",
|
|
)
|
|
|
|
monkeypatch.setattr(_mod.subprocess, "run", fake_run)
|
|
|
|
result = _mod.upload_evidence(
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
tmp_path,
|
|
"NousResearch/hermes-agent",
|
|
"bot-session-token",
|
|
)
|
|
|
|
assert result == {"shot.png": "https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc"}
|
|
assert calls[0][0] == ["gh", "image", "--repo", "NousResearch/hermes-agent", str(shot)]
|
|
assert calls[0][1]["env"]["GH_SESSION_TOKEN"] == "bot-session-token"
|
|
|
|
|
|
def test_upload_evidence_rejects_unexpected_gh_image_output(tmp_path, monkeypatch):
|
|
(tmp_path / "shot.png").write_bytes(_png())
|
|
|
|
def fake_run(args, **kwargs):
|
|
return _mod.subprocess.CompletedProcess(args, 0, stdout="https://example.invalid/shot.png\n")
|
|
|
|
monkeypatch.setattr(_mod.subprocess, "run", fake_run)
|
|
|
|
with pytest.raises(ValueError, match="invalid attachment reference"):
|
|
_mod.upload_evidence(
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
tmp_path,
|
|
"NousResearch/hermes-agent",
|
|
"bot-session-token",
|
|
)
|
|
|
|
|
|
def test_upload_evidence_reports_gh_image_error(tmp_path, monkeypatch, capsys):
|
|
shot = tmp_path / "shot.png"
|
|
shot.write_bytes(_png())
|
|
|
|
def fake_run(args, **kwargs):
|
|
raise _mod.subprocess.CalledProcessError(
|
|
1,
|
|
args,
|
|
output="upload output",
|
|
stderr="upload error",
|
|
)
|
|
|
|
monkeypatch.setattr(_mod.subprocess, "run", fake_run)
|
|
|
|
with pytest.raises(RuntimeError, match="Failed to upload shot.png.*upload error"):
|
|
_mod.upload_evidence(
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
tmp_path,
|
|
"NousResearch/hermes-agent",
|
|
"bot-session-token",
|
|
)
|
|
|
|
captured = capsys.readouterr()
|
|
assert "Failed to upload shot.png" in captured.err
|
|
assert "upload output" in captured.err
|
|
assert "upload error" in captured.err
|
|
|
|
|
|
def test_publish_marks_evidence_upload_failure_in_pr_comment(tmp_path, monkeypatch):
|
|
comment = {
|
|
"id": 123,
|
|
"body": "before\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->\nafter",
|
|
}
|
|
updates = []
|
|
|
|
monkeypatch.setattr(
|
|
_mod,
|
|
"load_evidence",
|
|
lambda evidence_dir: (
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
{},
|
|
),
|
|
)
|
|
monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: comment)
|
|
monkeypatch.setattr(
|
|
_mod,
|
|
"upload_evidence",
|
|
lambda *args: (_ for _ in ()).throw(
|
|
RuntimeError("Failed to upload shot.png: bad <response>")
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
_mod,
|
|
"_api_request",
|
|
lambda url, token, method, payload: updates.append((
|
|
url,
|
|
token,
|
|
method,
|
|
payload,
|
|
)),
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="Failed to upload shot.png"):
|
|
_mod.publish(
|
|
"github-token",
|
|
"NousResearch/hermes-agent",
|
|
tmp_path,
|
|
"69868",
|
|
"image-token",
|
|
)
|
|
|
|
assert updates == [
|
|
(
|
|
"https://api.github.com/repos/NousResearch/hermes-agent/issues/comments/123",
|
|
"github-token",
|
|
"PATCH",
|
|
{
|
|
"body": "before\n<!-- hermes-e2e-evidence:start -->\n<sub>inline evidence upload failed.</sub>\n\n<pre>Failed to upload shot.png: bad <response></pre>\n<!-- hermes-e2e-evidence:end -->\nafter"
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_find_review_comment_requires_the_evidence_marker():
|
|
pending = "<!-- hermes-ci-review-bot -->\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->"
|
|
|
|
assert _mod._find_review_comment([{"body": "<!-- hermes-ci-review-bot --> no evidence"}]) is None
|
|
assert _mod._find_review_comment([{"body": pending, "id": 123}]) == {"body": pending, "id": 123}
|
|
|
|
|
|
def test_replace_evidence_marker_requires_exactly_one_marker():
|
|
with pytest.raises(ValueError, match="does not contain one"):
|
|
_mod.replace_evidence_marker("no marker", "evidence")
|