diff --git a/scripts/ci/publish_e2e_evidence.py b/scripts/ci/publish_e2e_evidence.py index c381a5990df8..94ad6af66594 100644 --- a/scripts/ci/publish_e2e_evidence.py +++ b/scripts/ci/publish_e2e_evidence.py @@ -11,10 +11,12 @@ source PR's CI review comment with those attachment URLs. from __future__ import annotations import argparse +import html import json import os import re import subprocess +import sys import time import urllib.request from dataclasses import dataclass @@ -168,6 +170,17 @@ def render_evidence(files: list[EvidenceFile], attachment_urls: dict[str, str]) return "\n".join(blocks) +def render_upload_failure(error: Exception) -> str: + """Render an escaped upload error inside the review-comment marker.""" + return "\n".join(( + EVIDENCE_START, + "inline evidence upload failed.", + "", + f"
{html.escape(str(error))}",
+ EVIDENCE_END,
+ ))
+
+
def replace_evidence_marker(comment: str, evidence: str) -> str:
"""Replace exactly the pending-evidence region in a CI review comment."""
pattern = re.compile(f"{re.escape(EVIDENCE_START)}.*?{re.escape(EVIDENCE_END)}", re.DOTALL)
@@ -222,13 +235,31 @@ def upload_evidence(
environment["GH_SESSION_TOKEN"] = session_token
attachment_urls: dict[str, str] = {}
for item in files:
- result = subprocess.run(
- ["gh", "image", "--repo", source_repo, str(evidence_dir / item.filename)],
- check=True,
- capture_output=True,
- text=True,
- env=environment,
- )
+ try:
+ result = subprocess.run(
+ [
+ "gh",
+ "image",
+ "--repo",
+ source_repo,
+ str(evidence_dir / item.filename),
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ env=environment,
+ )
+ except subprocess.CalledProcessError as exc:
+ output = "; ".join(
+ value.strip()
+ for value in (exc.stdout, exc.stderr)
+ if value and value.strip()
+ )
+ message = f"Failed to upload {item.filename} with gh image (exit code {exc.returncode})"
+ if output:
+ message = f"{message}: {output}"
+ print(message, file=sys.stderr)
+ raise RuntimeError(message) from exc
match = _ATTACHMENT_URL.fullmatch(result.stdout.strip())
if match is None:
raise ValueError(f"gh-image returned an invalid attachment reference for {item.filename}")
@@ -249,7 +280,21 @@ def publish(
print("No inline E2E evidence to publish.")
return False
comment = _wait_for_review_comment(token, source_repo, pr_number)
- attachment_urls = upload_evidence(files, evidence_dir, source_repo, session_token)
+ try:
+ attachment_urls = upload_evidence(
+ files, evidence_dir, source_repo, session_token
+ )
+ except Exception as exc:
+ body = replace_evidence_marker(
+ str(comment.get("body", "")), render_upload_failure(exc)
+ )
+ _api_request(
+ f"{API_BASE}/repos/{source_repo}/issues/comments/{comment['id']}",
+ token,
+ method="PATCH",
+ payload={"body": body},
+ )
+ raise
evidence = render_evidence(files, attachment_urls)
body = replace_evidence_marker(str(comment.get("body", "")), evidence)
_api_request(
diff --git a/tests/ci/test_publish_e2e_evidence.py b/tests/ci/test_publish_e2e_evidence.py
index 23e492e762de..63d24dc61442 100644
--- a/tests/ci/test_publish_e2e_evidence.py
+++ b/tests/ci/test_publish_e2e_evidence.py
@@ -124,6 +124,89 @@ def test_upload_evidence_rejects_unexpected_gh_image_output(tmp_path, monkeypatc
)
+def test_upload_evidence_reports_gh_image_error(tmp_path, monkeypatch, capsys):
+ shot = tmp_path / "shot.png"
+ shot.write_bytes(_png())
+
+ def fake_run(args, **kwargs):
+ raise _mod.subprocess.CalledProcessError(
+ 1,
+ args,
+ output="upload output",
+ stderr="upload error",
+ )
+
+ monkeypatch.setattr(_mod.subprocess, "run", fake_run)
+
+ with pytest.raises(RuntimeError, match="Failed to upload shot.png.*upload error"):
+ _mod.upload_evidence(
+ [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+ tmp_path,
+ "NousResearch/hermes-agent",
+ "bot-session-token",
+ )
+
+ captured = capsys.readouterr()
+ assert "Failed to upload shot.png" in captured.err
+ assert "upload output" in captured.err
+ assert "upload error" in captured.err
+
+
+def test_publish_marks_evidence_upload_failure_in_pr_comment(tmp_path, monkeypatch):
+ comment = {
+ "id": 123,
+ "body": "before\n\npending\n\nafter",
+ }
+ updates = []
+
+ monkeypatch.setattr(
+ _mod,
+ "load_evidence",
+ lambda evidence_dir: (
+ [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+ {},
+ ),
+ )
+ monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: comment)
+ monkeypatch.setattr(
+ _mod,
+ "upload_evidence",
+ lambda *args: (_ for _ in ()).throw(
+ RuntimeError("Failed to upload shot.png: bad Failed to upload shot.png: bad <response>\n\nafter" + }, + ) + ] + + def test_find_review_comment_requires_the_evidence_marker(): pending = "\n\npending\n"