diff --git a/scripts/ci/publish_e2e_evidence.py b/scripts/ci/publish_e2e_evidence.py index c381a5990df8..94ad6af66594 100644 --- a/scripts/ci/publish_e2e_evidence.py +++ b/scripts/ci/publish_e2e_evidence.py @@ -11,10 +11,12 @@ source PR's CI review comment with those attachment URLs. from __future__ import annotations import argparse +import html import json import os import re import subprocess +import sys import time import urllib.request from dataclasses import dataclass @@ -168,6 +170,17 @@ def render_evidence(files: list[EvidenceFile], attachment_urls: dict[str, str]) return "\n".join(blocks) +def render_upload_failure(error: Exception) -> str: + """Render an escaped upload error inside the review-comment marker.""" + return "\n".join(( + EVIDENCE_START, + "inline evidence upload failed.", + "", + f"
{html.escape(str(error))}
", + EVIDENCE_END, + )) + + def replace_evidence_marker(comment: str, evidence: str) -> str: """Replace exactly the pending-evidence region in a CI review comment.""" pattern = re.compile(f"{re.escape(EVIDENCE_START)}.*?{re.escape(EVIDENCE_END)}", re.DOTALL) @@ -222,13 +235,31 @@ def upload_evidence( environment["GH_SESSION_TOKEN"] = session_token attachment_urls: dict[str, str] = {} for item in files: - result = subprocess.run( - ["gh", "image", "--repo", source_repo, str(evidence_dir / item.filename)], - check=True, - capture_output=True, - text=True, - env=environment, - ) + try: + result = subprocess.run( + [ + "gh", + "image", + "--repo", + source_repo, + str(evidence_dir / item.filename), + ], + check=True, + capture_output=True, + text=True, + env=environment, + ) + except subprocess.CalledProcessError as exc: + output = "; ".join( + value.strip() + for value in (exc.stdout, exc.stderr) + if value and value.strip() + ) + message = f"Failed to upload {item.filename} with gh image (exit code {exc.returncode})" + if output: + message = f"{message}: {output}" + print(message, file=sys.stderr) + raise RuntimeError(message) from exc match = _ATTACHMENT_URL.fullmatch(result.stdout.strip()) if match is None: raise ValueError(f"gh-image returned an invalid attachment reference for {item.filename}") @@ -249,7 +280,21 @@ def publish( print("No inline E2E evidence to publish.") return False comment = _wait_for_review_comment(token, source_repo, pr_number) - attachment_urls = upload_evidence(files, evidence_dir, source_repo, session_token) + try: + attachment_urls = upload_evidence( + files, evidence_dir, source_repo, session_token + ) + except Exception as exc: + body = replace_evidence_marker( + str(comment.get("body", "")), render_upload_failure(exc) + ) + _api_request( + f"{API_BASE}/repos/{source_repo}/issues/comments/{comment['id']}", + token, + method="PATCH", + payload={"body": body}, + ) + raise evidence = render_evidence(files, attachment_urls) body = replace_evidence_marker(str(comment.get("body", "")), evidence) _api_request( diff --git a/tests/ci/test_publish_e2e_evidence.py b/tests/ci/test_publish_e2e_evidence.py index 23e492e762de..63d24dc61442 100644 --- a/tests/ci/test_publish_e2e_evidence.py +++ b/tests/ci/test_publish_e2e_evidence.py @@ -124,6 +124,89 @@ def test_upload_evidence_rejects_unexpected_gh_image_output(tmp_path, monkeypatc ) +def test_upload_evidence_reports_gh_image_error(tmp_path, monkeypatch, capsys): + shot = tmp_path / "shot.png" + shot.write_bytes(_png()) + + def fake_run(args, **kwargs): + raise _mod.subprocess.CalledProcessError( + 1, + args, + output="upload output", + stderr="upload error", + ) + + monkeypatch.setattr(_mod.subprocess, "run", fake_run) + + with pytest.raises(RuntimeError, match="Failed to upload shot.png.*upload error"): + _mod.upload_evidence( + [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")], + tmp_path, + "NousResearch/hermes-agent", + "bot-session-token", + ) + + captured = capsys.readouterr() + assert "Failed to upload shot.png" in captured.err + assert "upload output" in captured.err + assert "upload error" in captured.err + + +def test_publish_marks_evidence_upload_failure_in_pr_comment(tmp_path, monkeypatch): + comment = { + "id": 123, + "body": "before\n\npending\n\nafter", + } + updates = [] + + monkeypatch.setattr( + _mod, + "load_evidence", + lambda evidence_dir: ( + [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")], + {}, + ), + ) + monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: comment) + monkeypatch.setattr( + _mod, + "upload_evidence", + lambda *args: (_ for _ in ()).throw( + RuntimeError("Failed to upload shot.png: bad ") + ), + ) + monkeypatch.setattr( + _mod, + "_api_request", + lambda url, token, method, payload: updates.append(( + url, + token, + method, + payload, + )), + ) + + with pytest.raises(RuntimeError, match="Failed to upload shot.png"): + _mod.publish( + "github-token", + "NousResearch/hermes-agent", + tmp_path, + "69868", + "image-token", + ) + + assert updates == [ + ( + "https://api.github.com/repos/NousResearch/hermes-agent/issues/comments/123", + "github-token", + "PATCH", + { + "body": "before\n\ninline evidence upload failed.\n\n
Failed to upload shot.png: bad <response>
\n\nafter" + }, + ) + ] + + def test_find_review_comment_requires_the_evidence_marker(): pending = "\n\npending\n"