fix(ci): report E2E evidence upload failures (#69901)

Print gh-image stdout and stderr on attachment failures, then replace the
pending inline-evidence marker in the PR review comment with an escaped
failure notice before preserving the failing workflow result.

scripts/ci/publish_e2e_evidence.py

@@ -11,10 +11,12 @@ source PR's CI review comment with those attachment URLs.
 from __future__ import annotations
 
 import argparse
+import html
 import json
 import os
 import re
 import subprocess
+import sys
 import time
 import urllib.request
 from dataclasses import dataclass
@@ -168,6 +170,17 @@ def render_evidence(files: list[EvidenceFile], attachment_urls: dict[str, str])
     return "\n".join(blocks)
 
 
+def render_upload_failure(error: Exception) -> str:
+    """Render an escaped upload error inside the review-comment marker."""
+    return "\n".join((
+        EVIDENCE_START,
+        "<sub>inline evidence upload failed.</sub>",
+        "",
+        f"<pre>{html.escape(str(error))}</pre>",
+        EVIDENCE_END,
+    ))
+
+
 def replace_evidence_marker(comment: str, evidence: str) -> str:
     """Replace exactly the pending-evidence region in a CI review comment."""
     pattern = re.compile(f"{re.escape(EVIDENCE_START)}.*?{re.escape(EVIDENCE_END)}", re.DOTALL)
@@ -222,13 +235,31 @@ def upload_evidence(
     environment["GH_SESSION_TOKEN"] = session_token
     attachment_urls: dict[str, str] = {}
     for item in files:
-        result = subprocess.run(
-            ["gh", "image", "--repo", source_repo, str(evidence_dir / item.filename)],
-            check=True,
-            capture_output=True,
-            text=True,
-            env=environment,
-        )
+        try:
+            result = subprocess.run(
+                [
+                    "gh",
+                    "image",
+                    "--repo",
+                    source_repo,
+                    str(evidence_dir / item.filename),
+                ],
+                check=True,
+                capture_output=True,
+                text=True,
+                env=environment,
+            )
+        except subprocess.CalledProcessError as exc:
+            output = "; ".join(
+                value.strip()
+                for value in (exc.stdout, exc.stderr)
+                if value and value.strip()
+            )
+            message = f"Failed to upload {item.filename} with gh image (exit code {exc.returncode})"
+            if output:
+                message = f"{message}: {output}"
+            print(message, file=sys.stderr)
+            raise RuntimeError(message) from exc
         match = _ATTACHMENT_URL.fullmatch(result.stdout.strip())
         if match is None:
             raise ValueError(f"gh-image returned an invalid attachment reference for {item.filename}")
@@ -249,7 +280,21 @@ def publish(
         print("No inline E2E evidence to publish.")
         return False
     comment = _wait_for_review_comment(token, source_repo, pr_number)
-    attachment_urls = upload_evidence(files, evidence_dir, source_repo, session_token)
+    try:
+        attachment_urls = upload_evidence(
+            files, evidence_dir, source_repo, session_token
+        )
+    except Exception as exc:
+        body = replace_evidence_marker(
+            str(comment.get("body", "")), render_upload_failure(exc)
+        )
+        _api_request(
+            f"{API_BASE}/repos/{source_repo}/issues/comments/{comment['id']}",
+            token,
+            method="PATCH",
+            payload={"body": body},
+        )
+        raise
     evidence = render_evidence(files, attachment_urls)
     body = replace_evidence_marker(str(comment.get("body", "")), evidence)
     _api_request(

tests/ci/test_publish_e2e_evidence.py

@@ -124,6 +124,89 @@ def test_upload_evidence_rejects_unexpected_gh_image_output(tmp_path, monkeypatc
         )
 
 
+def test_upload_evidence_reports_gh_image_error(tmp_path, monkeypatch, capsys):
+    shot = tmp_path / "shot.png"
+    shot.write_bytes(_png())
+
+    def fake_run(args, **kwargs):
+        raise _mod.subprocess.CalledProcessError(
+            1,
+            args,
+            output="upload output",
+            stderr="upload error",
+        )
+
+    monkeypatch.setattr(_mod.subprocess, "run", fake_run)
+
+    with pytest.raises(RuntimeError, match="Failed to upload shot.png.*upload error"):
+        _mod.upload_evidence(
+            [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+            tmp_path,
+            "NousResearch/hermes-agent",
+            "bot-session-token",
+        )
+
+    captured = capsys.readouterr()
+    assert "Failed to upload shot.png" in captured.err
+    assert "upload output" in captured.err
+    assert "upload error" in captured.err
+
+
+def test_publish_marks_evidence_upload_failure_in_pr_comment(tmp_path, monkeypatch):
+    comment = {
+        "id": 123,
+        "body": "before\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->\nafter",
+    }
+    updates = []
+
+    monkeypatch.setattr(
+        _mod,
+        "load_evidence",
+        lambda evidence_dir: (
+            [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+            {},
+        ),
+    )
+    monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: comment)
+    monkeypatch.setattr(
+        _mod,
+        "upload_evidence",
+        lambda *args: (_ for _ in ()).throw(
+            RuntimeError("Failed to upload shot.png: bad <response>")
+        ),
+    )
+    monkeypatch.setattr(
+        _mod,
+        "_api_request",
+        lambda url, token, method, payload: updates.append((
+            url,
+            token,
+            method,
+            payload,
+        )),
+    )
+
+    with pytest.raises(RuntimeError, match="Failed to upload shot.png"):
+        _mod.publish(
+            "github-token",
+            "NousResearch/hermes-agent",
+            tmp_path,
+            "69868",
+            "image-token",
+        )
+
+    assert updates == [
+        (
+            "https://api.github.com/repos/NousResearch/hermes-agent/issues/comments/123",
+            "github-token",
+            "PATCH",
+            {
+                "body": "before\n<!-- hermes-e2e-evidence:start -->\n<sub>inline evidence upload failed.</sub>\n\n<pre>Failed to upload shot.png: bad &lt;response&gt;</pre>\n<!-- hermes-e2e-evidence:end -->\nafter"
+            },
+        )
+    ]
+
+
 def test_find_review_comment_requires_the_evidence_marker():
     pending = "<!-- hermes-ci-review-bot -->\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->"