mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-10-20 05:15:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

fix(desktop): link artifacts in E2E summary + upload all screenshots

Browse source 
The visual diff summary told reviewers to 'download and open to compare'
instead of linking to the actual run's artifacts page. Now links directly
to the run's /artifacts page and lists both artifacts with descriptions.

Also, screenshots that matched their baselines were never written to
test-results/, so the artifact only contained screenshots that diffed.
Now the actual screenshot is always written to the output dir regardless
of match/diff, so CI artifacts include every screenshot.
This commit is contained in:
ethernet 2026-07-16 19:14:28 -04:00
parent 92eb59c99d
commit 8b1738904d
2 changed files with 21 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

19
.github/workflows/e2e-desktop.yml vendored
View file

@ -113,10 +113,10 @@ jobs:
DIFF_COUNT=$(find test-results -name '*-diff.png' 2>/dev/null | wc -l)
ACTUAL_COUNT=$(find test-results -name '*-actual.png' 2>/dev/null | wc -l)
if [ "$DIFF_COUNT" -eq 0 ] && [ "$ACTUAL_COUNT" -eq 0 ]; then
echo "✅ All screenshots matched their baselines (or no baselines existed yet)." >> "$GITHUB_STEP_SUMMARY"
if [ "$DIFF_COUNT" -eq 0 ]; then
echo "✅ All $ACTUAL_COUNT screenshot(s) matched their baselines (or no baselines existed yet)." >> "$GITHUB_STEP_SUMMARY"
else
echo "📸 **$DIFF_COUNT screenshot(s) differ from baseline:**" >> "$GITHUB_STEP_SUMMARY"
echo "📸 **$DIFF_COUNT of $ACTUAL_COUNT screenshot(s) differ from baseline:**" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "| Test | Diff | Actual | Expected |" >> "$GITHUB_STEP_SUMMARY"
echo "|------|------|--------|----------|" >> "$GITHUB_STEP_SUMMARY"
@ -127,12 +127,17 @@ jobs:
test_name=$(basename "$base")
echo "| $test_name | [diff]($diff) | [actual](${base}-actual.png) | [expected](${base}-expected.png) |" >> "$GITHUB_STEP_SUMMARY"
done
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "Diff images are in the \`playwright-test-results\` artifact. Download and open to compare." >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "**To update baselines:** merge to main (baselines auto-update on main runs) or run \`npx playwright test --update-snapshots\` locally." >> "$GITHUB_STEP_SUMMARY"
fi
echo "" >> "$GITHUB_STEP_SUMMARY"
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "📥 Artifacts on the [run page](${RUN_URL}/artifacts):" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "- \`playwright-test-results\` — all screenshots (actual + expected + diff) + traces" >> "$GITHUB_STEP_SUMMARY"
echo "- \`playwright-report\` — interactive HTML report" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "**To update baselines:** merge to main (baselines auto-update on main runs) or run \`npx playwright test --update-snapshots\` locally." >> "$GITHUB_STEP_SUMMARY"
# Also parse the JSON report for pass/fail counts
if [ -f playwright-report/results.json ]; then
echo "" >> "$GITHUB_STEP_SUMMARY"

13
apps/desktop/e2e/visual-snapshot.ts
View file

@ -7,10 +7,10 @@
* as artifacts. This keeps visual regressions visible without gating PRs
* on pixel-perfect matches.
*
* When a screenshot matches the baseline, nothing happens. When it
* differs, this helper writes three images to the test's output dir:
* The actual screenshot is always written to the test output dir so CI
* artifacts include every screenshot — not just the ones that diffed.
* When it differs, this helper also writes expected and diff images:
* <name>-actual.png, <name>-expected.png, <name>-diff.png
* These are picked up by the "Upload test results" artifact step.
*/
import fs from 'node:fs'
import path from 'node:path'
@ -83,6 +83,8 @@ export async function expectVisualSnapshot(
if (info.config.updateSnapshots === 'all' || info.config.updateSnapshots === 'changed') {
fs.mkdirSync(path.dirname(baselinePath), { recursive: true })
fs.writeFileSync(baselinePath, actual)
// Also write to the output dir so CI artifacts include the screenshot.
fs.writeFileSync(info.outputPath(`${outputName}-actual.png`), actual)
console.log(`[visual-baseline] updated ${baselinePath}`)
return
}
@ -132,11 +134,14 @@ export async function expectVisualSnapshot(
{ actual: actual.toString('base64'), expected: expected.toString('base64') },
)
// Always write the actual screenshot to the output dir so CI artifacts
// include every screenshot — not just the ones that diffed.
fs.writeFileSync(info.outputPath(`${outputName}-actual.png`), actual)
if (comparison.mismatchRatio <= 0.01) {
return
}
fs.writeFileSync(info.outputPath(`${outputName}-actual.png`), actual)
fs.writeFileSync(info.outputPath(`${outputName}-expected.png`), expected)
fs.writeFileSync(info.outputPath(`${outputName}-diff.png`), Buffer.from(comparison.diff, 'base64'))
console.log(