mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
d4e940f241
Swap all `runs-on: ubuntu-latest` to `runs-on: arc-runner-set` all jobs.
The ARM docker build job in docker.yml uses `${{ matrix.runner }}`
and is left untouched since the GKE runner pool is x86_64 only.
Runners are backed by ARC (Actions Runner Controller) on a GKE cluster
with a spot preemptible node pool that scales based on job demand.
Use the baked Electron dependencies for the desktop E2E job.
253 lines
11 KiB
YAML
253 lines
11 KiB
YAML
name: E2E Desktop
|
|
|
|
on:
|
|
workflow_call:
|
|
outputs:
|
|
review_status:
|
|
description: Screenshot and visual-diff status for the CI review comment.
|
|
value: ${{ jobs.e2e.outputs.review_status }}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: e2e-desktop-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
e2e:
|
|
name: Playwright E2E (Linux)
|
|
runs-on: arc-runner-set
|
|
timeout-minutes: 20
|
|
outputs:
|
|
review_status: ${{ steps.review-status.outputs.review_status }}
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
# ── System deps for Electron on headless Ubuntu ───────────────────
|
|
# xvfb and Electron's GTK/NSS/audio libraries are baked into
|
|
# nousresearch/nous-gke-runner so this works in the ARC runner pod
|
|
# without requiring passwordless sudo.
|
|
|
|
# ── Node ───────────────────────────────────────────────────────────
|
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
|
with:
|
|
node-version: 22
|
|
cache: npm
|
|
# Full npm ci (not --ignore-scripts): electron's postinstall
|
|
# downloads the binary we launch, and node-pty's native build is
|
|
# needed for the terminal pane.
|
|
- uses: ./.github/actions/retry
|
|
with:
|
|
command: npm ci
|
|
|
|
# ── Python (for the hermes serve backend) ──────────────────────────
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
|
|
with:
|
|
# Pin the uv version: unpinned, setup-uv resolves "latest" by
|
|
# fetching a manifest from raw.githubusercontent.com on EVERY job —
|
|
# a transient fetch failure fails the whole job (2026-07-28 slice-5
|
|
# incident). Pinned, the binary downloads directly; no manifest hop.
|
|
version: "0.9.28"
|
|
enable-cache: true
|
|
cache-dependency-glob: |
|
|
pyproject.toml
|
|
uv.lock
|
|
- name: Set up Python 3.11
|
|
run: uv python install 3.11
|
|
- name: Install Python dependencies
|
|
uses: ./.github/actions/retry
|
|
with:
|
|
command: uv sync --locked --python 3.11 --extra all --extra dev
|
|
|
|
# ── Build desktop app ─────────────────────────────────────────────
|
|
# The Playwright step below runs `npm run build` before testing so
|
|
# dist/ is always fresh — no separate build step needed here.
|
|
|
|
# ── Restore visual baseline screenshots from main ──────────────────
|
|
# Baselines are generated on main (via --update-snapshots) and cached.
|
|
# On PRs, we restore them so toHaveScreenshot has something to compare
|
|
# against. The cache key is keyed on the desktop source files so a
|
|
# UI change naturally invalidates it — but we fall back to the main
|
|
# cache to avoid cold starts on unrelated PRs.
|
|
- name: Restore visual baseline screenshots
|
|
id: restore-baselines
|
|
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
|
|
with:
|
|
path: apps/desktop/e2e/*-snapshots
|
|
key: visual-baselines-${{ github.ref_name }}
|
|
restore-keys: |
|
|
visual-baselines-main
|
|
|
|
# ── Run Playwright E2E under xvfb ─────────────────────────────────
|
|
# xvfb runs at a fixed 1280x1024 screen so the 1220x800 Electron
|
|
# window always has a consistent viewport for screenshot comparison.
|
|
# On main, we run with --update-snapshots to generate baselines.
|
|
# `npm run test:e2e` builds dist/ as a pretest hook so the renderer
|
|
# is always fresh — no separate build step needed.
|
|
- name: Run Playwright E2E tests
|
|
working-directory: apps/desktop
|
|
run: |
|
|
if [ "${{ github.ref_name }}" = "main" ]; then
|
|
echo "On main — generating/updating baseline screenshots"
|
|
npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \
|
|
npx playwright test --reporter=list --update-snapshots
|
|
else
|
|
echo "On PR — comparing against cached baselines"
|
|
npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \
|
|
npx playwright test --reporter=list
|
|
fi
|
|
env:
|
|
CI: "true"
|
|
# Ensure no real API keys leak into the test env.
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|
|
|
|
# ── Save updated baselines to cache (main only) ───────────────────
|
|
- name: Save updated baselines to cache
|
|
if: github.ref_name == 'main' && always()
|
|
uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
|
|
with:
|
|
path: apps/desktop/e2e/*-snapshots
|
|
key: visual-baselines-main
|
|
|
|
# ── Upload Playwright report (HTML + traces) ──────────────────────
|
|
- name: Upload Playwright report
|
|
id: upload-report
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-${{ github.sha }}
|
|
path: apps/desktop/playwright-report
|
|
retention-days: 14
|
|
overwrite: true
|
|
|
|
# ── Upload test results (screenshots, traces, diffs) ───────────────
|
|
- name: Upload test results
|
|
id: upload-results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-test-results-${{ github.sha }}
|
|
path: apps/desktop/test-results
|
|
retention-days: 14
|
|
overwrite: true
|
|
|
|
# ── Upload just the visual diffs (small, fast to review) ──────────
|
|
- name: Upload visual diffs
|
|
id: upload-diffs
|
|
if: always() && github.ref_name != 'main'
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: visual-diffs-${{ github.sha }}
|
|
path: |
|
|
apps/desktop/test-results/**/*-diff.png
|
|
apps/desktop/test-results/**/*-actual.png
|
|
apps/desktop/test-results/**/*-expected.png
|
|
retention-days: 14
|
|
overwrite: true
|
|
if-no-files-found: ignore
|
|
|
|
- name: Build screenshot review status
|
|
id: review-status
|
|
if: always()
|
|
working-directory: apps/desktop
|
|
env:
|
|
RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }}
|
|
run: |
|
|
python3 ../../scripts/ci/e2e_screenshot_status.py \
|
|
--results-dir test-results \
|
|
--manifest-output /tmp/e2e-screenshot-manifest.json \
|
|
--evidence-dir /tmp/e2e-evidence \
|
|
--artifact-url "$RESULTS_URL" \
|
|
--output /tmp/e2e-review-status.json
|
|
{
|
|
echo 'review_status<<__E2E_REVIEW_STATUS__'
|
|
cat /tmp/e2e-review-status.json
|
|
echo '__E2E_REVIEW_STATUS__'
|
|
} >> "$GITHUB_OUTPUT"
|
|
|
|
# The trusted workflow_run publisher consumes only this flat, bounded
|
|
# artifact. It turns selected images into GitHub attachment URLs; it
|
|
# never checks out or runs this PR's code.
|
|
- name: Upload inline E2E evidence
|
|
if: always() && github.ref_name != 'main'
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: e2e-evidence-${{ github.sha }}
|
|
path: /tmp/e2e-evidence
|
|
retention-days: 14
|
|
overwrite: true
|
|
if-no-files-found: error
|
|
|
|
# ── Generate step summary with visual diff info ───────────────────
|
|
# Parse the JSON report + scan for diff images, then post a summary
|
|
# to the GitHub Actions step output so reviewers can see what changed
|
|
# without downloading artifacts. Runs AFTER uploads so it can link
|
|
# the artifact download URLs from their step outputs.
|
|
- name: Generate visual diff summary
|
|
if: always()
|
|
working-directory: apps/desktop
|
|
env:
|
|
REPORT_URL: ${{ steps.upload-report.outputs.artifact-url }}
|
|
RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }}
|
|
DIFFS_URL: ${{ steps.upload-diffs.outputs.artifact-url }}
|
|
run: |
|
|
{
|
|
echo "## Desktop E2E — Visual Diff Report"
|
|
echo ""
|
|
|
|
# Count diff images (playwright writes *-diff.png on mismatch)
|
|
DIFF_COUNT=$(find test-results -name '*-diff.png' 2>/dev/null | wc -l)
|
|
ACTUAL_COUNT=$(find test-results -name '*-actual.png' 2>/dev/null | wc -l)
|
|
|
|
if [ "$DIFF_COUNT" -eq 0 ]; then
|
|
echo "✅ All $ACTUAL_COUNT screenshot(s) matched their baselines (or no baselines existed yet)."
|
|
else
|
|
echo "📸 **$DIFF_COUNT of $ACTUAL_COUNT screenshot(s) differ from baseline:**"
|
|
echo ""
|
|
echo "| Test | Diff | Actual | Expected |"
|
|
echo "|------|------|--------|----------|"
|
|
|
|
# List each diff image with a link to the artifact
|
|
for diff in $(find test-results -name '*-diff.png' 2>/dev/null | sort); do
|
|
base=${diff%-diff.png}
|
|
test_name=$(basename "$base")
|
|
echo "| $test_name | [diff]($diff) | [actual](${base}-actual.png) | [expected](${base}-expected.png) |"
|
|
done
|
|
fi
|
|
|
|
echo ""
|
|
echo "📥 **Artifacts:**"
|
|
echo ""
|
|
if [ -n "$RESULTS_URL" ]; then
|
|
echo "- [playwright-test-results]($RESULTS_URL) — all screenshots (actual + expected + diff) + traces"
|
|
fi
|
|
if [ -n "$REPORT_URL" ]; then
|
|
echo "- [playwright-report]($REPORT_URL) — interactive HTML report"
|
|
fi
|
|
if [ -n "$DIFFS_URL" ]; then
|
|
echo "- [visual-diffs]($DIFFS_URL) — just the diffed screenshots (small, fast to review)"
|
|
fi
|
|
echo ""
|
|
echo "**To update baselines:** merge to main (baselines auto-update on main runs) or run \`npx playwright test --update-snapshots\` locally."
|
|
|
|
# Also parse the JSON report for pass/fail counts
|
|
if [ -f playwright-report/results.json ]; then
|
|
echo ""
|
|
echo "### Test Results"
|
|
echo ""
|
|
node -e "
|
|
const r = require('./playwright-report/results.json');
|
|
const stats = r.stats || {};
|
|
console.log('| Status | Count |');
|
|
console.log('|--------|-------|');
|
|
console.log('| ✅ Passed | ' + (stats.expected || 0) + ' |');
|
|
console.log('| ❌ Failed | ' + (stats.unexpected || 0) + ' |');
|
|
console.log('| ⏭️ Skipped | ' + (stats.skipped || 0) + ' |');
|
|
console.log('| 🔄 Flaky | ' + (stats.flaky || 0) + ' |');
|
|
" 2>/dev/null || true
|
|
fi
|
|
} >> "$GITHUB_STEP_SUMMARY"
|