mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-22 16:25:58 +00:00
feat(desktop/e2e): Playwright E2E suite with visual regression diffs
Adds a full desktop Playwright E2E suite that launches the Electron app against a mock inference server, exercising the full boot chain: electron -> hermes serve -> mock provider -> renderer Includes: - Mock OpenAI-compatible inference server (mock-server.ts) - Shared fixtures with sandbox isolation (credentials, HERMES_HOME, userData, fixed window-state.json for reproducible screenshots) - Test specs: boot, boot-failure, onboarding, mock-backend-setup, chat, and packaged-app launch - Visual regression: expectVisualSnapshot() wraps toHaveScreenshot in try/catch so diffs are reported without failing the test suite - CI workflow: xvfb at 1280x1024, baseline cache from main (--update-snapshots on main, compare on PRs), step summary table with diff/actual/expected image links, dedicated visual-diffs artifact - dev:mock script for local fake-provider development - test:e2e:visual + test:e2e:update-snapshots scripts using cage - .gitignore: *-snapshots/ (baselines cached in CI, not committed)
This commit is contained in:
parent
c5111388c7
commit
0860ee4e5a
17 changed files with 1990 additions and 1 deletions
7
.github/workflows/ci.yml
vendored
7
.github/workflows/ci.yml
vendored
|
|
@ -87,6 +87,12 @@ jobs:
|
|||
uses: ./.github/workflows/js-tests.yml
|
||||
secrets: inherit
|
||||
|
||||
e2e-desktop:
|
||||
name: Desktop E2E
|
||||
needs: detect
|
||||
if: needs.detect.outputs.python == 'true' || needs.detect.outputs.frontend == 'true'
|
||||
uses: ./.github/workflows/e2e-desktop.yml
|
||||
|
||||
docs-site:
|
||||
name: Docs Site
|
||||
needs: detect
|
||||
|
|
@ -165,6 +171,7 @@ jobs:
|
|||
- tests
|
||||
- lint
|
||||
- js-tests
|
||||
- e2e-desktop
|
||||
- docs-site
|
||||
- history-check
|
||||
- contributor-check
|
||||
|
|
|
|||
193
.github/workflows/e2e-desktop.yml
vendored
Normal file
193
.github/workflows/e2e-desktop.yml
vendored
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
name: E2E Desktop
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: e2e-desktop-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
name: Playwright E2E (Linux)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# ── System deps for Electron on headless Ubuntu ───────────────────
|
||||
# Electron needs GTK, NSS,atk, etc. even under xvfb. Playwright's
|
||||
# install-deps covers browsers; for Electron we install the apt
|
||||
# packages directly.
|
||||
- name: Install system dependencies for Electron
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq \
|
||||
xvfb \
|
||||
libgtk-3-0 libnotify4 libnss3 libxss1 libxtst6 \
|
||||
xdg-utils libatspi2.0-0 libdrm2 libgbm1 libasound2t64
|
||||
|
||||
# ── Node ───────────────────────────────────────────────────────────
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
# Full npm ci (not --ignore-scripts): electron's postinstall
|
||||
# downloads the binary we launch, and node-pty's native build is
|
||||
# needed for the terminal pane.
|
||||
- uses: ./.github/actions/retry
|
||||
with:
|
||||
command: npm ci
|
||||
|
||||
# ── Python (for the hermes serve backend) ──────────────────────────
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
- name: Set up Python 3.11
|
||||
run: uv python install 3.11
|
||||
- name: Install Python dependencies
|
||||
uses: ./.github/actions/retry
|
||||
with:
|
||||
command: uv sync --locked --python 3.11 --extra all --extra dev
|
||||
|
||||
# ── Build desktop app ─────────────────────────────────────────────
|
||||
- run: npm run --prefix apps/desktop build
|
||||
|
||||
# ── Restore visual baseline screenshots from main ──────────────────
|
||||
# Baselines are generated on main (via --update-snapshots) and cached.
|
||||
# On PRs, we restore them so toHaveScreenshot has something to compare
|
||||
# against. The cache key is keyed on the desktop source files so a
|
||||
# UI change naturally invalidates it — but we fall back to the main
|
||||
# cache to avoid cold starts on unrelated PRs.
|
||||
- name: Restore visual baseline screenshots
|
||||
id: restore-baselines
|
||||
uses: actions/cache@0400d5f6a4f407c1b1b78f4ddd5bffb6548ef6f6 # v4.2.4
|
||||
with:
|
||||
path: apps/desktop/e2e/__screenshots__
|
||||
key: visual-baselines-${{ github.ref_name }}
|
||||
restore-keys: |
|
||||
visual-baselines-main
|
||||
|
||||
# ── Run Playwright E2E under xvfb ─────────────────────────────────
|
||||
# xvfb runs at a fixed 1280x1024 screen so the 1220x800 Electron
|
||||
# window always has a consistent viewport for screenshot comparison.
|
||||
# On main, we run with --update-snapshots to generate baselines.
|
||||
- name: Run Playwright E2E tests
|
||||
working-directory: apps/desktop
|
||||
run: |
|
||||
if [ "${{ github.ref_name }}" = "main" ]; then
|
||||
echo "On main — generating/updating baseline screenshots"
|
||||
xvfb-run -a --server-args="-screen 0 1280x1024x24" \
|
||||
npx playwright test --reporter=list --update-snapshots
|
||||
else
|
||||
echo "On PR — comparing against cached baselines"
|
||||
xvfb-run -a --server-args="-screen 0 1280x1024x24" \
|
||||
npx playwright test --reporter=list
|
||||
fi
|
||||
env:
|
||||
CI: "true"
|
||||
# Ensure no real API keys leak into the test env.
|
||||
OPENROUTER_API_KEY: ""
|
||||
OPENAI_API_KEY: ""
|
||||
NOUS_API_KEY: ""
|
||||
|
||||
# ── Generate step summary with visual diff info ───────────────────
|
||||
# Parse the JSON report + scan for diff images, then post a summary
|
||||
# to the GitHub Actions step output so reviewers can see what changed
|
||||
# without downloading artifacts.
|
||||
- name: Generate visual diff summary
|
||||
if: always()
|
||||
working-directory: apps/desktop
|
||||
run: |
|
||||
echo "## Desktop E2E — Visual Diff Report" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Count diff images (playwright writes *-diff.png on mismatch)
|
||||
DIFF_COUNT=$(find test-results -name '*-diff.png' 2>/dev/null | wc -l)
|
||||
ACTUAL_COUNT=$(find test-results -name '*-actual.png' 2>/dev/null | wc -l)
|
||||
|
||||
if [ "$DIFF_COUNT" -eq 0 ] && [ "$ACTUAL_COUNT" -eq 0 ]; then
|
||||
echo "✅ All screenshots matched their baselines (or no baselines existed yet)." >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "📸 **$DIFF_COUNT screenshot(s) differ from baseline:**" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Test | Diff | Actual | Expected |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "|------|------|--------|----------|" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# List each diff image with a link to the artifact
|
||||
for diff in $(find test-results -name '*-diff.png' 2>/dev/null | sort); do
|
||||
base=$(echo "$diff" | sed 's/-diff\.png$//')
|
||||
test_name=$(basename "$base")
|
||||
echo "| $test_name | [diff]($diff) | [actual](${base}-actual.png) | [expected](${base}-expected.png) |" >> "$GITHUB_STEP_SUMMARY"
|
||||
done
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Diff images are in the \`playwright-test-results\` artifact. Download and open to compare." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "**To update baselines:** merge to main (baselines auto-update on main runs) or run \`npx playwright test --update-snapshots\` locally." >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
# Also parse the JSON report for pass/fail counts
|
||||
if [ -f playwright-report/results.json ]; then
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "### Test Results" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
node -e "
|
||||
const r = require('./playwright-report/results.json');
|
||||
const stats = r.stats || {};
|
||||
console.log('| Status | Count |');
|
||||
console.log('|--------|-------|');
|
||||
console.log('| ✅ Passed | ' + (stats.expected || 0) + ' |');
|
||||
console.log('| ❌ Failed | ' + (stats.unexpected || 0) + ' |');
|
||||
console.log('| ⏭️ Skipped | ' + (stats.skipped || 0) + ' |');
|
||||
console.log('| 🔄 Flaky | ' + (stats.flaky || 0) + ' |');
|
||||
" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ── Save updated baselines to cache (main only) ───────────────────
|
||||
- name: Save updated baselines to cache
|
||||
if: github.ref_name == 'main' && always()
|
||||
uses: actions/cache/save@0400d5f6a4f407c1b1b78f4ddd5bffb6548ef6f6 # v4.2.4
|
||||
with:
|
||||
path: apps/desktop/e2e/__screenshots__
|
||||
key: visual-baselines-main
|
||||
|
||||
# ── Upload Playwright report (HTML + traces) ──────────────────────
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-report-${{ github.sha }}
|
||||
path: apps/desktop/playwright-report
|
||||
retention-days: 14
|
||||
overwrite: true
|
||||
|
||||
# ── Upload test results (screenshots, traces, diffs) ───────────────
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: playwright-test-results-${{ github.sha }}
|
||||
path: apps/desktop/test-results
|
||||
retention-days: 14
|
||||
overwrite: true
|
||||
|
||||
# ── Upload just the visual diffs (small, fast to review) ──────────
|
||||
- name: Upload visual diffs
|
||||
if: always() && github.ref_name != 'main'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: visual-diffs-${{ github.sha }}
|
||||
path: |
|
||||
apps/desktop/test-results/**/*-diff.png
|
||||
apps/desktop/test-results/**/*-actual.png
|
||||
apps/desktop/test-results/**/*-expected.png
|
||||
retention-days: 14
|
||||
overwrite: true
|
||||
if-no-files-found: ignore
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -4,6 +4,8 @@
|
|||
/_pycache/
|
||||
*.pyc*
|
||||
__pycache__/
|
||||
act/
|
||||
.act-sandbox-agent.*
|
||||
.venv/
|
||||
.venv
|
||||
.vscode/
|
||||
|
|
@ -54,6 +56,10 @@ __pycache__/
|
|||
hermes_agent.egg-info/
|
||||
wandb/
|
||||
testlogs
|
||||
playwright-report/
|
||||
test-results/
|
||||
# Playwright visual regression baselines — cached from main in CI, not committed
|
||||
*-snapshots/
|
||||
|
||||
# CLI config (may contain sensitive SSH paths)
|
||||
cli-config.yaml
|
||||
|
|
|
|||
52
apps/desktop/e2e/boot-failure.spec.ts
Normal file
52
apps/desktop/e2e/boot-failure.spec.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
/**
|
||||
* E2E boot-failure tests — verify the app shows an error overlay when the
|
||||
* backend can't reach the inference provider.
|
||||
*
|
||||
* Launches the app with a provider pointing at a dead endpoint (port 1).
|
||||
* The `hermes serve` backend starts, but when the renderer tries to connect
|
||||
* or when a runtime check fails, the app should show a boot failure or
|
||||
* onboarding error overlay.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*/
|
||||
|
||||
import { test } from '@playwright/test'
|
||||
|
||||
import {
|
||||
type DeadBackendFixture,
|
||||
setupDeadBackend,
|
||||
waitForBootFailure,
|
||||
} from './fixtures'
|
||||
import { expectVisualSnapshot } from './visual-snapshot'
|
||||
|
||||
let fixture: DeadBackendFixture | null = null
|
||||
|
||||
test.afterAll(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test.describe('boot failure with dead provider endpoint', () => {
|
||||
test('app shows error state or onboarding', async () => {
|
||||
fixture = await setupDeadBackend()
|
||||
|
||||
// With a dead provider endpoint, the app should eventually show either:
|
||||
// 1. A boot failure overlay (if the backend fails to start), or
|
||||
// 2. An onboarding overlay with an error (if the runtime check fails)
|
||||
// Both outcomes prove the app is handling provider failures gracefully.
|
||||
//
|
||||
// We give it a generous timeout — the backend needs to start, the
|
||||
// renderer needs to boot, and then the runtime check needs to fail.
|
||||
await waitForBootFailure(fixture.page, 90_000)
|
||||
})
|
||||
|
||||
test('screenshot of error state', async () => {
|
||||
if (!fixture) {
|
||||
test.skip(true, 'Previous test failed — no app running')
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
await expectVisualSnapshot(fixture!.page, { name: 'boot-failure-error-state', app: fixture.app })
|
||||
})
|
||||
})
|
||||
63
apps/desktop/e2e/boot.spec.ts
Normal file
63
apps/desktop/e2e/boot.spec.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
/**
|
||||
* E2E smoke tests for the dev-mode desktop app.
|
||||
*
|
||||
* These tests launch the Electron app from the built dist/ (not the
|
||||
* packaged binary) with a real `hermes serve` backend pointed at a mock
|
||||
* inference server. The full chain is exercised:
|
||||
*
|
||||
* electron → hermes serve (python) → mock provider → renderer
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
* Run from the nix devshell:
|
||||
* npm exec playwright test e2e/boot.spec.ts --reporter=list
|
||||
*/
|
||||
import { expect, test } from '@playwright/test'
|
||||
|
||||
import {
|
||||
type MockBackendFixture,
|
||||
setupMockBackend,
|
||||
waitForAppReady,
|
||||
} from './fixtures'
|
||||
import { expectVisualSnapshot } from './visual-snapshot'
|
||||
|
||||
let fixture: MockBackendFixture | null = null
|
||||
|
||||
test.beforeAll(async () => {
|
||||
fixture = await setupMockBackend()
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test.describe('dev-mode boot with mock backend', () => {
|
||||
test('window opens with Hermes title', async () => {
|
||||
const title = await fixture!.page.title()
|
||||
expect(title).toContain('Hermes')
|
||||
})
|
||||
|
||||
test('renderer mounts and shows DOM content', async () => {
|
||||
const page = fixture!.page
|
||||
// Wait for the React root to mount. The app renders into #root
|
||||
// (see src/main.tsx), but content may arrive through portals — so
|
||||
// check the body for any interactive content instead.
|
||||
await page.waitForSelector('body', { state: 'attached' })
|
||||
// Wait for the main app shell — the composer is always present.
|
||||
await page.waitForSelector('textarea, [contenteditable="true"]', {
|
||||
state: 'attached',
|
||||
timeout: 30_000,
|
||||
})
|
||||
})
|
||||
|
||||
test('backend boots and app becomes ready', async () => {
|
||||
// This is the big one — wait for the full boot chain to complete:
|
||||
// electron starts → hermes serve is spawned → WS connects → config
|
||||
// loaded → sessions loaded → boot overlay dismissed → composer visible.
|
||||
await waitForAppReady(fixture!, 120_000)
|
||||
})
|
||||
|
||||
test('screenshot after boot', async () => {
|
||||
await expectVisualSnapshot(fixture!.page, { name: 'boot-ready', app: fixture!.app })
|
||||
})
|
||||
})
|
||||
89
apps/desktop/e2e/chat.spec.ts
Normal file
89
apps/desktop/e2e/chat.spec.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
/**
|
||||
* E2E chat tests — send a message and verify a response appears.
|
||||
*
|
||||
* Requires the full boot chain to complete (hermes serve + mock inference
|
||||
* provider). The mock server returns a canned reply, so we verify the
|
||||
* response text shows up in the chat transcript.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*/
|
||||
|
||||
import { test } from '@playwright/test'
|
||||
|
||||
import {
|
||||
type MockBackendFixture,
|
||||
setupMockBackend,
|
||||
waitForAppReady,
|
||||
} from './fixtures'
|
||||
import { expectVisualSnapshot } from './visual-snapshot'
|
||||
|
||||
let fixture: MockBackendFixture | null = null
|
||||
|
||||
test.beforeAll(async () => {
|
||||
fixture = await setupMockBackend()
|
||||
await waitForAppReady(fixture!, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test.describe('chat interaction with mock backend', () => {
|
||||
test('send a message and receive a response', async () => {
|
||||
const page = fixture!.page
|
||||
|
||||
// Find the composer — it's a contenteditable textbox.
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
await composer.waitFor({ state: 'visible', timeout: 10_000 })
|
||||
|
||||
// Click to focus, then type the message character by character.
|
||||
// Using `type` instead of `fill` because the composer is a
|
||||
// contenteditable div with custom keydown handling that tracks
|
||||
// IME composition state — `fill` bypasses the event chain.
|
||||
await composer.click()
|
||||
await composer.type('Hello, can you hear me?', { delay: 20 })
|
||||
|
||||
// Submit with Enter — the composer's keydown handler intercepts
|
||||
// plain Enter (without Shift) and calls submitDraft().
|
||||
await page.keyboard.press('Enter')
|
||||
|
||||
// Wait for the user's message to appear in the transcript.
|
||||
// The message renders as an assistant-ui message in the chat view.
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const body = document.body
|
||||
|
||||
if (!body) {
|
||||
return false
|
||||
}
|
||||
|
||||
return (body.textContent ?? '').includes('Hello, can you hear me?')
|
||||
},
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
|
||||
// Wait for the mock response to appear. The canned reply is:
|
||||
// "Hello from the mock inference server! The full boot chain is working."
|
||||
// Give it a generous timeout — the inference request goes through the
|
||||
// gateway → hermes serve → mock server → streaming SSE back.
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const body = document.body
|
||||
|
||||
if (!body) {
|
||||
return false
|
||||
}
|
||||
|
||||
const text = body.textContent ?? ''
|
||||
|
||||
return text.includes('mock inference server') || text.includes('boot chain is working')
|
||||
},
|
||||
{ timeout: 60_000 },
|
||||
)
|
||||
})
|
||||
|
||||
test('screenshot of chat with messages', async () => {
|
||||
await expectVisualSnapshot(fixture!.page, { name: 'chat-with-messages', app: fixture!.app })
|
||||
})
|
||||
})
|
||||
66
apps/desktop/e2e/fix-electron-tracing.ts
Normal file
66
apps/desktop/e2e/fix-electron-tracing.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
/**
|
||||
* Monkey-patch: playwright's test runner never calls tracing.start() on
|
||||
* Electron's internal BrowserContext because:
|
||||
* 1. Playwright._allContexts() only returns [chromium, firefox, webkit]
|
||||
* contexts — Electron's context is excluded.
|
||||
* 2. ArtifactsRecorder.didCreateBrowserContext runs in willStartTest, before
|
||||
* beforeAll launches the electron app.
|
||||
* 3. The runAfterCreateBrowserContext hook doesn't exist on the Electron
|
||||
* class (only on BrowserType).
|
||||
*
|
||||
* As a result, trace screenshots (screencast) and DOM snapshots are never
|
||||
* captured for electron tests.
|
||||
*
|
||||
* This patch:
|
||||
* 1. Patches _allContexts() to include electron contexts, so the test
|
||||
* runner's didFinishTest() cleanup calls _stopTracing() → stopChunk()
|
||||
* on the electron context (saving the trace chunk + merging it into
|
||||
* the final trace.zip).
|
||||
* 2. Manually calls tracing.start() + startChunk() after launch.
|
||||
* 3. Wraps tracing.start to become startChunk after the first call,
|
||||
* so the test runner's willStartTest doesn't throw "already started".
|
||||
*
|
||||
* Imported from playwright.config.ts so it runs before any test.
|
||||
*/
|
||||
|
||||
import { _electron as electron, type BrowserContext } from '@playwright/test'
|
||||
import * as crypto from 'node:crypto'
|
||||
|
||||
const electronContexts = new Set<BrowserContext>()
|
||||
const originalLaunch = electron.launch.bind(electron)
|
||||
|
||||
electron.launch = async (options: any) => {
|
||||
const app = await originalLaunch(options)
|
||||
const ctx = app._context as BrowserContext
|
||||
electronContexts.add(ctx)
|
||||
ctx.once('close', () => electronContexts.delete(ctx))
|
||||
|
||||
// Patch _allContexts so the test runner sees the electron context
|
||||
// (didFinishTest cleanup → _stopTracing → stopChunk → merge into trace.zip).
|
||||
const pw = electron._playwright as any
|
||||
if (pw && !pw.__electronTracingPatched) {
|
||||
pw.__electronTracingPatched = true
|
||||
const original = pw._allContexts.bind(pw)
|
||||
pw._allContexts = () => [...original(), ...electronContexts]
|
||||
}
|
||||
|
||||
// Start tracing — mirrors ArtifactsRecorder.didCreateBrowserContext.
|
||||
const traceName = crypto.randomUUID()
|
||||
await ctx.tracing.start({
|
||||
screenshots: true,
|
||||
snapshots: true,
|
||||
sources: true,
|
||||
}).catch(() => {})
|
||||
await ctx.tracing.startChunk({ title: 'electron', name: traceName }).catch(() => {})
|
||||
|
||||
// Wrap tracing.start to redirect to startChunk after the first call.
|
||||
// The test runner's willStartTest calls tracing.start() on all contexts
|
||||
// in _allContexts(). Since we already started, redirect to startChunk
|
||||
// to avoid "Tracing has been already started" errors.
|
||||
const tracing = ctx.tracing as any
|
||||
tracing.start = async (opts: any) => {
|
||||
return tracing.startChunk(opts)
|
||||
}
|
||||
|
||||
return app
|
||||
}
|
||||
613
apps/desktop/e2e/fixtures.ts
Normal file
613
apps/desktop/e2e/fixtures.ts
Normal file
|
|
@ -0,0 +1,613 @@
|
|||
/**
|
||||
* Shared E2E fixtures for the Hermes desktop Playwright suite.
|
||||
*
|
||||
* Two fixture modes:
|
||||
*
|
||||
* 1. `mockBackend` — starts a mock inference server, writes a config.yaml
|
||||
* that points at it, and launches the desktop app so the full chain
|
||||
* (electron → hermes serve → provider → inference → renderer) is
|
||||
* exercised with a real backend but a fake LLM.
|
||||
*
|
||||
* 2. `noProvider` — launches the app with an empty config (no provider
|
||||
* configured). The onboarding overlay should appear. Used to test the
|
||||
* first-run flow without real credentials.
|
||||
*
|
||||
* Both modes launch the *dev* Electron app (`electron .` against the built
|
||||
* `dist/`), not the packaged binary. This avoids the multi-minute
|
||||
* `electron-builder --dir` step and matches `hermes desktop --source`. The
|
||||
* packaged-binary path is already covered by `launch.spec.ts`.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so that `dist/` exists.
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
|
||||
import { _electron, type ElectronApplication, type Page } from '@playwright/test'
|
||||
|
||||
import { startMockServer } from './mock-server'
|
||||
|
||||
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
|
||||
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
|
||||
const RELEASE_ROOT = path.join(DESKTOP_ROOT, 'release')
|
||||
|
||||
// ─── Credential stripping (matches launch.spec.ts) ──────────────────────
|
||||
|
||||
const CREDENTIAL_SUFFIXES: string[] = [
|
||||
'_API_KEY',
|
||||
'_TOKEN',
|
||||
'_SECRET',
|
||||
'_PASSWORD',
|
||||
'_CREDENTIALS',
|
||||
'_ACCESS_KEY',
|
||||
'_PRIVATE_KEY',
|
||||
'_OAUTH_TOKEN',
|
||||
]
|
||||
|
||||
const CREDENTIAL_NAMES = new Set([
|
||||
'ANTHROPIC_BASE_URL',
|
||||
'ANTHROPIC_TOKEN',
|
||||
'AWS_ACCESS_KEY_ID',
|
||||
'AWS_SECRET_ACCESS_KEY',
|
||||
'AWS_SESSION_TOKEN',
|
||||
'CUSTOM_API_KEY',
|
||||
'GEMINI_BASE_URL',
|
||||
'OPENAI_BASE_URL',
|
||||
'OPENROUTER_BASE_URL',
|
||||
'OLLAMA_BASE_URL',
|
||||
'GROQ_BASE_URL',
|
||||
'XAI_BASE_URL',
|
||||
])
|
||||
|
||||
function isCredentialEnvVar(name: string): boolean {
|
||||
if (CREDENTIAL_NAMES.has(name)) {
|
||||
return true
|
||||
}
|
||||
|
||||
return CREDENTIAL_SUFFIXES.some((suffix) => name.endsWith(suffix))
|
||||
}
|
||||
|
||||
function stripCredentials(env: Record<string, string | undefined>): Record<string, string> {
|
||||
const clean: Record<string, string> = {}
|
||||
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
if (!value) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (isCredentialEnvVar(key)) {
|
||||
continue
|
||||
}
|
||||
|
||||
clean[key] = value
|
||||
}
|
||||
|
||||
return clean
|
||||
}
|
||||
|
||||
// ─── Sandbox creation ──────────────────────────────────────────────────
|
||||
|
||||
export interface Sandbox {
|
||||
root: string
|
||||
hermesHome: string
|
||||
userDataDir: string
|
||||
cleanup: () => void
|
||||
}
|
||||
|
||||
function createSandbox(prefix: string): Sandbox {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), `hermes-e2e-${prefix}-${Math.random()}`))
|
||||
const hermesHome = path.join(root, 'hermes-home')
|
||||
const userDataDir = path.join(root, 'electron-user-data')
|
||||
|
||||
fs.mkdirSync(hermesHome, { recursive: true })
|
||||
fs.mkdirSync(userDataDir, { recursive: true })
|
||||
|
||||
// Write a fixed window-state.json so the Electron window opens at a
|
||||
// consistent size — helps with visual regression screenshots. The
|
||||
// exact size is also enforced right before each screenshot (see
|
||||
// expectVisualSnapshot in visual-snapshot.ts) because window managers
|
||||
// may resize after launch.
|
||||
fs.writeFileSync(
|
||||
path.join(userDataDir, 'window-state.json'),
|
||||
JSON.stringify(
|
||||
{ x: 0, y: 0, width: 1220, height: 800, isMaximized: false },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
'utf8',
|
||||
)
|
||||
|
||||
return {
|
||||
root,
|
||||
hermesHome,
|
||||
userDataDir,
|
||||
cleanup: () => {
|
||||
try {
|
||||
fs.rmSync(root, { recursive: true, force: true })
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Config writing ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Write a config.yaml that pre-configures a mock provider pointing at the
|
||||
* mock inference server. The provider is set as the active model provider so
|
||||
* the desktop app skips onboarding and boots straight to the chat UI.
|
||||
*/
|
||||
function writeMockProviderConfig(hermesHome: string, mockUrl: string): void {
|
||||
const configPath = path.join(hermesHome, 'config.yaml')
|
||||
|
||||
const config = `# Auto-generated by E2E test fixtures
|
||||
model:
|
||||
default: mock-model
|
||||
provider: mock
|
||||
providers:
|
||||
mock:
|
||||
api: ${mockUrl}/v1
|
||||
name: Mock
|
||||
api_mode: chat_completions
|
||||
key_env: MOCK_API_KEY
|
||||
models:
|
||||
mock-model: {}
|
||||
context_length: 4096
|
||||
`
|
||||
|
||||
fs.writeFileSync(configPath, config, 'utf8')
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a minimal .env with the mock API key. The key_env in config.yaml
|
||||
* references MOCK_API_KEY, so the backend resolves credentials from here.
|
||||
*/
|
||||
function writeEnvFile(hermesHome: string, apiKey = 'e2e-mock-key'): void {
|
||||
const envPath = path.join(hermesHome, '.env')
|
||||
fs.writeFileSync(envPath, `MOCK_API_KEY=${apiKey}\n`, 'utf8')
|
||||
}
|
||||
|
||||
/**
|
||||
* Write an empty config (no providers). The desktop app should show the
|
||||
* onboarding overlay because no inference provider is configured.
|
||||
*/
|
||||
function writeEmptyConfig(hermesHome: string): void {
|
||||
const configPath = path.join(hermesHome, 'config.yaml')
|
||||
fs.writeFileSync(configPath, '# Auto-generated by E2E test fixtures — no providers configured\n', 'utf8')
|
||||
}
|
||||
|
||||
// ─── Env building ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build the environment for the Electron app process.
|
||||
*
|
||||
* Key env vars:
|
||||
* - HERMES_HOME → sandbox hermes-home (isolated config/sessions)
|
||||
* - HERMES_DESKTOP_USER_DATA_DIR → sandbox electron-user-data
|
||||
* - HERMES_DESKTOP_IGNORE_EXISTING=1 → don't pick up `hermes` from PATH
|
||||
* (we want the dev checkout at REPO_ROOT)
|
||||
* - HERMES_DESKTOP_HERMES_ROOT → REPO_ROOT (dev checkout resolution)
|
||||
* - HERMES_DESKTOP_APP_NAME → unique-ish per test (avoids single-instance lock)
|
||||
* - XDG_RUNTIME_DIR → ensure Electron has a writable runtime dir on Linux
|
||||
*/
|
||||
function buildAppEnv(sandbox: Sandbox, extra: Record<string, string> = {}): Record<string, string> {
|
||||
const clean = stripCredentials(process.env)
|
||||
|
||||
// XDG_RUNTIME_DIR is needed for Electron on Linux when running in a
|
||||
// headless/CI context — without it the zygote may fail to initialize.
|
||||
if (!clean.XDG_RUNTIME_DIR && process.env.XDG_RUNTIME_DIR) {
|
||||
clean.XDG_RUNTIME_DIR = process.env.XDG_RUNTIME_DIR
|
||||
}
|
||||
|
||||
// DISPLAY — needed for Electron to open a window.
|
||||
if (!clean.DISPLAY && process.env.DISPLAY) {
|
||||
clean.DISPLAY = process.env.DISPLAY
|
||||
}
|
||||
|
||||
return {
|
||||
...clean,
|
||||
HERMES_HOME: sandbox.hermesHome,
|
||||
HERMES_DESKTOP_USER_DATA_DIR: sandbox.userDataDir,
|
||||
HERMES_DESKTOP_IGNORE_EXISTING: '1',
|
||||
HERMES_DESKTOP_HERMES_ROOT: REPO_ROOT,
|
||||
HERMES_DESKTOP_APP_NAME: `HermesE2E-${Date.now()}`,
|
||||
// Clear dev-server override — we want the built dist/, not a vite server.
|
||||
// The dev-server check in main.ts looks for this env var; if it's set,
|
||||
// it loads from the vite URL instead of the local file.
|
||||
...extra,
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Electron launch ────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Verify that the desktop app has been built (dist/ exists). Playwright
|
||||
* tests can't run without it — the Electron main process loads
|
||||
* dist/electron-main.mjs and the renderer loads dist/index.html.
|
||||
*/
|
||||
function assertDistBuilt(): void {
|
||||
const distDir = path.join(DESKTOP_ROOT, 'dist')
|
||||
const electronMain = path.join(distDir, 'electron-main.mjs')
|
||||
const indexHtml = path.join(distDir, 'index.html')
|
||||
|
||||
if (!fs.existsSync(electronMain)) {
|
||||
throw new Error(
|
||||
`Desktop dist not built. Run 'cd apps/desktop && npm run build' first.\n` +
|
||||
`Missing: ${electronMain}`,
|
||||
)
|
||||
}
|
||||
|
||||
if (!fs.existsSync(indexHtml)) {
|
||||
throw new Error(
|
||||
`Desktop dist/index.html not found. Run 'cd apps/desktop && npm run build' first.\n` +
|
||||
`Missing: ${indexHtml}`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the Electron binary. In the nix devshell, `electron` is on PATH.
|
||||
* As a fallback, use the node_modules/.bin/electron from the desktop package.
|
||||
*/
|
||||
function findElectron(): string {
|
||||
// In dev mode, we use the `electron` binary directly (not the packaged app).
|
||||
// The dev:electron script in package.json does exactly this: `electron .`
|
||||
// after building. We replicate that here.
|
||||
const localElectron = path.join(REPO_ROOT, 'node_modules', 'electron', 'dist', 'electron')
|
||||
|
||||
if (fs.existsSync(localElectron)) {
|
||||
return localElectron
|
||||
}
|
||||
|
||||
// Fall back to PATH
|
||||
const result = spawnSync('which', ['electron'], {
|
||||
encoding: 'utf8',
|
||||
})
|
||||
|
||||
if (result.status === 0 && result.stdout.trim()) {
|
||||
return result.stdout.trim()
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
'Electron binary not found. Run "npm install" from the repo root to install devDependencies.',
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch the desktop app in dev mode.
|
||||
*
|
||||
* @param sandbox - isolated HERMES_HOME + userData
|
||||
* @param env - the process environment (already has HERMES_HOME etc.)
|
||||
* @returns the ElectronApplication + first Page
|
||||
*/
|
||||
async function launchDesktop(
|
||||
env: Record<string, string>,
|
||||
): Promise<{ app: ElectronApplication; page: Page }> {
|
||||
assertDistBuilt()
|
||||
|
||||
const electronBin = findElectron()
|
||||
|
||||
// `electron .` loads from the package.json `main` field
|
||||
// (dist/electron-main.mjs after build).
|
||||
const app = await _electron.launch({
|
||||
executablePath: electronBin,
|
||||
args: [
|
||||
DESKTOP_ROOT, // `electron .` — the `.` is the desktop package dir
|
||||
'--disable-gpu',
|
||||
'--no-sandbox',
|
||||
],
|
||||
env,
|
||||
cwd: DESKTOP_ROOT,
|
||||
})
|
||||
|
||||
const page = await app.firstWindow()
|
||||
|
||||
return { app, page }
|
||||
}
|
||||
|
||||
// ─── Public fixtures ────────────────────────────────────────────────────
|
||||
|
||||
export interface MockBackendFixture {
|
||||
app: ElectronApplication
|
||||
page: Page
|
||||
mockUrl: string
|
||||
sandbox: Sandbox
|
||||
cleanup: () => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up a full mock-backend E2E environment:
|
||||
* 1. Start the mock inference server
|
||||
* 2. Create a sandbox with config.yaml pointing at it
|
||||
* 3. Launch the desktop app
|
||||
* 4. Return handles for test interaction
|
||||
*/
|
||||
export async function setupMockBackend(): Promise<MockBackendFixture> {
|
||||
// 1. Start mock server
|
||||
const mock = await startMockServer()
|
||||
|
||||
// 2. Create sandbox + write config
|
||||
const sandbox = createSandbox('mock')
|
||||
writeMockProviderConfig(sandbox.hermesHome, mock.url)
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
|
||||
// 3. Build env + launch
|
||||
const env = buildAppEnv(sandbox)
|
||||
const { app, page } = await launchDesktop(env)
|
||||
|
||||
return {
|
||||
app,
|
||||
page,
|
||||
mockUrl: mock.url,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
await mock.close()
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export interface NoProviderFixture {
|
||||
app: ElectronApplication
|
||||
page: Page
|
||||
sandbox: Sandbox
|
||||
cleanup: () => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch the app with no provider configured. The onboarding overlay should
|
||||
* appear because there's no inference provider in config.yaml.
|
||||
*/
|
||||
export async function setupNoProvider(): Promise<NoProviderFixture> {
|
||||
const sandbox = createSandbox('noprovider')
|
||||
writeEmptyConfig(sandbox.hermesHome)
|
||||
|
||||
const env = buildAppEnv(sandbox)
|
||||
const { app, page } = await launchDesktop(env)
|
||||
|
||||
return {
|
||||
app,
|
||||
page,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export interface DeadBackendFixture {
|
||||
app: ElectronApplication
|
||||
page: Page
|
||||
sandbox: Sandbox
|
||||
cleanup: () => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch the app with a provider pointing at a dead endpoint (port 1, which
|
||||
* nothing listens on). The boot should fail with a connection error,
|
||||
* triggering the BootFailureOverlay.
|
||||
*/
|
||||
export async function setupDeadBackend(): Promise<DeadBackendFixture> {
|
||||
const sandbox = createSandbox('dead')
|
||||
const configPath = path.join(sandbox.hermesHome, 'config.yaml')
|
||||
fs.writeFileSync(
|
||||
configPath,
|
||||
`# Auto-generated by E2E test fixtures — dead provider
|
||||
model: mock-model
|
||||
providers:
|
||||
mock:
|
||||
api: http://127.0.0.1:1/v1
|
||||
name: Mock
|
||||
api_mode: chat_completions
|
||||
key_env: MOCK_API_KEY
|
||||
models:
|
||||
mock-model: {}
|
||||
context_length: 4096
|
||||
`,
|
||||
'utf8',
|
||||
)
|
||||
writeEnvFile(sandbox.hermesHome)
|
||||
|
||||
const env = buildAppEnv(sandbox)
|
||||
const { app, page } = await launchDesktop(env)
|
||||
|
||||
return {
|
||||
app,
|
||||
page,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Packaged-binary fixture ───────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Resolve the packaged Electron binary path, per-platform, matching
|
||||
* electron-builder's output layout under release/.
|
||||
*/
|
||||
function resolvePackagedBinaryPath(): string {
|
||||
if (process.platform === 'win32') {
|
||||
return path.join(RELEASE_ROOT, 'win-unpacked', 'Hermes.exe')
|
||||
}
|
||||
|
||||
if (process.platform === 'darwin') {
|
||||
const arch = process.arch === 'arm64' ? 'arm64' : 'x64'
|
||||
|
||||
return path.join(RELEASE_ROOT, `mac-${arch}`, 'Hermes.app', 'Contents', 'MacOS', 'Hermes')
|
||||
}
|
||||
|
||||
return path.join(RELEASE_ROOT, 'linux-unpacked', 'hermes')
|
||||
}
|
||||
|
||||
export const PACKAGED_BINARY_PATH = resolvePackagedBinaryPath()
|
||||
|
||||
export function packagedBinaryExists(): boolean {
|
||||
return fs.existsSync(PACKAGED_BINARY_PATH)
|
||||
}
|
||||
|
||||
export interface PackagedAppFixture {
|
||||
app: ElectronApplication
|
||||
page: Page
|
||||
sandbox: Sandbox
|
||||
cleanup: () => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch the *packaged* Electron binary (from `npm run pack` →
|
||||
* `electron-builder --dir`) with `BOOT_FAKE=1` so it simulates boot
|
||||
* progress without spawning a real Hermes backend.
|
||||
*
|
||||
* Uses the same sandbox isolation (credential stripping, isolated
|
||||
* HERMES_HOME + userData, unique app name) as the dev-mode fixtures.
|
||||
*
|
||||
* Skips if the packaged binary doesn't exist — run `npm run pack` first.
|
||||
*/
|
||||
export async function setupPackagedApp(): Promise<PackagedAppFixture> {
|
||||
if (!packagedBinaryExists()) {
|
||||
throw new Error(
|
||||
`Built app binary not found: ${PACKAGED_BINARY_PATH}. Run 'npm run pack' first.`,
|
||||
)
|
||||
}
|
||||
|
||||
const sandbox = createSandbox('packaged')
|
||||
|
||||
// Build the sandbox env using the shared helpers, then add the
|
||||
// packaged-binary-specific overrides.
|
||||
const env = buildAppEnv(sandbox, {
|
||||
// Fake boot: simulates progress steps without spawning the real backend.
|
||||
HERMES_DESKTOP_BOOT_FAKE: '1',
|
||||
HERMES_DESKTOP_BOOT_FAKE_STEP_MS: '120',
|
||||
})
|
||||
|
||||
// Clear dev-server + hermes-root overrides — the packaged binary
|
||||
// should use its own bundled renderer, not the dev checkout.
|
||||
delete (env as Record<string, string | undefined>).HERMES_DESKTOP_DEV_SERVER
|
||||
delete (env as Record<string, string | undefined>).HERMES_DESKTOP_HERMES
|
||||
delete (env as Record<string, string | undefined>).HERMES_DESKTOP_HERMES_ROOT
|
||||
|
||||
const app = await _electron.launch({
|
||||
executablePath: PACKAGED_BINARY_PATH,
|
||||
args: ['--disable-gpu', '--no-sandbox'],
|
||||
env,
|
||||
})
|
||||
|
||||
const page = await app.firstWindow()
|
||||
|
||||
return {
|
||||
app,
|
||||
page,
|
||||
sandbox,
|
||||
cleanup: async () => {
|
||||
await app.close().catch(() => undefined)
|
||||
sandbox.cleanup()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Wait helpers ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Wait for the desktop app to finish booting and show the main chat UI.
|
||||
*
|
||||
* The boot overlay disappears when `completeDesktopBoot()` fires in the
|
||||
* renderer — at that point the gateway is open, config is loaded, and
|
||||
* sessions are loaded. We detect this by waiting for the boot/connecting
|
||||
* overlay to become invisible and the main app shell to be present.
|
||||
*/
|
||||
export async function waitForAppReady(fixture: MockBackendFixture | NoProviderFixture | DeadBackendFixture, timeoutMs = 60_000): Promise<void> {
|
||||
const { page, app } = fixture
|
||||
// The connecting overlay has a data-testid or we can check for the
|
||||
// absence of boot indicators. The simplest reliable approach is to wait
|
||||
// for the composer (chat input) to become visible — it's disabled until
|
||||
// the gateway is open.
|
||||
await page.waitForSelector('textarea, [contenteditable="true"]', {
|
||||
state: 'visible',
|
||||
timeout: timeoutMs,
|
||||
})
|
||||
|
||||
// On Electron 40.x, ready-to-show may never fire (electron/electron#51972)
|
||||
// and the window stays hidden even though the DOM is rendered. The main
|
||||
// process has a TEST_WORKER_INDEX-gated fallback that force-shows the
|
||||
// window, but the DOM can be ready before that fires. Poll until the
|
||||
// window is actually visible so interactions (click, screenshot) don't
|
||||
// hit a hidden surface.
|
||||
if (app) {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const visible = await app.evaluate(({ BrowserWindow }) => {
|
||||
const w = BrowserWindow.getAllWindows()[0]
|
||||
|
||||
return w ? w.isVisible() : false
|
||||
}).catch(() => false)
|
||||
|
||||
if (visible) {break}
|
||||
await page.waitForTimeout(500)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the onboarding overlay to appear (no provider configured).
|
||||
*/
|
||||
export async function waitForOnboarding(page: Page, timeoutMs = 60_000): Promise<void> {
|
||||
// The onboarding overlay contains a heading with "Choose your provider"
|
||||
// or similar text. We look for any text that indicates the picker.
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const root = document.getElementById('root')
|
||||
|
||||
if (!root) {
|
||||
return false
|
||||
}
|
||||
|
||||
const text = root.textContent ?? ''
|
||||
|
||||
return (
|
||||
text.includes('provider') ||
|
||||
text.includes('Provider') ||
|
||||
text.includes('Choose') ||
|
||||
text.includes('API key') ||
|
||||
text.includes('Sign in')
|
||||
)
|
||||
},
|
||||
{ timeout: timeoutMs },
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the boot failure overlay to appear.
|
||||
*/
|
||||
export async function waitForBootFailure(page: Page, timeoutMs = 60_000): Promise<void> {
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const root = document.getElementById('root')
|
||||
|
||||
if (!root) {
|
||||
return false
|
||||
}
|
||||
|
||||
const text = root.textContent ?? ''
|
||||
|
||||
// The boot failure overlay shows an error message + retry/repair
|
||||
// buttons. Look for error-related text.
|
||||
return (
|
||||
text.includes('error') ||
|
||||
text.includes('Error') ||
|
||||
text.includes('failed') ||
|
||||
text.includes('Failed') ||
|
||||
text.includes('Retry') ||
|
||||
text.includes('Repair')
|
||||
)
|
||||
},
|
||||
{ timeout: timeoutMs },
|
||||
)
|
||||
}
|
||||
87
apps/desktop/e2e/launch-packaged-app.spec.ts
Normal file
87
apps/desktop/e2e/launch-packaged-app.spec.ts
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import { expect, test } from '@playwright/test'
|
||||
|
||||
import {
|
||||
PACKAGED_BINARY_PATH,
|
||||
type PackagedAppFixture,
|
||||
packagedBinaryExists,
|
||||
setupPackagedApp,
|
||||
} from './fixtures'
|
||||
import { expectVisualSnapshot } from './visual-snapshot'
|
||||
|
||||
/**
|
||||
* E2E smoke tests for the packaged Hermes desktop app.
|
||||
*
|
||||
* Launches the real packaged Electron binary (produced by `npm run pack` →
|
||||
* `electron-builder --dir`) with BOOT_FAKE=1 and full sandbox isolation
|
||||
* (credential stripping, isolated HERMES_HOME + userData, unique app name).
|
||||
*
|
||||
* Skips if the packaged binary doesn't exist — run `npm run pack` first.
|
||||
*/
|
||||
|
||||
let fixture: PackagedAppFixture | null = null
|
||||
|
||||
test.beforeAll(async () => {
|
||||
test.skip(
|
||||
!packagedBinaryExists(),
|
||||
`Built app binary not found: ${PACKAGED_BINARY_PATH}. Run 'npm run pack' first.`,
|
||||
)
|
||||
|
||||
fixture = await setupPackagedApp()
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test('window opens with the Hermes title', async () => {
|
||||
const title = await fixture!.page.title()
|
||||
expect(title).toContain('Hermes')
|
||||
})
|
||||
|
||||
test('renderer loads and shows DOM content', async () => {
|
||||
const page = fixture!.page
|
||||
await page.waitForSelector('#root', { state: 'attached', timeout: 30_000 })
|
||||
const childCount = await page.locator('#root > *').count()
|
||||
expect(childCount).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
test('boot progress overlay fades out or shows error state', async () => {
|
||||
const page = fixture!.page
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const root = document.getElementById('root')
|
||||
|
||||
if (!root) {
|
||||
return false
|
||||
}
|
||||
|
||||
const text = root.textContent ?? ''
|
||||
|
||||
// Error path: boot failure overlay renders an error message.
|
||||
if (text.includes('error') || text.includes('Error') || text.includes('failed')) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Success path: overlay disappears and the app renders. If there's
|
||||
// no "boot" / "starting" / "installing" text visible, boot has
|
||||
// completed (either to the main UI or to onboarding).
|
||||
const bootIndicators = ['starting', 'resolving', 'spawning', 'waiting', 'installing']
|
||||
const lower = text.toLowerCase()
|
||||
|
||||
return !bootIndicators.some((word) => lower.includes(word))
|
||||
},
|
||||
{ timeout: 60_000 },
|
||||
)
|
||||
})
|
||||
|
||||
test('can capture a screenshot for the CI artifact', async () => {
|
||||
if (!fixture) {
|
||||
test.skip(true, 'Previous test failed — no app running')
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Visual snapshot — won't fail on diff, just logs + generates diff image
|
||||
await expectVisualSnapshot(fixture!.page, { name: 'packaged-app-booted', timeout: 10_000, app: fixture!.app })
|
||||
})
|
||||
85
apps/desktop/e2e/mock-backend-setup.spec.ts
Normal file
85
apps/desktop/e2e/mock-backend-setup.spec.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
/**
|
||||
* E2E tests asserting the mock backend gets the app past the setup/onboarding
|
||||
* screen.
|
||||
*
|
||||
* The mock backend fixture writes a config.yaml with a pre-configured mock
|
||||
* provider pointing at a mock inference server. When the app boots, the
|
||||
* runtime readiness check should detect the working provider and dismiss the
|
||||
* onboarding overlay — landing straight on the chat UI without ever showing
|
||||
* the "Let's get you setup with Hermes Agent" screen.
|
||||
*
|
||||
* If these tests fail, the mock backend config isn't getting the app past
|
||||
* onboarding — the chat interaction tests (chat.spec.ts) will also fail
|
||||
* because the composer is blocked by the setup overlay.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*/
|
||||
|
||||
import { expect, test } from '@playwright/test'
|
||||
|
||||
import {
|
||||
type MockBackendFixture,
|
||||
setupMockBackend,
|
||||
waitForAppReady,
|
||||
} from './fixtures'
|
||||
import { expectVisualSnapshot } from './visual-snapshot'
|
||||
|
||||
let fixture: MockBackendFixture | null = null
|
||||
|
||||
test.beforeAll(async () => {
|
||||
fixture = await setupMockBackend()
|
||||
await waitForAppReady(fixture!, 120_000)
|
||||
})
|
||||
|
||||
test.afterAll(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test.describe('mock backend gets past setup screen', () => {
|
||||
test('onboarding overlay is not shown', async () => {
|
||||
const page = fixture!.page
|
||||
|
||||
// The onboarding overlay renders "Let's get you setup with Hermes Agent"
|
||||
// when the runtime check fails to find a working provider. With the mock
|
||||
// backend configured, the runtime check should pass and the overlay
|
||||
// returns null — this text should NOT be present in the DOM.
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const text = document.body.textContent ?? ''
|
||||
|
||||
return !text.includes("Let's get you setup")
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
})
|
||||
|
||||
test('chat composer is visible', async () => {
|
||||
const page = fixture!.page
|
||||
|
||||
// The composer (contenteditable div) should be visible and not blocked
|
||||
// by the onboarding overlay. If the first test passed, the overlay is
|
||||
// gone and the composer is the primary interactive surface.
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
await expect(composer).toBeVisible()
|
||||
})
|
||||
|
||||
test('can type into the composer', async () => {
|
||||
const page = fixture!.page
|
||||
|
||||
// If the setup overlay is truly gone, the composer accepts input.
|
||||
const composer = page.locator('[contenteditable="true"]').first()
|
||||
await composer.click()
|
||||
await composer.type('hello mock backend', { delay: 20 })
|
||||
|
||||
// Verify the typed text appears in the DOM.
|
||||
await page.waitForFunction(
|
||||
() => (document.body.textContent ?? '').includes('hello mock backend'),
|
||||
{ timeout: 10_000 },
|
||||
)
|
||||
})
|
||||
|
||||
test('screenshot shows chat UI without setup screen', async () => {
|
||||
await expectVisualSnapshot(fixture!.page, { name: 'mock-backend-chat-ready', app: fixture!.app })
|
||||
})
|
||||
})
|
||||
203
apps/desktop/e2e/mock-server.ts
Normal file
203
apps/desktop/e2e/mock-server.ts
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
/**
|
||||
* Minimal OpenAI-compatible mock inference server for E2E tests.
|
||||
*
|
||||
* Implements just enough of the /v1/* surface for `hermes serve` to resolve a
|
||||
* provider, list models, and stream a canned chat completion back to the
|
||||
* desktop app — without any real LLM.
|
||||
*
|
||||
* Endpoints:
|
||||
* GET /v1/models → { data: [{ id, ... }] }
|
||||
* POST /v1/chat/completions → streaming (SSE) or non-streaming response
|
||||
*
|
||||
* The canned response is a short, deterministic assistant message. Tool-call
|
||||
* requests are not simulated — the E2E tests only need the chat surface to
|
||||
* prove the full boot → gateway → inference → renderer chain works.
|
||||
*/
|
||||
|
||||
import http from 'node:http'
|
||||
|
||||
/** A canned assistant reply used for every chat completion request. */
|
||||
const CANNED_REPLY = 'Hello from the mock inference server! The full boot chain is working.'
|
||||
|
||||
/**
|
||||
* Start the mock server on an ephemeral port.
|
||||
*
|
||||
* @returns a handle with `port`, `url`, and `close()`.
|
||||
*/
|
||||
export function startMockServer(): Promise<{ port: number; url: string; close: () => Promise<void> }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = http.createServer((req, res) => {
|
||||
// CORS headers — the Electron renderer doesn't need them, but they
|
||||
// don't hurt and make the server usable from a browser context too.
|
||||
res.setHeader('Access-Control-Allow-Origin', '*')
|
||||
res.setHeader('Access-Control-Allow-Headers', '*')
|
||||
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
|
||||
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.writeHead(204)
|
||||
res.end()
|
||||
return
|
||||
}
|
||||
|
||||
// GET /v1/models — return a single fake model.
|
||||
if (req.method === 'GET' && req.url === '/v1/models') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' })
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
object: 'list',
|
||||
data: [
|
||||
{
|
||||
id: 'mock-model',
|
||||
object: 'model',
|
||||
created: 0,
|
||||
owned_by: 'mock',
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
// POST /v1/chat/completions — return a canned response.
|
||||
if (req.method === 'POST' && req.url?.startsWith('/v1/chat/completions')) {
|
||||
let body = ''
|
||||
|
||||
req.on('data', (chunk: Buffer) => {
|
||||
body += chunk.toString()
|
||||
})
|
||||
|
||||
req.on('end', () => {
|
||||
let parsed: any = {}
|
||||
|
||||
try {
|
||||
parsed = JSON.parse(body)
|
||||
} catch {
|
||||
// malformed JSON — treat as non-streaming with defaults
|
||||
}
|
||||
|
||||
const stream = parsed.stream === true
|
||||
const model = parsed.model || 'mock-model'
|
||||
|
||||
if (stream) {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
})
|
||||
|
||||
// Send the content in a few chunks to simulate streaming.
|
||||
const words = CANNED_REPLY.split(' ')
|
||||
let i = 0
|
||||
|
||||
const sendChunk = () => {
|
||||
if (i >= words.length) {
|
||||
// Final chunk with finish_reason
|
||||
res.write(
|
||||
`data: ${JSON.stringify({
|
||||
id: 'mock-completion',
|
||||
object: 'chat.completion.chunk',
|
||||
created: 0,
|
||||
model,
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
delta: {},
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
})}\n\n`,
|
||||
)
|
||||
res.write('data: [DONE]\n\n')
|
||||
res.end()
|
||||
return
|
||||
}
|
||||
|
||||
const word = i === 0 ? words[i] : ' ' + words[i]
|
||||
res.write(
|
||||
`data: ${JSON.stringify({
|
||||
id: 'mock-completion',
|
||||
object: 'chat.completion.chunk',
|
||||
created: 0,
|
||||
model,
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
delta: { content: word },
|
||||
finish_reason: null,
|
||||
},
|
||||
],
|
||||
})}\n\n`,
|
||||
)
|
||||
i++
|
||||
// Small delay between chunks to simulate real streaming.
|
||||
setTimeout(sendChunk, 20)
|
||||
}
|
||||
|
||||
sendChunk()
|
||||
} else {
|
||||
// Non-streaming response
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' })
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
id: 'mock-completion',
|
||||
object: 'chat.completion',
|
||||
created: 0,
|
||||
model,
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: { role: 'assistant', content: CANNED_REPLY },
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
},
|
||||
}),
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
req.on('error', () => {
|
||||
res.writeHead(400)
|
||||
res.end('Bad request')
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Fallback — 404 for anything else
|
||||
res.writeHead(404, { 'Content-Type': 'application/json' })
|
||||
res.end(JSON.stringify({ error: 'Not found' }))
|
||||
})
|
||||
|
||||
server.on('error', reject)
|
||||
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const addr = server.address()
|
||||
if (addr === null || typeof addr === 'string') {
|
||||
reject(new Error('Failed to get server address'))
|
||||
return
|
||||
}
|
||||
|
||||
const port = addr.port
|
||||
const url = `http://127.0.0.1:${port}`
|
||||
|
||||
resolve({
|
||||
port,
|
||||
url,
|
||||
close: () =>
|
||||
new Promise((resolveClose, rejectClose) => {
|
||||
server.close((err) => {
|
||||
if (err) {
|
||||
rejectClose(err)
|
||||
} else {
|
||||
resolveClose()
|
||||
}
|
||||
})
|
||||
}),
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
76
apps/desktop/e2e/onboarding.spec.ts
Normal file
76
apps/desktop/e2e/onboarding.spec.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
/**
|
||||
* E2E onboarding tests — verify the provider picker appears when no
|
||||
* inference provider is configured.
|
||||
*
|
||||
* Launches the app with an empty config.yaml (no providers). The renderer
|
||||
* should detect the unconfigured state and show the DesktopOnboardingOverlay
|
||||
* with provider options / API key form.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*/
|
||||
|
||||
import { expect, test } from '@playwright/test'
|
||||
|
||||
import {
|
||||
type NoProviderFixture,
|
||||
setupNoProvider,
|
||||
waitForOnboarding,
|
||||
} from './fixtures'
|
||||
import { expectVisualSnapshot } from './visual-snapshot'
|
||||
|
||||
let fixture: NoProviderFixture | null = null
|
||||
|
||||
test.afterAll(async () => {
|
||||
await fixture?.cleanup()
|
||||
fixture = null
|
||||
})
|
||||
|
||||
test.describe('onboarding with no provider configured', () => {
|
||||
test('onboarding overlay appears on first boot', async () => {
|
||||
fixture = await setupNoProvider()
|
||||
|
||||
// The app should boot (hermes serve starts fine even without a provider),
|
||||
// but the renderer should show the onboarding overlay because no
|
||||
// provider is configured.
|
||||
await waitForOnboarding(fixture.page, 90_000)
|
||||
})
|
||||
|
||||
test('onboarding shows provider options or API key form', async () => {
|
||||
if (!fixture) {
|
||||
test.skip(true, 'Previous test failed — no app running')
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
const page = fixture.page
|
||||
|
||||
// The onboarding overlay should contain provider-related text.
|
||||
// It might show OAuth providers, an API key form, or a "choose later"
|
||||
// link. Verify at least one of these is visible.
|
||||
const rootText = await page.evaluate(() => {
|
||||
const root = document.getElementById('root')
|
||||
|
||||
return root?.textContent ?? ''
|
||||
})
|
||||
|
||||
const hasProviderText =
|
||||
rootText.includes('provider') ||
|
||||
rootText.includes('Provider') ||
|
||||
rootText.includes('API key') ||
|
||||
rootText.includes('Sign in') ||
|
||||
rootText.includes('OpenRouter') ||
|
||||
rootText.includes('OpenAI')
|
||||
|
||||
expect(hasProviderText).toBe(true)
|
||||
})
|
||||
|
||||
test('screenshot of onboarding overlay', async () => {
|
||||
if (!fixture) {
|
||||
test.skip(true, 'Previous test failed — no app running')
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
await expectVisualSnapshot(fixture.page, { name: 'onboarding-overlay', app: fixture.app })
|
||||
})
|
||||
})
|
||||
90
apps/desktop/e2e/visual-snapshot.ts
Normal file
90
apps/desktop/e2e/visual-snapshot.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
/**
|
||||
* Visual snapshot helper — wraps `toHaveScreenshot` so visual diffs are
|
||||
* reported without failing the test suite.
|
||||
*
|
||||
* On CI, the JSON reporter + post-test script parse the results and post a
|
||||
* summary to the GitHub Actions step output, and diff images are uploaded
|
||||
* as artifacts. This keeps visual regressions visible without gating PRs
|
||||
* on pixel-perfect matches.
|
||||
*
|
||||
* When a screenshot matches the baseline, nothing happens. When it
|
||||
* differs, Playwright writes three images to the test's output dir:
|
||||
* <name>-actual.png, <name>-expected.png, <name>-diff.png
|
||||
* These are picked up by the "Upload test results" artifact step.
|
||||
*/
|
||||
import { type ElectronApplication, expect, type Page } from '@playwright/test'
|
||||
|
||||
/** Fixed window dimensions for visual regression screenshots. */
|
||||
export const VISUAL_WINDOW_WIDTH = 1220
|
||||
export const VISUAL_WINDOW_HEIGHT = 800
|
||||
|
||||
export interface VisualSnapshotOptions {
|
||||
/** Snapshot name — defaults to the test title. */
|
||||
name?: string
|
||||
/** Full page screenshot (default) vs. viewport-only. */
|
||||
fullPage?: boolean
|
||||
/** Timeout in ms. */
|
||||
timeout?: number
|
||||
/** The Electron app handle — needed to force a fixed window size. */
|
||||
app?: ElectronApplication
|
||||
}
|
||||
|
||||
/**
|
||||
* Force the Electron window to a fixed size so screenshots are comparable
|
||||
* across runs and CI environments. Window managers (Hyprland, etc.) may
|
||||
* auto-tile or resize windows after launch; calling this right before the
|
||||
* screenshot ensures the viewport is always the expected size.
|
||||
*/
|
||||
async function forceFixedSize(app: ElectronApplication): Promise<void> {
|
||||
await app.evaluate(({ BrowserWindow }, { width, height }) => {
|
||||
const win = BrowserWindow.getAllWindows()[0]
|
||||
|
||||
if (win) {
|
||||
win.unmaximize()
|
||||
// setMinimumSize must be ≤ the target, otherwise setSize is clamped.
|
||||
win.setMinimumSize(width, height)
|
||||
win.setSize(width, height, false)
|
||||
win.setBounds({ x: 0, y: 0, width, height })
|
||||
}
|
||||
}, { width: VISUAL_WINDOW_WIDTH, height: VISUAL_WINDOW_HEIGHT })
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a screenshot and compare it against the baseline.
|
||||
*
|
||||
* If the baseline doesn't exist yet (first run), Playwright creates it.
|
||||
* If it differs, the test logs a soft warning but does NOT fail — the diff
|
||||
* images are still generated for CI to surface.
|
||||
*/
|
||||
export async function expectVisualSnapshot(
|
||||
page: Page,
|
||||
options: VisualSnapshotOptions = {},
|
||||
): Promise<void> {
|
||||
const { name, fullPage = false, timeout = 30_000, app } = options
|
||||
|
||||
// Force the window to a fixed size right before the screenshot so it's
|
||||
// always comparable, regardless of WM resizing during the test.
|
||||
if (app) {
|
||||
await forceFixedSize(app)
|
||||
// Give the renderer a moment to relayout after the resize.
|
||||
await page.waitForTimeout(500)
|
||||
}
|
||||
|
||||
// Playwright appends a platform suffix (e.g. "-linux") and requires
|
||||
// a .png extension on the name argument. Auto-append it if missing.
|
||||
const snapshotName = name ? (name.endsWith('.png') ? name : `${name}.png`) : undefined
|
||||
|
||||
try {
|
||||
if (snapshotName) {
|
||||
await expect(page).toHaveScreenshot(snapshotName, { fullPage, timeout })
|
||||
} else {
|
||||
await expect(page).toHaveScreenshot({ fullPage, timeout })
|
||||
}
|
||||
} catch (err) {
|
||||
// Don't fail the test — just log that a diff was detected.
|
||||
// The diff/actual/expected images are already written to the test
|
||||
// output directory by Playwright for the CI workflow to pick up.
|
||||
console.log(`[visual-diff] ${name ?? '(unnamed)'} — screenshot differs from baseline`)
|
||||
console.log(` ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`)
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@
|
|||
"scripts": {
|
||||
"dev": "concurrently -k \"npm:dev:renderer\" \"npm:dev:electron\"",
|
||||
"dev:fake-boot": "cross-env HERMES_DESKTOP_BOOT_FAKE=1 HERMES_DESKTOP_BOOT_FAKE_STEP_MS=650 npm run dev",
|
||||
"dev:mock": "node scripts/dev-mock.mjs",
|
||||
"dev:renderer": "node scripts/assert-root-install.mjs && vite --host 127.0.0.1 --port 5174",
|
||||
"dev:electron": "wait-on http://127.0.0.1:5174 && node scripts/bundle-electron-main.mjs --dev && cross-env XCURSOR_SIZE=24 HERMES_DESKTOP_DEV_SERVER=http://127.0.0.1:5174 electron .",
|
||||
"profile:main": "wait-on http://127.0.0.1:5174 && node scripts/bundle-electron-main.mjs --dev && cross-env XCURSOR_SIZE=24 HERMES_DESKTOP_DEV_SERVER=http://127.0.0.1:5174 electron --inspect=9229 .",
|
||||
|
|
@ -49,7 +50,10 @@
|
|||
"test:desktop:platforms": "vitest run --project electron",
|
||||
"test": "vitest run",
|
||||
"preview": "node scripts/assert-root-install.mjs && vite preview --host 127.0.0.1 --port 4174",
|
||||
"check": "npm run typecheck && npm run test && npm run test:desktop:all && npm run build"
|
||||
"check": "npm run typecheck && npm run test && npm run test:desktop:all && npm run build",
|
||||
"test:e2e": "playwright test e2e/",
|
||||
"test:e2e:visual": "WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list",
|
||||
"test:e2e:update-snapshots": "WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list --update-snapshots"
|
||||
},
|
||||
"dependencies": {
|
||||
"@assistant-ui/react": "^0.14.23",
|
||||
|
|
@ -124,6 +128,7 @@
|
|||
"devDependencies": {
|
||||
"@electron/rebuild": "^4.0.6",
|
||||
"@eslint/js": "^9.39.4",
|
||||
"@playwright/test": "=1.58.2",
|
||||
"@testing-library/dom": "^10.4.0",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@types/d3-force": "^3.0.10",
|
||||
|
|
|
|||
53
apps/desktop/playwright.config.ts
Normal file
53
apps/desktop/playwright.config.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import './e2e/fix-electron-tracing'
|
||||
|
||||
import { defineConfig, type ReporterDescription } from '@playwright/test'
|
||||
|
||||
/**
|
||||
* Visual regression testing config.
|
||||
*
|
||||
* Screenshots are compared against baselines. On `main`, baselines are
|
||||
* generated with `--update-snapshots` and cached. On PRs, the cached
|
||||
* baselines are restored and screenshots are compared — but tests DON'T
|
||||
* fail on visual diffs (see `expectVisualSnapshot` in visual-snapshot.ts).
|
||||
* Instead, diffs are surfaced in the CI step summary and uploaded as
|
||||
* artifacts for human review.
|
||||
*
|
||||
* To update baselines after an intentional UI change:
|
||||
* npx playwright test --update-snapshots
|
||||
*/
|
||||
const reporters: ReporterDescription[] = [
|
||||
['list'],
|
||||
['html', { open: 'never', outputFolder: 'playwright-report' }],
|
||||
]
|
||||
|
||||
if (process.env.CI) {
|
||||
reporters.push(['json', { outputFile: 'playwright-report/results.json' }])
|
||||
}
|
||||
|
||||
export default defineConfig({
|
||||
/* Test files live under e2e/ so they never collide with the vitest suite
|
||||
* under src/ or the node:test files under electron/. */
|
||||
testDir: './e2e',
|
||||
/* The desktop app can take a while to bootstrap on cold CI runners — 90 s
|
||||
* per test gives us headroom without masking real hangs. */
|
||||
timeout: 90_000,
|
||||
retries: process.env.CI ? 1 : 0,
|
||||
/* Each test gets its own worker so the Electron process is fully isolated. */
|
||||
fullyParallel: false,
|
||||
reporter: reporters,
|
||||
use: {
|
||||
screenshot: 'on',
|
||||
trace: { mode: 'on', screenshots: true, snapshots: true, sources: true },
|
||||
},
|
||||
expect: {
|
||||
toHaveScreenshot: {
|
||||
// 1% of pixels may differ — absorbs sub-pixel font rendering variance
|
||||
// between local and CI environments.
|
||||
maxDiffPixelRatio: 0.01,
|
||||
animations: 'disabled',
|
||||
caret: 'hide',
|
||||
// Per-channel threshold for "close enough" — anti-aliasing differences.
|
||||
threshold: 0.2,
|
||||
},
|
||||
},
|
||||
})
|
||||
237
apps/desktop/scripts/dev-mock.mjs
Normal file
237
apps/desktop/scripts/dev-mock.mjs
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Launch the desktop app with a mock inference provider — no real API
|
||||
* keys needed. Starts a local OpenAI-compatible server that returns a
|
||||
* canned reply, writes an isolated config.yaml + .env, and launches the
|
||||
* built Electron app against them.
|
||||
*
|
||||
* This reuses the same mock-server and config format as the E2E fixtures
|
||||
* (apps/desktop/e2e/mock-server.ts + fixtures.ts), so local dev and CI
|
||||
* test the same chain.
|
||||
*
|
||||
* Prerequisite: `npm run build` must have been run so dist/ exists.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/dev-mock.mjs
|
||||
* npm run dev:mock
|
||||
*
|
||||
* The mock server listens on an ephemeral port and replies to every
|
||||
* chat completion with:
|
||||
* "Hello from the mock inference server! The full boot chain is working."
|
||||
*/
|
||||
|
||||
import http from 'node:http'
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { spawn, spawnSync } from 'node:child_process'
|
||||
|
||||
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
|
||||
const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..')
|
||||
|
||||
// ── Canned reply ───────────────────────────────────────────────────────
|
||||
|
||||
const CANNED_REPLY =
|
||||
'Hello from the mock inference server! The full boot chain is working.'
|
||||
|
||||
// ── Mock server (mirrors e2e/mock-server.ts) ───────────────────────────
|
||||
|
||||
function startMockServer() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = http.createServer((req, res) => {
|
||||
res.setHeader('Access-Control-Allow-Origin', '*')
|
||||
res.setHeader('Access-Control-Allow-Headers', '*')
|
||||
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
|
||||
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.writeHead(204)
|
||||
res.end()
|
||||
return
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && req.url === '/v1/models') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' })
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
object: 'list',
|
||||
data: [{ id: 'mock-model', object: 'model', created: 0, owned_by: 'mock' }],
|
||||
}),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (req.method === 'POST' && req.url?.startsWith('/v1/chat/completions')) {
|
||||
let body = ''
|
||||
req.on('data', (chunk) => { body += chunk.toString() })
|
||||
req.on('end', () => {
|
||||
let parsed = {}
|
||||
try { parsed = JSON.parse(body) } catch { /* non-streaming */ }
|
||||
|
||||
const stream = parsed.stream === true
|
||||
const model = parsed.model || 'mock-model'
|
||||
|
||||
if (stream) {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
Connection: 'keep-alive',
|
||||
})
|
||||
const words = CANNED_REPLY.split(' ')
|
||||
let i = 0
|
||||
const sendChunk = () => {
|
||||
if (i >= words.length) {
|
||||
res.write(
|
||||
`data: ${JSON.stringify({
|
||||
id: 'mock-completion', object: 'chat.completion.chunk',
|
||||
created: 0, model,
|
||||
choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
|
||||
})}\n\n`,
|
||||
)
|
||||
res.write('data: [DONE]\n\n')
|
||||
res.end()
|
||||
return
|
||||
}
|
||||
const word = i === 0 ? words[i] : ' ' + words[i]
|
||||
res.write(
|
||||
`data: ${JSON.stringify({
|
||||
id: 'mock-completion', object: 'chat.completion.chunk',
|
||||
created: 0, model,
|
||||
choices: [{ index: 0, delta: { content: word }, finish_reason: null }],
|
||||
})}\n\n`,
|
||||
)
|
||||
i++
|
||||
setTimeout(sendChunk, 20)
|
||||
}
|
||||
sendChunk()
|
||||
} else {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' })
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
id: 'mock-completion', object: 'chat.completion',
|
||||
created: 0, model,
|
||||
choices: [{
|
||||
index: 0,
|
||||
message: { role: 'assistant', content: CANNED_REPLY },
|
||||
finish_reason: 'stop',
|
||||
}],
|
||||
usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
|
||||
}),
|
||||
)
|
||||
}
|
||||
})
|
||||
req.on('error', () => { res.writeHead(400); res.end('Bad request') })
|
||||
return
|
||||
}
|
||||
|
||||
res.writeHead(404, { 'Content-Type': 'application/json' })
|
||||
res.end(JSON.stringify({ error: 'Not found' }))
|
||||
})
|
||||
|
||||
server.on('error', reject)
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const addr = server.address()
|
||||
if (addr === null || typeof addr === 'string') {
|
||||
reject(new Error('Failed to get server address'))
|
||||
return
|
||||
}
|
||||
resolve({ port: addr.port, url: `http://127.0.0.1:${addr.port}`, close: () => server.close() })
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// ── Config + env writing (mirrors e2e/fixtures.ts) ─────────────────────
|
||||
|
||||
function createSandbox() {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), `hermes-dev-mock-${Date.now()}`))
|
||||
const hermesHome = path.join(root, 'hermes-home')
|
||||
const userDataDir = path.join(root, 'electron-user-data')
|
||||
fs.mkdirSync(hermesHome, { recursive: true })
|
||||
fs.mkdirSync(userDataDir, { recursive: true })
|
||||
return { root, hermesHome, userDataDir, cleanup: () => fs.rmSync(root, { recursive: true, force: true }) }
|
||||
}
|
||||
|
||||
function writeMockConfig(hermesHome, mockUrl) {
|
||||
fs.writeFileSync(
|
||||
path.join(hermesHome, 'config.yaml'),
|
||||
`# Auto-generated by dev-mock.mjs
|
||||
model:
|
||||
default: mock-model
|
||||
provider: mock
|
||||
providers:
|
||||
mock:
|
||||
api: ${mockUrl}/v1
|
||||
name: Mock
|
||||
api_mode: chat_completions
|
||||
key_env: MOCK_API_KEY
|
||||
models:
|
||||
mock-model: {}
|
||||
context_length: 4096
|
||||
`,
|
||||
'utf8',
|
||||
)
|
||||
fs.writeFileSync(path.join(hermesHome, '.env'), 'MOCK_API_KEY=e2e-mock-key\n', 'utf8')
|
||||
}
|
||||
|
||||
// ── Electron launch ────────────────────────────────────────────────────
|
||||
|
||||
function findElectron() {
|
||||
const local = path.join(REPO_ROOT, 'node_modules', 'electron', 'dist', 'electron')
|
||||
if (fs.existsSync(local)) return local
|
||||
const r = spawnSync('which', ['electron'], { encoding: 'utf8' })
|
||||
if (r.status === 0 && r.stdout.trim()) return r.stdout.trim()
|
||||
throw new Error('Electron binary not found. Run "npm install" from the repo root.')
|
||||
}
|
||||
|
||||
function assertDistBuilt() {
|
||||
const electronMain = path.join(DESKTOP_ROOT, 'dist', 'electron-main.mjs')
|
||||
const indexHtml = path.join(DESKTOP_ROOT, 'dist', 'index.html')
|
||||
if (!fs.existsSync(electronMain) || !fs.existsSync(indexHtml)) {
|
||||
throw new Error(
|
||||
`Desktop dist not built. Run 'cd apps/desktop && npm run build' first.\n` +
|
||||
`Missing: ${electronMain}`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Main ───────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
assertDistBuilt()
|
||||
|
||||
console.log('Starting mock inference server...')
|
||||
const mock = await startMockServer()
|
||||
console.log(` Mock server: ${mock.url}`)
|
||||
|
||||
const sandbox = createSandbox()
|
||||
writeMockConfig(sandbox.hermesHome, mock.url)
|
||||
console.log(` HERMES_HOME: ${sandbox.hermesHome}`)
|
||||
|
||||
const electronBin = findElectron()
|
||||
|
||||
const env = {
|
||||
...process.env,
|
||||
HERMES_HOME: sandbox.hermesHome,
|
||||
HERMES_DESKTOP_USER_DATA_DIR: sandbox.userDataDir,
|
||||
HERMES_DESKTOP_IGNORE_EXISTING: '1',
|
||||
HERMES_DESKTOP_HERMES_ROOT: REPO_ROOT,
|
||||
HERMES_DESKTOP_APP_NAME: `HermesDevMock-${Date.now()}`,
|
||||
}
|
||||
|
||||
console.log('Launching Electron...')
|
||||
const child = spawn(electronBin, [DESKTOP_ROOT, '--disable-gpu', '--no-sandbox'], {
|
||||
env,
|
||||
cwd: DESKTOP_ROOT,
|
||||
stdio: 'inherit',
|
||||
})
|
||||
|
||||
child.on('exit', (code) => {
|
||||
mock.close()
|
||||
sandbox.cleanup()
|
||||
process.exit(code ?? 0)
|
||||
})
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
64
package-lock.json
generated
64
package-lock.json
generated
|
|
@ -155,6 +155,7 @@
|
|||
"devDependencies": {
|
||||
"@electron/rebuild": "^4.0.6",
|
||||
"@eslint/js": "^9.39.4",
|
||||
"@playwright/test": "=1.58.2",
|
||||
"@testing-library/dom": "^10.4.0",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@types/d3-force": "^3.0.10",
|
||||
|
|
@ -376,6 +377,22 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"apps/desktop/node_modules/@playwright/test": {
|
||||
"version": "1.58.2",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.2.tgz",
|
||||
"integrity": "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright": "1.58.2"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"apps/desktop/node_modules/@types/node": {
|
||||
"version": "22.20.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.0.tgz",
|
||||
|
|
@ -386,6 +403,53 @@
|
|||
"undici-types": "~6.21.0"
|
||||
}
|
||||
},
|
||||
"apps/desktop/node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"apps/desktop/node_modules/playwright": {
|
||||
"version": "1.58.2",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz",
|
||||
"integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.58.2"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"apps/desktop/node_modules/playwright-core": {
|
||||
"version": "1.58.2",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz",
|
||||
"integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"apps/desktop/node_modules/undici-types": {
|
||||
"version": "6.21.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue