diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faae3b6f2704..c9dd08790778 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,12 @@ jobs: uses: ./.github/workflows/js-tests.yml secrets: inherit + e2e-desktop: + name: Desktop E2E + needs: detect + if: needs.detect.outputs.python == 'true' || needs.detect.outputs.frontend == 'true' + uses: ./.github/workflows/e2e-desktop.yml + docs-site: name: Docs Site needs: detect @@ -165,6 +171,7 @@ jobs: - tests - lint - js-tests + - e2e-desktop - docs-site - history-check - contributor-check diff --git a/.github/workflows/e2e-desktop.yml b/.github/workflows/e2e-desktop.yml new file mode 100644 index 000000000000..b34369fa98a6 --- /dev/null +++ b/.github/workflows/e2e-desktop.yml @@ -0,0 +1,193 @@ +name: E2E Desktop + +on: + workflow_call: + +permissions: + contents: read + +concurrency: + group: e2e-desktop-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e: + name: Playwright E2E (Linux) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # ── System deps for Electron on headless Ubuntu ─────────────────── + # Electron needs GTK, NSS,atk, etc. even under xvfb. Playwright's + # install-deps covers browsers; for Electron we install the apt + # packages directly. + - name: Install system dependencies for Electron + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq \ + xvfb \ + libgtk-3-0 libnotify4 libnss3 libxss1 libxtst6 \ + xdg-utils libatspi2.0-0 libdrm2 libgbm1 libasound2t64 + + # ── Node ─────────────────────────────────────────────────────────── + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + cache: npm + # Full npm ci (not --ignore-scripts): electron's postinstall + # downloads the binary we launch, and node-pty's native build is + # needed for the terminal pane. + - uses: ./.github/actions/retry + with: + command: npm ci + + # ── Python (for the hermes serve backend) ────────────────────────── + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + - name: Set up Python 3.11 + run: uv python install 3.11 + - name: Install Python dependencies + uses: ./.github/actions/retry + with: + command: uv sync --locked --python 3.11 --extra all --extra dev + + # ── Build desktop app ───────────────────────────────────────────── + - run: npm run --prefix apps/desktop build + + # ── Restore visual baseline screenshots from main ────────────────── + # Baselines are generated on main (via --update-snapshots) and cached. + # On PRs, we restore them so toHaveScreenshot has something to compare + # against. The cache key is keyed on the desktop source files so a + # UI change naturally invalidates it — but we fall back to the main + # cache to avoid cold starts on unrelated PRs. + - name: Restore visual baseline screenshots + id: restore-baselines + uses: actions/cache@0400d5f6a4f407c1b1b78f4ddd5bffb6548ef6f6 # v4.2.4 + with: + path: apps/desktop/e2e/__screenshots__ + key: visual-baselines-${{ github.ref_name }} + restore-keys: | + visual-baselines-main + + # ── Run Playwright E2E under xvfb ───────────────────────────────── + # xvfb runs at a fixed 1280x1024 screen so the 1220x800 Electron + # window always has a consistent viewport for screenshot comparison. + # On main, we run with --update-snapshots to generate baselines. + - name: Run Playwright E2E tests + working-directory: apps/desktop + run: | + if [ "${{ github.ref_name }}" = "main" ]; then + echo "On main — generating/updating baseline screenshots" + xvfb-run -a --server-args="-screen 0 1280x1024x24" \ + npx playwright test --reporter=list --update-snapshots + else + echo "On PR — comparing against cached baselines" + xvfb-run -a --server-args="-screen 0 1280x1024x24" \ + npx playwright test --reporter=list + fi + env: + CI: "true" + # Ensure no real API keys leak into the test env. + OPENROUTER_API_KEY: "" + OPENAI_API_KEY: "" + NOUS_API_KEY: "" + + # ── Generate step summary with visual diff info ─────────────────── + # Parse the JSON report + scan for diff images, then post a summary + # to the GitHub Actions step output so reviewers can see what changed + # without downloading artifacts. + - name: Generate visual diff summary + if: always() + working-directory: apps/desktop + run: | + echo "## Desktop E2E — Visual Diff Report" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + + # Count diff images (playwright writes *-diff.png on mismatch) + DIFF_COUNT=$(find test-results -name '*-diff.png' 2>/dev/null | wc -l) + ACTUAL_COUNT=$(find test-results -name '*-actual.png' 2>/dev/null | wc -l) + + if [ "$DIFF_COUNT" -eq 0 ] && [ "$ACTUAL_COUNT" -eq 0 ]; then + echo "✅ All screenshots matched their baselines (or no baselines existed yet)." >> "$GITHUB_STEP_SUMMARY" + else + echo "📸 **$DIFF_COUNT screenshot(s) differ from baseline:**" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Test | Diff | Actual | Expected |" >> "$GITHUB_STEP_SUMMARY" + echo "|------|------|--------|----------|" >> "$GITHUB_STEP_SUMMARY" + + # List each diff image with a link to the artifact + for diff in $(find test-results -name '*-diff.png' 2>/dev/null | sort); do + base=$(echo "$diff" | sed 's/-diff\.png$//') + test_name=$(basename "$base") + echo "| $test_name | [diff]($diff) | [actual](${base}-actual.png) | [expected](${base}-expected.png) |" >> "$GITHUB_STEP_SUMMARY" + done + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Diff images are in the \`playwright-test-results\` artifact. Download and open to compare." >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**To update baselines:** merge to main (baselines auto-update on main runs) or run \`npx playwright test --update-snapshots\` locally." >> "$GITHUB_STEP_SUMMARY" + fi + + # Also parse the JSON report for pass/fail counts + if [ -f playwright-report/results.json ]; then + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "### Test Results" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + node -e " + const r = require('./playwright-report/results.json'); + const stats = r.stats || {}; + console.log('| Status | Count |'); + console.log('|--------|-------|'); + console.log('| ✅ Passed | ' + (stats.expected || 0) + ' |'); + console.log('| ❌ Failed | ' + (stats.unexpected || 0) + ' |'); + console.log('| ⏭️ Skipped | ' + (stats.skipped || 0) + ' |'); + console.log('| 🔄 Flaky | ' + (stats.flaky || 0) + ' |'); + " >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + fi + + # ── Save updated baselines to cache (main only) ─────────────────── + - name: Save updated baselines to cache + if: github.ref_name == 'main' && always() + uses: actions/cache/save@0400d5f6a4f407c1b1b78f4ddd5bffb6548ef6f6 # v4.2.4 + with: + path: apps/desktop/e2e/__screenshots__ + key: visual-baselines-main + + # ── Upload Playwright report (HTML + traces) ────────────────────── + - name: Upload Playwright report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: playwright-report-${{ github.sha }} + path: apps/desktop/playwright-report + retention-days: 14 + overwrite: true + + # ── Upload test results (screenshots, traces, diffs) ─────────────── + - name: Upload test results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: playwright-test-results-${{ github.sha }} + path: apps/desktop/test-results + retention-days: 14 + overwrite: true + + # ── Upload just the visual diffs (small, fast to review) ────────── + - name: Upload visual diffs + if: always() && github.ref_name != 'main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: visual-diffs-${{ github.sha }} + path: | + apps/desktop/test-results/**/*-diff.png + apps/desktop/test-results/**/*-actual.png + apps/desktop/test-results/**/*-expected.png + retention-days: 14 + overwrite: true + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 6f1b3be6d92b..c4fb20049ea4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ /_pycache/ *.pyc* __pycache__/ +act/ +.act-sandbox-agent.* .venv/ .venv .vscode/ @@ -54,6 +56,10 @@ __pycache__/ hermes_agent.egg-info/ wandb/ testlogs +playwright-report/ +test-results/ +# Playwright visual regression baselines — cached from main in CI, not committed +*-snapshots/ # CLI config (may contain sensitive SSH paths) cli-config.yaml diff --git a/apps/desktop/e2e/boot-failure.spec.ts b/apps/desktop/e2e/boot-failure.spec.ts new file mode 100644 index 000000000000..91433c3ccc89 --- /dev/null +++ b/apps/desktop/e2e/boot-failure.spec.ts @@ -0,0 +1,52 @@ +/** + * E2E boot-failure tests — verify the app shows an error overlay when the + * backend can't reach the inference provider. + * + * Launches the app with a provider pointing at a dead endpoint (port 1). + * The `hermes serve` backend starts, but when the renderer tries to connect + * or when a runtime check fails, the app should show a boot failure or + * onboarding error overlay. + * + * Prerequisite: `npm run build` must have been run so dist/ exists. + */ + +import { test } from '@playwright/test' + +import { + type DeadBackendFixture, + setupDeadBackend, + waitForBootFailure, +} from './fixtures' +import { expectVisualSnapshot } from './visual-snapshot' + +let fixture: DeadBackendFixture | null = null + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test.describe('boot failure with dead provider endpoint', () => { + test('app shows error state or onboarding', async () => { + fixture = await setupDeadBackend() + + // With a dead provider endpoint, the app should eventually show either: + // 1. A boot failure overlay (if the backend fails to start), or + // 2. An onboarding overlay with an error (if the runtime check fails) + // Both outcomes prove the app is handling provider failures gracefully. + // + // We give it a generous timeout — the backend needs to start, the + // renderer needs to boot, and then the runtime check needs to fail. + await waitForBootFailure(fixture.page, 90_000) + }) + + test('screenshot of error state', async () => { + if (!fixture) { + test.skip(true, 'Previous test failed — no app running') + + return + } + + await expectVisualSnapshot(fixture!.page, { name: 'boot-failure-error-state', app: fixture.app }) + }) +}) diff --git a/apps/desktop/e2e/boot.spec.ts b/apps/desktop/e2e/boot.spec.ts new file mode 100644 index 000000000000..0fd73d845117 --- /dev/null +++ b/apps/desktop/e2e/boot.spec.ts @@ -0,0 +1,63 @@ +/** + * E2E smoke tests for the dev-mode desktop app. + * + * These tests launch the Electron app from the built dist/ (not the + * packaged binary) with a real `hermes serve` backend pointed at a mock + * inference server. The full chain is exercised: + * + * electron → hermes serve (python) → mock provider → renderer + * + * Prerequisite: `npm run build` must have been run so dist/ exists. + * Run from the nix devshell: + * npm exec playwright test e2e/boot.spec.ts --reporter=list + */ +import { expect, test } from '@playwright/test' + +import { + type MockBackendFixture, + setupMockBackend, + waitForAppReady, +} from './fixtures' +import { expectVisualSnapshot } from './visual-snapshot' + +let fixture: MockBackendFixture | null = null + +test.beforeAll(async () => { + fixture = await setupMockBackend() +}) + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test.describe('dev-mode boot with mock backend', () => { + test('window opens with Hermes title', async () => { + const title = await fixture!.page.title() + expect(title).toContain('Hermes') + }) + + test('renderer mounts and shows DOM content', async () => { + const page = fixture!.page + // Wait for the React root to mount. The app renders into #root + // (see src/main.tsx), but content may arrive through portals — so + // check the body for any interactive content instead. + await page.waitForSelector('body', { state: 'attached' }) + // Wait for the main app shell — the composer is always present. + await page.waitForSelector('textarea, [contenteditable="true"]', { + state: 'attached', + timeout: 30_000, + }) + }) + + test('backend boots and app becomes ready', async () => { + // This is the big one — wait for the full boot chain to complete: + // electron starts → hermes serve is spawned → WS connects → config + // loaded → sessions loaded → boot overlay dismissed → composer visible. + await waitForAppReady(fixture!, 120_000) + }) + + test('screenshot after boot', async () => { + await expectVisualSnapshot(fixture!.page, { name: 'boot-ready', app: fixture!.app }) + }) +}) diff --git a/apps/desktop/e2e/chat.spec.ts b/apps/desktop/e2e/chat.spec.ts new file mode 100644 index 000000000000..fe626847e83c --- /dev/null +++ b/apps/desktop/e2e/chat.spec.ts @@ -0,0 +1,89 @@ +/** + * E2E chat tests — send a message and verify a response appears. + * + * Requires the full boot chain to complete (hermes serve + mock inference + * provider). The mock server returns a canned reply, so we verify the + * response text shows up in the chat transcript. + * + * Prerequisite: `npm run build` must have been run so dist/ exists. + */ + +import { test } from '@playwright/test' + +import { + type MockBackendFixture, + setupMockBackend, + waitForAppReady, +} from './fixtures' +import { expectVisualSnapshot } from './visual-snapshot' + +let fixture: MockBackendFixture | null = null + +test.beforeAll(async () => { + fixture = await setupMockBackend() + await waitForAppReady(fixture!, 120_000) +}) + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test.describe('chat interaction with mock backend', () => { + test('send a message and receive a response', async () => { + const page = fixture!.page + + // Find the composer — it's a contenteditable textbox. + const composer = page.locator('[contenteditable="true"]').first() + await composer.waitFor({ state: 'visible', timeout: 10_000 }) + + // Click to focus, then type the message character by character. + // Using `type` instead of `fill` because the composer is a + // contenteditable div with custom keydown handling that tracks + // IME composition state — `fill` bypasses the event chain. + await composer.click() + await composer.type('Hello, can you hear me?', { delay: 20 }) + + // Submit with Enter — the composer's keydown handler intercepts + // plain Enter (without Shift) and calls submitDraft(). + await page.keyboard.press('Enter') + + // Wait for the user's message to appear in the transcript. + // The message renders as an assistant-ui message in the chat view. + await page.waitForFunction( + () => { + const body = document.body + + if (!body) { + return false + } + + return (body.textContent ?? '').includes('Hello, can you hear me?') + }, + { timeout: 15_000 }, + ) + + // Wait for the mock response to appear. The canned reply is: + // "Hello from the mock inference server! The full boot chain is working." + // Give it a generous timeout — the inference request goes through the + // gateway → hermes serve → mock server → streaming SSE back. + await page.waitForFunction( + () => { + const body = document.body + + if (!body) { + return false + } + + const text = body.textContent ?? '' + + return text.includes('mock inference server') || text.includes('boot chain is working') + }, + { timeout: 60_000 }, + ) + }) + + test('screenshot of chat with messages', async () => { + await expectVisualSnapshot(fixture!.page, { name: 'chat-with-messages', app: fixture!.app }) + }) +}) diff --git a/apps/desktop/e2e/fix-electron-tracing.ts b/apps/desktop/e2e/fix-electron-tracing.ts new file mode 100644 index 000000000000..a59825e98444 --- /dev/null +++ b/apps/desktop/e2e/fix-electron-tracing.ts @@ -0,0 +1,66 @@ +/** + * Monkey-patch: playwright's test runner never calls tracing.start() on + * Electron's internal BrowserContext because: + * 1. Playwright._allContexts() only returns [chromium, firefox, webkit] + * contexts — Electron's context is excluded. + * 2. ArtifactsRecorder.didCreateBrowserContext runs in willStartTest, before + * beforeAll launches the electron app. + * 3. The runAfterCreateBrowserContext hook doesn't exist on the Electron + * class (only on BrowserType). + * + * As a result, trace screenshots (screencast) and DOM snapshots are never + * captured for electron tests. + * + * This patch: + * 1. Patches _allContexts() to include electron contexts, so the test + * runner's didFinishTest() cleanup calls _stopTracing() → stopChunk() + * on the electron context (saving the trace chunk + merging it into + * the final trace.zip). + * 2. Manually calls tracing.start() + startChunk() after launch. + * 3. Wraps tracing.start to become startChunk after the first call, + * so the test runner's willStartTest doesn't throw "already started". + * + * Imported from playwright.config.ts so it runs before any test. + */ + +import { _electron as electron, type BrowserContext } from '@playwright/test' +import * as crypto from 'node:crypto' + +const electronContexts = new Set() +const originalLaunch = electron.launch.bind(electron) + +electron.launch = async (options: any) => { + const app = await originalLaunch(options) + const ctx = app._context as BrowserContext + electronContexts.add(ctx) + ctx.once('close', () => electronContexts.delete(ctx)) + + // Patch _allContexts so the test runner sees the electron context + // (didFinishTest cleanup → _stopTracing → stopChunk → merge into trace.zip). + const pw = electron._playwright as any + if (pw && !pw.__electronTracingPatched) { + pw.__electronTracingPatched = true + const original = pw._allContexts.bind(pw) + pw._allContexts = () => [...original(), ...electronContexts] + } + + // Start tracing — mirrors ArtifactsRecorder.didCreateBrowserContext. + const traceName = crypto.randomUUID() + await ctx.tracing.start({ + screenshots: true, + snapshots: true, + sources: true, + }).catch(() => {}) + await ctx.tracing.startChunk({ title: 'electron', name: traceName }).catch(() => {}) + + // Wrap tracing.start to redirect to startChunk after the first call. + // The test runner's willStartTest calls tracing.start() on all contexts + // in _allContexts(). Since we already started, redirect to startChunk + // to avoid "Tracing has been already started" errors. + const tracing = ctx.tracing as any + tracing.start = async (opts: any) => { + return tracing.startChunk(opts) + } + + return app +} diff --git a/apps/desktop/e2e/fixtures.ts b/apps/desktop/e2e/fixtures.ts new file mode 100644 index 000000000000..2ce46f9bc957 --- /dev/null +++ b/apps/desktop/e2e/fixtures.ts @@ -0,0 +1,613 @@ +/** + * Shared E2E fixtures for the Hermes desktop Playwright suite. + * + * Two fixture modes: + * + * 1. `mockBackend` — starts a mock inference server, writes a config.yaml + * that points at it, and launches the desktop app so the full chain + * (electron → hermes serve → provider → inference → renderer) is + * exercised with a real backend but a fake LLM. + * + * 2. `noProvider` — launches the app with an empty config (no provider + * configured). The onboarding overlay should appear. Used to test the + * first-run flow without real credentials. + * + * Both modes launch the *dev* Electron app (`electron .` against the built + * `dist/`), not the packaged binary. This avoids the multi-minute + * `electron-builder --dir` step and matches `hermes desktop --source`. The + * packaged-binary path is already covered by `launch.spec.ts`. + * + * Prerequisite: `npm run build` must have been run so that `dist/` exists. + */ + +import { spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' + +import { _electron, type ElectronApplication, type Page } from '@playwright/test' + +import { startMockServer } from './mock-server' + +const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..') +const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..') +const RELEASE_ROOT = path.join(DESKTOP_ROOT, 'release') + +// ─── Credential stripping (matches launch.spec.ts) ────────────────────── + +const CREDENTIAL_SUFFIXES: string[] = [ + '_API_KEY', + '_TOKEN', + '_SECRET', + '_PASSWORD', + '_CREDENTIALS', + '_ACCESS_KEY', + '_PRIVATE_KEY', + '_OAUTH_TOKEN', +] + +const CREDENTIAL_NAMES = new Set([ + 'ANTHROPIC_BASE_URL', + 'ANTHROPIC_TOKEN', + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SESSION_TOKEN', + 'CUSTOM_API_KEY', + 'GEMINI_BASE_URL', + 'OPENAI_BASE_URL', + 'OPENROUTER_BASE_URL', + 'OLLAMA_BASE_URL', + 'GROQ_BASE_URL', + 'XAI_BASE_URL', +]) + +function isCredentialEnvVar(name: string): boolean { + if (CREDENTIAL_NAMES.has(name)) { + return true + } + + return CREDENTIAL_SUFFIXES.some((suffix) => name.endsWith(suffix)) +} + +function stripCredentials(env: Record): Record { + const clean: Record = {} + + for (const [key, value] of Object.entries(env)) { + if (!value) { + continue + } + + if (isCredentialEnvVar(key)) { + continue + } + + clean[key] = value + } + + return clean +} + +// ─── Sandbox creation ────────────────────────────────────────────────── + +export interface Sandbox { + root: string + hermesHome: string + userDataDir: string + cleanup: () => void +} + +function createSandbox(prefix: string): Sandbox { + const root = fs.mkdtempSync(path.join(os.tmpdir(), `hermes-e2e-${prefix}-${Math.random()}`)) + const hermesHome = path.join(root, 'hermes-home') + const userDataDir = path.join(root, 'electron-user-data') + + fs.mkdirSync(hermesHome, { recursive: true }) + fs.mkdirSync(userDataDir, { recursive: true }) + + // Write a fixed window-state.json so the Electron window opens at a + // consistent size — helps with visual regression screenshots. The + // exact size is also enforced right before each screenshot (see + // expectVisualSnapshot in visual-snapshot.ts) because window managers + // may resize after launch. + fs.writeFileSync( + path.join(userDataDir, 'window-state.json'), + JSON.stringify( + { x: 0, y: 0, width: 1220, height: 800, isMaximized: false }, + null, + 2, + ), + 'utf8', + ) + + return { + root, + hermesHome, + userDataDir, + cleanup: () => { + try { + fs.rmSync(root, { recursive: true, force: true }) + } catch { + // best-effort + } + }, + } +} + +// ─── Config writing ───────────────────────────────────────────────────── + +/** + * Write a config.yaml that pre-configures a mock provider pointing at the + * mock inference server. The provider is set as the active model provider so + * the desktop app skips onboarding and boots straight to the chat UI. + */ +function writeMockProviderConfig(hermesHome: string, mockUrl: string): void { + const configPath = path.join(hermesHome, 'config.yaml') + + const config = `# Auto-generated by E2E test fixtures +model: + default: mock-model + provider: mock +providers: + mock: + api: ${mockUrl}/v1 + name: Mock + api_mode: chat_completions + key_env: MOCK_API_KEY + models: + mock-model: {} + context_length: 4096 +` + + fs.writeFileSync(configPath, config, 'utf8') +} + +/** + * Write a minimal .env with the mock API key. The key_env in config.yaml + * references MOCK_API_KEY, so the backend resolves credentials from here. + */ +function writeEnvFile(hermesHome: string, apiKey = 'e2e-mock-key'): void { + const envPath = path.join(hermesHome, '.env') + fs.writeFileSync(envPath, `MOCK_API_KEY=${apiKey}\n`, 'utf8') +} + +/** + * Write an empty config (no providers). The desktop app should show the + * onboarding overlay because no inference provider is configured. + */ +function writeEmptyConfig(hermesHome: string): void { + const configPath = path.join(hermesHome, 'config.yaml') + fs.writeFileSync(configPath, '# Auto-generated by E2E test fixtures — no providers configured\n', 'utf8') +} + +// ─── Env building ────────────────────────────────────────────────────── + +/** + * Build the environment for the Electron app process. + * + * Key env vars: + * - HERMES_HOME → sandbox hermes-home (isolated config/sessions) + * - HERMES_DESKTOP_USER_DATA_DIR → sandbox electron-user-data + * - HERMES_DESKTOP_IGNORE_EXISTING=1 → don't pick up `hermes` from PATH + * (we want the dev checkout at REPO_ROOT) + * - HERMES_DESKTOP_HERMES_ROOT → REPO_ROOT (dev checkout resolution) + * - HERMES_DESKTOP_APP_NAME → unique-ish per test (avoids single-instance lock) + * - XDG_RUNTIME_DIR → ensure Electron has a writable runtime dir on Linux + */ +function buildAppEnv(sandbox: Sandbox, extra: Record = {}): Record { + const clean = stripCredentials(process.env) + + // XDG_RUNTIME_DIR is needed for Electron on Linux when running in a + // headless/CI context — without it the zygote may fail to initialize. + if (!clean.XDG_RUNTIME_DIR && process.env.XDG_RUNTIME_DIR) { + clean.XDG_RUNTIME_DIR = process.env.XDG_RUNTIME_DIR + } + + // DISPLAY — needed for Electron to open a window. + if (!clean.DISPLAY && process.env.DISPLAY) { + clean.DISPLAY = process.env.DISPLAY + } + + return { + ...clean, + HERMES_HOME: sandbox.hermesHome, + HERMES_DESKTOP_USER_DATA_DIR: sandbox.userDataDir, + HERMES_DESKTOP_IGNORE_EXISTING: '1', + HERMES_DESKTOP_HERMES_ROOT: REPO_ROOT, + HERMES_DESKTOP_APP_NAME: `HermesE2E-${Date.now()}`, + // Clear dev-server override — we want the built dist/, not a vite server. + // The dev-server check in main.ts looks for this env var; if it's set, + // it loads from the vite URL instead of the local file. + ...extra, + } +} + +// ─── Electron launch ──────────────────────────────────────────────────── + +/** + * Verify that the desktop app has been built (dist/ exists). Playwright + * tests can't run without it — the Electron main process loads + * dist/electron-main.mjs and the renderer loads dist/index.html. + */ +function assertDistBuilt(): void { + const distDir = path.join(DESKTOP_ROOT, 'dist') + const electronMain = path.join(distDir, 'electron-main.mjs') + const indexHtml = path.join(distDir, 'index.html') + + if (!fs.existsSync(electronMain)) { + throw new Error( + `Desktop dist not built. Run 'cd apps/desktop && npm run build' first.\n` + + `Missing: ${electronMain}`, + ) + } + + if (!fs.existsSync(indexHtml)) { + throw new Error( + `Desktop dist/index.html not found. Run 'cd apps/desktop && npm run build' first.\n` + + `Missing: ${indexHtml}`, + ) + } +} + +/** + * Find the Electron binary. In the nix devshell, `electron` is on PATH. + * As a fallback, use the node_modules/.bin/electron from the desktop package. + */ +function findElectron(): string { + // In dev mode, we use the `electron` binary directly (not the packaged app). + // The dev:electron script in package.json does exactly this: `electron .` + // after building. We replicate that here. + const localElectron = path.join(REPO_ROOT, 'node_modules', 'electron', 'dist', 'electron') + + if (fs.existsSync(localElectron)) { + return localElectron + } + + // Fall back to PATH + const result = spawnSync('which', ['electron'], { + encoding: 'utf8', + }) + + if (result.status === 0 && result.stdout.trim()) { + return result.stdout.trim() + } + + throw new Error( + 'Electron binary not found. Run "npm install" from the repo root to install devDependencies.', + ) +} + +/** + * Launch the desktop app in dev mode. + * + * @param sandbox - isolated HERMES_HOME + userData + * @param env - the process environment (already has HERMES_HOME etc.) + * @returns the ElectronApplication + first Page + */ +async function launchDesktop( + env: Record, +): Promise<{ app: ElectronApplication; page: Page }> { + assertDistBuilt() + + const electronBin = findElectron() + + // `electron .` loads from the package.json `main` field + // (dist/electron-main.mjs after build). + const app = await _electron.launch({ + executablePath: electronBin, + args: [ + DESKTOP_ROOT, // `electron .` — the `.` is the desktop package dir + '--disable-gpu', + '--no-sandbox', + ], + env, + cwd: DESKTOP_ROOT, + }) + + const page = await app.firstWindow() + + return { app, page } +} + +// ─── Public fixtures ──────────────────────────────────────────────────── + +export interface MockBackendFixture { + app: ElectronApplication + page: Page + mockUrl: string + sandbox: Sandbox + cleanup: () => Promise +} + +/** + * Set up a full mock-backend E2E environment: + * 1. Start the mock inference server + * 2. Create a sandbox with config.yaml pointing at it + * 3. Launch the desktop app + * 4. Return handles for test interaction + */ +export async function setupMockBackend(): Promise { + // 1. Start mock server + const mock = await startMockServer() + + // 2. Create sandbox + write config + const sandbox = createSandbox('mock') + writeMockProviderConfig(sandbox.hermesHome, mock.url) + writeEnvFile(sandbox.hermesHome) + + // 3. Build env + launch + const env = buildAppEnv(sandbox) + const { app, page } = await launchDesktop(env) + + return { + app, + page, + mockUrl: mock.url, + sandbox, + cleanup: async () => { + await app.close().catch(() => undefined) + await mock.close() + sandbox.cleanup() + }, + } +} + +export interface NoProviderFixture { + app: ElectronApplication + page: Page + sandbox: Sandbox + cleanup: () => Promise +} + +/** + * Launch the app with no provider configured. The onboarding overlay should + * appear because there's no inference provider in config.yaml. + */ +export async function setupNoProvider(): Promise { + const sandbox = createSandbox('noprovider') + writeEmptyConfig(sandbox.hermesHome) + + const env = buildAppEnv(sandbox) + const { app, page } = await launchDesktop(env) + + return { + app, + page, + sandbox, + cleanup: async () => { + await app.close().catch(() => undefined) + sandbox.cleanup() + }, + } +} + +export interface DeadBackendFixture { + app: ElectronApplication + page: Page + sandbox: Sandbox + cleanup: () => Promise +} + +/** + * Launch the app with a provider pointing at a dead endpoint (port 1, which + * nothing listens on). The boot should fail with a connection error, + * triggering the BootFailureOverlay. + */ +export async function setupDeadBackend(): Promise { + const sandbox = createSandbox('dead') + const configPath = path.join(sandbox.hermesHome, 'config.yaml') + fs.writeFileSync( + configPath, + `# Auto-generated by E2E test fixtures — dead provider +model: mock-model +providers: + mock: + api: http://127.0.0.1:1/v1 + name: Mock + api_mode: chat_completions + key_env: MOCK_API_KEY + models: + mock-model: {} + context_length: 4096 +`, + 'utf8', + ) + writeEnvFile(sandbox.hermesHome) + + const env = buildAppEnv(sandbox) + const { app, page } = await launchDesktop(env) + + return { + app, + page, + sandbox, + cleanup: async () => { + await app.close().catch(() => undefined) + sandbox.cleanup() + }, + } +} + +// ─── Packaged-binary fixture ─────────────────────────────────────────── + +/** + * Resolve the packaged Electron binary path, per-platform, matching + * electron-builder's output layout under release/. + */ +function resolvePackagedBinaryPath(): string { + if (process.platform === 'win32') { + return path.join(RELEASE_ROOT, 'win-unpacked', 'Hermes.exe') + } + + if (process.platform === 'darwin') { + const arch = process.arch === 'arm64' ? 'arm64' : 'x64' + + return path.join(RELEASE_ROOT, `mac-${arch}`, 'Hermes.app', 'Contents', 'MacOS', 'Hermes') + } + + return path.join(RELEASE_ROOT, 'linux-unpacked', 'hermes') +} + +export const PACKAGED_BINARY_PATH = resolvePackagedBinaryPath() + +export function packagedBinaryExists(): boolean { + return fs.existsSync(PACKAGED_BINARY_PATH) +} + +export interface PackagedAppFixture { + app: ElectronApplication + page: Page + sandbox: Sandbox + cleanup: () => Promise +} + +/** + * Launch the *packaged* Electron binary (from `npm run pack` → + * `electron-builder --dir`) with `BOOT_FAKE=1` so it simulates boot + * progress without spawning a real Hermes backend. + * + * Uses the same sandbox isolation (credential stripping, isolated + * HERMES_HOME + userData, unique app name) as the dev-mode fixtures. + * + * Skips if the packaged binary doesn't exist — run `npm run pack` first. + */ +export async function setupPackagedApp(): Promise { + if (!packagedBinaryExists()) { + throw new Error( + `Built app binary not found: ${PACKAGED_BINARY_PATH}. Run 'npm run pack' first.`, + ) + } + + const sandbox = createSandbox('packaged') + + // Build the sandbox env using the shared helpers, then add the + // packaged-binary-specific overrides. + const env = buildAppEnv(sandbox, { + // Fake boot: simulates progress steps without spawning the real backend. + HERMES_DESKTOP_BOOT_FAKE: '1', + HERMES_DESKTOP_BOOT_FAKE_STEP_MS: '120', + }) + + // Clear dev-server + hermes-root overrides — the packaged binary + // should use its own bundled renderer, not the dev checkout. + delete (env as Record).HERMES_DESKTOP_DEV_SERVER + delete (env as Record).HERMES_DESKTOP_HERMES + delete (env as Record).HERMES_DESKTOP_HERMES_ROOT + + const app = await _electron.launch({ + executablePath: PACKAGED_BINARY_PATH, + args: ['--disable-gpu', '--no-sandbox'], + env, + }) + + const page = await app.firstWindow() + + return { + app, + page, + sandbox, + cleanup: async () => { + await app.close().catch(() => undefined) + sandbox.cleanup() + }, + } +} + +// ─── Wait helpers ────────────────────────────────────────────────────── + +/** + * Wait for the desktop app to finish booting and show the main chat UI. + * + * The boot overlay disappears when `completeDesktopBoot()` fires in the + * renderer — at that point the gateway is open, config is loaded, and + * sessions are loaded. We detect this by waiting for the boot/connecting + * overlay to become invisible and the main app shell to be present. + */ +export async function waitForAppReady(fixture: MockBackendFixture | NoProviderFixture | DeadBackendFixture, timeoutMs = 60_000): Promise { + const { page, app } = fixture + // The connecting overlay has a data-testid or we can check for the + // absence of boot indicators. The simplest reliable approach is to wait + // for the composer (chat input) to become visible — it's disabled until + // the gateway is open. + await page.waitForSelector('textarea, [contenteditable="true"]', { + state: 'visible', + timeout: timeoutMs, + }) + + // On Electron 40.x, ready-to-show may never fire (electron/electron#51972) + // and the window stays hidden even though the DOM is rendered. The main + // process has a TEST_WORKER_INDEX-gated fallback that force-shows the + // window, but the DOM can be ready before that fires. Poll until the + // window is actually visible so interactions (click, screenshot) don't + // hit a hidden surface. + if (app) { + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + const visible = await app.evaluate(({ BrowserWindow }) => { + const w = BrowserWindow.getAllWindows()[0] + + return w ? w.isVisible() : false + }).catch(() => false) + + if (visible) {break} + await page.waitForTimeout(500) + } + } +} + +/** + * Wait for the onboarding overlay to appear (no provider configured). + */ +export async function waitForOnboarding(page: Page, timeoutMs = 60_000): Promise { + // The onboarding overlay contains a heading with "Choose your provider" + // or similar text. We look for any text that indicates the picker. + await page.waitForFunction( + () => { + const root = document.getElementById('root') + + if (!root) { + return false + } + + const text = root.textContent ?? '' + + return ( + text.includes('provider') || + text.includes('Provider') || + text.includes('Choose') || + text.includes('API key') || + text.includes('Sign in') + ) + }, + { timeout: timeoutMs }, + ) +} + +/** + * Wait for the boot failure overlay to appear. + */ +export async function waitForBootFailure(page: Page, timeoutMs = 60_000): Promise { + await page.waitForFunction( + () => { + const root = document.getElementById('root') + + if (!root) { + return false + } + + const text = root.textContent ?? '' + + // The boot failure overlay shows an error message + retry/repair + // buttons. Look for error-related text. + return ( + text.includes('error') || + text.includes('Error') || + text.includes('failed') || + text.includes('Failed') || + text.includes('Retry') || + text.includes('Repair') + ) + }, + { timeout: timeoutMs }, + ) +} diff --git a/apps/desktop/e2e/launch-packaged-app.spec.ts b/apps/desktop/e2e/launch-packaged-app.spec.ts new file mode 100644 index 000000000000..5f04e422e34a --- /dev/null +++ b/apps/desktop/e2e/launch-packaged-app.spec.ts @@ -0,0 +1,87 @@ +import { expect, test } from '@playwright/test' + +import { + PACKAGED_BINARY_PATH, + type PackagedAppFixture, + packagedBinaryExists, + setupPackagedApp, +} from './fixtures' +import { expectVisualSnapshot } from './visual-snapshot' + +/** + * E2E smoke tests for the packaged Hermes desktop app. + * + * Launches the real packaged Electron binary (produced by `npm run pack` → + * `electron-builder --dir`) with BOOT_FAKE=1 and full sandbox isolation + * (credential stripping, isolated HERMES_HOME + userData, unique app name). + * + * Skips if the packaged binary doesn't exist — run `npm run pack` first. + */ + +let fixture: PackagedAppFixture | null = null + +test.beforeAll(async () => { + test.skip( + !packagedBinaryExists(), + `Built app binary not found: ${PACKAGED_BINARY_PATH}. Run 'npm run pack' first.`, + ) + + fixture = await setupPackagedApp() +}) + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test('window opens with the Hermes title', async () => { + const title = await fixture!.page.title() + expect(title).toContain('Hermes') +}) + +test('renderer loads and shows DOM content', async () => { + const page = fixture!.page + await page.waitForSelector('#root', { state: 'attached', timeout: 30_000 }) + const childCount = await page.locator('#root > *').count() + expect(childCount).toBeGreaterThan(0) +}) + +test('boot progress overlay fades out or shows error state', async () => { + const page = fixture!.page + await page.waitForFunction( + () => { + const root = document.getElementById('root') + + if (!root) { + return false + } + + const text = root.textContent ?? '' + + // Error path: boot failure overlay renders an error message. + if (text.includes('error') || text.includes('Error') || text.includes('failed')) { + return true + } + + // Success path: overlay disappears and the app renders. If there's + // no "boot" / "starting" / "installing" text visible, boot has + // completed (either to the main UI or to onboarding). + const bootIndicators = ['starting', 'resolving', 'spawning', 'waiting', 'installing'] + const lower = text.toLowerCase() + + return !bootIndicators.some((word) => lower.includes(word)) + }, + { timeout: 60_000 }, + ) +}) + +test('can capture a screenshot for the CI artifact', async () => { + if (!fixture) { + test.skip(true, 'Previous test failed — no app running') + + return + } + + // Visual snapshot — won't fail on diff, just logs + generates diff image + await expectVisualSnapshot(fixture!.page, { name: 'packaged-app-booted', timeout: 10_000, app: fixture!.app }) +}) diff --git a/apps/desktop/e2e/mock-backend-setup.spec.ts b/apps/desktop/e2e/mock-backend-setup.spec.ts new file mode 100644 index 000000000000..0c7840c322d7 --- /dev/null +++ b/apps/desktop/e2e/mock-backend-setup.spec.ts @@ -0,0 +1,85 @@ +/** + * E2E tests asserting the mock backend gets the app past the setup/onboarding + * screen. + * + * The mock backend fixture writes a config.yaml with a pre-configured mock + * provider pointing at a mock inference server. When the app boots, the + * runtime readiness check should detect the working provider and dismiss the + * onboarding overlay — landing straight on the chat UI without ever showing + * the "Let's get you setup with Hermes Agent" screen. + * + * If these tests fail, the mock backend config isn't getting the app past + * onboarding — the chat interaction tests (chat.spec.ts) will also fail + * because the composer is blocked by the setup overlay. + * + * Prerequisite: `npm run build` must have been run so dist/ exists. + */ + +import { expect, test } from '@playwright/test' + +import { + type MockBackendFixture, + setupMockBackend, + waitForAppReady, +} from './fixtures' +import { expectVisualSnapshot } from './visual-snapshot' + +let fixture: MockBackendFixture | null = null + +test.beforeAll(async () => { + fixture = await setupMockBackend() + await waitForAppReady(fixture!, 120_000) +}) + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test.describe('mock backend gets past setup screen', () => { + test('onboarding overlay is not shown', async () => { + const page = fixture!.page + + // The onboarding overlay renders "Let's get you setup with Hermes Agent" + // when the runtime check fails to find a working provider. With the mock + // backend configured, the runtime check should pass and the overlay + // returns null — this text should NOT be present in the DOM. + await page.waitForFunction( + () => { + const text = document.body.textContent ?? '' + + return !text.includes("Let's get you setup") + }, + { timeout: 30_000 }, + ) + }) + + test('chat composer is visible', async () => { + const page = fixture!.page + + // The composer (contenteditable div) should be visible and not blocked + // by the onboarding overlay. If the first test passed, the overlay is + // gone and the composer is the primary interactive surface. + const composer = page.locator('[contenteditable="true"]').first() + await expect(composer).toBeVisible() + }) + + test('can type into the composer', async () => { + const page = fixture!.page + + // If the setup overlay is truly gone, the composer accepts input. + const composer = page.locator('[contenteditable="true"]').first() + await composer.click() + await composer.type('hello mock backend', { delay: 20 }) + + // Verify the typed text appears in the DOM. + await page.waitForFunction( + () => (document.body.textContent ?? '').includes('hello mock backend'), + { timeout: 10_000 }, + ) + }) + + test('screenshot shows chat UI without setup screen', async () => { + await expectVisualSnapshot(fixture!.page, { name: 'mock-backend-chat-ready', app: fixture!.app }) + }) +}) diff --git a/apps/desktop/e2e/mock-server.ts b/apps/desktop/e2e/mock-server.ts new file mode 100644 index 000000000000..680cba30e7af --- /dev/null +++ b/apps/desktop/e2e/mock-server.ts @@ -0,0 +1,203 @@ +/** + * Minimal OpenAI-compatible mock inference server for E2E tests. + * + * Implements just enough of the /v1/* surface for `hermes serve` to resolve a + * provider, list models, and stream a canned chat completion back to the + * desktop app — without any real LLM. + * + * Endpoints: + * GET /v1/models → { data: [{ id, ... }] } + * POST /v1/chat/completions → streaming (SSE) or non-streaming response + * + * The canned response is a short, deterministic assistant message. Tool-call + * requests are not simulated — the E2E tests only need the chat surface to + * prove the full boot → gateway → inference → renderer chain works. + */ + +import http from 'node:http' + +/** A canned assistant reply used for every chat completion request. */ +const CANNED_REPLY = 'Hello from the mock inference server! The full boot chain is working.' + +/** + * Start the mock server on an ephemeral port. + * + * @returns a handle with `port`, `url`, and `close()`. + */ +export function startMockServer(): Promise<{ port: number; url: string; close: () => Promise }> { + return new Promise((resolve, reject) => { + const server = http.createServer((req, res) => { + // CORS headers — the Electron renderer doesn't need them, but they + // don't hurt and make the server usable from a browser context too. + res.setHeader('Access-Control-Allow-Origin', '*') + res.setHeader('Access-Control-Allow-Headers', '*') + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') + + if (req.method === 'OPTIONS') { + res.writeHead(204) + res.end() + return + } + + // GET /v1/models — return a single fake model. + if (req.method === 'GET' && req.url === '/v1/models') { + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end( + JSON.stringify({ + object: 'list', + data: [ + { + id: 'mock-model', + object: 'model', + created: 0, + owned_by: 'mock', + }, + ], + }), + ) + return + } + + // POST /v1/chat/completions — return a canned response. + if (req.method === 'POST' && req.url?.startsWith('/v1/chat/completions')) { + let body = '' + + req.on('data', (chunk: Buffer) => { + body += chunk.toString() + }) + + req.on('end', () => { + let parsed: any = {} + + try { + parsed = JSON.parse(body) + } catch { + // malformed JSON — treat as non-streaming with defaults + } + + const stream = parsed.stream === true + const model = parsed.model || 'mock-model' + + if (stream) { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }) + + // Send the content in a few chunks to simulate streaming. + const words = CANNED_REPLY.split(' ') + let i = 0 + + const sendChunk = () => { + if (i >= words.length) { + // Final chunk with finish_reason + res.write( + `data: ${JSON.stringify({ + id: 'mock-completion', + object: 'chat.completion.chunk', + created: 0, + model, + choices: [ + { + index: 0, + delta: {}, + finish_reason: 'stop', + }, + ], + })}\n\n`, + ) + res.write('data: [DONE]\n\n') + res.end() + return + } + + const word = i === 0 ? words[i] : ' ' + words[i] + res.write( + `data: ${JSON.stringify({ + id: 'mock-completion', + object: 'chat.completion.chunk', + created: 0, + model, + choices: [ + { + index: 0, + delta: { content: word }, + finish_reason: null, + }, + ], + })}\n\n`, + ) + i++ + // Small delay between chunks to simulate real streaming. + setTimeout(sendChunk, 20) + } + + sendChunk() + } else { + // Non-streaming response + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end( + JSON.stringify({ + id: 'mock-completion', + object: 'chat.completion', + created: 0, + model, + choices: [ + { + index: 0, + message: { role: 'assistant', content: CANNED_REPLY }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + }, + }), + ) + } + }) + + req.on('error', () => { + res.writeHead(400) + res.end('Bad request') + }) + return + } + + // Fallback — 404 for anything else + res.writeHead(404, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ error: 'Not found' })) + }) + + server.on('error', reject) + + server.listen(0, '127.0.0.1', () => { + const addr = server.address() + if (addr === null || typeof addr === 'string') { + reject(new Error('Failed to get server address')) + return + } + + const port = addr.port + const url = `http://127.0.0.1:${port}` + + resolve({ + port, + url, + close: () => + new Promise((resolveClose, rejectClose) => { + server.close((err) => { + if (err) { + rejectClose(err) + } else { + resolveClose() + } + }) + }), + }) + }) + }) +} diff --git a/apps/desktop/e2e/onboarding.spec.ts b/apps/desktop/e2e/onboarding.spec.ts new file mode 100644 index 000000000000..952178a3e063 --- /dev/null +++ b/apps/desktop/e2e/onboarding.spec.ts @@ -0,0 +1,76 @@ +/** + * E2E onboarding tests — verify the provider picker appears when no + * inference provider is configured. + * + * Launches the app with an empty config.yaml (no providers). The renderer + * should detect the unconfigured state and show the DesktopOnboardingOverlay + * with provider options / API key form. + * + * Prerequisite: `npm run build` must have been run so dist/ exists. + */ + +import { expect, test } from '@playwright/test' + +import { + type NoProviderFixture, + setupNoProvider, + waitForOnboarding, +} from './fixtures' +import { expectVisualSnapshot } from './visual-snapshot' + +let fixture: NoProviderFixture | null = null + +test.afterAll(async () => { + await fixture?.cleanup() + fixture = null +}) + +test.describe('onboarding with no provider configured', () => { + test('onboarding overlay appears on first boot', async () => { + fixture = await setupNoProvider() + + // The app should boot (hermes serve starts fine even without a provider), + // but the renderer should show the onboarding overlay because no + // provider is configured. + await waitForOnboarding(fixture.page, 90_000) + }) + + test('onboarding shows provider options or API key form', async () => { + if (!fixture) { + test.skip(true, 'Previous test failed — no app running') + + return + } + + const page = fixture.page + + // The onboarding overlay should contain provider-related text. + // It might show OAuth providers, an API key form, or a "choose later" + // link. Verify at least one of these is visible. + const rootText = await page.evaluate(() => { + const root = document.getElementById('root') + + return root?.textContent ?? '' + }) + + const hasProviderText = + rootText.includes('provider') || + rootText.includes('Provider') || + rootText.includes('API key') || + rootText.includes('Sign in') || + rootText.includes('OpenRouter') || + rootText.includes('OpenAI') + + expect(hasProviderText).toBe(true) + }) + + test('screenshot of onboarding overlay', async () => { + if (!fixture) { + test.skip(true, 'Previous test failed — no app running') + + return + } + + await expectVisualSnapshot(fixture.page, { name: 'onboarding-overlay', app: fixture.app }) + }) +}) diff --git a/apps/desktop/e2e/visual-snapshot.ts b/apps/desktop/e2e/visual-snapshot.ts new file mode 100644 index 000000000000..570d62053456 --- /dev/null +++ b/apps/desktop/e2e/visual-snapshot.ts @@ -0,0 +1,90 @@ +/** + * Visual snapshot helper — wraps `toHaveScreenshot` so visual diffs are + * reported without failing the test suite. + * + * On CI, the JSON reporter + post-test script parse the results and post a + * summary to the GitHub Actions step output, and diff images are uploaded + * as artifacts. This keeps visual regressions visible without gating PRs + * on pixel-perfect matches. + * + * When a screenshot matches the baseline, nothing happens. When it + * differs, Playwright writes three images to the test's output dir: + * -actual.png, -expected.png, -diff.png + * These are picked up by the "Upload test results" artifact step. + */ +import { type ElectronApplication, expect, type Page } from '@playwright/test' + +/** Fixed window dimensions for visual regression screenshots. */ +export const VISUAL_WINDOW_WIDTH = 1220 +export const VISUAL_WINDOW_HEIGHT = 800 + +export interface VisualSnapshotOptions { + /** Snapshot name — defaults to the test title. */ + name?: string + /** Full page screenshot (default) vs. viewport-only. */ + fullPage?: boolean + /** Timeout in ms. */ + timeout?: number + /** The Electron app handle — needed to force a fixed window size. */ + app?: ElectronApplication +} + +/** + * Force the Electron window to a fixed size so screenshots are comparable + * across runs and CI environments. Window managers (Hyprland, etc.) may + * auto-tile or resize windows after launch; calling this right before the + * screenshot ensures the viewport is always the expected size. + */ +async function forceFixedSize(app: ElectronApplication): Promise { + await app.evaluate(({ BrowserWindow }, { width, height }) => { + const win = BrowserWindow.getAllWindows()[0] + + if (win) { + win.unmaximize() + // setMinimumSize must be ≤ the target, otherwise setSize is clamped. + win.setMinimumSize(width, height) + win.setSize(width, height, false) + win.setBounds({ x: 0, y: 0, width, height }) + } + }, { width: VISUAL_WINDOW_WIDTH, height: VISUAL_WINDOW_HEIGHT }) +} + +/** + * Take a screenshot and compare it against the baseline. + * + * If the baseline doesn't exist yet (first run), Playwright creates it. + * If it differs, the test logs a soft warning but does NOT fail — the diff + * images are still generated for CI to surface. + */ +export async function expectVisualSnapshot( + page: Page, + options: VisualSnapshotOptions = {}, +): Promise { + const { name, fullPage = false, timeout = 30_000, app } = options + + // Force the window to a fixed size right before the screenshot so it's + // always comparable, regardless of WM resizing during the test. + if (app) { + await forceFixedSize(app) + // Give the renderer a moment to relayout after the resize. + await page.waitForTimeout(500) + } + + // Playwright appends a platform suffix (e.g. "-linux") and requires + // a .png extension on the name argument. Auto-append it if missing. + const snapshotName = name ? (name.endsWith('.png') ? name : `${name}.png`) : undefined + + try { + if (snapshotName) { + await expect(page).toHaveScreenshot(snapshotName, { fullPage, timeout }) + } else { + await expect(page).toHaveScreenshot({ fullPage, timeout }) + } + } catch (err) { + // Don't fail the test — just log that a diff was detected. + // The diff/actual/expected images are already written to the test + // output directory by Playwright for the CI workflow to pick up. + console.log(`[visual-diff] ${name ?? '(unnamed)'} — screenshot differs from baseline`) + console.log(` ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`) + } +} diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 1cde03bf64d1..a54b7182ee1a 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -13,6 +13,7 @@ "scripts": { "dev": "concurrently -k \"npm:dev:renderer\" \"npm:dev:electron\"", "dev:fake-boot": "cross-env HERMES_DESKTOP_BOOT_FAKE=1 HERMES_DESKTOP_BOOT_FAKE_STEP_MS=650 npm run dev", + "dev:mock": "node scripts/dev-mock.mjs", "dev:renderer": "node scripts/assert-root-install.mjs && vite --host 127.0.0.1 --port 5174", "dev:electron": "wait-on http://127.0.0.1:5174 && node scripts/bundle-electron-main.mjs --dev && cross-env XCURSOR_SIZE=24 HERMES_DESKTOP_DEV_SERVER=http://127.0.0.1:5174 electron .", "profile:main": "wait-on http://127.0.0.1:5174 && node scripts/bundle-electron-main.mjs --dev && cross-env XCURSOR_SIZE=24 HERMES_DESKTOP_DEV_SERVER=http://127.0.0.1:5174 electron --inspect=9229 .", @@ -49,7 +50,10 @@ "test:desktop:platforms": "vitest run --project electron", "test": "vitest run", "preview": "node scripts/assert-root-install.mjs && vite preview --host 127.0.0.1 --port 4174", - "check": "npm run typecheck && npm run test && npm run test:desktop:all && npm run build" + "check": "npm run typecheck && npm run test && npm run test:desktop:all && npm run build", + "test:e2e": "playwright test e2e/", + "test:e2e:visual": "WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list", + "test:e2e:update-snapshots": "WLR_BACKENDS=headless WLR_NO_HARDWARE_CURSORS=1 cage -- npx playwright test e2e/ --reporter=list --update-snapshots" }, "dependencies": { "@assistant-ui/react": "^0.14.23", @@ -124,6 +128,7 @@ "devDependencies": { "@electron/rebuild": "^4.0.6", "@eslint/js": "^9.39.4", + "@playwright/test": "=1.58.2", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.3.2", "@types/d3-force": "^3.0.10", diff --git a/apps/desktop/playwright.config.ts b/apps/desktop/playwright.config.ts new file mode 100644 index 000000000000..9ed34b2fea26 --- /dev/null +++ b/apps/desktop/playwright.config.ts @@ -0,0 +1,53 @@ +import './e2e/fix-electron-tracing' + +import { defineConfig, type ReporterDescription } from '@playwright/test' + +/** + * Visual regression testing config. + * + * Screenshots are compared against baselines. On `main`, baselines are + * generated with `--update-snapshots` and cached. On PRs, the cached + * baselines are restored and screenshots are compared — but tests DON'T + * fail on visual diffs (see `expectVisualSnapshot` in visual-snapshot.ts). + * Instead, diffs are surfaced in the CI step summary and uploaded as + * artifacts for human review. + * + * To update baselines after an intentional UI change: + * npx playwright test --update-snapshots + */ +const reporters: ReporterDescription[] = [ + ['list'], + ['html', { open: 'never', outputFolder: 'playwright-report' }], +] + +if (process.env.CI) { + reporters.push(['json', { outputFile: 'playwright-report/results.json' }]) +} + +export default defineConfig({ + /* Test files live under e2e/ so they never collide with the vitest suite + * under src/ or the node:test files under electron/. */ + testDir: './e2e', + /* The desktop app can take a while to bootstrap on cold CI runners — 90 s + * per test gives us headroom without masking real hangs. */ + timeout: 90_000, + retries: process.env.CI ? 1 : 0, + /* Each test gets its own worker so the Electron process is fully isolated. */ + fullyParallel: false, + reporter: reporters, + use: { + screenshot: 'on', + trace: { mode: 'on', screenshots: true, snapshots: true, sources: true }, + }, + expect: { + toHaveScreenshot: { + // 1% of pixels may differ — absorbs sub-pixel font rendering variance + // between local and CI environments. + maxDiffPixelRatio: 0.01, + animations: 'disabled', + caret: 'hide', + // Per-channel threshold for "close enough" — anti-aliasing differences. + threshold: 0.2, + }, + }, +}) diff --git a/apps/desktop/scripts/dev-mock.mjs b/apps/desktop/scripts/dev-mock.mjs new file mode 100644 index 000000000000..7b523d88b3c3 --- /dev/null +++ b/apps/desktop/scripts/dev-mock.mjs @@ -0,0 +1,237 @@ +#!/usr/bin/env node +/** + * Launch the desktop app with a mock inference provider — no real API + * keys needed. Starts a local OpenAI-compatible server that returns a + * canned reply, writes an isolated config.yaml + .env, and launches the + * built Electron app against them. + * + * This reuses the same mock-server and config format as the E2E fixtures + * (apps/desktop/e2e/mock-server.ts + fixtures.ts), so local dev and CI + * test the same chain. + * + * Prerequisite: `npm run build` must have been run so dist/ exists. + * + * Usage: + * node scripts/dev-mock.mjs + * npm run dev:mock + * + * The mock server listens on an ephemeral port and replies to every + * chat completion with: + * "Hello from the mock inference server! The full boot chain is working." + */ + +import http from 'node:http' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { spawn, spawnSync } from 'node:child_process' + +const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..') +const REPO_ROOT = path.resolve(DESKTOP_ROOT, '..', '..') + +// ── Canned reply ─────────────────────────────────────────────────────── + +const CANNED_REPLY = + 'Hello from the mock inference server! The full boot chain is working.' + +// ── Mock server (mirrors e2e/mock-server.ts) ─────────────────────────── + +function startMockServer() { + return new Promise((resolve, reject) => { + const server = http.createServer((req, res) => { + res.setHeader('Access-Control-Allow-Origin', '*') + res.setHeader('Access-Control-Allow-Headers', '*') + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') + + if (req.method === 'OPTIONS') { + res.writeHead(204) + res.end() + return + } + + if (req.method === 'GET' && req.url === '/v1/models') { + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end( + JSON.stringify({ + object: 'list', + data: [{ id: 'mock-model', object: 'model', created: 0, owned_by: 'mock' }], + }), + ) + return + } + + if (req.method === 'POST' && req.url?.startsWith('/v1/chat/completions')) { + let body = '' + req.on('data', (chunk) => { body += chunk.toString() }) + req.on('end', () => { + let parsed = {} + try { parsed = JSON.parse(body) } catch { /* non-streaming */ } + + const stream = parsed.stream === true + const model = parsed.model || 'mock-model' + + if (stream) { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }) + const words = CANNED_REPLY.split(' ') + let i = 0 + const sendChunk = () => { + if (i >= words.length) { + res.write( + `data: ${JSON.stringify({ + id: 'mock-completion', object: 'chat.completion.chunk', + created: 0, model, + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + })}\n\n`, + ) + res.write('data: [DONE]\n\n') + res.end() + return + } + const word = i === 0 ? words[i] : ' ' + words[i] + res.write( + `data: ${JSON.stringify({ + id: 'mock-completion', object: 'chat.completion.chunk', + created: 0, model, + choices: [{ index: 0, delta: { content: word }, finish_reason: null }], + })}\n\n`, + ) + i++ + setTimeout(sendChunk, 20) + } + sendChunk() + } else { + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end( + JSON.stringify({ + id: 'mock-completion', object: 'chat.completion', + created: 0, model, + choices: [{ + index: 0, + message: { role: 'assistant', content: CANNED_REPLY }, + finish_reason: 'stop', + }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }), + ) + } + }) + req.on('error', () => { res.writeHead(400); res.end('Bad request') }) + return + } + + res.writeHead(404, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ error: 'Not found' })) + }) + + server.on('error', reject) + server.listen(0, '127.0.0.1', () => { + const addr = server.address() + if (addr === null || typeof addr === 'string') { + reject(new Error('Failed to get server address')) + return + } + resolve({ port: addr.port, url: `http://127.0.0.1:${addr.port}`, close: () => server.close() }) + }) + }) +} + +// ── Config + env writing (mirrors e2e/fixtures.ts) ───────────────────── + +function createSandbox() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), `hermes-dev-mock-${Date.now()}`)) + const hermesHome = path.join(root, 'hermes-home') + const userDataDir = path.join(root, 'electron-user-data') + fs.mkdirSync(hermesHome, { recursive: true }) + fs.mkdirSync(userDataDir, { recursive: true }) + return { root, hermesHome, userDataDir, cleanup: () => fs.rmSync(root, { recursive: true, force: true }) } +} + +function writeMockConfig(hermesHome, mockUrl) { + fs.writeFileSync( + path.join(hermesHome, 'config.yaml'), + `# Auto-generated by dev-mock.mjs +model: + default: mock-model + provider: mock +providers: + mock: + api: ${mockUrl}/v1 + name: Mock + api_mode: chat_completions + key_env: MOCK_API_KEY + models: + mock-model: {} + context_length: 4096 +`, + 'utf8', + ) + fs.writeFileSync(path.join(hermesHome, '.env'), 'MOCK_API_KEY=e2e-mock-key\n', 'utf8') +} + +// ── Electron launch ──────────────────────────────────────────────────── + +function findElectron() { + const local = path.join(REPO_ROOT, 'node_modules', 'electron', 'dist', 'electron') + if (fs.existsSync(local)) return local + const r = spawnSync('which', ['electron'], { encoding: 'utf8' }) + if (r.status === 0 && r.stdout.trim()) return r.stdout.trim() + throw new Error('Electron binary not found. Run "npm install" from the repo root.') +} + +function assertDistBuilt() { + const electronMain = path.join(DESKTOP_ROOT, 'dist', 'electron-main.mjs') + const indexHtml = path.join(DESKTOP_ROOT, 'dist', 'index.html') + if (!fs.existsSync(electronMain) || !fs.existsSync(indexHtml)) { + throw new Error( + `Desktop dist not built. Run 'cd apps/desktop && npm run build' first.\n` + + `Missing: ${electronMain}`, + ) + } +} + +// ── Main ─────────────────────────────────────────────────────────────── + +async function main() { + assertDistBuilt() + + console.log('Starting mock inference server...') + const mock = await startMockServer() + console.log(` Mock server: ${mock.url}`) + + const sandbox = createSandbox() + writeMockConfig(sandbox.hermesHome, mock.url) + console.log(` HERMES_HOME: ${sandbox.hermesHome}`) + + const electronBin = findElectron() + + const env = { + ...process.env, + HERMES_HOME: sandbox.hermesHome, + HERMES_DESKTOP_USER_DATA_DIR: sandbox.userDataDir, + HERMES_DESKTOP_IGNORE_EXISTING: '1', + HERMES_DESKTOP_HERMES_ROOT: REPO_ROOT, + HERMES_DESKTOP_APP_NAME: `HermesDevMock-${Date.now()}`, + } + + console.log('Launching Electron...') + const child = spawn(electronBin, [DESKTOP_ROOT, '--disable-gpu', '--no-sandbox'], { + env, + cwd: DESKTOP_ROOT, + stdio: 'inherit', + }) + + child.on('exit', (code) => { + mock.close() + sandbox.cleanup() + process.exit(code ?? 0) + }) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/package-lock.json b/package-lock.json index 0b5dfdb169d8..3236f2e7310a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -155,6 +155,7 @@ "devDependencies": { "@electron/rebuild": "^4.0.6", "@eslint/js": "^9.39.4", + "@playwright/test": "=1.58.2", "@testing-library/dom": "^10.4.0", "@testing-library/react": "^16.3.2", "@types/d3-force": "^3.0.10", @@ -376,6 +377,22 @@ } } }, + "apps/desktop/node_modules/@playwright/test": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.2.tgz", + "integrity": "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.58.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "apps/desktop/node_modules/@types/node": { "version": "22.20.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.0.tgz", @@ -386,6 +403,53 @@ "undici-types": "~6.21.0" } }, + "apps/desktop/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "apps/desktop/node_modules/playwright": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", + "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.58.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "apps/desktop/node_modules/playwright-core": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", + "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "apps/desktop/node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",