From 2c184ed3d1b3bbe90abc55758847c727a91a88f9 Mon Sep 17 00:00:00 2001 From: ethernet Date: Thu, 16 Jul 2026 15:11:42 -0400 Subject: [PATCH] fix(desktop): keep visual E2E diffs advisory --- .github/workflows/e2e-desktop.yml | 4 +- apps/desktop/e2e/chat.spec.ts | 2 + apps/desktop/e2e/fixtures.ts | 2 + apps/desktop/e2e/launch-packaged-app.spec.ts | 1 + apps/desktop/e2e/mock-backend-setup.spec.ts | 2 + apps/desktop/e2e/visual-snapshot.ts | 101 ++++++++++++++----- 6 files changed, 87 insertions(+), 25 deletions(-) diff --git a/.github/workflows/e2e-desktop.yml b/.github/workflows/e2e-desktop.yml index a80d8d21721..98b1444d207 100644 --- a/.github/workflows/e2e-desktop.yml +++ b/.github/workflows/e2e-desktop.yml @@ -70,7 +70,7 @@ jobs: id: restore-baselines uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: - path: apps/desktop/e2e/__screenshots__ + path: apps/desktop/e2e/*-snapshots key: visual-baselines-${{ github.ref_name }} restore-keys: | visual-baselines-main @@ -155,7 +155,7 @@ jobs: if: github.ref_name == 'main' && always() uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: - path: apps/desktop/e2e/__screenshots__ + path: apps/desktop/e2e/*-snapshots key: visual-baselines-main # ── Upload Playwright report (HTML + traces) ────────────────────── diff --git a/apps/desktop/e2e/chat.spec.ts b/apps/desktop/e2e/chat.spec.ts index fe626847e83..0d12003df81 100644 --- a/apps/desktop/e2e/chat.spec.ts +++ b/apps/desktop/e2e/chat.spec.ts @@ -60,6 +60,7 @@ test.describe('chat interaction with mock backend', () => { return (body.textContent ?? '').includes('Hello, can you hear me?') }, + undefined, { timeout: 15_000 }, ) @@ -79,6 +80,7 @@ test.describe('chat interaction with mock backend', () => { return text.includes('mock inference server') || text.includes('boot chain is working') }, + undefined, { timeout: 60_000 }, ) }) diff --git a/apps/desktop/e2e/fixtures.ts b/apps/desktop/e2e/fixtures.ts index 2ce46f9bc95..00321c7b132 100644 --- a/apps/desktop/e2e/fixtures.ts +++ b/apps/desktop/e2e/fixtures.ts @@ -579,6 +579,7 @@ export async function waitForOnboarding(page: Page, timeoutMs = 60_000): Promise text.includes('Sign in') ) }, + undefined, { timeout: timeoutMs }, ) } @@ -608,6 +609,7 @@ export async function waitForBootFailure(page: Page, timeoutMs = 60_000): Promis text.includes('Repair') ) }, + undefined, { timeout: timeoutMs }, ) } diff --git a/apps/desktop/e2e/launch-packaged-app.spec.ts b/apps/desktop/e2e/launch-packaged-app.spec.ts index 5f04e422e34..a404a01cf55 100644 --- a/apps/desktop/e2e/launch-packaged-app.spec.ts +++ b/apps/desktop/e2e/launch-packaged-app.spec.ts @@ -71,6 +71,7 @@ test('boot progress overlay fades out or shows error state', async () => { return !bootIndicators.some((word) => lower.includes(word)) }, + undefined, { timeout: 60_000 }, ) }) diff --git a/apps/desktop/e2e/mock-backend-setup.spec.ts b/apps/desktop/e2e/mock-backend-setup.spec.ts index 0c7840c322d..1f068f20203 100644 --- a/apps/desktop/e2e/mock-backend-setup.spec.ts +++ b/apps/desktop/e2e/mock-backend-setup.spec.ts @@ -50,6 +50,7 @@ test.describe('mock backend gets past setup screen', () => { return !text.includes("Let's get you setup") }, + undefined, { timeout: 30_000 }, ) }) @@ -75,6 +76,7 @@ test.describe('mock backend gets past setup screen', () => { // Verify the typed text appears in the DOM. await page.waitForFunction( () => (document.body.textContent ?? '').includes('hello mock backend'), + undefined, { timeout: 10_000 }, ) }) diff --git a/apps/desktop/e2e/visual-snapshot.ts b/apps/desktop/e2e/visual-snapshot.ts index 570d6205345..f84864ed2dd 100644 --- a/apps/desktop/e2e/visual-snapshot.ts +++ b/apps/desktop/e2e/visual-snapshot.ts @@ -8,11 +8,14 @@ * on pixel-perfect matches. * * When a screenshot matches the baseline, nothing happens. When it - * differs, Playwright writes three images to the test's output dir: + * differs, this helper writes three images to the test's output dir: * -actual.png, -expected.png, -diff.png * These are picked up by the "Upload test results" artifact step. */ -import { type ElectronApplication, expect, type Page } from '@playwright/test' +import fs from 'node:fs' +import path from 'node:path' + +import { type ElectronApplication, type Page, test } from '@playwright/test' /** Fixed window dimensions for visual regression screenshots. */ export const VISUAL_WINDOW_WIDTH = 1220 @@ -21,12 +24,12 @@ export const VISUAL_WINDOW_HEIGHT = 800 export interface VisualSnapshotOptions { /** Snapshot name — defaults to the test title. */ name?: string - /** Full page screenshot (default) vs. viewport-only. */ + /** Full page screenshot vs. viewport-only (default). */ fullPage?: boolean /** Timeout in ms. */ timeout?: number - /** The Electron app handle — needed to force a fixed window size. */ - app?: ElectronApplication + /** The Electron app handle — used to size and decode screenshots. */ + app: ElectronApplication } /** @@ -58,33 +61,85 @@ async function forceFixedSize(app: ElectronApplication): Promise { */ export async function expectVisualSnapshot( page: Page, - options: VisualSnapshotOptions = {}, + options: VisualSnapshotOptions, ): Promise { const { name, fullPage = false, timeout = 30_000, app } = options // Force the window to a fixed size right before the screenshot so it's // always comparable, regardless of WM resizing during the test. - if (app) { - await forceFixedSize(app) - // Give the renderer a moment to relayout after the resize. - await page.waitForTimeout(500) - } + await forceFixedSize(app) + // Give the renderer a moment to relayout after the resize. + await page.waitForTimeout(500) // Playwright appends a platform suffix (e.g. "-linux") and requires // a .png extension on the name argument. Auto-append it if missing. const snapshotName = name ? (name.endsWith('.png') ? name : `${name}.png`) : undefined - try { - if (snapshotName) { - await expect(page).toHaveScreenshot(snapshotName, { fullPage, timeout }) - } else { - await expect(page).toHaveScreenshot({ fullPage, timeout }) - } - } catch (err) { - // Don't fail the test — just log that a diff was detected. - // The diff/actual/expected images are already written to the test - // output directory by Playwright for the CI workflow to pick up. - console.log(`[visual-diff] ${name ?? '(unnamed)'} — screenshot differs from baseline`) - console.log(` ${err instanceof Error ? err.message.split('\n')[0] : String(err)}`) + const info = test.info() + const actual = await page.screenshot({ animations: 'disabled', caret: 'hide', fullPage, timeout }) + const baselinePath = info.snapshotPath(snapshotName ?? `${info.title}.png`) + const outputName = (snapshotName ?? 'snapshot.png').replace(/\.png$/, '') + + if (info.config.updateSnapshots === 'all' || info.config.updateSnapshots === 'changed') { + fs.mkdirSync(path.dirname(baselinePath), { recursive: true }) + fs.writeFileSync(baselinePath, actual) + console.log(`[visual-baseline] updated ${baselinePath}`) + return } + + if (!fs.existsSync(baselinePath)) { + fs.writeFileSync(info.outputPath(`${outputName}-actual.png`), actual) + console.log(`[visual-diff] ${name ?? '(unnamed)'} — no baseline available`) + return + } + + const expected = fs.readFileSync(baselinePath) + const comparison = await app.evaluate( + ({ nativeImage }, images) => { + const actualImage = nativeImage.createFromBuffer(Buffer.from(images.actual, 'base64')) + const expectedImage = nativeImage.createFromBuffer(Buffer.from(images.expected, 'base64')) + const actualSize = actualImage.getSize() + const expectedSize = expectedImage.getSize() + + if (actualSize.width !== expectedSize.width || actualSize.height !== expectedSize.height) { + return { mismatchRatio: 1, diff: images.actual } + } + + const actualPixels = actualImage.toBitmap() + const expectedPixels = expectedImage.toBitmap() + const diffPixels = Buffer.alloc(actualPixels.length) + let mismatched = 0 + + for (let i = 0; i < actualPixels.length; i += 4) { + const different = + Math.abs(actualPixels[i] - expectedPixels[i]) > 51 || + Math.abs(actualPixels[i + 1] - expectedPixels[i + 1]) > 51 || + Math.abs(actualPixels[i + 2] - expectedPixels[i + 2]) > 51 || + Math.abs(actualPixels[i + 3] - expectedPixels[i + 3]) > 51 + + if (different) { + mismatched++ + diffPixels[i + 2] = 255 + } + diffPixels[i + 3] = 255 + } + + return { + mismatchRatio: mismatched / (actualPixels.length / 4), + diff: nativeImage.createFromBitmap(diffPixels, actualSize).toPNG().toString('base64'), + } + }, + { actual: actual.toString('base64'), expected: expected.toString('base64') }, + ) + + if (comparison.mismatchRatio <= 0.01) { + return + } + + fs.writeFileSync(info.outputPath(`${outputName}-actual.png`), actual) + fs.writeFileSync(info.outputPath(`${outputName}-expected.png`), expected) + fs.writeFileSync(info.outputPath(`${outputName}-diff.png`), Buffer.from(comparison.diff, 'base64')) + console.log( + `[visual-diff] ${name ?? '(unnamed)'} — ${(comparison.mismatchRatio * 100).toFixed(2)}% of pixels differ`, + ) }