From b1fb3c528582d1282c7393f72ebb816c86bdc13c Mon Sep 17 00:00:00 2001 From: brooklyn! Date: Sun, 19 Jul 2026 18:52:39 -0400 Subject: [PATCH] =?UTF-8?q?bench(desktop):=20measure=20the=20full=20pictur?= =?UTF-8?q?e=20=E2=80=94=20prod=20build,=20cold-start,=20first-token=20(#6?= =?UTF-8?q?7697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop drip-feeding scenarios: extend the harness to cover the latencies that actually dominate perceived speed, and measure them on a REAL production build. - --prod: build a production renderer with the probe included (VITE_PERF_PROBE=1, off in normal builds) and launch it from dist/. Measures minified React, so numbers are representative shipped figures instead of ~3x-inflated dev ones. - cold-start scenario (tier "cold"): launch → CDP → driver → first paint, via a fresh isolated spawn per run. Captures spawn_to_cdp_ms, spawn_to_driver_ms, fcp_ms. - first-token scenario (backend tier): Enter → first assistant token painted — the TTFT latency an agent app is uniquely judged on. - run.mjs gained --prod (build once), cold-start fresh-spawn loop, and gates ci+cold tiers against the baseline. Baseline re-captured on a PRODUCTION build (median of 5), darwin-arm64 — all green. Representative numbers: cold-start spawn→interactive ~1.6s, FCP ~0.5s stream frame p95 22ms, 1 longtask keystroke p50 2ms, p95 8.7ms transcript mount 145ms, 82ms longtask (400-msg open) The prod build also settled the open question from the dev numbers: the transcript-mount "lead" (221ms longtask in dev) is only ~72-82ms in prod — not actionable. Measurement did its job. --- apps/desktop/scripts/perf/README.md | 22 +- apps/desktop/scripts/perf/baseline.json | 35 ++-- apps/desktop/scripts/perf/lib/launch.mjs | 195 +++++++++++------- apps/desktop/scripts/perf/run.mjs | 120 +++++++---- .../scripts/perf/scenarios/cold-start.mjs | 18 ++ .../scripts/perf/scenarios/first-token.mjs | 84 ++++++++ apps/desktop/scripts/perf/scenarios/index.mjs | 4 + apps/desktop/src/main.tsx | 6 +- 8 files changed, 358 insertions(+), 126 deletions(-) create mode 100644 apps/desktop/scripts/perf/scenarios/cold-start.mjs create mode 100644 apps/desktop/scripts/perf/scenarios/first-token.mjs diff --git a/apps/desktop/scripts/perf/README.md b/apps/desktop/scripts/perf/README.md index 56673c467348..e8cfc5752312 100644 --- a/apps/desktop/scripts/perf/README.md +++ b/apps/desktop/scripts/perf/README.md @@ -19,10 +19,21 @@ npm run perf # attaches, runs the CI suite, gates on baseline # One scenario, with a CPU profile: npm run perf -- stream --cpuprofile --tokens 800 +# Representative PRODUCTION numbers (minified React, not the ~3x-slower dev build): +npm run perf -- cold-start stream keystroke transcript --spawn --prod + # Re-capture the baseline on your reference device, then commit baseline.json: -npm run perf -- --update-baseline +npm run perf -- cold-start stream keystroke transcript --spawn --prod --update-baseline ``` +## Dev vs prod + +By default the harness measures the **dev** renderer (fast to spin up, good for +relative regression checks). Pass `--prod` (with `--spawn`) to build a +production renderer *with the probe included* (`VITE_PERF_PROBE=1`) and measure +minified React — the representative shipped numbers. The committed baseline is +captured with `--prod`. + ## Why isolation matters The measurement this harness exists to run was historically blocked: a running @@ -40,13 +51,16 @@ directly via `window.__PERF_DRIVE__`, so no LLM credits are spent. | `stream --real` | backend | same, from a real LLM stream | measure-real-stream, profile-real-stream | | `keystroke` | ci | composer keystroke → paint latency | measure-latency, profile-typing, leak-typing | | `transcript` | ci | large-transcript mount + paint cost | (new) | +| `cold-start` | cold | launch → CDP → driver → first paint (fresh spawn/run) | (new) | +| `first-token` | backend | Enter → first assistant token painted (TTFT) | (new) | | `submit` | backend | Enter → cleared → user msg painted, scroll jump | measure-submit, measure-jump | | `session-switch` | backend | route → first-paint → settle | profile-session-switch | | `profile-switch` | backend | rail click → sidebar settled | measure-profile-switch | -`ci` scenarios need no backend/credits and are gated against `baseline.json`. -`backend` scenarios need a live backend (and `--spawn` or a real session) and -are report-only. +`ci` + `cold` scenarios need no backend/credits and are gated against +`baseline.json` (`cold-start` requires `--spawn` since it measures a fresh +launch, and must be run in its own invocation). `backend` scenarios need a live +backend (and `--spawn` or a real session/credits) and are report-only. CPU profiling is a cross-cutting `--cpuprofile` flag on any scenario (it wraps the run in `Profiler.start/stop` and prints a top-self-time table), replacing diff --git a/apps/desktop/scripts/perf/baseline.json b/apps/desktop/scripts/perf/baseline.json index bbe15b56288d..1de3fea43b90 100644 --- a/apps/desktop/scripts/perf/baseline.json +++ b/apps/desktop/scripts/perf/baseline.json @@ -1,9 +1,9 @@ { "_meta": { - "note": "Median of 5 runs, darwin-arm64, `npm run perf -- --spawn` (dev renderer, gateway waited-for-connected, realistic block-settling stream chunk). Absolute values are dev-build (React dev mode is ~3x prod) so treat them as regression guards, not shipped numbers; re-baseline per device with `npm run perf -- --update-baseline`. Tolerances are loose to absorb cross-machine variance.", + "note": "Median of 5 runs, darwin-arm64, `npm run perf -- cold-start stream keystroke transcript --spawn --prod` — a PRODUCTION renderer (minified React), so these are representative shipped numbers, not dev-inflated. Re-baseline per device with the same command + `--update-baseline`. Tolerances are loose to absorb cross-machine variance; cold-start especially varies with disk/OS state.", "platform": "darwin-arm64", "node": "v24.11.0", - "updated": "2026-07-19T21:24:19.047Z" + "updated": "2026-07-19T21:38:19.701Z" }, "scenarios": { "stream": { @@ -13,11 +13,11 @@ }, "metrics": { "longtasks_n": 1, - "longtask_max_ms": 145, - "frame_p95_ms": 23.5, - "frame_p99_ms": 26.9, + "longtask_max_ms": 67, + "frame_p95_ms": 22, + "frame_p99_ms": 23.7, "slow_frames_33": 1, - "intermut_p95_ms": 48.2 + "intermut_p95_ms": 36.1 } }, "keystroke": { @@ -26,9 +26,9 @@ "tolAbs": 4 }, "metrics": { - "keystroke_p50_ms": 2, - "keystroke_p95_ms": 8.2, - "keystroke_p99_ms": 17.4, + "keystroke_p50_ms": 2.1, + "keystroke_p95_ms": 8.7, + "keystroke_p99_ms": 16.9, "keystroke_slow_16": 2 } }, @@ -38,9 +38,20 @@ "tolAbs": 40 }, "metrics": { - "transcript_mount_ms": 280.1, - "transcript_longtask_ms": 431, - "transcript_longtask_max_ms": 221 + "transcript_mount_ms": 145, + "transcript_longtask_ms": 82, + "transcript_longtask_max_ms": 82 + } + }, + "cold-start": { + "tolerance": { + "tolFrac": 0.6, + "tolAbs": 150 + }, + "metrics": { + "spawn_to_cdp_ms": 326, + "spawn_to_driver_ms": 1647, + "fcp_ms": 500 } } } diff --git a/apps/desktop/scripts/perf/lib/launch.mjs b/apps/desktop/scripts/perf/lib/launch.mjs index bc56626c0604..c95da3f28545 100644 --- a/apps/desktop/scripts/perf/lib/launch.mjs +++ b/apps/desktop/scripts/perf/lib/launch.mjs @@ -105,17 +105,32 @@ async function waitForConnected(cdp, timeoutMs) { return false } -function runNode(scriptRelPath, args = []) { +function runProcess(command, args, { env } = {}) { return new Promise((resolveRun, reject) => { - const child = spawn(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args], { + const child = spawn(command, args, { cwd: DESKTOP_DIR, - stdio: 'inherit' + stdio: 'inherit', + env: env ? { ...process.env, ...env } : process.env }) child.on('error', reject) - child.on('exit', code => (code === 0 ? resolveRun() : reject(new Error(`${scriptRelPath} exited ${code}`)))) + child.on('exit', code => (code === 0 ? resolveRun() : reject(new Error(`${command} ${args[0]} exited ${code}`)))) }) } +function runNode(scriptRelPath, args = []) { + return runProcess(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args]) +} + +// Build a production renderer WITH the perf probe included (VITE_PERF_PROBE=1), +// plus the prod electron-main bundle, so the harness can measure a real, +// minified React build instead of the ~3x-slower dev build. Slow (a full vite +// build); do it once, then run/attach many times. +export async function buildProdRenderer() { + const viteBin = resolveViteBin() + await runProcess(process.execPath, [viteBin, 'build'], { env: { VITE_PERF_PROBE: '1' } }) + await runNode('scripts/bundle-electron-main.mjs') +} + /** Attach to a renderer already listening on `port` (launched via perf:serve or with --remote-debugging-port). */ export async function attach({ port = 9222, match } = {}) { const cdp = await CDP.connect({ port, match }) @@ -129,9 +144,29 @@ export async function attach({ port = 9222, match } = {}) { * and return `{ cdp, teardown, devUrl, port }`. `teardown` kills both children * and removes any temp dirs it created. */ +// Chromium switches that stop frame-production throttling for a window that +// isn't foregrounded (the perf window usually sits behind the IDE/terminal). +const ANTI_THROTTLE_FLAGS = [ + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + '--disable-backgrounding-occluded-windows', + '--disable-features=CalculateNativeWinOcclusion' +] + +/** + * Spawn an isolated instance and connect the perf driver. Two render modes: + * · dev (default): vite dev server + dev electron-main bundle. + * · prod (`prod: true`): a production build (call buildProdRenderer first); + * electron loads dist/index.html — representative, minified React. + * `coldStart: true` skips the gateway-connect wait and settle (for launch-time + * measurement) and returns `timings` (spawn→CDP, spawn→driver) plus renderer + * boot marks (FCP, time-to-composer). + */ export async function startIsolatedInstance({ port = 9222, devPort = 5174, + prod = false, + coldStart = false, hermesHome, userDataDir, seedConfig = true, @@ -151,9 +186,8 @@ export async function startIsolatedInstance({ const home = hermesHome ?? mkTemp('hermes-perf-home-') const userData = userDataDir ?? mkTemp('hermes-perf-ud-') - const devUrl = `http://127.0.0.1:${devPort}` + const devUrl = prod ? null : `http://127.0.0.1:${devPort}` - // Only seed a temp home we created — never scribble into a user-provided one. if (seedConfig && !hermesHome) { seedConfigFrom(join(homedir(), '.hermes'), home) } @@ -177,61 +211,56 @@ export async function startIsolatedInstance({ } try { - // 1. Renderer: reuse an already-running dev server, else start one. - if (!(await reachable(devUrl))) { - const viteBin = resolveViteBin() - const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], { - cwd: DESKTOP_DIR, - stdio: ['ignore', 'inherit', 'inherit'] - }) - children.push(vite) - await waitFor(() => reachable(devUrl), { timeoutMs: 60000, label: `vite dev server on :${devPort}` }) + if (prod) { + // Renderer + main are expected pre-built (buildProdRenderer). Cheap to + // re-bundle main so an isolated run always matches current source. + await runNode('scripts/bundle-electron-main.mjs') + } else { + if (!(await reachable(devUrl))) { + const viteBin = resolveViteBin() + const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], { + cwd: DESKTOP_DIR, + stdio: ['ignore', 'inherit', 'inherit'] + }) + children.push(vite) + await waitFor(() => reachable(devUrl), { timeoutMs: 60000, label: `vite dev server on :${devPort}` }) + } + + await runNode('scripts/bundle-electron-main.mjs', ['--dev']) } - // 2. Electron main bundle (dev variant) — same step the dev script runs. - await runNode('scripts/bundle-electron-main.mjs', ['--dev']) - - // 3. Isolated Electron. --user-data-dir gives it its own single-instance - // lock scope; HERMES_HOME gives it its own backend + sessions. + // Isolated Electron: own --user-data-dir (single-instance lock scope) + own + // HERMES_HOME (backend + sessions). No DEV_SERVER env in prod → dist load. const electronBin = require('electron') + const env = { + ...process.env, + HERMES_HOME: home, + HERMES_DESKTOP_BOOT_FAKE: '1', + HERMES_DESKTOP_BOOT_FAKE_STEP_MS: String(bootFakeStepMs), + XCURSOR_SIZE: '24' + } + + if (devUrl) { + env.HERMES_DESKTOP_DEV_SERVER = devUrl + } + + const spawnAt = Date.now() const electron = spawn( electronBin, - [ - '.', - `--user-data-dir=${userData}`, - `--remote-debugging-port=${port}`, - // The perf window usually opens behind the user's other windows, and - // Chromium throttles frame production for backgrounded/occluded windows - // (~17fps), which shows up as choppy frames with ZERO longtasks and - // wrecks the stream frame-pacing metric. Disable every throttle path so - // measurements reflect real render cost regardless of window state - // (CalculateNativeWinOcclusion is the macOS/Windows occlusion detector). - '--disable-background-timer-throttling', - '--disable-renderer-backgrounding', - '--disable-backgrounding-occluded-windows', - '--disable-features=CalculateNativeWinOcclusion' - ], - { - cwd: DESKTOP_DIR, - stdio: ['ignore', 'inherit', 'inherit'], - env: { - ...process.env, - HERMES_HOME: home, - HERMES_DESKTOP_DEV_SERVER: devUrl, - HERMES_DESKTOP_BOOT_FAKE: '1', - HERMES_DESKTOP_BOOT_FAKE_STEP_MS: String(bootFakeStepMs), - XCURSOR_SIZE: '24' - } - } + ['.', `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`, ...ANTI_THROTTLE_FLAGS], + { cwd: DESKTOP_DIR, stdio: ['ignore', 'inherit', 'inherit'], env } ) children.push(electron) - // 4. Wait for the renderer + the perf driver to be live. + // Wait for the renderer + perf driver. In prod the target URL is file://, + // so don't match on the dev port. let cdp = null + let cdpAt = 0 await waitFor( async () => { try { - cdp = await CDP.connect({ port, match: String(devPort), timeoutMs: 2000 }) + cdp = await CDP.connect({ port, match: devUrl ? String(devPort) : undefined, timeoutMs: 2000 }) + cdpAt = cdpAt || Date.now() return await cdp.eval('!!(window.__PERF_DRIVE__ && window.__PERF_DRIVE__.stream)') } catch { @@ -245,41 +274,46 @@ export async function startIsolatedInstance({ }, { timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' } ) + const driverAt = Date.now() - // Electron throttles rAF/timers for a window that isn't foregrounded - // (per-window backgroundThrottling, which the Chromium CLI flags above don't - // override). Focus emulation makes the renderer behave as if focused so - // frame-pacing measurements are real even though the perf window sits behind - // the user's other windows — WITHOUT actually stealing OS focus. try { - // Behave as if focused so frame-pacing isn't throttled while the perf - // window sits behind the user's IDE/terminal — WITHOUT stealing OS focus. await cdp.send('Emulation.setFocusEmulationEnabled', { enabled: true }) } catch { - // Older CDP / not supported — fall back to the anti-throttle flags above. + // Older CDP / not supported — fall back to the anti-throttle flags. } - // Wait for the gateway socket to actually open. A booting/absent backend - // retries on a 1–15s backoff, and that churn contaminates frame-pacing - // (the `stream` scenario). Best-effort: proceed after the timeout so the - // backend-independent scenarios (keystroke, transcript) still run. - const connected = await waitForConnected(cdp, connectTimeoutMs) - - if (!connected) { - console.warn( - `[perf] gateway did not connect within ${connectTimeoutMs}ms — ` + - 'stream/frame numbers may be inflated by reconnect churn.' - ) + // Renderer-side boot marks (relative to its own navigation start). + const bootMarks = await readBootMarks(cdp) + const timings = { + spawn_to_cdp_ms: cdpAt ? cdpAt - spawnAt : null, + spawn_to_driver_ms: driverAt - spawnAt, + ...bootMarks } - // Let residual cold-start work (vite dep pre-bundling, initial paint) drain. - await sleep(settleMs) + let connected = true + + if (!coldStart) { + // Steady-state scenarios: wait for the gateway to connect (reconnect churn + // contaminates frame pacing) and let residual cold-start work drain. + connected = await waitForConnected(cdp, connectTimeoutMs) + + if (!connected) { + console.warn( + `[perf] gateway did not connect within ${connectTimeoutMs}ms — ` + + 'stream/frame numbers may be inflated by reconnect churn.' + ) + } + + await sleep(settleMs) + } return { connected, cdp, devUrl, port, + prod, + timings, teardown: () => { cdp?.close() teardown() @@ -291,4 +325,25 @@ export async function startIsolatedInstance({ } } +// Read First Contentful Paint + time-to-composer from the renderer, relative to +// its navigation start (the process-spawn deltas live in `timings`). +async function readBootMarks(cdp) { + try { + return await cdp.eval(`(() => { + const paints = performance.getEntriesByType('paint') + const fcp = paints.find(p => p.name === 'first-contentful-paint') + const composer = document.querySelector('[data-slot="composer-rich-input"]') + return { + fcp_ms: fcp ? Math.round(fcp.startTime) : null, + // performance.now() at read time ≈ time since nav start; only meaningful + // right after boot (cold-start reads it immediately). + nav_to_read_ms: Math.round(performance.now()), + composer_present: !!composer + } + })()`) + } catch { + return { fcp_ms: null, nav_to_read_ms: null, composer_present: false } + } +} + export { DESKTOP_DIR } diff --git a/apps/desktop/scripts/perf/run.mjs b/apps/desktop/scripts/perf/run.mjs index 570e1b9e096f..2bf0c0649a2f 100644 --- a/apps/desktop/scripts/perf/run.mjs +++ b/apps/desktop/scripts/perf/run.mjs @@ -29,7 +29,7 @@ import { fileURLToPath } from 'node:url' import { withCpuProfile } from './lib/cdp.mjs' import { compareScenario, loadBaseline, updateBaseline } from './lib/baseline.mjs' -import { attach, startIsolatedInstance } from './lib/launch.mjs' +import { attach, buildProdRenderer, startIsolatedInstance } from './lib/launch.mjs' import { cpuProfileTopSelf, median } from './lib/stats.mjs' import { CI_SCENARIOS, SCENARIOS } from './scenarios/index.mjs' @@ -111,52 +111,94 @@ async function main() { const runs = Number(flags.runs ?? 1) const port = Number(flags.port ?? 9222) const devPort = Number(flags['dev-port'] ?? 5174) + const prod = 'prod' in flags const cpuProfile = 'cpuprofile' in flags const cpuProfileDir = typeof flags.cpuprofile === 'string' ? flags.cpuprofile : HERE - const connection = flags.spawn - ? await startIsolatedInstance({ port, devPort }) - : await attach({ port, match: String(devPort) }) + const coldNames = names.filter(n => SCENARIOS[n].tier === 'cold') + const liveNames = names.filter(n => SCENARIOS[n].tier !== 'cold') - const { cdp, teardown } = connection + // ci + cold metrics are stable enough to gate against the baseline; backend + // scenarios vary too much with the live environment, so they're report-only. + const GATED = new Set(['ci', 'cold']) + const baseline = loadBaseline(BASELINE_PATH) const results = [] let regressed = false - try { - const baseline = loadBaseline(BASELINE_PATH) + const record = (name, tier, metrics, detail) => { + const comparison = GATED.has(tier) ? compareScenario(name, metrics, baseline) : null + regressed = regressed || Boolean(comparison?.regressed) + results.push({ name, tier, metrics, detail }) + printMetrics(name, metrics, comparison) + } - for (const name of names) { - const scenario = SCENARIOS[name] - const perRun = [] - let detail = null - - for (let i = 0; i < runs; i++) { - if (cpuProfile && i === 0) { - const { result, profile } = await withCpuProfile(cdp, () => scenario.run(cdp, flags)) - const out = join(cpuProfileDir, `${name}-${Date.now()}.cpuprofile`) - writeFileSync(out, JSON.stringify(profile)) - console.log(`\n[cpuprofile] wrote ${out}`) - console.log('[cpuprofile] top self-time (ms):') - for (const r of cpuProfileTopSelf(profile, 15)) { - console.log(` ${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(38)} ${r.url}:${r.line}`) - } - perRun.push(result.metrics) - detail = result.detail - } else { - const result = await scenario.run(cdp, flags) - perRun.push(result.metrics) - detail = result.detail - } - } - - const metrics = medianMetrics(perRun) - const comparison = scenario.tier === 'ci' ? compareScenario(name, metrics, baseline) : null - regressed = regressed || Boolean(comparison?.regressed) - results.push({ name, tier: scenario.tier, metrics, detail }) - printMetrics(name, metrics, comparison) + if (prod) { + if (!flags.spawn) { + console.error('--prod requires --spawn (it builds and launches an isolated production renderer)') + process.exit(2) + } + + console.log('[perf] building production renderer with the probe (VITE_PERF_PROBE=1)…') + await buildProdRenderer() + } + + // Cold start measures the launch itself → a fresh spawn per run. + if (coldNames.length) { + if (!flags.spawn) { + console.error('cold-start requires --spawn (it measures a fresh launch)') + process.exit(2) + } + + const perRun = [] + + for (let i = 0; i < runs; i++) { + const inst = await startIsolatedInstance({ port, devPort, prod, coldStart: true }) + const t = inst.timings + perRun.push({ spawn_to_cdp_ms: t.spawn_to_cdp_ms, spawn_to_driver_ms: t.spawn_to_driver_ms, fcp_ms: t.fcp_ms }) + inst.teardown() + } + + record('cold-start', 'cold', medianMetrics(perRun), { runs }) + } + + // Steady-state scenarios share one persistent connection. + if (liveNames.length) { + const connection = flags.spawn + ? await startIsolatedInstance({ port, devPort, prod }) + : await attach({ port, match: prod ? undefined : String(devPort) }) + + const { cdp, teardown } = connection + + try { + for (const name of liveNames) { + const scenario = SCENARIOS[name] + const perRun = [] + let detail = null + + for (let i = 0; i < runs; i++) { + if (cpuProfile && i === 0) { + const { result, profile } = await withCpuProfile(cdp, () => scenario.run(cdp, flags)) + const out = join(cpuProfileDir, `${name}-${Date.now()}.cpuprofile`) + writeFileSync(out, JSON.stringify(profile)) + console.log(`\n[cpuprofile] wrote ${out}`) + console.log('[cpuprofile] top self-time (ms):') + for (const r of cpuProfileTopSelf(profile, 15)) { + console.log(` ${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(38)} ${r.url}:${r.line}`) + } + perRun.push(result.metrics) + detail = result.detail + } else { + const result = await scenario.run(cdp, flags) + perRun.push(result.metrics) + detail = result.detail + } + } + + record(name, scenario.tier, medianMetrics(perRun), detail) + } + } finally { + teardown() } - } finally { - teardown() } if (flags.json) { @@ -165,7 +207,7 @@ async function main() { } if (flags['update-baseline']) { - updateBaseline(BASELINE_PATH, results.filter(r => r.tier === 'ci')) + updateBaseline(BASELINE_PATH, results.filter(r => GATED.has(r.tier))) console.log(`\nupdated ${BASELINE_PATH}`) return } diff --git a/apps/desktop/scripts/perf/scenarios/cold-start.mjs b/apps/desktop/scripts/perf/scenarios/cold-start.mjs new file mode 100644 index 000000000000..0d67833c7b1a --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/cold-start.mjs @@ -0,0 +1,18 @@ +// Cold start — launch → renderer → interactive. Unlike the other scenarios this +// measures the LAUNCH itself, so it can't run against an already-up instance: +// the runner spawns a fresh isolated instance per run (requires --spawn) and +// reads the timings/boot-marks the launcher captures. Registered here so it's a +// known name with a baseline entry; the actual measurement lives in run.mjs. +// +// Metrics (lower is better): +// spawn_to_cdp_ms process spawn → CDP page target reachable (electron/V8 up) +// spawn_to_driver_ms process spawn → renderer mounted + perf driver present +// fcp_ms renderer nav start → first contentful paint +export default { + name: 'cold-start', + tier: 'cold', + description: 'Launch → first paint → interactive (fresh spawn per run).', + run() { + throw new Error('cold-start is measured by the runner via fresh spawns; use `--spawn`.') + } +} diff --git a/apps/desktop/scripts/perf/scenarios/first-token.mjs b/apps/desktop/scripts/perf/scenarios/first-token.mjs new file mode 100644 index 000000000000..55477c824be8 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/first-token.mjs @@ -0,0 +1,84 @@ +// Time-to-first-token — Enter → first assistant token painted. The latency an +// agent app is uniquely judged on, spanning the desktop submit path AND the +// backend/agent-loop first-token time. Backend tier: fires a REAL prompt, needs +// a live backend (and credits). Report-only. +// +// node scripts/perf/run.mjs first-token --spawn --prompt "hi" + +import { SELECTORS, sleep, typeIntoComposer } from '../lib/cdp.mjs' +import { summarize } from '../lib/stats.mjs' + +export default { + name: 'first-token', + tier: 'backend', + description: 'Enter → first assistant token painted (real backend).', + async run(cdp, opts = {}) { + const rounds = Number(opts.rounds ?? 3) + const prompt = opts.prompt ?? 'reply with a single short sentence' + const timeoutMs = Number(opts.timeoutMs ?? 60000) + + await cdp.send('Runtime.enable') + + const firstTokens = [] + + for (let i = 0; i < rounds; i++) { + const baseText = await cdp.eval(`(() => { + const a = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}) + return a.length ? a[a.length - 1].textContent.length : 0 + })()`) + const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`) + + await typeIntoComposer(cdp, `${prompt} (${i})`, { cps: 60 }) + const submitAt = Date.now() + await cdp.eval(`(() => { + const el = document.querySelector(${JSON.stringify(SELECTORS.composer)}) + el && el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true })) + })()`) + + const deadline = Date.now() + timeoutMs + let firstTokenMs = null + + while (Date.now() < deadline) { + await sleep(25) + const grown = await cdp.eval(`(() => { + const a = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}) + if (a.length > ${baseCount}) return true + return a.length ? a[a.length - 1].textContent.length > ${baseText} : false + })()`) + + if (grown) { + firstTokenMs = Date.now() - submitAt + break + } + } + + if (firstTokenMs !== null) { + firstTokens.push(firstTokenMs) + } + + // Let the turn finish before the next round. + const turnDeadline = Date.now() + timeoutMs + while (Date.now() < turnDeadline) { + await sleep(250) + const busy = await cdp.eval(`!!document.querySelector('[data-status="running"], [data-busy="true"]')`) + + if (!busy) { + break + } + } + + await sleep(500) + } + + if (!firstTokens.length) { + throw new Error('no first token observed — is a backend with credits connected?') + } + + const s = summarize(firstTokens) + + return { + metrics: { first_token_p50_ms: s.p50, first_token_p95_ms: s.p95 }, + detail: { rounds, samples: firstTokens, summary: s } + } + } +} diff --git a/apps/desktop/scripts/perf/scenarios/index.mjs b/apps/desktop/scripts/perf/scenarios/index.mjs index 02771b2f0d98..d9aeece1effd 100644 --- a/apps/desktop/scripts/perf/scenarios/index.mjs +++ b/apps/desktop/scripts/perf/scenarios/index.mjs @@ -1,6 +1,8 @@ // Scenario registry. Add a scenario module here and it's automatically // available to the runner, the default suite (tier 'ci'), and the baseline gate. +import coldStart from './cold-start.mjs' +import firstToken from './first-token.mjs' import keystroke from './keystroke.mjs' import profileSwitch from './profile-switch.mjs' import sessionSwitch from './session-switch.mjs' @@ -12,6 +14,8 @@ export const SCENARIOS = { [stream.name]: stream, [keystroke.name]: keystroke, [transcript.name]: transcript, + [coldStart.name]: coldStart, + [firstToken.name]: firstToken, [submit.name]: submit, [sessionSwitch.name]: sessionSwitch, [profileSwitch.name]: profileSwitch diff --git a/apps/desktop/src/main.tsx b/apps/desktop/src/main.tsx index 343816b8e024..b1dd657655ba 100644 --- a/apps/desktop/src/main.tsx +++ b/apps/desktop/src/main.tsx @@ -17,7 +17,11 @@ import { ThemeProvider } from './themes/context' installClipboardShim() -if (import.meta.env.MODE !== 'production') { +// The perf probe ships in dev, and in a production build ONLY when explicitly +// opted in (VITE_PERF_PROBE=1) — this lets the perf harness measure a real, +// minified production renderer for representative absolute numbers. Normal +// `npm run build` leaves the flag unset, so the probe never reaches users. +if (import.meta.env.MODE !== 'production' || import.meta.env.VITE_PERF_PROBE === '1') { import('./app/chat/perf-probe') }