diff --git a/apps/desktop/scripts/perf/baseline.json b/apps/desktop/scripts/perf/baseline.json index b0cb9ef00303..bbe15b56288d 100644 --- a/apps/desktop/scripts/perf/baseline.json +++ b/apps/desktop/scripts/perf/baseline.json @@ -1,37 +1,46 @@ { "_meta": { - "note": "Captured on darwin-arm64 via `npm run perf -- --spawn`. keystroke + transcript are clean medians. stream = a clean single-run capture; under --spawn the isolated backend may not connect, and its reconnect churn inflates frame-pacing, so re-capture stream on a backend-CONNECTED instance (perf:serve against a live profile, or attach) for tighter tolerances. Re-baseline with `npm run perf -- --update-baseline`.", + "note": "Median of 5 runs, darwin-arm64, `npm run perf -- --spawn` (dev renderer, gateway waited-for-connected, realistic block-settling stream chunk). Absolute values are dev-build (React dev mode is ~3x prod) so treat them as regression guards, not shipped numbers; re-baseline per device with `npm run perf -- --update-baseline`. Tolerances are loose to absorb cross-machine variance.", "platform": "darwin-arm64", - "node": null, - "updated": "2026-07-19" + "node": "v24.11.0", + "updated": "2026-07-19T21:24:19.047Z" }, "scenarios": { "stream": { - "tolerance": { "tolFrac": 0.6, "tolAbs": 5 }, + "tolerance": { + "tolFrac": 0.6, + "tolAbs": 5 + }, "metrics": { - "longtasks_n": 2, - "longtask_max_ms": 97, - "frame_p95_ms": 18.4, - "frame_p99_ms": 28.5, - "slow_frames_33": 4, - "intermut_p95_ms": 37.1 + "longtasks_n": 1, + "longtask_max_ms": 145, + "frame_p95_ms": 23.5, + "frame_p99_ms": 26.9, + "slow_frames_33": 1, + "intermut_p95_ms": 48.2 } }, "keystroke": { - "tolerance": { "tolFrac": 0.6, "tolAbs": 4 }, + "tolerance": { + "tolFrac": 0.6, + "tolAbs": 4 + }, "metrics": { - "keystroke_p50_ms": 2.7, - "keystroke_p95_ms": 9.9, - "keystroke_p99_ms": 17.7, + "keystroke_p50_ms": 2, + "keystroke_p95_ms": 8.2, + "keystroke_p99_ms": 17.4, "keystroke_slow_16": 2 } }, "transcript": { - "tolerance": { "tolFrac": 0.75, "tolAbs": 40 }, + "tolerance": { + "tolFrac": 0.75, + "tolAbs": 40 + }, "metrics": { - "transcript_mount_ms": 287.3, - "transcript_longtask_ms": 559, - "transcript_longtask_max_ms": 295 + "transcript_mount_ms": 280.1, + "transcript_longtask_ms": 431, + "transcript_longtask_max_ms": 221 } } } diff --git a/apps/desktop/scripts/perf/lib/launch.mjs b/apps/desktop/scripts/perf/lib/launch.mjs index 64490fcbe7e9..bc56626c0604 100644 --- a/apps/desktop/scripts/perf/lib/launch.mjs +++ b/apps/desktop/scripts/perf/lib/launch.mjs @@ -83,6 +83,28 @@ function resolveViteBin() { return join(dirname(pkgPath), rel) } +// Poll the perf driver's `connected()` until the gateway socket is open. +// Returns false if the probe predates this helper or the timeout elapses. +async function waitForConnected(cdp, timeoutMs) { + const hasProbe = await cdp.eval('typeof window.__PERF_DRIVE__.connected === "function"') + + if (!hasProbe) { + return false + } + + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + if (await cdp.eval('window.__PERF_DRIVE__.connected()')) { + return true + } + + await sleep(500) + } + + return false +} + function runNode(scriptRelPath, args = []) { return new Promise((resolveRun, reject) => { const child = spawn(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args], { @@ -114,7 +136,8 @@ export async function startIsolatedInstance({ userDataDir, seedConfig = true, bootFakeStepMs = 120, - settleMs = 2500 + settleMs = 2500, + connectTimeoutMs = 90000 } = {}) { const children = [] const tempDirs = [] @@ -173,7 +196,21 @@ export async function startIsolatedInstance({ const electronBin = require('electron') const electron = spawn( electronBin, - ['.', `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`], + [ + '.', + `--user-data-dir=${userData}`, + `--remote-debugging-port=${port}`, + // The perf window usually opens behind the user's other windows, and + // Chromium throttles frame production for backgrounded/occluded windows + // (~17fps), which shows up as choppy frames with ZERO longtasks and + // wrecks the stream frame-pacing metric. Disable every throttle path so + // measurements reflect real render cost regardless of window state + // (CalculateNativeWinOcclusion is the macOS/Windows occlusion detector). + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + '--disable-backgrounding-occluded-windows', + '--disable-features=CalculateNativeWinOcclusion' + ], { cwd: DESKTOP_DIR, stdio: ['ignore', 'inherit', 'inherit'], @@ -209,12 +246,37 @@ export async function startIsolatedInstance({ { timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' } ) - // Let cold-start contention (vite dep pre-bundling, first backend-connect - // attempts, initial paint) drain before scenarios measure, so numbers - // reflect steady state rather than launch noise. + // Electron throttles rAF/timers for a window that isn't foregrounded + // (per-window backgroundThrottling, which the Chromium CLI flags above don't + // override). Focus emulation makes the renderer behave as if focused so + // frame-pacing measurements are real even though the perf window sits behind + // the user's other windows — WITHOUT actually stealing OS focus. + try { + // Behave as if focused so frame-pacing isn't throttled while the perf + // window sits behind the user's IDE/terminal — WITHOUT stealing OS focus. + await cdp.send('Emulation.setFocusEmulationEnabled', { enabled: true }) + } catch { + // Older CDP / not supported — fall back to the anti-throttle flags above. + } + + // Wait for the gateway socket to actually open. A booting/absent backend + // retries on a 1–15s backoff, and that churn contaminates frame-pacing + // (the `stream` scenario). Best-effort: proceed after the timeout so the + // backend-independent scenarios (keystroke, transcript) still run. + const connected = await waitForConnected(cdp, connectTimeoutMs) + + if (!connected) { + console.warn( + `[perf] gateway did not connect within ${connectTimeoutMs}ms — ` + + 'stream/frame numbers may be inflated by reconnect churn.' + ) + } + + // Let residual cold-start work (vite dep pre-bundling, initial paint) drain. await sleep(settleMs) return { + connected, cdp, devUrl, port, diff --git a/apps/desktop/scripts/perf/scenarios/stream.mjs b/apps/desktop/scripts/perf/scenarios/stream.mjs index a1c567934e9a..66ae3ca58d53 100644 --- a/apps/desktop/scripts/perf/scenarios/stream.mjs +++ b/apps/desktop/scripts/perf/scenarios/stream.mjs @@ -10,10 +10,16 @@ import { frameHistogram, percentile } from '../lib/stats.mjs' const RECORDERS = ` (() => { + // Generation guard: a prior run's rAF loop re-reads window.__FT__ each frame, + // so simply reassigning it would leave the old loop running and pushing into + // the new array (overlapping recorders inflate frame intervals on run 2+). + // Bumping the generation makes every stale loop exit on its next tick. + window.__FT_GEN__ = (window.__FT_GEN__ || 0) + 1 + const ftGen = window.__FT_GEN__ window.__FT__ = { times: [], stop: false } let last = performance.now() const tick = () => { - if (window.__FT__.stop) return + if (window.__FT_GEN__ !== ftGen || window.__FT__.stop) return const now = performance.now() window.__FT__.times.push(now - last) last = now @@ -113,9 +119,14 @@ export default { const tokens = Number(opts.tokens ?? 400) const intervalMs = Number(opts.intervalMs ?? 16) const flushMinMs = Number(opts.flushMinMs ?? 33) - // No raw autolink in the default chunk — a resolvable URL triggers link - // embeds / DNS lookups that add noise unrelated to render cost. - const chunk = opts.chunk ?? '**word** in _italic_ with `code` and a bit of ordinary prose. ' + // Realistic default: a short markdown paragraph ending in a blank line, so + // blocks SETTLE as they stream — exactly how real LLM output behaves, and + // what block-memoization is designed for (only the growing tail re-renders). + // A chunk with NO paragraph break (e.g. `--chunk 'word '`) instead grows one + // ever-larger block that re-renders fully every flush — a useful worst-case + // stress, but not the typical number. No raw autolink (avoids DNS/link-embed + // noise unrelated to render cost). + const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n' const real = Boolean(opts.real) await cdp.send('Runtime.enable') diff --git a/apps/desktop/src/app/chat/perf-probe.tsx b/apps/desktop/src/app/chat/perf-probe.tsx index ad881450eca2..987d89fb428d 100644 --- a/apps/desktop/src/app/chat/perf-probe.tsx +++ b/apps/desktop/src/app/chat/perf-probe.tsx @@ -1,5 +1,6 @@ import { Profiler, type ProfilerOnRenderCallback, type ReactNode } from 'react' +import { $gateway } from '@/store/gateway' import { $messages, setBusy, setMessages } from '@/store/session' type Sample = { @@ -31,6 +32,12 @@ declare global { * proxy). Used by the `transcript` perf scenario. `reset()` restores. */ loadTranscript: (turns?: number) => Promise + /** + * Whether the active gateway socket is open. The perf harness waits on + * this before measuring so background reconnect churn (a booting/absent + * backend) doesn't contaminate frame-pacing numbers. + */ + connected: () => boolean reset: () => void snapshotMsgs: () => number } @@ -152,6 +159,13 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) { window.__PERF_DRIVE__ = { snapshotMsgs: () => $messages.get().length, + connected: () => { + try { + return $gateway.get()?.connectionState === 'open' + } catch { + return false + } + }, loadTranscript: (turns = 200) => { if (!baseline) { baseline = $messages.get()