From 9173f589c6705291369ae28664785d11de130628 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 26 Jul 2026 00:20:50 -0500 Subject: [PATCH] perf(desktop): add a multitab scenario to the perf harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit N session tiles stacked as tabs, all mounted (keep-alive) and all streaming concurrently through the real publishSessionState path — the "several PR reviews at once" workload. Frame pacing + longtask metrics, no backend or credits needed; the workload that exposed both fixes above and the regression gate that keeps them fixed. --- apps/desktop/scripts/perf/scenarios/index.mjs | 2 + .../scripts/perf/scenarios/multitab.mjs | 246 ++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 apps/desktop/scripts/perf/scenarios/multitab.mjs diff --git a/apps/desktop/scripts/perf/scenarios/index.mjs b/apps/desktop/scripts/perf/scenarios/index.mjs index d9aeece1eff..ad667262a6c 100644 --- a/apps/desktop/scripts/perf/scenarios/index.mjs +++ b/apps/desktop/scripts/perf/scenarios/index.mjs @@ -4,6 +4,7 @@ import coldStart from './cold-start.mjs' import firstToken from './first-token.mjs' import keystroke from './keystroke.mjs' +import multitab from './multitab.mjs' import profileSwitch from './profile-switch.mjs' import sessionSwitch from './session-switch.mjs' import stream from './stream.mjs' @@ -14,6 +15,7 @@ export const SCENARIOS = { [stream.name]: stream, [keystroke.name]: keystroke, [transcript.name]: transcript, + [multitab.name]: multitab, [coldStart.name]: coldStart, [firstToken.name]: firstToken, [submit.name]: submit, diff --git a/apps/desktop/scripts/perf/scenarios/multitab.mjs b/apps/desktop/scripts/perf/scenarios/multitab.mjs new file mode 100644 index 00000000000..d93116f1285 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/multitab.mjs @@ -0,0 +1,246 @@ +// Multi-tab working sessions: N session tiles stacked as tabs in the main +// zone, EVERY tab mounted (keep-alive), all streaming concurrently — the +// "5 tabs doing PR review" workload. Measures frame pacing + longtasks while +// the whole stack streams, which is where multitab renderers crawl. +// +// Drives the real pipeline synthetically (no backend, no credits): +// publishSessionState per session per flush — exactly what the gateway's +// delta flush does — via the __HERMES_SESSION_TILES__ hook. +// +// node scripts/perf/run.mjs multitab --spawn [--tiles 5] [--tokens 240] + +import { sleep } from '../lib/cdp.mjs' +import { frameHistogram, percentile } from '../lib/stats.mjs' + +// Same recorder pattern as stream.mjs (generation-guarded rAF + longtasks). +const RECORDERS = ` + (() => { + window.__FT_GEN__ = (window.__FT_GEN__ || 0) + 1 + const ftGen = window.__FT_GEN__ + window.__FT__ = { times: [], stop: false } + let last = performance.now() + const tick = () => { + if (window.__FT_GEN__ !== ftGen || window.__FT__.stop) return + const now = performance.now() + window.__FT__.times.push(now - last) + last = now + requestAnimationFrame(tick) + } + requestAnimationFrame(tick) + + window.__LT__ = { entries: [], stop: false } + try { + const po = new PerformanceObserver((list) => { + if (window.__LT__.stop) return + for (const e of list.getEntries()) window.__LT__.entries.push({ duration: e.duration, startTime: e.startTime }) + }) + po.observe({ entryTypes: ['longtask'] }) + window.__LT__.po = po + } catch {} + return 'armed' + })() +` + +const COLLECT = ` + (() => { + window.__FT__.stop = true + window.__LT__.stop = true + try { window.__LT__.po && window.__LT__.po.disconnect() } catch {} + return JSON.stringify({ frames: window.__FT__.times, longtasks: window.__LT__.entries }) + })() +` + +/** Page-side setup: open `tiles` session tiles stacked into the main zone, + * bind fake runtime ids, and seed each with a realistic transcript. */ +const setup = (tiles, seedTurns, streamSeed) => ` + (() => { + const hook = window.__HERMES_SESSION_TILES__ + if (!hook) return 'no-hook' + + const turn = (sid, i) => ([ + { id: sid + '-u' + i, role: 'user', timestamp: Date.now(), + parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff in module ' + i + ' handle the error path?' }] }, + { id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false, + parts: [{ type: 'text', text: [ + '## Finding ' + i, '', + 'The handler swallows the rejection. Key points for hunk \\\`' + i + '\\\`:', '', + '- The catch block drops the original error.', + '- Retries are unbounded — see [the loop](https://example.com/loop).', '', + '\\\`\\\`\\\`ts', + 'async function retry' + i + '(fn: () => Promise) {', + ' for (;;) { try { return await fn() } catch {} }', + '}', + '\\\`\\\`\\\`', '', + '| path | covered |', '|---|---|', '| happy | yes |', '| error | no |', '' + ].join('\\n') }] } + ]) + + const state = (sid, rid) => { + const messages = [] + for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i)) + // Streaming tail the driver grows (--code seeds an open fence). + messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true, + parts: [{ type: 'text', text: ${JSON.stringify(streamSeed)} }] }) + return { + storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '', + reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '', + busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true, + pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false, + needsInput: false, turnStartedAt: Date.now(), usage: null + } + } + + window.__MT__ = { ids: [], timer: null } + for (let n = 1; n <= ${tiles}; n++) { + const sid = 'perf-tile-' + n + const rid = 'perf-rt-' + n + window.__MT__.ids.push({ sid, rid }) + hook.open(sid, 'center') + hook.patch(sid, { runtimeId: rid }) + hook.publish(rid, state(sid, rid)) + } + return 'ok' + })() +` + +// Activate every tab once so keep-alive mounts the full stack (lazy mount: +// a never-activated tab stays unmounted, which would understate the cost). +const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})` + +/** Page-side driver: grow every tile's streaming tail by `chunk` each + * `intervalMs`, through the same publish path the gateway flush uses. */ +const drive = (chunk, intervalMs, totalTokens) => ` + (() => { + const hook = window.__HERMES_SESSION_TILES__ + let pushed = 0 + const tick = () => { + const states = hook.states() + for (const { rid } of window.__MT__.ids) { + const prev = states[rid] + if (!prev) continue + const messages = prev.messages.map(m => { + if (m.id !== prev.streamId) return m + const head = m.parts.slice(0, -1) + const last = m.parts[m.parts.length - 1] + return { ...m, parts: [...head, { type: 'text', text: last.text + ${JSON.stringify(chunk)} }] } + }) + hook.publish(rid, { ...prev, messages }) + } + pushed += 1 + if (pushed < ${totalTokens}) window.__MT__.timer = setTimeout(tick, ${intervalMs}) + else window.__MT__.done = true + } + window.__MT__.timer = setTimeout(tick, ${intervalMs}) + return 'driving' + })() +` + +const CLEANUP = ` + (() => { + if (window.__MT__) { + clearTimeout(window.__MT__.timer) + for (const { sid, rid } of window.__MT__.ids) { + window.__HERMES_SESSION_TILES__.publish(rid, { + ...window.__HERMES_SESSION_TILES__.states()[rid], busy: false, streamId: null + }) + window.__HERMES_SESSION_TILES__.close(sid) + } + window.__MT__ = null + } + return 'cleaned' + })() +` + +export default { + name: 'multitab', + tier: 'ci', + description: 'N mounted session-tile tabs all streaming: frame pacing + longtasks.', + async run(cdp, opts = {}) { + const tiles = Number(opts.tiles ?? 5) + const seedTurns = Number(opts.turns ?? 20) + const tokens = Number(opts.tokens ?? 240) + // Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush. + const intervalMs = Number(opts.intervalMs ?? 33) + // --code: every tile grows ONE giant fenced code block with no settle + // boundaries — what a coding agent streams. The block re-parses and + // re-renders fully every flush (block memoization can't settle it), the + // documented worst case and the "5 tabs all coding" crawl. + const chunk = opts.code + ? ' const value = await resolve(ctx, { retry: true }) // step\n' + : (opts.chunk ?? 'A streamed review sentence with **bold**, `code`, and ordinary prose.\n\n') + const streamSeed = opts.code ? '```ts\n' : '' + + await cdp.send('Runtime.enable') + + const ok = await cdp.eval(setup(tiles, seedTurns, streamSeed)) + + if (ok !== 'ok') { + throw new Error(`multitab setup failed (${ok}) — dev hooks missing? (needs a dev/probe renderer)`) + } + + // Mount every tab (keep-alive mounts on first activation), then settle. + for (let n = 1; n <= tiles; n++) { + await cdp.eval(reveal(`perf-tile-${n}`)) + await sleep(350) + } + + await sleep(1000) + await cdp.eval(RECORDERS) + await cdp.eval(drive(chunk, intervalMs, tokens)) + await sleep(tokens * intervalMs + 1500) + + const data = JSON.parse(await cdp.eval(COLLECT)) + await cdp.eval(CLEANUP) + + // Drop the first 500ms (recorder install + settle). + const frames = [] + let acc = 0 + + for (const f of data.frames) { + acc += f + + if (acc >= 500) { + frames.push(f) + } + } + + const ltDurations = data.longtasks.map(e => e.duration) + const windowS = frames.reduce((a, b) => a + b, 0) / 1000 + // The felt numbers: sustained fps over the window, and the fps of the + // worst 1-second slice (a 333ms frame IS "3fps" even if the average looks + // fine). Worst slice = max summed frame time in any sliding 1s window. + const avgFps = windowS ? frames.length / windowS : 0 + let worstFps = avgFps + + for (let i = 0, j = 0, sum = 0; j < frames.length; j++) { + sum += frames[j] + + while (sum > 1000) { + sum -= frames[i++] + } + + // Only a window that actually spans ~1s counts; short prefixes don't. + if (sum >= 900) { + worstFps = Math.min(worstFps, ((j - i + 1) / sum) * 1000) + } + } + + return { + metrics: { + longtasks_n: data.longtasks.length, + longtask_max_ms: Math.round((ltDurations.length ? Math.max(...ltDurations) : 0) * 10) / 10, + frame_p95_ms: Math.round(percentile(frames, 0.95) * 10) / 10, + frame_p99_ms: Math.round(percentile(frames, 0.99) * 10) / 10, + slow_frames_33: frames.filter(f => f > 33).length + }, + detail: { + tiles, + code: Boolean(opts.code), + windowS: Math.round(windowS * 10) / 10, + avgFps: Math.round(avgFps * 10) / 10, + worstSecondFps: Math.round(worstFps * 10) / 10, + frameHistogram: frameHistogram(frames) + } + } + } +}