From 20fcef5207f46a5626c78cc886068eb2185616aa Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 26 Jul 2026 03:15:48 -0500 Subject: [PATCH] perf(desktop): add a stream-history scenario to the perf harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing `stream` scenario measures streaming into an empty thread, which is exactly the case where per-delta transcript work does not show up. This adds `--historyTurns`, which mounts a settled transcript and lets it drain BEFORE the recorders start, so the measurement window contains streaming work only — and `stream-history`, a report-only preset that turns it on. Report-only (tier: manual) rather than gated: how much history a host can mount varies, so the absolute number is not comparable across machines. It is meant for same-machine before/after runs. Co-authored-by: Jakub Wolniewicz --- apps/desktop/scripts/perf/scenarios/index.mjs | 2 ++ .../scripts/perf/scenarios/stream-history.mjs | 18 +++++++++++ .../desktop/scripts/perf/scenarios/stream.mjs | 30 +++++++++++++++++-- 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 apps/desktop/scripts/perf/scenarios/stream-history.mjs diff --git a/apps/desktop/scripts/perf/scenarios/index.mjs b/apps/desktop/scripts/perf/scenarios/index.mjs index ad667262a6c..d69ecce2fbc 100644 --- a/apps/desktop/scripts/perf/scenarios/index.mjs +++ b/apps/desktop/scripts/perf/scenarios/index.mjs @@ -8,11 +8,13 @@ import multitab from './multitab.mjs' import profileSwitch from './profile-switch.mjs' import sessionSwitch from './session-switch.mjs' import stream from './stream.mjs' +import streamHistory from './stream-history.mjs' import submit from './submit.mjs' import transcript from './transcript.mjs' export const SCENARIOS = { [stream.name]: stream, + [streamHistory.name]: streamHistory, [keystroke.name]: keystroke, [transcript.name]: transcript, [multitab.name]: multitab, diff --git a/apps/desktop/scripts/perf/scenarios/stream-history.mjs b/apps/desktop/scripts/perf/scenarios/stream-history.mjs new file mode 100644 index 00000000000..d807f3f3573 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/stream-history.mjs @@ -0,0 +1,18 @@ +// Streaming into an ALREADY-LONG transcript. Same measurement as `stream`, but +// the history is mounted and allowed to settle before the recorders start, so +// what it captures is the per-delta cost that scales with transcript length — +// the regression reported in #69120. +// +// Report-only (tier: manual): the number depends on how much history the host +// can mount, so it is not gated against the committed baseline. + +import stream from './stream.mjs' + +export default { + name: 'stream-history', + tier: 'manual', + description: 'Streaming cost with a long settled transcript already mounted.', + run(cdp, opts = {}) { + return stream.run(cdp, { ...opts, historyTurns: Number(opts.historyTurns ?? 200) }) + } +} diff --git a/apps/desktop/scripts/perf/scenarios/stream.mjs b/apps/desktop/scripts/perf/scenarios/stream.mjs index 66ae3ca58d5..f32a4831b1e 100644 --- a/apps/desktop/scripts/perf/scenarios/stream.mjs +++ b/apps/desktop/scripts/perf/scenarios/stream.mjs @@ -70,7 +70,7 @@ const COLLECT = ` })() ` -function analyze(data, warmupMs) { +function analyze(data, warmupMs, extra = {}) { // Drop warm-up frames (recorder installs before the stream starts). const frames = [] let acc = 0 @@ -102,6 +102,7 @@ function analyze(data, warmupMs) { intermut_p95_ms: Math.round(percentile(interMut, 0.95) * 10) / 10 }, detail: { + ...extra, windowS: Math.round(windowS * 10) / 10, avgFps: windowS ? Math.round((frames.length / windowS) * 10) / 10 : 0, frameHistogram: frameHistogram(frames), @@ -128,13 +129,36 @@ export default { // noise unrelated to render cost). const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n' const real = Boolean(opts.real) + const historyTurns = Number(opts.historyTurns ?? 0) + const historySettleMs = Number(opts.historySettleMs ?? 1500) await cdp.send('Runtime.enable') + + // Mount the settled history BEFORE the recorders start, so the measurement + // window contains only streaming work — not the one-off mount cost. + if (historyTurns > 0) { + if (real) { + throw new Error('--historyTurns is only supported by the synthetic stream path') + } + + await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${historyTurns})`) + await sleep(historySettleMs) + + const mounted = Number(await cdp.eval('window.__PERF_DRIVE__.snapshotMsgs()')) + const expected = historyTurns * 2 + + if (mounted !== expected) { + throw new Error(`expected ${expected} preloaded history messages, got ${mounted}`) + } + } + await cdp.eval(RECORDERS) if (real) { // Backend path: fire a real prompt and wait for the stream to appear. - const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`) + const baseCount = await cdp.eval( + `document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length` + ) await typeIntoComposer(cdp, opts.prompt ?? 'count from 1 to 80, one number per line', { cps: 40 }) await cdp.eval(`(() => { const el = document.querySelector(${JSON.stringify(SELECTORS.composer)}) @@ -186,6 +210,6 @@ export default { await cdp.eval('window.__PERF_DRIVE__.reset()') } - return analyze(data, real ? 0 : 500) + return analyze(data, real ? 0 : 500, { historyTurns }) } }