perf(desktop): add a stream-history scenario to the perf harness

The existing `stream` scenario measures streaming into an empty thread, which
is exactly the case where per-delta transcript work does not show up. This adds
`--historyTurns`, which mounts a settled transcript and lets it drain BEFORE
the recorders start, so the measurement window contains streaming work only —
and `stream-history`, a report-only preset that turns it on.

Report-only (tier: manual) rather than gated: how much history a host can mount
varies, so the absolute number is not comparable across machines. It is meant
for same-machine before/after runs.

Co-authored-by: Jakub Wolniewicz <frizikk@users.noreply.github.com>
This commit is contained in:
Brooklyn Nicholson 2026-07-26 03:15:48 -05:00
parent ad09bf3872
commit 20fcef5207
3 changed files with 47 additions and 3 deletions

View file

@ -8,11 +8,13 @@ import multitab from './multitab.mjs'
import profileSwitch from './profile-switch.mjs'
import sessionSwitch from './session-switch.mjs'
import stream from './stream.mjs'
import streamHistory from './stream-history.mjs'
import submit from './submit.mjs'
import transcript from './transcript.mjs'
export const SCENARIOS = {
[stream.name]: stream,
[streamHistory.name]: streamHistory,
[keystroke.name]: keystroke,
[transcript.name]: transcript,
[multitab.name]: multitab,

View file

@ -0,0 +1,18 @@
// Streaming into an ALREADY-LONG transcript. Same measurement as `stream`, but
// the history is mounted and allowed to settle before the recorders start, so
// what it captures is the per-delta cost that scales with transcript length —
// the regression reported in #69120.
//
// Report-only (tier: manual): the number depends on how much history the host
// can mount, so it is not gated against the committed baseline.
import stream from './stream.mjs'
export default {
name: 'stream-history',
tier: 'manual',
description: 'Streaming cost with a long settled transcript already mounted.',
run(cdp, opts = {}) {
return stream.run(cdp, { ...opts, historyTurns: Number(opts.historyTurns ?? 200) })
}
}

View file

@ -70,7 +70,7 @@ const COLLECT = `
})()
`
function analyze(data, warmupMs) {
function analyze(data, warmupMs, extra = {}) {
// Drop warm-up frames (recorder installs before the stream starts).
const frames = []
let acc = 0
@ -102,6 +102,7 @@ function analyze(data, warmupMs) {
intermut_p95_ms: Math.round(percentile(interMut, 0.95) * 10) / 10
},
detail: {
...extra,
windowS: Math.round(windowS * 10) / 10,
avgFps: windowS ? Math.round((frames.length / windowS) * 10) / 10 : 0,
frameHistogram: frameHistogram(frames),
@ -128,13 +129,36 @@ export default {
// noise unrelated to render cost).
const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n'
const real = Boolean(opts.real)
const historyTurns = Number(opts.historyTurns ?? 0)
const historySettleMs = Number(opts.historySettleMs ?? 1500)
await cdp.send('Runtime.enable')
// Mount the settled history BEFORE the recorders start, so the measurement
// window contains only streaming work — not the one-off mount cost.
if (historyTurns > 0) {
if (real) {
throw new Error('--historyTurns is only supported by the synthetic stream path')
}
await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${historyTurns})`)
await sleep(historySettleMs)
const mounted = Number(await cdp.eval('window.__PERF_DRIVE__.snapshotMsgs()'))
const expected = historyTurns * 2
if (mounted !== expected) {
throw new Error(`expected ${expected} preloaded history messages, got ${mounted}`)
}
}
await cdp.eval(RECORDERS)
if (real) {
// Backend path: fire a real prompt and wait for the stream to appear.
const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`)
const baseCount = await cdp.eval(
`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`
)
await typeIntoComposer(cdp, opts.prompt ?? 'count from 1 to 80, one number per line', { cps: 40 })
await cdp.eval(`(() => {
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
@ -186,6 +210,6 @@ export default {
await cdp.eval('window.__PERF_DRIVE__.reset()')
}
return analyze(data, real ? 0 : 500)
return analyze(data, real ? 0 : 500, { historyTurns })
}
}