feat(desktop): render-churn perf scenario

Drives the same synthetic multi-tab streaming workload as `multitab`
(publishSessionState per session per flush, no backend, no credits) but
reports render attribution instead of frame pacing — sidebar_renders,
wasted_renders, and wasted_notifies.

Answers 'does the sidebar re-render while an agent is typing' directly.
This commit is contained in:
Brooklyn Nicholson 2026-07-26 06:11:00 -05:00
parent 44211f3608
commit 37ac8ae76a
3 changed files with 220 additions and 0 deletions

View file

@ -51,6 +51,7 @@ directly via `window.__PERF_DRIVE__`, so no LLM credits are spent.
| `stream --real` | backend | same, from a real LLM stream | measure-real-stream, profile-real-stream |
| `keystroke` | ci | composer keystroke → paint latency | measure-latency, profile-typing, leak-typing |
| `transcript` | ci | large-transcript mount + paint cost | (new) |
| `render-churn` | ci | per-component render attribution + store churn while N tabs stream | (new) |
| `cold-start` | cold | launch → CDP → driver → first paint (fresh spawn/run) | (new) |
| `first-token` | backend | Enter → first assistant token painted (TTFT) | (new) |
| `submit` | backend | Enter → cleared → user msg painted, scroll jump | measure-submit, measure-jump |

View file

@ -6,6 +6,7 @@ import firstToken from './first-token.mjs'
import keystroke from './keystroke.mjs'
import multitab from './multitab.mjs'
import profileSwitch from './profile-switch.mjs'
import renderChurn from './render-churn.mjs'
import sessionSwitch from './session-switch.mjs'
import stream from './stream.mjs'
import streamHistory from './stream-history.mjs'
@ -18,6 +19,7 @@ export const SCENARIOS = {
[keystroke.name]: keystroke,
[transcript.name]: transcript,
[multitab.name]: multitab,
[renderChurn.name]: renderChurn,
[coldStart.name]: coldStart,
[firstToken.name]: firstToken,
[submit.name]: submit,

View file

@ -0,0 +1,217 @@
// Render churn during multi-tab streaming: WHAT re-rendered and WHY, and which
// store published the update. Frame pacing (see `multitab`) tells you the cost;
// this tells you the cause.
//
// Drives the same synthetic pipeline as `multitab` — publishSessionState per
// session per flush via `__HERMES_SESSION_TILES__`, no backend, no credits —
// then reads the dev-only counters installed by `src/debug/`:
//
// window.__RENDER_COUNTS__ — per-component renders, attributed to
// props / hook state / parent-only ("wasted")
// window.__ATOM_CHURN__ — per-store notifications, listener fan-out, and
// notifications whose value was deep-equal to the
// previous one ("wasted")
//
// The headline metric is `sidebar_renders`: how many times the sidebar tree
// re-rendered while agents were typing in other tabs. It should be 0.
//
// node scripts/perf/run.mjs render-churn --spawn [--tiles 5] [--tokens 240]
import { sleep } from '../lib/cdp.mjs'
/** Components that make up the sidebar tree. A render of any of these while
* a background tab streams is work the user cannot see. */
const SIDEBAR_COMPONENTS = [
'ChatSidebar',
'SidebarSurface',
'SessionRow',
'SessionsSection',
'CronJobsSection',
'ProfileSwitcher',
'VirtualSessionList',
'WorkspaceGroup',
'OverviewRow',
'SessionStatusDot'
]
/** Page-side setup: open `tiles` session tiles, seed each with a transcript.
* Mirrors `multitab.mjs` so the two scenarios measure the same workload. */
const setup = (tiles, seedTurns) => `
(() => {
const hook = window.__HERMES_SESSION_TILES__
if (!hook) return 'no-hook'
if (!window.__RENDER_COUNTS__) return 'no-render-counter'
if (!window.__ATOM_CHURN__) return 'no-atom-churn'
const turn = (sid, i) => ([
{ id: sid + '-u' + i, role: 'user', timestamp: Date.now(),
parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff handle the error path?' }] },
{ id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false,
parts: [{ type: 'text', text: '## Finding ' + i + '\\n\\nThe handler swallows the rejection.\\n\\n- The catch block drops the error.\\n- Retries are unbounded.\\n' }] }
])
const state = (sid) => {
const messages = []
for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i))
messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true,
parts: [{ type: 'text', text: '' }] })
return {
storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '',
reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '',
busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true,
pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false,
needsInput: false, turnStartedAt: Date.now(), usage: null
}
}
window.__RC__ = { ids: [], timer: null }
for (let n = 1; n <= ${tiles}; n++) {
const sid = 'churn-tile-' + n
const rid = 'churn-rt-' + n
window.__RC__.ids.push({ sid, rid })
hook.open(sid, 'center')
hook.patch(sid, { runtimeId: rid })
hook.publish(rid, state(sid))
}
return 'ok'
})()
`
const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})`
/** Grow every tile's streaming tail by `chunk` each `intervalMs`, through the
* same publish path the gateway's delta flush uses. */
const drive = (chunk, intervalMs, totalTokens) => `
(() => {
const hook = window.__HERMES_SESSION_TILES__
let pushed = 0
const tick = () => {
const states = hook.states()
for (const { rid } of window.__RC__.ids) {
const prev = states[rid]
if (!prev) continue
const messages = prev.messages.map(m => {
if (m.id !== prev.streamId) return m
const head = m.parts.slice(0, -1)
const last = m.parts[m.parts.length - 1]
return { ...m, parts: [...head, { type: 'text', text: last.text + ${JSON.stringify(chunk)} }] }
})
hook.publish(rid, { ...prev, messages })
}
pushed += 1
if (pushed < ${totalTokens}) window.__RC__.timer = setTimeout(tick, ${intervalMs})
else window.__RC__.done = true
}
window.__RC__.timer = setTimeout(tick, ${intervalMs})
return 'driving'
})()
`
const START = `
(() => {
window.__RENDER_COUNTS__.start()
window.__ATOM_CHURN__.start()
return 'recording'
})()
`
const COLLECT = `
(() => {
window.__RENDER_COUNTS__.stop()
window.__ATOM_CHURN__.stop()
return JSON.stringify({
commits: window.__RENDER_COUNTS__.commits(),
renders: window.__RENDER_COUNTS__.report(200),
atoms: window.__ATOM_CHURN__.report(200)
})
})()
`
const CLEANUP = `
(() => {
if (window.__RC__) {
clearTimeout(window.__RC__.timer)
for (const { sid, rid } of window.__RC__.ids) {
const states = window.__HERMES_SESSION_TILES__.states()
window.__HERMES_SESSION_TILES__.publish(rid, { ...states[rid], busy: false, streamId: null })
window.__HERMES_SESSION_TILES__.close(sid)
}
window.__RC__ = null
}
window.__RENDER_COUNTS__.clear()
window.__ATOM_CHURN__.clear()
return 'cleaned'
})()
`
export default {
name: 'render-churn',
tier: 'ci',
description: 'N streaming tabs: per-component render attribution + store churn.',
async run(cdp, opts = {}) {
const tiles = Number(opts.tiles ?? 5)
const seedTurns = Number(opts.turns ?? 20)
const tokens = Number(opts.tokens ?? 240)
// Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush.
const intervalMs = Number(opts.intervalMs ?? 33)
const chunk = opts.chunk ?? 'A streamed review sentence with **bold** and `code`.\n\n'
await cdp.send('Runtime.enable')
const ok = await cdp.eval(setup(tiles, seedTurns))
if (ok !== 'ok') {
throw new Error(
`render-churn setup failed (${ok}) — needs a dev renderer with src/debug installed ` +
'(the counters are aliased out of production builds unless VITE_PERF_PROBE=1).'
)
}
// Mount every tab (keep-alive mounts on first activation), then settle.
for (let n = 1; n <= tiles; n++) {
await cdp.eval(reveal(`churn-tile-${n}`))
await sleep(350)
}
await sleep(1000)
// Start recording AFTER mount so the numbers are steady-state streaming
// cost, not one-off mount cost.
await cdp.eval(START)
await cdp.eval(drive(chunk, intervalMs, tokens))
await sleep(tokens * intervalMs + 1500)
const data = JSON.parse(await cdp.eval(COLLECT))
await cdp.eval(CLEANUP)
const byName = new Map(data.renders.map(r => [r.name, r]))
const sidebarRows = SIDEBAR_COMPONENTS.map(n => byName.get(n)).filter(Boolean)
const sidebarRenders = sidebarRows.reduce((a, r) => a + r.renders, 0)
const sidebarWasted = sidebarRows.reduce((a, r) => a + r.wasted, 0)
const totalRenders = data.renders.reduce((a, r) => a + r.renders, 0)
const totalWasted = data.renders.reduce((a, r) => a + r.wasted, 0)
const atomWasted = data.atoms.reduce((a, r) => a + r.wasted, 0)
return {
metrics: {
// The hypothesis, as a number: sidebar renders while background tabs
// stream. Should be 0.
sidebar_renders: sidebarRenders,
sidebar_wasted: sidebarWasted,
// Renders with no changed props and no changed hook state — pure
// parent-driven work, across the whole tree.
wasted_renders: totalWasted,
total_renders: totalRenders,
commits: data.commits,
// Store notifications that published a value equal to the last one.
wasted_notifies: atomWasted
},
detail: {
tiles,
tokens,
sidebar: sidebarRows,
topRenders: data.renders.slice(0, 15),
topAtoms: data.atoms.slice(0, 15)
}
}
}
}