From 37ac8ae76a165a4e5a27582b917a9b91a6b91ea7 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sun, 26 Jul 2026 06:11:00 -0500 Subject: [PATCH] feat(desktop): render-churn perf scenario MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drives the same synthetic multi-tab streaming workload as `multitab` (publishSessionState per session per flush, no backend, no credits) but reports render attribution instead of frame pacing — sidebar_renders, wasted_renders, and wasted_notifies. Answers 'does the sidebar re-render while an agent is typing' directly. --- apps/desktop/scripts/perf/README.md | 1 + apps/desktop/scripts/perf/scenarios/index.mjs | 2 + .../scripts/perf/scenarios/render-churn.mjs | 217 ++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 apps/desktop/scripts/perf/scenarios/render-churn.mjs diff --git a/apps/desktop/scripts/perf/README.md b/apps/desktop/scripts/perf/README.md index e8cfc5752312..4c74de4983dc 100644 --- a/apps/desktop/scripts/perf/README.md +++ b/apps/desktop/scripts/perf/README.md @@ -51,6 +51,7 @@ directly via `window.__PERF_DRIVE__`, so no LLM credits are spent. | `stream --real` | backend | same, from a real LLM stream | measure-real-stream, profile-real-stream | | `keystroke` | ci | composer keystroke → paint latency | measure-latency, profile-typing, leak-typing | | `transcript` | ci | large-transcript mount + paint cost | (new) | +| `render-churn` | ci | per-component render attribution + store churn while N tabs stream | (new) | | `cold-start` | cold | launch → CDP → driver → first paint (fresh spawn/run) | (new) | | `first-token` | backend | Enter → first assistant token painted (TTFT) | (new) | | `submit` | backend | Enter → cleared → user msg painted, scroll jump | measure-submit, measure-jump | diff --git a/apps/desktop/scripts/perf/scenarios/index.mjs b/apps/desktop/scripts/perf/scenarios/index.mjs index d69ecce2fbcf..d92526b0fcfa 100644 --- a/apps/desktop/scripts/perf/scenarios/index.mjs +++ b/apps/desktop/scripts/perf/scenarios/index.mjs @@ -6,6 +6,7 @@ import firstToken from './first-token.mjs' import keystroke from './keystroke.mjs' import multitab from './multitab.mjs' import profileSwitch from './profile-switch.mjs' +import renderChurn from './render-churn.mjs' import sessionSwitch from './session-switch.mjs' import stream from './stream.mjs' import streamHistory from './stream-history.mjs' @@ -18,6 +19,7 @@ export const SCENARIOS = { [keystroke.name]: keystroke, [transcript.name]: transcript, [multitab.name]: multitab, + [renderChurn.name]: renderChurn, [coldStart.name]: coldStart, [firstToken.name]: firstToken, [submit.name]: submit, diff --git a/apps/desktop/scripts/perf/scenarios/render-churn.mjs b/apps/desktop/scripts/perf/scenarios/render-churn.mjs new file mode 100644 index 000000000000..d1e940cbed7e --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/render-churn.mjs @@ -0,0 +1,217 @@ +// Render churn during multi-tab streaming: WHAT re-rendered and WHY, and which +// store published the update. Frame pacing (see `multitab`) tells you the cost; +// this tells you the cause. +// +// Drives the same synthetic pipeline as `multitab` — publishSessionState per +// session per flush via `__HERMES_SESSION_TILES__`, no backend, no credits — +// then reads the dev-only counters installed by `src/debug/`: +// +// window.__RENDER_COUNTS__ — per-component renders, attributed to +// props / hook state / parent-only ("wasted") +// window.__ATOM_CHURN__ — per-store notifications, listener fan-out, and +// notifications whose value was deep-equal to the +// previous one ("wasted") +// +// The headline metric is `sidebar_renders`: how many times the sidebar tree +// re-rendered while agents were typing in other tabs. It should be 0. +// +// node scripts/perf/run.mjs render-churn --spawn [--tiles 5] [--tokens 240] + +import { sleep } from '../lib/cdp.mjs' + +/** Components that make up the sidebar tree. A render of any of these while + * a background tab streams is work the user cannot see. */ +const SIDEBAR_COMPONENTS = [ + 'ChatSidebar', + 'SidebarSurface', + 'SessionRow', + 'SessionsSection', + 'CronJobsSection', + 'ProfileSwitcher', + 'VirtualSessionList', + 'WorkspaceGroup', + 'OverviewRow', + 'SessionStatusDot' +] + +/** Page-side setup: open `tiles` session tiles, seed each with a transcript. + * Mirrors `multitab.mjs` so the two scenarios measure the same workload. */ +const setup = (tiles, seedTurns) => ` + (() => { + const hook = window.__HERMES_SESSION_TILES__ + if (!hook) return 'no-hook' + if (!window.__RENDER_COUNTS__) return 'no-render-counter' + if (!window.__ATOM_CHURN__) return 'no-atom-churn' + + const turn = (sid, i) => ([ + { id: sid + '-u' + i, role: 'user', timestamp: Date.now(), + parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff handle the error path?' }] }, + { id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false, + parts: [{ type: 'text', text: '## Finding ' + i + '\\n\\nThe handler swallows the rejection.\\n\\n- The catch block drops the error.\\n- Retries are unbounded.\\n' }] } + ]) + + const state = (sid) => { + const messages = [] + for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i)) + messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true, + parts: [{ type: 'text', text: '' }] }) + return { + storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '', + reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '', + busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true, + pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false, + needsInput: false, turnStartedAt: Date.now(), usage: null + } + } + + window.__RC__ = { ids: [], timer: null } + for (let n = 1; n <= ${tiles}; n++) { + const sid = 'churn-tile-' + n + const rid = 'churn-rt-' + n + window.__RC__.ids.push({ sid, rid }) + hook.open(sid, 'center') + hook.patch(sid, { runtimeId: rid }) + hook.publish(rid, state(sid)) + } + return 'ok' + })() +` + +const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})` + +/** Grow every tile's streaming tail by `chunk` each `intervalMs`, through the + * same publish path the gateway's delta flush uses. */ +const drive = (chunk, intervalMs, totalTokens) => ` + (() => { + const hook = window.__HERMES_SESSION_TILES__ + let pushed = 0 + const tick = () => { + const states = hook.states() + for (const { rid } of window.__RC__.ids) { + const prev = states[rid] + if (!prev) continue + const messages = prev.messages.map(m => { + if (m.id !== prev.streamId) return m + const head = m.parts.slice(0, -1) + const last = m.parts[m.parts.length - 1] + return { ...m, parts: [...head, { type: 'text', text: last.text + ${JSON.stringify(chunk)} }] } + }) + hook.publish(rid, { ...prev, messages }) + } + pushed += 1 + if (pushed < ${totalTokens}) window.__RC__.timer = setTimeout(tick, ${intervalMs}) + else window.__RC__.done = true + } + window.__RC__.timer = setTimeout(tick, ${intervalMs}) + return 'driving' + })() +` + +const START = ` + (() => { + window.__RENDER_COUNTS__.start() + window.__ATOM_CHURN__.start() + return 'recording' + })() +` + +const COLLECT = ` + (() => { + window.__RENDER_COUNTS__.stop() + window.__ATOM_CHURN__.stop() + return JSON.stringify({ + commits: window.__RENDER_COUNTS__.commits(), + renders: window.__RENDER_COUNTS__.report(200), + atoms: window.__ATOM_CHURN__.report(200) + }) + })() +` + +const CLEANUP = ` + (() => { + if (window.__RC__) { + clearTimeout(window.__RC__.timer) + for (const { sid, rid } of window.__RC__.ids) { + const states = window.__HERMES_SESSION_TILES__.states() + window.__HERMES_SESSION_TILES__.publish(rid, { ...states[rid], busy: false, streamId: null }) + window.__HERMES_SESSION_TILES__.close(sid) + } + window.__RC__ = null + } + window.__RENDER_COUNTS__.clear() + window.__ATOM_CHURN__.clear() + return 'cleaned' + })() +` + +export default { + name: 'render-churn', + tier: 'ci', + description: 'N streaming tabs: per-component render attribution + store churn.', + async run(cdp, opts = {}) { + const tiles = Number(opts.tiles ?? 5) + const seedTurns = Number(opts.turns ?? 20) + const tokens = Number(opts.tokens ?? 240) + // Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush. + const intervalMs = Number(opts.intervalMs ?? 33) + const chunk = opts.chunk ?? 'A streamed review sentence with **bold** and `code`.\n\n' + + await cdp.send('Runtime.enable') + + const ok = await cdp.eval(setup(tiles, seedTurns)) + + if (ok !== 'ok') { + throw new Error( + `render-churn setup failed (${ok}) — needs a dev renderer with src/debug installed ` + + '(the counters are aliased out of production builds unless VITE_PERF_PROBE=1).' + ) + } + + // Mount every tab (keep-alive mounts on first activation), then settle. + for (let n = 1; n <= tiles; n++) { + await cdp.eval(reveal(`churn-tile-${n}`)) + await sleep(350) + } + + await sleep(1000) + // Start recording AFTER mount so the numbers are steady-state streaming + // cost, not one-off mount cost. + await cdp.eval(START) + await cdp.eval(drive(chunk, intervalMs, tokens)) + await sleep(tokens * intervalMs + 1500) + + const data = JSON.parse(await cdp.eval(COLLECT)) + await cdp.eval(CLEANUP) + + const byName = new Map(data.renders.map(r => [r.name, r])) + const sidebarRows = SIDEBAR_COMPONENTS.map(n => byName.get(n)).filter(Boolean) + const sidebarRenders = sidebarRows.reduce((a, r) => a + r.renders, 0) + const sidebarWasted = sidebarRows.reduce((a, r) => a + r.wasted, 0) + const totalRenders = data.renders.reduce((a, r) => a + r.renders, 0) + const totalWasted = data.renders.reduce((a, r) => a + r.wasted, 0) + const atomWasted = data.atoms.reduce((a, r) => a + r.wasted, 0) + + return { + metrics: { + // The hypothesis, as a number: sidebar renders while background tabs + // stream. Should be 0. + sidebar_renders: sidebarRenders, + sidebar_wasted: sidebarWasted, + // Renders with no changed props and no changed hook state — pure + // parent-driven work, across the whole tree. + wasted_renders: totalWasted, + total_renders: totalRenders, + commits: data.commits, + // Store notifications that published a value equal to the last one. + wasted_notifies: atomWasted + }, + detail: { + tiles, + tokens, + sidebar: sidebarRows, + topRenders: data.renders.slice(0, 15), + topAtoms: data.atoms.slice(0, 15) + } + } + } +}