mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-27 17:58:07 +00:00
feat(desktop): render-churn perf scenario
Drives the same synthetic multi-tab streaming workload as `multitab` (publishSessionState per session per flush, no backend, no credits) but reports render attribution instead of frame pacing — sidebar_renders, wasted_renders, and wasted_notifies. Answers 'does the sidebar re-render while an agent is typing' directly.
This commit is contained in:
parent
44211f3608
commit
37ac8ae76a
3 changed files with 220 additions and 0 deletions
|
|
@ -51,6 +51,7 @@ directly via `window.__PERF_DRIVE__`, so no LLM credits are spent.
|
|||
| `stream --real` | backend | same, from a real LLM stream | measure-real-stream, profile-real-stream |
|
||||
| `keystroke` | ci | composer keystroke → paint latency | measure-latency, profile-typing, leak-typing |
|
||||
| `transcript` | ci | large-transcript mount + paint cost | (new) |
|
||||
| `render-churn` | ci | per-component render attribution + store churn while N tabs stream | (new) |
|
||||
| `cold-start` | cold | launch → CDP → driver → first paint (fresh spawn/run) | (new) |
|
||||
| `first-token` | backend | Enter → first assistant token painted (TTFT) | (new) |
|
||||
| `submit` | backend | Enter → cleared → user msg painted, scroll jump | measure-submit, measure-jump |
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import firstToken from './first-token.mjs'
|
|||
import keystroke from './keystroke.mjs'
|
||||
import multitab from './multitab.mjs'
|
||||
import profileSwitch from './profile-switch.mjs'
|
||||
import renderChurn from './render-churn.mjs'
|
||||
import sessionSwitch from './session-switch.mjs'
|
||||
import stream from './stream.mjs'
|
||||
import streamHistory from './stream-history.mjs'
|
||||
|
|
@ -18,6 +19,7 @@ export const SCENARIOS = {
|
|||
[keystroke.name]: keystroke,
|
||||
[transcript.name]: transcript,
|
||||
[multitab.name]: multitab,
|
||||
[renderChurn.name]: renderChurn,
|
||||
[coldStart.name]: coldStart,
|
||||
[firstToken.name]: firstToken,
|
||||
[submit.name]: submit,
|
||||
|
|
|
|||
217
apps/desktop/scripts/perf/scenarios/render-churn.mjs
Normal file
217
apps/desktop/scripts/perf/scenarios/render-churn.mjs
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
// Render churn during multi-tab streaming: WHAT re-rendered and WHY, and which
|
||||
// store published the update. Frame pacing (see `multitab`) tells you the cost;
|
||||
// this tells you the cause.
|
||||
//
|
||||
// Drives the same synthetic pipeline as `multitab` — publishSessionState per
|
||||
// session per flush via `__HERMES_SESSION_TILES__`, no backend, no credits —
|
||||
// then reads the dev-only counters installed by `src/debug/`:
|
||||
//
|
||||
// window.__RENDER_COUNTS__ — per-component renders, attributed to
|
||||
// props / hook state / parent-only ("wasted")
|
||||
// window.__ATOM_CHURN__ — per-store notifications, listener fan-out, and
|
||||
// notifications whose value was deep-equal to the
|
||||
// previous one ("wasted")
|
||||
//
|
||||
// The headline metric is `sidebar_renders`: how many times the sidebar tree
|
||||
// re-rendered while agents were typing in other tabs. It should be 0.
|
||||
//
|
||||
// node scripts/perf/run.mjs render-churn --spawn [--tiles 5] [--tokens 240]
|
||||
|
||||
import { sleep } from '../lib/cdp.mjs'
|
||||
|
||||
/** Components that make up the sidebar tree. A render of any of these while
|
||||
* a background tab streams is work the user cannot see. */
|
||||
const SIDEBAR_COMPONENTS = [
|
||||
'ChatSidebar',
|
||||
'SidebarSurface',
|
||||
'SessionRow',
|
||||
'SessionsSection',
|
||||
'CronJobsSection',
|
||||
'ProfileSwitcher',
|
||||
'VirtualSessionList',
|
||||
'WorkspaceGroup',
|
||||
'OverviewRow',
|
||||
'SessionStatusDot'
|
||||
]
|
||||
|
||||
/** Page-side setup: open `tiles` session tiles, seed each with a transcript.
|
||||
* Mirrors `multitab.mjs` so the two scenarios measure the same workload. */
|
||||
const setup = (tiles, seedTurns) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
if (!hook) return 'no-hook'
|
||||
if (!window.__RENDER_COUNTS__) return 'no-render-counter'
|
||||
if (!window.__ATOM_CHURN__) return 'no-atom-churn'
|
||||
|
||||
const turn = (sid, i) => ([
|
||||
{ id: sid + '-u' + i, role: 'user', timestamp: Date.now(),
|
||||
parts: [{ type: 'text', text: 'Review question ' + i + ': does the diff handle the error path?' }] },
|
||||
{ id: sid + '-a' + i, role: 'assistant', timestamp: Date.now(), pending: false,
|
||||
parts: [{ type: 'text', text: '## Finding ' + i + '\\n\\nThe handler swallows the rejection.\\n\\n- The catch block drops the error.\\n- Retries are unbounded.\\n' }] }
|
||||
])
|
||||
|
||||
const state = (sid) => {
|
||||
const messages = []
|
||||
for (let i = 0; i < ${seedTurns}; i++) messages.push(...turn(sid, i))
|
||||
messages.push({ id: sid + '-stream', role: 'assistant', timestamp: Date.now(), pending: true,
|
||||
parts: [{ type: 'text', text: '' }] })
|
||||
return {
|
||||
storedSessionId: sid, messages, branch: '', cwd: '', model: '', provider: '',
|
||||
reasoningEffort: '', serviceTier: '', fast: false, yolo: false, personality: '',
|
||||
busy: true, awaitingResponse: false, streamId: sid + '-stream', sawAssistantPayload: true,
|
||||
pendingBranchGroup: null, interrupted: false, interimBoundaryPending: false,
|
||||
needsInput: false, turnStartedAt: Date.now(), usage: null
|
||||
}
|
||||
}
|
||||
|
||||
window.__RC__ = { ids: [], timer: null }
|
||||
for (let n = 1; n <= ${tiles}; n++) {
|
||||
const sid = 'churn-tile-' + n
|
||||
const rid = 'churn-rt-' + n
|
||||
window.__RC__.ids.push({ sid, rid })
|
||||
hook.open(sid, 'center')
|
||||
hook.patch(sid, { runtimeId: rid })
|
||||
hook.publish(rid, state(sid))
|
||||
}
|
||||
return 'ok'
|
||||
})()
|
||||
`
|
||||
|
||||
const reveal = sid => `window.__HERMES_LAYOUT_TREE__.reveal(${JSON.stringify(`session-tile:${sid}`)})`
|
||||
|
||||
/** Grow every tile's streaming tail by `chunk` each `intervalMs`, through the
|
||||
* same publish path the gateway's delta flush uses. */
|
||||
const drive = (chunk, intervalMs, totalTokens) => `
|
||||
(() => {
|
||||
const hook = window.__HERMES_SESSION_TILES__
|
||||
let pushed = 0
|
||||
const tick = () => {
|
||||
const states = hook.states()
|
||||
for (const { rid } of window.__RC__.ids) {
|
||||
const prev = states[rid]
|
||||
if (!prev) continue
|
||||
const messages = prev.messages.map(m => {
|
||||
if (m.id !== prev.streamId) return m
|
||||
const head = m.parts.slice(0, -1)
|
||||
const last = m.parts[m.parts.length - 1]
|
||||
return { ...m, parts: [...head, { type: 'text', text: last.text + ${JSON.stringify(chunk)} }] }
|
||||
})
|
||||
hook.publish(rid, { ...prev, messages })
|
||||
}
|
||||
pushed += 1
|
||||
if (pushed < ${totalTokens}) window.__RC__.timer = setTimeout(tick, ${intervalMs})
|
||||
else window.__RC__.done = true
|
||||
}
|
||||
window.__RC__.timer = setTimeout(tick, ${intervalMs})
|
||||
return 'driving'
|
||||
})()
|
||||
`
|
||||
|
||||
const START = `
|
||||
(() => {
|
||||
window.__RENDER_COUNTS__.start()
|
||||
window.__ATOM_CHURN__.start()
|
||||
return 'recording'
|
||||
})()
|
||||
`
|
||||
|
||||
const COLLECT = `
|
||||
(() => {
|
||||
window.__RENDER_COUNTS__.stop()
|
||||
window.__ATOM_CHURN__.stop()
|
||||
return JSON.stringify({
|
||||
commits: window.__RENDER_COUNTS__.commits(),
|
||||
renders: window.__RENDER_COUNTS__.report(200),
|
||||
atoms: window.__ATOM_CHURN__.report(200)
|
||||
})
|
||||
})()
|
||||
`
|
||||
|
||||
const CLEANUP = `
|
||||
(() => {
|
||||
if (window.__RC__) {
|
||||
clearTimeout(window.__RC__.timer)
|
||||
for (const { sid, rid } of window.__RC__.ids) {
|
||||
const states = window.__HERMES_SESSION_TILES__.states()
|
||||
window.__HERMES_SESSION_TILES__.publish(rid, { ...states[rid], busy: false, streamId: null })
|
||||
window.__HERMES_SESSION_TILES__.close(sid)
|
||||
}
|
||||
window.__RC__ = null
|
||||
}
|
||||
window.__RENDER_COUNTS__.clear()
|
||||
window.__ATOM_CHURN__.clear()
|
||||
return 'cleaned'
|
||||
})()
|
||||
`
|
||||
|
||||
export default {
|
||||
name: 'render-churn',
|
||||
tier: 'ci',
|
||||
description: 'N streaming tabs: per-component render attribution + store churn.',
|
||||
async run(cdp, opts = {}) {
|
||||
const tiles = Number(opts.tiles ?? 5)
|
||||
const seedTurns = Number(opts.turns ?? 20)
|
||||
const tokens = Number(opts.tokens ?? 240)
|
||||
// Matches STREAM_DELTA_FLUSH_MS — one publish per session per real flush.
|
||||
const intervalMs = Number(opts.intervalMs ?? 33)
|
||||
const chunk = opts.chunk ?? 'A streamed review sentence with **bold** and `code`.\n\n'
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const ok = await cdp.eval(setup(tiles, seedTurns))
|
||||
|
||||
if (ok !== 'ok') {
|
||||
throw new Error(
|
||||
`render-churn setup failed (${ok}) — needs a dev renderer with src/debug installed ` +
|
||||
'(the counters are aliased out of production builds unless VITE_PERF_PROBE=1).'
|
||||
)
|
||||
}
|
||||
|
||||
// Mount every tab (keep-alive mounts on first activation), then settle.
|
||||
for (let n = 1; n <= tiles; n++) {
|
||||
await cdp.eval(reveal(`churn-tile-${n}`))
|
||||
await sleep(350)
|
||||
}
|
||||
|
||||
await sleep(1000)
|
||||
// Start recording AFTER mount so the numbers are steady-state streaming
|
||||
// cost, not one-off mount cost.
|
||||
await cdp.eval(START)
|
||||
await cdp.eval(drive(chunk, intervalMs, tokens))
|
||||
await sleep(tokens * intervalMs + 1500)
|
||||
|
||||
const data = JSON.parse(await cdp.eval(COLLECT))
|
||||
await cdp.eval(CLEANUP)
|
||||
|
||||
const byName = new Map(data.renders.map(r => [r.name, r]))
|
||||
const sidebarRows = SIDEBAR_COMPONENTS.map(n => byName.get(n)).filter(Boolean)
|
||||
const sidebarRenders = sidebarRows.reduce((a, r) => a + r.renders, 0)
|
||||
const sidebarWasted = sidebarRows.reduce((a, r) => a + r.wasted, 0)
|
||||
const totalRenders = data.renders.reduce((a, r) => a + r.renders, 0)
|
||||
const totalWasted = data.renders.reduce((a, r) => a + r.wasted, 0)
|
||||
const atomWasted = data.atoms.reduce((a, r) => a + r.wasted, 0)
|
||||
|
||||
return {
|
||||
metrics: {
|
||||
// The hypothesis, as a number: sidebar renders while background tabs
|
||||
// stream. Should be 0.
|
||||
sidebar_renders: sidebarRenders,
|
||||
sidebar_wasted: sidebarWasted,
|
||||
// Renders with no changed props and no changed hook state — pure
|
||||
// parent-driven work, across the whole tree.
|
||||
wasted_renders: totalWasted,
|
||||
total_renders: totalRenders,
|
||||
commits: data.commits,
|
||||
// Store notifications that published a value equal to the last one.
|
||||
wasted_notifies: atomWasted
|
||||
},
|
||||
detail: {
|
||||
tiles,
|
||||
tokens,
|
||||
sidebar: sidebarRows,
|
||||
topRenders: data.renders.slice(0, 15),
|
||||
topAtoms: data.atoms.slice(0, 15)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue