diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 556a05b6d897..1cde03bf64d1 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -32,6 +32,8 @@ "dist:win:msi": "npm run build && npm run builder -- --win msi", "dist:win:nsis": "npm run build && npm run builder -- --win nsis", "dist:linux": "npm run build && npm run builder -- --linux AppImage deb rpm", + "perf": "node scripts/perf/run.mjs", + "perf:serve": "node scripts/perf/serve.mjs", "test:desktop": "node scripts/test-desktop.mjs", "test:desktop:all": "node scripts/test-desktop.mjs all", "test:desktop:dmg": "node scripts/test-desktop.mjs dmg", diff --git a/apps/desktop/scripts/leak-typing.mjs b/apps/desktop/scripts/leak-typing.mjs deleted file mode 100644 index d43a84782787..000000000000 --- a/apps/desktop/scripts/leak-typing.mjs +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env node -// Leak-detection harness — measure detached DOM, listener count, and FiberNode -// growth as a function of keystrokes typed. -// -// Workflow: -// 1. Open session, focus composer -// 2. forceGC; capture baseline counts -// 3. Repeat N rounds: type M chars, forceGC, capture counts, clear composer -// 4. Print growth-per-round table -// -// Usage: -// node apps/desktop/scripts/leak-typing.mjs [--rounds=6] [--chars=200] [--cps=40] [--port=9222] - -import { writeFileSync } from 'node:fs' - -const args = Object.fromEntries( - process.argv.slice(2).flatMap(s => { - const m = s.match(/^--([^=]+)(?:=(.*))?$/) - return m ? [[m[1], m[2] ?? true]] : [] - }) -) -const PORT = Number(args.port ?? 9222) -const ROUNDS = Number(args.rounds ?? 6) -const CHARS = Number(args.chars ?? 200) -const CPS = Number(args.cps ?? 40) - -const log = (...m) => console.log('[leak]', ...m) - -async function pickRenderer() { - const list = await (await fetch(`http://127.0.0.1:${PORT}/json/list`)).json() - return list.find(t => t.type === 'page' && t.url.startsWith('http')) -} - -function connect(url) { - return new Promise((resolve, reject) => { - const ws = new WebSocket(url) - let id = 0 - const pending = new Map() - const events = new Map() - ws.addEventListener('open', () => - resolve({ - send(method, params = {}) { - const myId = ++id - ws.send(JSON.stringify({ id: myId, method, params })) - return new Promise((res, rej) => pending.set(myId, { res, rej })) - }, - on(method, h) { - if (!events.has(method)) events.set(method, []) - events.get(method).push(h) - }, - close: () => ws.close() - }) - ) - ws.addEventListener('error', reject) - ws.addEventListener('message', ev => { - const m = JSON.parse(typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8')) - if (m.id != null) { - const p = pending.get(m.id) - if (!p) return - pending.delete(m.id) - m.error ? p.rej(new Error(m.error.message)) : p.res(m.result) - } else if (m.method) { - ;(events.get(m.method) ?? []).forEach(h => h(m.params)) - } - }) - }) -} - -async function evalInPage(cdp, expr) { - const r = await cdp.send('Runtime.evaluate', { expression: expr, returnByValue: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.text) - return r.result.value -} - -async function forceGCAndSettle(cdp) { - for (let i = 0; i < 3; i++) { - await cdp.send('HeapProfiler.collectGarbage') - await new Promise(r => setTimeout(r, 60)) - } -} - -async function focusComposer(cdp) { - return await evalInPage( - cdp, - `(() => { - const el = document.querySelector('[data-slot="composer-rich-input"]') - if (!el) return false - el.focus() - const range = document.createRange() - range.selectNodeContents(el) - range.collapse(false) - const sel = window.getSelection() - sel.removeAllRanges() - sel.addRange(range) - return true - })()` - ) -} - -async function clearComposer(cdp) { - await evalInPage( - cdp, - `(() => { - const el = document.querySelector('[data-slot="composer-rich-input"]') - if (!el) return false - // Clear via the same path as the composer's clear flow: - // dispatch a single Backspace until empty would be N round-trips; quicker - // to directly assign empty text and fire input. - el.innerHTML = '' - el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContentBackward' })) - el.focus() - return el.innerText.length === 0 - })()` - ) -} - -async function snapshotCounts(cdp) { - // Counts via Runtime.evaluate using internal V8 counters where possible. - // For DOM stats we directly query the document. - // Performance metrics include JSHeapUsedSize, Nodes, JSEventListeners, etc. - const { metrics } = await cdp.send('Performance.getMetrics') - const byName = Object.fromEntries(metrics.map(m => [m.name, m.value])) - // Total nodes in document - const docNodes = await evalInPage( - cdp, - `document.getElementsByTagName('*').length + document.querySelectorAll('*').length / 2` - ) - return { - heapUsedMB: (byName.JSHeapUsedSize / 1024 / 1024) || 0, - heapTotalMB: (byName.JSHeapTotalSize / 1024 / 1024) || 0, - nodes: byName.Nodes || 0, - jsListeners: byName.JSEventListeners || 0, - docNodes, - layoutCount: byName.LayoutCount || 0, - recalcStyleCount: byName.RecalcStyleCount || 0, - fps: byName.FramesPerSecond || 0 - } -} - -async function typeChars(cdp, text, cps) { - const intervalMs = Math.max(1, Math.round(1000 / cps)) - const start = Date.now() - for (let i = 0; i < text.length; i++) { - await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: text[i], unmodifiedText: text[i] }) - const expected = start + (i + 1) * intervalMs - const wait = expected - Date.now() - if (wait > 0) await new Promise(r => setTimeout(r, wait)) - } -} - -const lorem = - 'the quick brown fox jumps over the lazy dog while the agent thinks really hard about why typing into this composer feels like wading through molasses on a hot afternoon ' -function genText(n) { - let s = '' - while (s.length < n) s += lorem - return s.slice(0, n) -} - -async function main() { - log(`port ${PORT} · ${ROUNDS} rounds × ${CHARS} chars @ ${CPS} cps`) - const tgt = await pickRenderer() - log(`target ${tgt.url}`) - const cdp = await connect(tgt.webSocketDebuggerUrl) - await cdp.send('Runtime.enable') - await cdp.send('Performance.enable') - await cdp.send('DOM.enable') - - const focused = await focusComposer(cdp) - if (!focused) { - console.error('composer not focusable') - process.exit(2) - } - - await forceGCAndSettle(cdp) - const baseline = await snapshotCounts(cdp) - log('baseline:', JSON.stringify(baseline)) - - const text = genText(CHARS) - const history = [{ round: 0, ...baseline, charsTyped: 0 }] - - for (let r = 1; r <= ROUNDS; r++) { - await typeChars(cdp, text, CPS) - await new Promise(res => setTimeout(res, 200)) - await clearComposer(cdp) - await forceGCAndSettle(cdp) - const snap = await snapshotCounts(cdp) - snap.charsTyped = r * CHARS - snap.round = r - history.push(snap) - log( - `round ${r}: heap=${snap.heapUsedMB.toFixed(1)}MB ` + - `nodes=${snap.nodes} listeners=${snap.jsListeners} ` + - `domNodes=${Math.round(snap.docNodes)} ` + - `layoutCount=${snap.layoutCount} ` + - `Δheap=+${(snap.heapUsedMB - baseline.heapUsedMB).toFixed(2)}MB ` + - `Δnodes=+${snap.nodes - baseline.nodes} ` + - `Δlisteners=+${snap.jsListeners - baseline.jsListeners}` - ) - } - - console.log('\n=== GROWTH PER ROUND (averaged over last 5 rounds) ===') - const tail = history.slice(-5) - const first = tail[0] - const last = tail[tail.length - 1] - const rounds = last.round - first.round - const cells = ['heapUsedMB', 'nodes', 'jsListeners', 'docNodes', 'layoutCount'] - for (const c of cells) { - const delta = last[c] - first[c] - const per = delta / Math.max(1, rounds) - const perChar = delta / Math.max(1, rounds * CHARS) - console.log(` ${c.padEnd(16)} Δtotal=${delta.toFixed(2).padStart(10)} /round=${per.toFixed(2).padStart(8)} /char=${perChar.toFixed(4).padStart(8)}`) - } - - writeFileSync('/tmp/hermes-leak-history.json', JSON.stringify(history, null, 2)) - log('wrote /tmp/hermes-leak-history.json') - cdp.close() -} - -main().catch(e => { - console.error('[leak] fatal:', e.stack ?? e.message) - process.exit(1) -}) diff --git a/apps/desktop/scripts/measure-jump.mjs b/apps/desktop/scripts/measure-jump.mjs deleted file mode 100644 index 1b5d88f722b9..000000000000 --- a/apps/desktop/scripts/measure-jump.mjs +++ /dev/null @@ -1,108 +0,0 @@ -// Measure scroll position before and after Enter on a long thread. -// The user's complaint: pressing Enter to submit makes the view "jump up". -// -// Steps: -// 1. Scroll to the bottom of the thread -// 2. Type a short message -// 3. Record scroll position -// 4. Hit Enter -// 5. Record scroll position every 10ms for 1.5s after Enter -// 6. Report deltas -// -// Usage: node apps/desktop/scripts/measure-jump.mjs - -const list = await (await fetch('http://127.0.0.1:9222/json/list')).json() -const tgt = list.find(t => t.type === 'page' && t.url.startsWith('http')) -const ws = new WebSocket(tgt.webSocketDebuggerUrl) -let id = 0 -const pending = new Map() -ws.addEventListener('message', ev => { - const m = JSON.parse(ev.data) - if (m.id != null && pending.has(m.id)) { - pending.get(m.id)(m) - pending.delete(m.id) - } -}) -await new Promise(r => ws.addEventListener('open', r)) -const send = (m, p = {}) => - new Promise(r => { - const i = ++id - pending.set(i, r) - ws.send(JSON.stringify({ id: i, method: m, params: p })) - }) -const evalP = async expr => { - const r = await send('Runtime.evaluate', { expression: expr, returnByValue: true }) - if (r.result?.exceptionDetails) throw new Error(r.result.exceptionDetails.text) - return r.result.result.value -} - -// Scroll to bottom -await evalP(`(() => { - const v = document.querySelector('[data-slot="aui_thread-viewport"]') - if (v) v.scrollTop = v.scrollHeight -})()`) -await new Promise(r => setTimeout(r, 300)) - -// Focus composer and type -await evalP(`(() => { - const el = document.querySelector('[data-slot="composer-rich-input"]') - el.focus() - const r = document.createRange(); r.selectNodeContents(el); r.collapse(false) - window.getSelection().removeAllRanges(); window.getSelection().addRange(r) -})()`) - -const text = 'short follow-up message' -for (const c of text) { - await send('Input.dispatchKeyEvent', { type: 'char', text: c, unmodifiedText: c }) - await new Promise(r => setTimeout(r, 10)) -} -await new Promise(r => setTimeout(r, 300)) - -// Set up sampling — sample scroll position every animation frame -await evalP(`(() => { - const v = document.querySelector('[data-slot="aui_thread-viewport"]') - window.__jumpSamples = [] - window.__jumpStart = performance.now() - const tick = () => { - if (!v) return - window.__jumpSamples.push({ - t: performance.now() - window.__jumpStart, - scrollTop: v.scrollTop, - scrollHeight: v.scrollHeight, - clientHeight: v.clientHeight, - distFromBottom: v.scrollHeight - v.scrollTop - v.clientHeight - }) - if (performance.now() - window.__jumpStart < 2000) { - requestAnimationFrame(tick) - } - } - requestAnimationFrame(tick) -})()`) - -// Fire Enter -await send('Input.dispatchKeyEvent', { - type: 'rawKeyDown', windowsVirtualKeyCode: 13, key: 'Enter', code: 'Enter', text: '\r', unmodifiedText: '\r' -}) -await send('Input.dispatchKeyEvent', { type: 'keyUp', windowsVirtualKeyCode: 13, key: 'Enter', code: 'Enter' }) - -await new Promise(r => setTimeout(r, 2200)) - -const samples = JSON.parse(await evalP(`JSON.stringify(window.__jumpSamples || [])`)) -console.log(`\n${samples.length} samples over 2s`) -console.log(`\n t(ms) scrollTop scrollHeight clientHeight distFromBottom`) -let prev = null -for (const s of samples) { - const marker = prev && Math.abs(s.scrollTop - prev.scrollTop) > 5 ? ' ← jump' : '' - console.log(` ${String(s.t.toFixed(0)).padStart(5)} ${String(s.scrollTop).padStart(9)} ${String(s.scrollHeight).padStart(12)} ${String(s.clientHeight).padStart(12)} ${String(s.distFromBottom).padStart(14)}${marker}`) - prev = s -} - -// Cancel any running agent -await evalP(`(() => { - for (const b of document.querySelectorAll('button')) { - if ((b.getAttribute('aria-label') || '').toLowerCase().includes('stop')) { b.click(); return 'stopped' } - } - return 'no-stop' -})()`).then(r => console.log('\ncancel:', r)) - -ws.close() diff --git a/apps/desktop/scripts/measure-latency.mjs b/apps/desktop/scripts/measure-latency.mjs deleted file mode 100644 index c3f3da1302ce..000000000000 --- a/apps/desktop/scripts/measure-latency.mjs +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env node -// Measure end-to-end keystroke→paint latency in the Electron renderer. -// -// For each synthetic keystroke we record: -// t0 = Input.dispatchKeyEvent send time -// t1 = first observed mutation of [data-slot="composer-rich-input"] childList/character data -// t2 = first requestAnimationFrame callback after t1 (proxy for next paint) -// -// We use Page.startScreencast briefly to also get frame-presentation timestamps; -// alternatively rely on rAF timing which is close enough for typing UX. -// -// Output: per-char latency histogram (min/p50/p95/p99/max) + samples > 16ms. -// -// Usage: -// node apps/desktop/scripts/measure-latency.mjs [--chars=100] [--cps=15] [--port=9222] - -import { writeFileSync } from 'node:fs' - -const args = Object.fromEntries( - process.argv.slice(2).flatMap(s => { - const m = s.match(/^--([^=]+)(?:=(.*))?$/) - return m ? [[m[1], m[2] ?? true]] : [] - }) -) -const PORT = Number(args.port ?? 9222) -const CHARS = Number(args.chars ?? 100) -const CPS = Number(args.cps ?? 15) - -const log = (...m) => console.log('[latency]', ...m) - -async function pickRenderer() { - const list = await (await fetch(`http://127.0.0.1:${PORT}/json/list`)).json() - return list.find(t => t.type === 'page' && t.url.startsWith('http')) -} - -function connect(url) { - return new Promise((resolve, reject) => { - const ws = new WebSocket(url) - let id = 0 - const pending = new Map() - const events = new Map() - ws.addEventListener('open', () => - resolve({ - send(method, params = {}) { - const myId = ++id - ws.send(JSON.stringify({ id: myId, method, params })) - return new Promise((res, rej) => pending.set(myId, { res, rej })) - }, - on(method, h) { - if (!events.has(method)) events.set(method, []) - events.get(method).push(h) - }, - close: () => ws.close() - }) - ) - ws.addEventListener('error', reject) - ws.addEventListener('message', ev => { - const m = JSON.parse(typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8')) - if (m.id != null) { - const p = pending.get(m.id) - if (!p) return - pending.delete(m.id) - m.error ? p.rej(new Error(m.error.message)) : p.res(m.result) - } else if (m.method) { - ;(events.get(m.method) ?? []).forEach(h => h(m.params)) - } - }) - }) -} - -async function evalInPage(cdp, expr) { - const r = await cdp.send('Runtime.evaluate', { expression: expr, returnByValue: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.text) - return r.result.value -} - -async function main() { - const tgt = await pickRenderer() - log(`target ${tgt.url}`) - const cdp = await connect(tgt.webSocketDebuggerUrl) - await cdp.send('Runtime.enable') - - await evalInPage( - cdp, - `(() => { - const el = document.querySelector('[data-slot="composer-rich-input"]') - if (!el) return false - el.focus() - const range = document.createRange() - range.selectNodeContents(el) - range.collapse(false) - const sel = window.getSelection() - sel.removeAllRanges() - sel.addRange(range) - window.__keypressTimings = [] - window.__pendingKey = null - // Observe the composer for content/text changes; record the time relative - // to the most recent simulated keypress timestamp set on window.__pendingKey. - const obs = new MutationObserver(() => { - const start = window.__pendingKey - if (start === null) return - const mutationT = performance.now() - window.__pendingKey = null - requestAnimationFrame(() => { - const paintT = performance.now() - window.__keypressTimings.push({ - start, mutationT, paintT, - mutationLatency: mutationT - start, - paintLatency: paintT - start - }) - }) - }) - obs.observe(el, { childList: true, subtree: true, characterData: true }) - window.__keystrokeObserver = obs - return true - })()` - ) - - const lorem = - 'the quick brown fox jumps over the lazy dog while typing into this composer feels like wading through molasses on a hot afternoon. ' - let text = '' - while (text.length < CHARS) text += lorem - text = text.slice(0, CHARS) - - const intervalMs = Math.max(1, Math.round(1000 / CPS)) - const start = Date.now() - for (let i = 0; i < text.length; i++) { - // Mark the keypress time inside the page so it's measured from the same clock. - await evalInPage(cdp, `window.__pendingKey = performance.now()`) - await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: text[i], unmodifiedText: text[i] }) - const expected = start + (i + 1) * intervalMs - const wait = expected - Date.now() - if (wait > 0) await new Promise(r => setTimeout(r, wait)) - } - - await new Promise(r => setTimeout(r, 500)) - const samples = await evalInPage(cdp, `window.__keypressTimings`) - log(`${samples.length} keystroke samples measured out of ${text.length} typed`) - - // Clear composer for next run - await evalInPage(cdp, ` - (() => { - const el = document.querySelector('[data-slot="composer-rich-input"]') - if (el) { el.innerHTML = ''; el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContentBackward' })) } - window.__keystrokeObserver?.disconnect() - })() - `) - - const mutLat = samples.map(s => s.mutationLatency).sort((a, b) => a - b) - const paintLat = samples.map(s => s.paintLatency).sort((a, b) => a - b) - const stat = arr => ({ - n: arr.length, - min: arr[0]?.toFixed(2), - p50: arr[Math.floor(arr.length * 0.5)]?.toFixed(2), - p90: arr[Math.floor(arr.length * 0.9)]?.toFixed(2), - p95: arr[Math.floor(arr.length * 0.95)]?.toFixed(2), - p99: arr[Math.floor(arr.length * 0.99)]?.toFixed(2), - max: arr[arr.length - 1]?.toFixed(2), - mean: arr.length ? (arr.reduce((s, x) => s + x, 0) / arr.length).toFixed(2) : 0 - }) - - console.log('\n=== keypress → mutation latency (ms) ===') - console.log(' ', stat(mutLat)) - console.log('\n=== keypress → next rAF (≈paint) latency (ms) ===') - console.log(' ', stat(paintLat)) - - const slow = samples.filter(s => s.paintLatency > 16) - console.log(`\n=== ${slow.length}/${samples.length} keystrokes >16ms (one frame) ===`) - if (slow.length) { - const slowSorted = [...slow].sort((a, b) => b.paintLatency - a.paintLatency).slice(0, 10) - for (const s of slowSorted) { - console.log(` paint=${s.paintLatency.toFixed(1)}ms mut=${s.mutationLatency.toFixed(1)}ms at t=${s.start.toFixed(0)}`) - } - } - - writeFileSync('/tmp/hermes-latency-samples.json', JSON.stringify(samples, null, 2)) - - cdp.close() -} - -main().catch(e => { - console.error('[latency] fatal:', e.stack ?? e.message) - process.exit(1) -}) diff --git a/apps/desktop/scripts/measure-profile-switch.mjs b/apps/desktop/scripts/measure-profile-switch.mjs deleted file mode 100644 index 2f221f00de4c..000000000000 --- a/apps/desktop/scripts/measure-profile-switch.mjs +++ /dev/null @@ -1,159 +0,0 @@ -// Measure a profile switch end-to-end: click a profile square in the rail, -// then break the wall time into the phases the renderer can observe: -// - getConnection IPC (Electron: pool backend spawn / reuse + readiness) -// - gateway WS connect -// - swap-target clear ($gatewaySwapTarget → sidebar loader gone) -// - sidebar session rows for the new profile painted -// -// Instruments window.hermesDesktop.getConnection + WebSocket to timestamp the -// phases without touching app code. -// -// Usage: -// node apps/desktop/scripts/measure-profile-switch.mjs [settleTimeoutMs] - -const CDP_HTTP = 'http://127.0.0.1:9222' -const PROFILE = process.argv[2] -const SETTLE_TIMEOUT = Number(process.argv[3] || 60000) - -if (!PROFILE) { - console.error('usage: measure-profile-switch.mjs ') - process.exit(1) -} - -class CDP { - constructor(ws) { this.ws = ws; this.id = 0; this.pending = new Map() } - static async open(url) { - const ws = new WebSocket(url) - await new Promise((r) => ws.addEventListener('open', r, { once: true })) - const cdp = new CDP(ws) - ws.addEventListener('message', (ev) => { - const m = JSON.parse(ev.data.toString()) - if (m.id != null && cdp.pending.has(m.id)) { - const { resolve, reject } = cdp.pending.get(m.id) - cdp.pending.delete(m.id) - if (m.error) reject(new Error(m.error.message)) - else resolve(m.result) - } - }) - ws.addEventListener('close', () => { - for (const { reject } of cdp.pending.values()) reject(new Error('CDP socket closed')) - cdp.pending.clear() - }) - return cdp - } - send(method, params) { - const id = ++this.id - return new Promise((res, rej) => { - this.pending.set(id, { resolve: res, reject: rej }) - this.ws.send(JSON.stringify({ id, method, params })) - }) - } - async eval(expr) { - const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'eval failed') - return r.result.value - } - close() { this.ws.close() } -} - -async function main() { - const list = await (await fetch(`${CDP_HTTP}/json`)).json() - const target = list.find((t) => t.type === 'page' && /5174/.test(t.url)) - if (!target) { console.error('renderer not found on 9222'); process.exit(1) } - const cdp = await CDP.open(target.webSocketDebuggerUrl) - - // Instrument getConnection + WebSocket once. - await cdp.eval(`(() => { - if (window.__PROFILE_SWITCH_OBS__) return 'already' - const obs = { events: [] } - const mark = (name, extra) => obs.events.push({ name, t: performance.now(), ...(extra || {}) }) - window.__PROFILE_SWITCH_OBS__ = obs - window.__psMark = mark - - const desktop = window.hermesDesktop - if (desktop && desktop.getConnection) { - const orig = desktop.getConnection.bind(desktop) - desktop.getConnection = async (profile) => { - mark('getConnection:start', { profile }) - try { - const res = await orig(profile) - mark('getConnection:done', { profile }) - return res - } catch (e) { - mark('getConnection:error', { profile, error: String(e).slice(0, 120) }) - throw e - } - } - } - - const OrigWS = window.WebSocket - window.WebSocket = function (url, ...rest) { - const ws = new OrigWS(url, ...rest) - if (String(url).includes('/api/ws')) { - mark('ws:new', { url: String(url).replace(/token=[^&]+/, 'token=…').slice(0, 90) }) - ws.addEventListener('open', () => mark('ws:open')) - } - return ws - } - window.WebSocket.prototype = OrigWS.prototype - Object.assign(window.WebSocket, OrigWS) - return 'installed' - })()`) - - const before = await cdp.eval(`(() => { - const rail = document.querySelector('[data-slot="profile-rail"]') - return { - railButtons: rail ? [...rail.querySelectorAll('[role="tab"], button')].map(b => (b.getAttribute('aria-label') || b.title || b.textContent || '').slice(0, 30)) : [], - sessions: document.querySelectorAll('[data-slot="sidebar-session-row"], [data-session-id]').length - } - })()`) - console.log('rail buttons:', JSON.stringify(before.railButtons)) - - const clicked = await cdp.eval(`(() => { - window.__psMark('click', { profile: ${JSON.stringify(PROFILE)} }) - const rail = document.querySelector('[data-slot="profile-rail"]') - if (!rail) return 'no-rail' - const target = [...rail.querySelectorAll('button, [role="tab"]')].find(b => - ((b.getAttribute('aria-label') || '') + ' ' + (b.title || '') + ' ' + (b.textContent || '')).toLowerCase().includes(${JSON.stringify(PROFILE.toLowerCase())})) - if (!target) return 'not-found' - target.click() - return 'clicked' - })()`) - console.log('click:', clicked) - if (clicked !== 'clicked') { cdp.close(); process.exit(2) } - - // Poll until the swap settles: loader gone + session rows painted (or empty - // list settled) + active profile pill shows the target. - const t0 = Date.now() - let settled = null - while (Date.now() - t0 < SETTLE_TIMEOUT) { - await new Promise((r) => setTimeout(r, 100)) - const s = await cdp.eval(`(() => { - // The swap overlay stays mounted at opacity-0 after the swap — check the - // computed opacity of the container that holds the "Waking up …" label. - const label = [...document.querySelectorAll('div[aria-hidden]')].find(el => /waking up/i.test(el.textContent || '')) - const overlayVisible = label ? Number(getComputedStyle(label).opacity) > 0.05 : false - return { - t: performance.now(), - overlayVisible, - sessions: document.querySelectorAll('[data-slot="row-button"]').length - } - })()`) - if (!s.overlayVisible && s.sessions > 0) { settled = s; break } - } - - await new Promise((r) => setTimeout(r, 400)) - const obs = await cdp.eval('window.__PROFILE_SWITCH_OBS__') - const events = obs.events - const click = events.find((e) => e.name === 'click' && e.profile === PROFILE) - console.log('\n=== PHASES (ms after click) ===') - for (const e of events) { - if (e.t < click.t - 5) continue - console.log(`${(e.t - click.t).toFixed(0).padStart(7)} ${e.name}${e.profile ? ' [' + e.profile + ']' : ''}${e.error ? ' ' + e.error : ''}${e.url ? ' ' + e.url : ''}`) - } - console.log(settled ? `\nsettled (loader gone + rows painted) at ~${Date.now() - t0} ms wall` : '\nTIMEOUT waiting for settle') - - cdp.close() -} - -main().catch((e) => { console.error(e); process.exit(1) }) diff --git a/apps/desktop/scripts/measure-real-stream.mjs b/apps/desktop/scripts/measure-real-stream.mjs deleted file mode 100644 index 57eee502d12e..000000000000 --- a/apps/desktop/scripts/measure-real-stream.mjs +++ /dev/null @@ -1,252 +0,0 @@ -// REAL streaming measurement — no React internals. -// -// Measures: -// 1) rAF frame intervals during a verified live stream (long-frame histogram) -// 2) MutationObserver: how often does the live assistant message mutate, what's the budget per mutation -// 3) Text length growth rate (chars/sec) -// 4) PerformanceObserver `longtask` entries (any task > 50ms blocks input) -// -// Detects REAL stream by waiting for assistant-message DOM count to grow past baseline. -// Does NOT cancel — lets the stream run to completion or hits TIMEOUT_MS. - -const CDP_HTTP = 'http://127.0.0.1:9222' -const PROMPT = process.env.PROMPT || 'count from 1 to 80, one number per line' -const TIMEOUT_MS = Number(process.env.TIMEOUT_MS || 60000) - -async function getTarget() { - const list = await (await fetch(`${CDP_HTTP}/json`)).json() - const t = list.find((t) => t.type === 'page' && /5174/.test(t.url)) - if (!t) throw new Error('renderer not found') - return t -} - -class CDP { - constructor(ws) { this.ws = ws; this.id = 0; this.pending = new Map() } - static async open(url) { - const ws = new WebSocket(url) - await new Promise((r, j) => { - ws.addEventListener('open', r, { once: true }) - ws.addEventListener('error', (e) => j(e), { once: true }) - }) - const cdp = new CDP(ws) - ws.addEventListener('message', (event) => { - const m = JSON.parse(event.data.toString()) - if (m.id != null && cdp.pending.has(m.id)) { - const { resolve, reject } = cdp.pending.get(m.id) - cdp.pending.delete(m.id) - if (m.error) reject(new Error(m.error.message)) - else resolve(m.result) - } - }) - return cdp - } - send(method, params) { - const id = ++this.id - return new Promise((res, rej) => { - this.pending.set(id, { resolve: res, reject: rej }) - this.ws.send(JSON.stringify({ id, method, params })) - }) - } - async eval(expr) { - const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'eval') - return r.result.value - } - close() { this.ws.close() } -} - -async function main() { - const target = await getTarget() - const cdp = await CDP.open(target.webSocketDebuggerUrl) - - // Install recorders. - await cdp.eval(` - (() => { - // rAF frame intervals - window.__FT__ = { times: [], stop: false } - let last = performance.now() - const tick = () => { - if (window.__FT__.stop) return - const now = performance.now() - window.__FT__.times.push(now - last) - last = now - requestAnimationFrame(tick) - } - requestAnimationFrame(tick) - - // longtask observer - window.__LT__ = { entries: [], stop: false } - try { - const po = new PerformanceObserver((list) => { - if (window.__LT__.stop) return - for (const e of list.getEntries()) { - window.__LT__.entries.push({ name: e.name, duration: e.duration, startTime: e.startTime }) - } - }) - po.observe({ entryTypes: ['longtask'] }) - window.__LT__.po = po - } catch {} - - // mutation observer on streaming message - window.__MO__ = { mutations: [], stop: false, currentMsg: null } - const tryArm = () => { - const all = document.querySelectorAll('[data-slot="aui_assistant-message-root"]') - const last = all[all.length - 1] - if (!last || last === window.__MO__.currentMsg) return - window.__MO__.currentMsg = last - if (window.__MO__.obs) window.__MO__.obs.disconnect() - const obs = new MutationObserver((muts) => { - if (window.__MO__.stop) return - const t = performance.now() - window.__MO__.mutations.push({ t, count: muts.length, len: last.textContent.length }) - }) - obs.observe(last, { childList: true, subtree: true, characterData: true }) - window.__MO__.obs = obs - } - window.__MO__.arm = tryArm - return 'recorders armed' - })() - `) - - // Baseline - const base = JSON.parse(await cdp.eval(` - JSON.stringify({ - assistantCount: document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length, - busy: !!document.querySelector('[data-status="running"], [data-busy="true"]'), - hasComposer: !!document.querySelector('[contenteditable="true"]'), - }) - `)) - console.log('baseline:', base) - if (!base.hasComposer) { console.error('no composer'); cdp.close(); return } - - // Type + submit - await cdp.eval(` - (() => { - const ed = document.querySelector('[contenteditable="true"]') - ed.focus() - document.execCommand('insertText', false, ${JSON.stringify(PROMPT)}) - return 'typed' - })() - `) - const submitT0 = Date.now() - await cdp.eval(` - (() => { - const ed = document.querySelector('[contenteditable="true"]') - ed.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true })) - return 'submitted' - })() - `) - - // Poll for REAL stream (assistant count > baseline). 30 seconds — accommodates - // slow first-token latencies on big providers. - let realStreamT = null - for (let i = 0; i < 600; i++) { - await new Promise((r) => setTimeout(r, 50)) - const s = JSON.parse(await cdp.eval(` - JSON.stringify({ - n: document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length, - busy: !!document.querySelector('[data-status="running"], [data-busy="true"]'), - text: (() => { const a = document.querySelectorAll('[data-slot="aui_assistant-message-root"]'); return a.length ? a[a.length-1].textContent.length : 0 })() - }) - `)) - if (s.n > base.assistantCount) { - realStreamT = Date.now() - console.log('REAL stream started after', realStreamT - submitT0, 'ms — busy=', s.busy, 'text=', s.text) - // Arm mutation observer on the new message - await cdp.eval('window.__MO__.arm()') - break - } - } - if (!realStreamT) { - console.error('REAL STREAM NEVER STARTED') - cdp.close() - return - } - - // Sample length growth, wait for completion or timeout - const samples = [] - const start = Date.now() - while (Date.now() - start < TIMEOUT_MS) { - await new Promise((r) => setTimeout(r, 250)) - const s = JSON.parse(await cdp.eval(` - JSON.stringify({ - t: performance.now(), - len: (() => { const a = document.querySelectorAll('[data-slot="aui_assistant-message-root"]'); return a.length ? a[a.length-1].textContent.length : 0 })(), - busy: !!document.querySelector('[data-status="running"], [data-busy="true"]') - }) - `)) - samples.push(s) - if (!s.busy && samples.length > 4) { - await new Promise((r) => setTimeout(r, 300)) - break - } - } - - // Pull recordings - const data = JSON.parse(await cdp.eval(` - (() => { - window.__FT__.stop = true - window.__LT__.stop = true - window.__MO__.stop = true - try { window.__LT__.po && window.__LT__.po.disconnect() } catch {} - try { window.__MO__.obs && window.__MO__.obs.disconnect() } catch {} - return JSON.stringify({ - frames: window.__FT__.times, - longtasks: window.__LT__.entries, - mutations: window.__MO__.mutations, - }) - })() - `)) - - const { frames, longtasks, mutations } = data - - // Frame histogram (filter to stream window) - const buckets = { '<=16.7': 0, '16.7-33': 0, '33-50': 0, '50-100': 0, '100-200': 0, '>200': 0 } - let frameTotal = 0 - let maxFrame = 0 - for (const f of frames) { - frameTotal += f - if (f > maxFrame) maxFrame = f - if (f <= 16.7) buckets['<=16.7']++ - else if (f <= 33) buckets['16.7-33']++ - else if (f <= 50) buckets['33-50']++ - else if (f <= 100) buckets['50-100']++ - else if (f <= 200) buckets['100-200']++ - else buckets['>200']++ - } - const avgFps = frames.length ? (frames.length / (frameTotal / 1000)).toFixed(1) : 'n/a' - const slowFrames = frames.filter((f) => f > 33).length - const veryslowFrames = frames.filter((f) => f > 100).length - - // Longtask summary - const ltMs = longtasks.reduce((a, b) => a + b.duration, 0) - const ltMax = longtasks.length ? Math.max(...longtasks.map((e) => e.duration)) : 0 - - // Mutation rate - let mutTotal = mutations.length - let mutDurs = [] - for (let i = 1; i < mutations.length; i++) { - mutDurs.push(mutations[i].t - mutations[i - 1].t) - } - mutDurs.sort((a, b) => a - b) - const mutP50 = mutDurs[Math.floor(mutDurs.length * 0.5)] ?? 0 - const mutP95 = mutDurs[Math.floor(mutDurs.length * 0.95)] ?? 0 - - // Growth rate - const firstLen = samples[0]?.len ?? 0 - const lastLen = samples[samples.length - 1]?.len ?? 0 - const elapsedS = samples.length ? (samples[samples.length - 1].t - samples[0].t) / 1000 : 0 - const charsPerSec = elapsedS ? ((lastLen - firstLen) / elapsedS).toFixed(1) : 'n/a' - - console.log('\n=== STREAM RESULTS ===') - console.log('window:', (frameTotal / 1000).toFixed(1), 's | frames:', frames.length, '| avgFps:', avgFps, '| maxFrame:', maxFrame.toFixed(1), 'ms') - console.log('frame histogram:', buckets) - console.log('slow frames (>33ms):', slowFrames, '| very slow (>100ms):', veryslowFrames) - console.log('longtasks:', longtasks.length, 'total', ltMs.toFixed(0), 'ms — max', ltMax.toFixed(1), 'ms') - console.log('text grew', firstLen, '→', lastLen, 'chars (', charsPerSec, 'char/s )') - console.log('mutations on streaming msg:', mutTotal, '| inter-mutation p50:', mutP50.toFixed(1), 'ms', 'p95:', mutP95.toFixed(1), 'ms') - - cdp.close() -} - -main().catch((e) => { console.error(e); process.exit(1) }) diff --git a/apps/desktop/scripts/measure-submit.mjs b/apps/desktop/scripts/measure-submit.mjs deleted file mode 100644 index 6c89c44e34d6..000000000000 --- a/apps/desktop/scripts/measure-submit.mjs +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env node -// Measure submit (Enter) latency in the composer. -// -// For each round: -// 1. Focus composer, type N chars of stub text -// 2. Mark a timestamp, fire Enter via Input.dispatchKeyEvent -// 3. Observe: time until the composer becomes empty (submit accepted), -// time until the user message renders in the thread viewport, -// time until the optional "running…" indicator appears, -// time until the next frame is painted after the message renders. -// -// Pre-condition: a session is loaded (load via click-session.mjs first). -// Note: this DOES talk to the real gateway/agent, so each round triggers -// a real prompt submission. Don't run this on a live conversation -// you care about — use a throwaway session. - -import { writeFileSync } from 'node:fs' - -const args = Object.fromEntries( - process.argv.slice(2).flatMap(s => { - const m = s.match(/^--([^=]+)(?:=(.*))?$/) - return m ? [[m[1], m[2] ?? true]] : [] - }) -) -const PORT = Number(args.port ?? 9222) -const ROUNDS = Number(args.rounds ?? 3) - -async function pickRenderer() { - const list = await (await fetch(`http://127.0.0.1:${PORT}/json/list`)).json() - return list.find(t => t.type === 'page' && t.url.startsWith('http')) -} - -function connect(url) { - return new Promise((resolve, reject) => { - const ws = new WebSocket(url) - let id = 0 - const pending = new Map() - ws.addEventListener('open', () => - resolve({ - send(method, params = {}) { - const myId = ++id - ws.send(JSON.stringify({ id: myId, method, params })) - return new Promise((res, rej) => pending.set(myId, { res, rej })) - }, - close: () => ws.close() - }) - ) - ws.addEventListener('error', reject) - ws.addEventListener('message', ev => { - const m = JSON.parse(typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8')) - if (m.id != null) { - const p = pending.get(m.id) - if (!p) return - pending.delete(m.id) - m.error ? p.rej(new Error(m.error.message)) : p.res(m.result) - } - }) - }) -} - -async function evalP(cdp, expr) { - const r = await cdp.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.text) - return r.result.value -} - -async function focusAndType(cdp, text) { - await evalP(cdp, ` - (() => { - const el = document.querySelector('[data-slot="composer-rich-input"]') - if (!el) return - el.focus() - const range = document.createRange() - range.selectNodeContents(el) - range.collapse(false) - const sel = window.getSelection() - sel.removeAllRanges() - sel.addRange(range) - })() - `) - for (const c of text) { - await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: c, unmodifiedText: c }) - await new Promise(r => setTimeout(r, 8)) - } -} - -async function submitAndMeasure(cdp, timeoutMs = 5000) { - // Install observers, record submit time as performance.now() inside the page, - // and wait for all milestones. - return await evalP(cdp, ` - new Promise((resolve) => { - const composer = document.querySelector('[data-slot="composer-rich-input"]') - const threadRoot = document.querySelector('[data-slot="aui_thread-content"]') || - document.querySelector('[data-slot="aui_thread-viewport"]') - const startMessageCount = threadRoot ? threadRoot.querySelectorAll('[data-slot="aui_turn-pair"], [data-slot="aui_message"]').length : 0 - const startComposerText = composer ? composer.innerText : '' - - const milestones = { start: performance.now() } - let done = false - const finish = (reason) => { - if (done) return - done = true - clearInterval(poll); clearTimeout(timer) - composerObs.disconnect() - threadObs?.disconnect() - milestones.reason = reason - milestones.end = performance.now() - milestones.totalMs = milestones.end - milestones.start - resolve(milestones) - } - - const composerObs = new MutationObserver(() => { - if (!milestones.composerClearedMs && composer && composer.innerText.length === 0) { - milestones.composerClearedMs = performance.now() - milestones.start - } - }) - composer && composerObs.observe(composer, { childList: true, subtree: true, characterData: true }) - - let threadObs = null - if (threadRoot) { - threadObs = new MutationObserver(() => { - const c = threadRoot.querySelectorAll('[data-slot="aui_turn-pair"], [data-slot="aui_message"]').length - if (!milestones.userMessageRenderedMs && c > startMessageCount) { - milestones.userMessageRenderedMs = performance.now() - milestones.start - requestAnimationFrame(() => { - milestones.userMessagePaintMs = performance.now() - milestones.start - finish('paint') - }) - } - }) - threadObs.observe(threadRoot, { childList: true, subtree: true }) - } - - const poll = setInterval(() => { - if (milestones.composerClearedMs && !milestones.userMessageRenderedMs && - performance.now() - milestones.start > 2000) { - finish('timeout-after-clear') - } - }, 100) - const timer = setTimeout(() => finish('timeout-overall'), ${timeoutMs}) - - // Send Enter immediately - window.dispatchEvent(new KeyboardEvent('keydown')) // no-op marker - const enterEv = new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true }) - composer?.dispatchEvent(enterEv) - }) - `) -} - -async function main() { - const tgt = await pickRenderer() - console.log('target', tgt.url) - const cdp = await connect(tgt.webSocketDebuggerUrl) - await cdp.send('Runtime.enable') - - const samples = [] - for (let i = 1; i <= ROUNDS; i++) { - await focusAndType(cdp, `latency test ${i} ${'x'.repeat(40)}`) - await new Promise(r => setTimeout(r, 300)) - const result = await submitAndMeasure(cdp, 4000) - samples.push({ round: i, ...result }) - console.log( - `r${i}: clear=${(result.composerClearedMs ?? -1).toFixed?.(0) ?? '?'}ms ` + - `userMsg=${(result.userMessageRenderedMs ?? -1).toFixed?.(0) ?? '?'}ms ` + - `paint=${(result.userMessagePaintMs ?? -1).toFixed?.(0) ?? '?'}ms ` + - `reason=${result.reason}` - ) - // wait for any agent activity to finish before next round so we're not piling up - await new Promise(r => setTimeout(r, 4000)) - } - writeFileSync('/tmp/hermes-submit-latency.json', JSON.stringify(samples, null, 2)) - console.log('\nwrote /tmp/hermes-submit-latency.json') - cdp.close() -} - -main().catch(e => { - console.error('fatal:', e.stack ?? e.message) - process.exit(1) -}) diff --git a/apps/desktop/scripts/measure-synthetic-stream.mjs b/apps/desktop/scripts/measure-synthetic-stream.mjs deleted file mode 100644 index 3b8afb297497..000000000000 --- a/apps/desktop/scripts/measure-synthetic-stream.mjs +++ /dev/null @@ -1,322 +0,0 @@ -// Measure render cost of a synthetic stream driven through the live $messages atom. -// -// Why synthetic: the user's LLM credits are depleted; we can't fire a real stream. -// The synthetic stream exercises the exact same React pipeline (assistant-ui runtime → -// repository.addOrUpdateMessage → MessagePrimitive re-render → markdown reflow) as a -// real stream. The only thing it does NOT exercise is the gateway → SSE → optimistic- -// merge path, which is orthogonal to the rendering question. -// -// What we record: -// 1) rAF frame intervals (long-frame histogram; >33ms = perceived jank, >100ms = bad) -// 2) PerformanceObserver `longtask` entries (task >50ms blocks input) -// 3) MutationObserver: per-message mutation count & inter-mutation latency -// 4) Optional: typing latency overlay — typing into composer while streaming -// -// Output is plain text suitable for terminal + a JSON sidecar for diffing across runs. - -import { writeFileSync } from 'node:fs' - -const CDP_HTTP = 'http://127.0.0.1:9222' -const TOKENS = Number(process.env.TOKENS || 300) -const INTERVAL_MS = Number(process.env.INTERVAL_MS || 16) -// Upstream flush throttle to apply in the synthetic driver. Mirrors what the -// real gateway path does in `use-message-stream.scheduleDeltaFlush`. 0 -// disables (worst-case, every token = one React commit). -const FLUSH_MIN_MS = Number(process.env.FLUSH_MIN_MS || 0) -const CHUNK = process.env.CHUNK || 'lorem ipsum ' -const TYPE_WHILE_STREAMING = process.env.TYPE_WHILE_STREAMING === '1' -const LABEL = process.env.LABEL || 'baseline' -const OUT = process.env.OUT || `frame-times-${LABEL}.json` - -async function getTarget() { - const list = await (await fetch(`${CDP_HTTP}/json`)).json() - const t = list.find((t) => t.type === 'page' && /5174/.test(t.url)) - if (!t) throw new Error('renderer not found') - return t -} - -class CDP { - constructor(ws) { this.ws = ws; this.id = 0; this.pending = new Map() } - static async open(url) { - const ws = new WebSocket(url) - await new Promise((r, j) => { - ws.addEventListener('open', r, { once: true }) - ws.addEventListener('error', (e) => j(e), { once: true }) - }) - const cdp = new CDP(ws) - ws.addEventListener('message', (ev) => { - const m = JSON.parse(ev.data.toString()) - if (m.id != null && cdp.pending.has(m.id)) { - const { resolve, reject } = cdp.pending.get(m.id) - cdp.pending.delete(m.id) - if (m.error) reject(new Error(m.error.message)) - else resolve(m.result) - } - }) - return cdp - } - send(method, params) { - const id = ++this.id - return new Promise((res, rej) => { - this.pending.set(id, { resolve: res, reject: rej }) - this.ws.send(JSON.stringify({ id, method, params })) - }) - } - async eval(expr) { - const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'eval') - return r.result.value - } - close() { this.ws.close() } -} - -function pct(arr, p) { - if (!arr.length) return 0 - const i = Math.min(arr.length - 1, Math.floor(arr.length * p)) - return arr[i] -} - -async function main() { - const target = await getTarget() - const cdp = await CDP.open(target.webSocketDebuggerUrl) - - // Sanity check driver is loaded. - const probeOk = await cdp.eval('!!window.__PERF_DRIVE__ && !!window.__PERF_DRIVE__.stream') - if (!probeOk) { - console.error('__PERF_DRIVE__ not on window — did you reload the renderer after editing perf-probe.tsx?') - cdp.close() - process.exit(2) - } - - // Install recorders. - await cdp.eval(` - (() => { - window.__FT__ = { times: [], stop: false } - let last = performance.now() - const tick = () => { - if (window.__FT__.stop) return - const now = performance.now() - window.__FT__.times.push(now - last) - last = now - requestAnimationFrame(tick) - } - requestAnimationFrame(tick) - - window.__LT__ = { entries: [], stop: false } - try { - const po = new PerformanceObserver((list) => { - if (window.__LT__.stop) return - for (const e of list.getEntries()) { - window.__LT__.entries.push({ name: e.name, duration: e.duration, startTime: e.startTime }) - } - }) - po.observe({ entryTypes: ['longtask'] }) - window.__LT__.po = po - } catch {} - - window.__MO__ = { mutations: [], stop: false, currentMsg: null } - const arm = () => { - const all = document.querySelectorAll('[data-slot="aui_assistant-message-root"]') - const last = all[all.length - 1] - if (!last || last === window.__MO__.currentMsg) return - window.__MO__.currentMsg = last - if (window.__MO__.obs) window.__MO__.obs.disconnect() - const obs = new MutationObserver((muts) => { - if (window.__MO__.stop) return - const t = performance.now() - window.__MO__.mutations.push({ t, count: muts.length, len: last.textContent.length }) - }) - obs.observe(last, { childList: true, subtree: true, characterData: true }) - window.__MO__.obs = obs - } - window.__MO__.arm = arm - - // Optional: typing observer — fires keystroke timings if asked. - window.__TYP__ = { times: [], stop: false, lastKey: 0 } - return 'recorders armed' - })() - `) - - // Baseline state. - const base = JSON.parse(await cdp.eval(` - JSON.stringify({ - assistantCount: document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length, - atomCount: window.__PERF_DRIVE__.snapshotMsgs() - }) - `)) - console.log('baseline:', base) - - // Drive a synthetic stream. - const streamStart = Date.now() - await cdp.eval(`window.__PERF_DRIVE__.stream({ chunk: ${JSON.stringify(CHUNK)}, intervalMs: ${INTERVAL_MS}, totalTokens: ${TOKENS}, flushMinMs: ${FLUSH_MIN_MS} })`) - - // After the first paint, arm MO on the new message. - await new Promise((r) => setTimeout(r, 200)) - await cdp.eval('window.__MO__.arm()') - - // Optional: type while streaming. - if (TYPE_WHILE_STREAMING) { - await new Promise((r) => setTimeout(r, 400)) - await cdp.eval(`(() => { - const ed = document.querySelector('[contenteditable="true"]') - ed.focus() - window.__TYP__.startedAt = performance.now() - const text = 'the quick brown fox jumps over the lazy dog ' - let i = 0 - const tick = () => { - if (i >= text.length) return - const t0 = performance.now() - document.execCommand('insertText', false, text[i]) - // requestAnimationFrame to wait for next paint - requestAnimationFrame(() => { - window.__TYP__.times.push(performance.now() - t0) - }) - i++ - setTimeout(tick, 60) - } - tick() - return 'typing' - })()`) - } - - // Wait for stream to complete + small grace. - const expectedMs = TOKENS * INTERVAL_MS + 1500 - await new Promise((r) => setTimeout(r, expectedMs)) - - // Pull recordings. - const data = JSON.parse(await cdp.eval(` - (() => { - window.__FT__.stop = true - window.__LT__.stop = true - window.__MO__.stop = true - window.__TYP__.stop = true - try { window.__LT__.po && window.__LT__.po.disconnect() } catch {} - try { window.__MO__.obs && window.__MO__.obs.disconnect() } catch {} - return JSON.stringify({ - frames: window.__FT__.times, - longtasks: window.__LT__.entries, - mutations: window.__MO__.mutations, - typing: window.__TYP__.times, - finalText: (() => { const a = document.querySelectorAll('[data-slot="aui_assistant-message-root"]'); return a.length ? a[a.length-1].textContent.length : 0 })() - }) - })() - `)) - - // Reset DOM back to baseline so we don't accumulate fake messages. - await cdp.eval('window.__PERF_DRIVE__.reset()') - - // Analysis (trim warm-up: drop frames before first mutation timestamp). - const firstMut = data.mutations[0]?.t - const frames = data.frames - - // Sum durations to figure out when each frame happened (relative to recorder start). - const frameTimeline = [] - let acc = 0 - for (const f of frames) { acc += f; frameTimeline.push(acc) } - - // Mutations are in performance.now() ms; frames started recording when we installed - // the recorder (before stream). To align: compute total stream window from frames - // after mutation activity began. Simpler heuristic: drop first 500ms of frames as warm-up. - const WARMUP_MS = 500 - let dropIdx = 0 - for (let i = 0; i < frames.length; i++) { - if (frameTimeline[i] >= WARMUP_MS) { dropIdx = i; break } - } - const streamFrames = frames.slice(dropIdx) - - const buckets = { '<=16.7': 0, '16.7-33': 0, '33-50': 0, '50-100': 0, '100-200': 0, '>200': 0 } - let frameTotal = 0 - let maxFrame = 0 - for (const f of streamFrames) { - frameTotal += f - if (f > maxFrame) maxFrame = f - if (f <= 16.7) buckets['<=16.7']++ - else if (f <= 33) buckets['16.7-33']++ - else if (f <= 50) buckets['33-50']++ - else if (f <= 100) buckets['50-100']++ - else if (f <= 200) buckets['100-200']++ - else buckets['>200']++ - } - const sortedFrames = [...streamFrames].sort((a, b) => a - b) - const fAvgFps = streamFrames.length ? (streamFrames.length / (frameTotal / 1000)).toFixed(1) : 'n/a' - const fP50 = pct(sortedFrames, 0.5).toFixed(1) - const fP95 = pct(sortedFrames, 0.95).toFixed(1) - const fP99 = pct(sortedFrames, 0.99).toFixed(1) - const slowFrames = streamFrames.filter((f) => f > 33).length - const veryslowFrames = streamFrames.filter((f) => f > 100).length - - const ltDur = data.longtasks.map((e) => e.duration).sort((a, b) => a - b) - const ltMs = ltDur.reduce((a, b) => a + b, 0) - const ltMax = ltDur.length ? ltDur[ltDur.length - 1] : 0 - const ltP95 = pct(ltDur, 0.95) - - // Mutation cadence. - const mutDurs = [] - for (let i = 1; i < data.mutations.length; i++) mutDurs.push(data.mutations[i].t - data.mutations[i - 1].t) - mutDurs.sort((a, b) => a - b) - const mutP50 = pct(mutDurs, 0.5) - const mutP95 = pct(mutDurs, 0.95) - const mutMax = mutDurs.length ? mutDurs[mutDurs.length - 1] : 0 - - // Typing latency (optional). - let typingSummary = null - if (TYPE_WHILE_STREAMING && data.typing.length) { - const t = [...data.typing].sort((a, b) => a - b) - typingSummary = { - n: t.length, - p50: pct(t, 0.5).toFixed(1), - p95: pct(t, 0.95).toFixed(1), - max: t[t.length - 1].toFixed(1) - } - } - - const result = { - label: LABEL, - timestamp: new Date().toISOString(), - config: { TOKENS, INTERVAL_MS, CHUNK, TYPE_WHILE_STREAMING, FLUSH_MIN_MS }, - streamWallMs: Date.now() - streamStart, - frames: { - total: streamFrames.length, - avgFps: fAvgFps, - windowS: (frameTotal / 1000).toFixed(1), - p50: fP50, - p95: fP95, - p99: fP99, - max: maxFrame.toFixed(1), - slow33: slowFrames, - veryslow100: veryslowFrames, - histogram: buckets - }, - longtasks: { - n: data.longtasks.length, - totalMs: ltMs.toFixed(0), - maxMs: ltMax.toFixed(1), - p95Ms: ltP95.toFixed(1) - }, - mutations: { - n: data.mutations.length, - finalTextLen: data.finalText, - interMutP50ms: mutP50.toFixed(1), - interMutP95ms: mutP95.toFixed(1), - interMutMaxMs: mutMax.toFixed(1) - }, - typing: typingSummary - } - - writeFileSync(OUT, JSON.stringify(result, null, 2)) - - console.log('\n=== SYNTHETIC STREAM RESULTS ===') - console.log('label:', LABEL, '| tokens:', TOKENS, '@', INTERVAL_MS, 'ms') - console.log('streamWallMs:', result.streamWallMs) - console.log('FRAMES: avgFps', fAvgFps, '| p50', fP50, 'ms | p95', fP95, 'ms | p99', fP99, 'ms | max', maxFrame.toFixed(1), 'ms') - console.log('FRAMES histogram:', buckets) - console.log('FRAMES slow(>33):', slowFrames, '/ veryslow(>100):', veryslowFrames, 'of', streamFrames.length) - console.log('LONGTASKS:', data.longtasks.length, '| total', ltMs.toFixed(0), 'ms | max', ltMax.toFixed(1), 'ms | p95', ltP95.toFixed(1), 'ms') - console.log('MUTATIONS:', data.mutations.length, '| finalLen', data.finalText, 'chars | inter p50', mutP50.toFixed(1), 'ms | p95', mutP95.toFixed(1), 'ms') - if (typingSummary) console.log('TYPING-WHILE-STREAMING latency: p50', typingSummary.p50, 'ms | p95', typingSummary.p95, 'ms | n=', typingSummary.n) - console.log('written to', OUT) - - cdp.close() -} - -main().catch((e) => { console.error(e); process.exit(1) }) diff --git a/apps/desktop/scripts/perf/README.md b/apps/desktop/scripts/perf/README.md new file mode 100644 index 000000000000..56673c467348 --- /dev/null +++ b/apps/desktop/scripts/perf/README.md @@ -0,0 +1,75 @@ +# Desktop perf harness + +One systematized way to measure desktop rendering/interaction performance, +diff it against a committed baseline, and fail on regressions. It replaces the +dozen one-off `measure-*` / `profile-*` scripts that each reinvented the CDP +client, arg parsing, stats, and output (and never had a baseline). + +## Quick start + +```bash +# Isolated instance (recommended) — no running app or LLM credits needed. +# Its own --user-data-dir + HERMES_HOME means it never collides with `hgui`. +npm run perf -- --spawn + +# Or: launch an isolated instance once, attach repeatedly (faster iteration). +npm run perf:serve # leaves an instance on :9222 +npm run perf # attaches, runs the CI suite, gates on baseline + +# One scenario, with a CPU profile: +npm run perf -- stream --cpuprofile --tokens 800 + +# Re-capture the baseline on your reference device, then commit baseline.json: +npm run perf -- --update-baseline +``` + +## Why isolation matters + +The measurement this harness exists to run was historically blocked: a running +`hgui` holds the Electron single-instance lock, so a second instance quit +immediately. `--spawn` / `perf:serve` launch with their own `--user-data-dir` +(separate lock scope), their own `HERMES_HOME` (separate backend + sessions), +and their own `--remote-debugging-port`. Synthetic scenarios drive `$messages` +directly via `window.__PERF_DRIVE__`, so no LLM credits are spent. + +## Scenarios + +| scenario | tier | measures | replaces | +|---|---|---|---| +| `stream` | ci | streaming longtasks, frame p95/p99, mutation cadence | measure-synthetic-stream, profile-synth-stream, profile-long-stream | +| `stream --real` | backend | same, from a real LLM stream | measure-real-stream, profile-real-stream | +| `keystroke` | ci | composer keystroke → paint latency | measure-latency, profile-typing, leak-typing | +| `transcript` | ci | large-transcript mount + paint cost | (new) | +| `submit` | backend | Enter → cleared → user msg painted, scroll jump | measure-submit, measure-jump | +| `session-switch` | backend | route → first-paint → settle | profile-session-switch | +| `profile-switch` | backend | rail click → sidebar settled | measure-profile-switch | + +`ci` scenarios need no backend/credits and are gated against `baseline.json`. +`backend` scenarios need a live backend (and `--spawn` or a real session) and +are report-only. + +CPU profiling is a cross-cutting `--cpuprofile` flag on any scenario (it wraps +the run in `Profiler.start/stop` and prints a top-self-time table), replacing +every standalone `profile-*` script. + +## Adding a scenario + +Create `scenarios/.mjs` exporting `{ name, tier, description, run(cdp, opts) }` +where `run` returns `{ metrics, detail }` (metrics = flat numbers, lower is +better), then register it in `scenarios/index.mjs`. If it's `ci`, add a +`baseline.json` entry (or run `--update-baseline`). + +## Layout + +- `lib/cdp.mjs` — the one CDP client + target discovery + typing + CPU-profile wrapper + DOM selectors. +- `lib/stats.mjs` — percentiles, histograms, CPU-profile self-time ranking. +- `lib/baseline.mjs` — load/compare/update the baseline + regression gate. +- `lib/launch.mjs` — attach, or spawn a fully isolated instance. +- `scenarios/` — one module per measurement. +- `run.mjs` — entrypoint. `serve.mjs` — standalone isolated launcher. + +## Not migrated (kept as dev utilities) + +`eval.mjs`, `reload.mjs`, `reload-renderer.mjs`, `probe-renderer.mjs`, +`probe-thread.mjs`, `click-session.mjs`, `diag-*.mjs` are interactive dev +helpers, not benchmarks. They can adopt `lib/cdp.mjs` in a follow-up. diff --git a/apps/desktop/scripts/perf/baseline.json b/apps/desktop/scripts/perf/baseline.json new file mode 100644 index 000000000000..0999aa0f7fe7 --- /dev/null +++ b/apps/desktop/scripts/perf/baseline.json @@ -0,0 +1,38 @@ +{ + "_meta": { + "note": "SEED baseline only. Values are approximations from apps/desktop/scripts/profile-typing-lag.md (May 2026, 34MB session, PRE incremental-lex). Run `npm run perf -- --update-baseline` on YOUR reference device to capture real numbers, then commit. Tolerances are intentionally loose until then.", + "platform": null, + "node": null, + "updated": null + }, + "scenarios": { + "stream": { + "tolerance": { "tolFrac": 0.5, "tolAbs": 2 }, + "metrics": { + "longtasks_n": 2, + "longtask_max_ms": 127, + "frame_p95_ms": 25.6, + "frame_p99_ms": 31.4, + "slow_frames_33": 6, + "intermut_p95_ms": 45 + } + }, + "keystroke": { + "tolerance": { "tolFrac": 0.5, "tolAbs": 3 }, + "metrics": { + "keystroke_p50_ms": 8, + "keystroke_p95_ms": 17, + "keystroke_p99_ms": 28, + "keystroke_slow_16": 6 + } + }, + "transcript": { + "tolerance": { "tolFrac": 0.5, "tolAbs": 20 }, + "metrics": { + "transcript_mount_ms": 450, + "transcript_longtask_ms": 350, + "transcript_longtask_max_ms": 160 + } + } + } +} diff --git a/apps/desktop/scripts/perf/lib/baseline.mjs b/apps/desktop/scripts/perf/lib/baseline.mjs new file mode 100644 index 000000000000..5ff270f8f136 --- /dev/null +++ b/apps/desktop/scripts/perf/lib/baseline.mjs @@ -0,0 +1,84 @@ +// Baseline + regression gate. This is the capability the old one-off scripts +// never had: measured numbers are compared against a committed baseline so a +// PR that regresses streaming/typing/mount cost fails loudly instead of +// silently drifting. +// +// Every tracked metric is "lower is better" (longtask counts, frame/keystroke +// percentiles, mount ms). A metric regresses when it exceeds +// `baseline * (1 + tolFrac) + tolAbs`. tolAbs absorbs sub-millisecond jitter on +// already-fast metrics so they don't false-positive. + +import { readFileSync, writeFileSync } from 'node:fs' + +const DEFAULT_TOLERANCE = { tolFrac: 0.25, tolAbs: 1 } + +export function loadBaseline(path) { + try { + return JSON.parse(readFileSync(path, 'utf8')) + } catch { + return { _meta: {}, scenarios: {} } + } +} + +/** + * Compare a scenario's measured metrics against the baseline. + * @returns {{ rows: Array, regressed: boolean }} + */ +export function compareScenario(name, measured, baseline) { + const base = baseline.scenarios?.[name] + const tol = { ...DEFAULT_TOLERANCE, ...(base?.tolerance ?? {}) } + const rows = [] + let regressed = false + + for (const [metric, value] of Object.entries(measured)) { + if (typeof value !== 'number') { + continue + } + + const baseValue = base?.metrics?.[metric] + + if (typeof baseValue !== 'number') { + rows.push({ metric, measured: value, baseline: null, limit: null, status: 'new' }) + + continue + } + + const limit = baseValue * (1 + tol.tolFrac) + tol.tolAbs + const over = value > limit + regressed = regressed || over + + rows.push({ + metric, + measured: value, + baseline: baseValue, + limit: Math.round(limit * 100) / 100, + deltaPct: baseValue ? Math.round(((value - baseValue) / baseValue) * 1000) / 10 : null, + status: over ? 'REGRESSED' : 'ok' + }) + } + + return { rows, regressed } +} + +/** Write measured metrics back as the new baseline for the given scenarios. */ +export function updateBaseline(path, results) { + const baseline = loadBaseline(path) + baseline.scenarios ??= {} + + for (const { name, metrics } of results) { + const numeric = Object.fromEntries(Object.entries(metrics).filter(([, v]) => typeof v === 'number')) + const prev = baseline.scenarios[name] ?? {} + baseline.scenarios[name] = { ...prev, metrics: numeric } + } + + baseline._meta = { + ...baseline._meta, + updated: new Date().toISOString(), + platform: `${process.platform}-${process.arch}`, + node: process.version + } + + writeFileSync(path, `${JSON.stringify(baseline, null, 2)}\n`) +} + +export { DEFAULT_TOLERANCE } diff --git a/apps/desktop/scripts/perf/lib/cdp.mjs b/apps/desktop/scripts/perf/lib/cdp.mjs new file mode 100644 index 000000000000..edba451ddcfe --- /dev/null +++ b/apps/desktop/scripts/perf/lib/cdp.mjs @@ -0,0 +1,202 @@ +// The one Chrome DevTools Protocol client for the desktop perf harness. +// +// Before this, every measure-*/profile-* script shipped its own copy-pasted +// `CDP` class (four subtly different implementations), its own `/json` vs +// `/json/list` target discovery, and its own Profiler ranking. Scenarios now +// import from here so there is a single place to fix a protocol bug. + +const DEFAULT_PORT = 9222 + +// Stable DOM hooks the renderer exposes. Centralised so a component refactor +// updates one constant instead of a dozen scattered querySelector strings. +export const SELECTORS = { + composer: '[data-slot="composer-rich-input"]', + threadViewport: '[data-slot="aui_thread-viewport"]', + threadContent: '[data-slot="aui_thread-content"]', + assistantMessage: '[data-slot="aui_assistant-message-root"]', + turnPair: '[data-slot="aui_turn-pair"]', + profileRail: '[data-slot="profile-rail"]', + rowButton: '[data-slot="row-button"]' +} + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) + +/** + * Poll the CDP HTTP endpoint until a page target is available. + * @param {object} [opts] + * @param {number} [opts.port] remote-debugging-port (default 9222). + * @param {string} [opts.match] substring the target URL must contain (e.g. a dev-server port). + * @param {number} [opts.timeoutMs] how long to wait for a target. + */ +export async function discoverTarget({ port = DEFAULT_PORT, match, timeoutMs = 30000 } = {}) { + const deadline = Date.now() + timeoutMs + + for (;;) { + try { + const list = await (await fetch(`http://127.0.0.1:${port}/json/list`)).json() + const pages = list.filter(t => t.type === 'page' && typeof t.webSocketDebuggerUrl === 'string') + const target = match + ? pages.find(t => String(t.url).includes(match)) + : pages.find(t => String(t.url).startsWith('http')) ?? pages[0] + + if (target) { + return target + } + } catch { + // debug port not up yet — keep polling until the deadline. + } + + if (Date.now() >= deadline) { + throw new Error(`no CDP page target on :${port}${match ? ` matching "${match}"` : ''} within ${timeoutMs}ms`) + } + + await sleep(250) + } +} + +export class CDP { + constructor(ws) { + this.ws = ws + this.id = 0 + this.pending = new Map() + this.listeners = new Map() + } + + static async open(url) { + const ws = new WebSocket(url) + + await new Promise((resolve, reject) => { + ws.addEventListener('open', resolve, { once: true }) + ws.addEventListener('error', reject, { once: true }) + }) + + const cdp = new CDP(ws) + + ws.addEventListener('message', ev => { + const m = JSON.parse(typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8')) + + if (m.id != null && cdp.pending.has(m.id)) { + const { resolve, reject } = cdp.pending.get(m.id) + cdp.pending.delete(m.id) + + if (m.error) { + reject(new Error(m.error.message)) + } else { + resolve(m.result) + } + } else if (m.method) { + for (const handler of cdp.listeners.get(m.method) ?? []) { + handler(m.params) + } + } + }) + + ws.addEventListener('close', () => { + for (const { reject } of cdp.pending.values()) { + reject(new Error('CDP socket closed')) + } + + cdp.pending.clear() + }) + + return cdp + } + + /** Connect straight to a discovered target. */ + static async connect(opts) { + const target = await discoverTarget(opts) + + return CDP.open(target.webSocketDebuggerUrl) + } + + send(method, params = {}) { + const id = ++this.id + + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }) + this.ws.send(JSON.stringify({ id, method, params })) + }) + } + + on(method, handler) { + if (!this.listeners.has(method)) { + this.listeners.set(method, []) + } + + this.listeners.get(method).push(handler) + } + + /** Evaluate an expression in the page and return its value (awaits promises). */ + async eval(expression) { + const r = await this.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }) + + if (r.exceptionDetails) { + throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text || 'eval failed') + } + + return r.result.value + } + + close() { + this.ws.close() + } +} + +/** Assert the renderer has the dev-only `__PERF_DRIVE__` harness attached. */ +export async function requireDriver(cdp) { + const ok = await cdp.eval('!!(window.__PERF_DRIVE__ && window.__PERF_DRIVE__.stream)') + + if (!ok) { + throw new Error( + '__PERF_DRIVE__ not on window. The perf harness needs a DEV renderer ' + + '(perf-probe.tsx is excluded from production builds). Launch with `npm run perf:serve`.' + ) + } +} + +/** Type real key events into the composer, one char at a time, at `cps` chars/sec. */ +export async function typeIntoComposer(cdp, text, { cps = 15 } = {}) { + await cdp.eval(`(() => { + const el = document.querySelector(${JSON.stringify(SELECTORS.composer)}) + if (!el) return false + el.focus() + const range = document.createRange() + range.selectNodeContents(el) + range.collapse(false) + const sel = window.getSelection() + sel.removeAllRanges() + sel.addRange(range) + return true + })()`) + + const intervalMs = Math.max(1, Math.round(1000 / cps)) + + for (const ch of text) { + await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: ch, unmodifiedText: ch }) + await sleep(intervalMs) + } +} + +/** + * Run `body()` while a V8 CPU profile is recording. Returns + * `{ result, profile }`; the caller decides whether to write the .cpuprofile. + */ +export async function withCpuProfile(cdp, body, { samplingIntervalUs = 100 } = {}) { + await cdp.send('Profiler.enable') + await cdp.send('Profiler.setSamplingInterval', { interval: samplingIntervalUs }) + await cdp.send('Profiler.start') + + let result + let stopped + + try { + result = await body() + } finally { + // Always stop so a scenario error can't leave the profiler running. + stopped = await cdp.send('Profiler.stop') + } + + return { result, profile: stopped.profile } +} + +export { sleep } diff --git a/apps/desktop/scripts/perf/lib/launch.mjs b/apps/desktop/scripts/perf/lib/launch.mjs new file mode 100644 index 000000000000..fbafff95931a --- /dev/null +++ b/apps/desktop/scripts/perf/lib/launch.mjs @@ -0,0 +1,212 @@ +// Connect the harness to a renderer — either an already-running debug instance +// (`attach`) or a freshly spawned, fully isolated one (`startIsolatedInstance`). +// +// The isolated instance is what makes the harness self-contained and unblocks +// the measurement that the single-instance lock used to prevent: +// · its own --user-data-dir → its own Electron single-instance lock, so it +// never collides with (or steals focus from) the user's running `hgui`. +// · its own HERMES_HOME → its own backend + sessions, no shared state. +// · its own --remote-debugging-port → a private CDP endpoint. +// · HERMES_DESKTOP_BOOT_FAKE=1 → deterministic boot overlay. +// The synthetic scenarios drive `$messages` directly, so no LLM credits are +// spent regardless of the isolated backend. + +import { spawn } from 'node:child_process' +import { copyFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs' +import { createRequire } from 'node:module' +import { homedir, tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { CDP, requireDriver, sleep } from './cdp.mjs' + +const require = createRequire(import.meta.url) +const DESKTOP_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..', '..') + +async function reachable(url) { + try { + await fetch(url) + + return true + } catch { + return false + } +} + +async function waitFor(fn, { timeoutMs, label }) { + const deadline = Date.now() + timeoutMs + + while (Date.now() < deadline) { + if (await fn()) { + return + } + + await sleep(300) + } + + throw new Error(`timed out after ${timeoutMs}ms waiting for ${label}`) +} + +// Seed an isolated HERMES_HOME with just enough config (NOT sessions) so the +// spawned instance reaches an empty chat view instead of the onboarding wizard. +// A separate HERMES_HOME dir means a separate gateway lock — no collision with +// the user's running app, which keeps its own sessions DB and state. +function seedConfigFrom(sourceHome, targetHome) { + if (!existsSync(sourceHome)) { + return + } + + for (const name of ['config.yaml', '.env', 'auth.json']) { + const from = join(sourceHome, name) + + if (existsSync(from)) { + try { + copyFileSync(from, join(targetHome, name)) + } catch { + // best-effort — a missing file just means onboarding may appear. + } + } + } +} + +function runNode(scriptRelPath, args = []) { + return new Promise((resolveRun, reject) => { + const child = spawn(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args], { + cwd: DESKTOP_DIR, + stdio: 'inherit' + }) + child.on('error', reject) + child.on('exit', code => (code === 0 ? resolveRun() : reject(new Error(`${scriptRelPath} exited ${code}`)))) + }) +} + +/** Attach to a renderer already listening on `port` (launched via perf:serve or with --remote-debugging-port). */ +export async function attach({ port = 9222, match } = {}) { + const cdp = await CDP.connect({ port, match }) + await requireDriver(cdp) + + return { cdp, teardown: () => cdp.close() } +} + +/** + * Spawn an isolated dev instance (vite + electron), wait for the perf driver, + * and return `{ cdp, teardown, devUrl, port }`. `teardown` kills both children + * and removes any temp dirs it created. + */ +export async function startIsolatedInstance({ + port = 9222, + devPort = 5174, + hermesHome, + userDataDir, + seedConfig = true, + bootFakeStepMs = 120 +} = {}) { + const children = [] + const tempDirs = [] + + const mkTemp = prefix => { + const dir = mkdtempSync(join(tmpdir(), prefix)) + tempDirs.push(dir) + + return dir + } + + const home = hermesHome ?? mkTemp('hermes-perf-home-') + const userData = userDataDir ?? mkTemp('hermes-perf-ud-') + const devUrl = `http://127.0.0.1:${devPort}` + + // Only seed a temp home we created — never scribble into a user-provided one. + if (seedConfig && !hermesHome) { + seedConfigFrom(join(homedir(), '.hermes'), home) + } + + const teardown = () => { + for (const child of children) { + try { + child.kill('SIGTERM') + } catch { + // already gone + } + } + + for (const dir of tempDirs) { + try { + rmSync(dir, { recursive: true, force: true }) + } catch { + // best-effort + } + } + } + + try { + // 1. Renderer: reuse an already-running dev server, else start one. + if (!(await reachable(devUrl))) { + const viteBin = require.resolve('vite/bin/vite.js') + const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], { + cwd: DESKTOP_DIR, + stdio: ['ignore', 'inherit', 'inherit'] + }) + children.push(vite) + await waitFor(() => reachable(devUrl), { timeoutMs: 60000, label: `vite dev server on :${devPort}` }) + } + + // 2. Electron main bundle (dev variant) — same step the dev script runs. + await runNode('scripts/bundle-electron-main.mjs', ['--dev']) + + // 3. Isolated Electron. --user-data-dir gives it its own single-instance + // lock scope; HERMES_HOME gives it its own backend + sessions. + const electronBin = require('electron') + const electron = spawn( + electronBin, + ['.', `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`], + { + cwd: DESKTOP_DIR, + stdio: ['ignore', 'inherit', 'inherit'], + env: { + ...process.env, + HERMES_HOME: home, + HERMES_DESKTOP_DEV_SERVER: devUrl, + HERMES_DESKTOP_BOOT_FAKE: '1', + HERMES_DESKTOP_BOOT_FAKE_STEP_MS: String(bootFakeStepMs), + XCURSOR_SIZE: '24' + } + } + ) + children.push(electron) + + // 4. Wait for the renderer + the perf driver to be live. + let cdp = null + await waitFor( + async () => { + try { + cdp = await CDP.connect({ port, match: String(devPort), timeoutMs: 2000 }) + + return await cdp.eval('!!(window.__PERF_DRIVE__ && window.__PERF_DRIVE__.stream)') + } catch { + if (cdp) { + cdp.close() + cdp = null + } + + return false + } + }, + { timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' } + ) + + return { + cdp, + devUrl, + port, + teardown: () => { + cdp?.close() + teardown() + } + } + } catch (err) { + teardown() + throw err + } +} + +export { DESKTOP_DIR } diff --git a/apps/desktop/scripts/perf/lib/stats.mjs b/apps/desktop/scripts/perf/lib/stats.mjs new file mode 100644 index 000000000000..d82dce40a5cd --- /dev/null +++ b/apps/desktop/scripts/perf/lib/stats.mjs @@ -0,0 +1,89 @@ +// Shared numeric helpers for perf scenarios. Every measure-*/profile-* script +// used to carry its own copy of these. + +/** Nearest-rank percentile over an UNSORTED array. p in [0,1]. */ +export function percentile(values, p) { + if (!values.length) { + return 0 + } + + const sorted = [...values].sort((a, b) => a - b) + const idx = Math.min(sorted.length - 1, Math.floor(sorted.length * p)) + + return sorted[idx] +} + +/** min/p50/p90/p95/p99/max/mean over a sample array (rounded to 2dp). */ +export function summarize(values) { + const round = n => Math.round(n * 100) / 100 + + if (!values.length) { + return { n: 0, min: 0, p50: 0, p90: 0, p95: 0, p99: 0, max: 0, mean: 0 } + } + + const sorted = [...values].sort((a, b) => a - b) + const mean = values.reduce((a, b) => a + b, 0) / values.length + + return { + n: values.length, + min: round(sorted[0]), + p50: round(percentile(sorted, 0.5)), + p90: round(percentile(sorted, 0.9)), + p95: round(percentile(sorted, 0.95)), + p99: round(percentile(sorted, 0.99)), + max: round(sorted[sorted.length - 1]), + mean: round(mean) + } +} + +/** Median of a numeric array (used to reduce N repeated runs to one number). */ +export function median(values) { + return percentile(values, 0.5) +} + +/** Frame-interval histogram matching the buckets the stream scripts reported. */ +export function frameHistogram(frames) { + const buckets = { '<=16.7': 0, '16.7-33': 0, '33-50': 0, '50-100': 0, '100-200': 0, '>200': 0 } + + for (const f of frames) { + if (f <= 16.7) buckets['<=16.7']++ + else if (f <= 33) buckets['16.7-33']++ + else if (f <= 50) buckets['33-50']++ + else if (f <= 100) buckets['50-100']++ + else if (f <= 200) buckets['100-200']++ + else buckets['>200']++ + } + + return buckets +} + +/** + * Rank functions by self-time from a V8 CPU profile (Profiler.stop output). + * Returns the top `limit` entries as { ms, name, url, line }. + */ +export function cpuProfileTopSelf(profile, limit = 30) { + const samples = profile.samples || [] + const timeDeltas = profile.timeDeltas || [] + const nodes = new Map(profile.nodes.map(n => [n.id, n])) + const selfUs = new Map() + + for (let i = 0; i < samples.length; i++) { + const id = samples[i] + selfUs.set(id, (selfUs.get(id) || 0) + (timeDeltas[i] ?? 0)) + } + + return [...selfUs.entries()] + .map(([id, us]) => { + const cf = nodes.get(id)?.callFrame || {} + + return { + ms: us / 1000, + name: cf.functionName || '(anonymous)', + url: String(cf.url || '').slice(-70), + line: cf.lineNumber + } + }) + .filter(x => !/\(root\)|\(idle\)|\(garbage collector\)|\(program\)/.test(x.name)) + .sort((a, b) => b.ms - a.ms) + .slice(0, limit) +} diff --git a/apps/desktop/scripts/perf/run.mjs b/apps/desktop/scripts/perf/run.mjs new file mode 100644 index 000000000000..570e1b9e096f --- /dev/null +++ b/apps/desktop/scripts/perf/run.mjs @@ -0,0 +1,184 @@ +// Desktop perf harness entrypoint. +// +// node scripts/perf/run.mjs [scenarios...] [flags] +// +// Default (no scenarios): runs the CI suite (stream, keystroke, transcript) +// against a renderer on :9222 and diffs the committed baseline. +// +// Flags: +// --spawn launch a fully isolated instance (own user-data-dir + +// HERMES_HOME + debug port) instead of attaching +// --port CDP port to attach to (default 9222) +// --dev-port vite dev-server port to match / spawn (default 5174) +// --runs repeat each scenario n times, report the median (default 1) +// --cpuprofile [dir] also record a V8 CPU profile per scenario (top-30 self time) +// --update-baseline overwrite baseline.json with this run's numbers +// --json write the full results JSON here +// --tier run all scenarios of a tier +// ...scenario opts e.g. --tokens 600, --turns 400, --real, --a --b , --profile +// +// Examples: +// npm run perf # attach to :9222, run CI suite, gate on baseline +// npm run perf -- --spawn # isolated instance, no running app needed +// npm run perf -- stream --cpuprofile --tokens 800 +// npm run perf -- --update-baseline + +import { writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { withCpuProfile } from './lib/cdp.mjs' +import { compareScenario, loadBaseline, updateBaseline } from './lib/baseline.mjs' +import { attach, startIsolatedInstance } from './lib/launch.mjs' +import { cpuProfileTopSelf, median } from './lib/stats.mjs' +import { CI_SCENARIOS, SCENARIOS } from './scenarios/index.mjs' + +const HERE = dirname(fileURLToPath(import.meta.url)) +const BASELINE_PATH = join(HERE, 'baseline.json') + +function parseArgs(argv) { + const positional = [] + const flags = {} + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i] + + if (arg.startsWith('--')) { + const [key, inlineValue] = arg.slice(2).split(/=(.*)/s) + const next = argv[i + 1] + + if (inlineValue !== undefined) { + flags[key] = inlineValue + } else if (next === undefined || next.startsWith('--')) { + flags[key] = true + } else { + flags[key] = next + i++ + } + } else { + positional.push(arg) + } + } + + return { positional, flags } +} + +function medianMetrics(runs) { + const keys = new Set(runs.flatMap(r => Object.keys(r))) + const out = {} + + for (const key of keys) { + const values = runs.map(r => r[key]).filter(v => typeof v === 'number') + out[key] = values.length ? Math.round(median(values) * 10) / 10 : runs[0][key] + } + + return out +} + +function printMetrics(name, metrics, comparison) { + console.log(`\n● ${name}`) + const byMetric = new Map((comparison?.rows ?? []).map(r => [r.metric, r])) + + for (const [metric, value] of Object.entries(metrics)) { + const row = byMetric.get(metric) + + if (!row || row.baseline === null) { + console.log(` ${metric.padEnd(26)} ${String(value).padStart(9)}${row ? ' (new)' : ''}`) + } else { + const tag = row.status === 'REGRESSED' ? ' ✗ REGRESSED' : ' ✓' + const delta = row.deltaPct === null ? '' : ` (${row.deltaPct > 0 ? '+' : ''}${row.deltaPct}%)` + console.log( + ` ${metric.padEnd(26)} ${String(value).padStart(9)} vs ${String(row.baseline).padStart(9)}${delta}${tag}` + ) + } + } +} + +async function main() { + const { positional, flags } = parseArgs(process.argv.slice(2)) + + let names = positional + if (!names.length) { + names = flags.tier ? Object.values(SCENARIOS).filter(s => s.tier === flags.tier).map(s => s.name) : CI_SCENARIOS + } + + const unknown = names.filter(n => !SCENARIOS[n]) + if (unknown.length) { + console.error(`unknown scenario(s): ${unknown.join(', ')}\nknown: ${Object.keys(SCENARIOS).join(', ')}`) + process.exit(2) + } + + const runs = Number(flags.runs ?? 1) + const port = Number(flags.port ?? 9222) + const devPort = Number(flags['dev-port'] ?? 5174) + const cpuProfile = 'cpuprofile' in flags + const cpuProfileDir = typeof flags.cpuprofile === 'string' ? flags.cpuprofile : HERE + + const connection = flags.spawn + ? await startIsolatedInstance({ port, devPort }) + : await attach({ port, match: String(devPort) }) + + const { cdp, teardown } = connection + const results = [] + let regressed = false + + try { + const baseline = loadBaseline(BASELINE_PATH) + + for (const name of names) { + const scenario = SCENARIOS[name] + const perRun = [] + let detail = null + + for (let i = 0; i < runs; i++) { + if (cpuProfile && i === 0) { + const { result, profile } = await withCpuProfile(cdp, () => scenario.run(cdp, flags)) + const out = join(cpuProfileDir, `${name}-${Date.now()}.cpuprofile`) + writeFileSync(out, JSON.stringify(profile)) + console.log(`\n[cpuprofile] wrote ${out}`) + console.log('[cpuprofile] top self-time (ms):') + for (const r of cpuProfileTopSelf(profile, 15)) { + console.log(` ${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(38)} ${r.url}:${r.line}`) + } + perRun.push(result.metrics) + detail = result.detail + } else { + const result = await scenario.run(cdp, flags) + perRun.push(result.metrics) + detail = result.detail + } + } + + const metrics = medianMetrics(perRun) + const comparison = scenario.tier === 'ci' ? compareScenario(name, metrics, baseline) : null + regressed = regressed || Boolean(comparison?.regressed) + results.push({ name, tier: scenario.tier, metrics, detail }) + printMetrics(name, metrics, comparison) + } + } finally { + teardown() + } + + if (flags.json) { + writeFileSync(resolve(String(flags.json)), `${JSON.stringify({ timestamp: new Date().toISOString(), results }, null, 2)}\n`) + console.log(`\nwrote ${flags.json}`) + } + + if (flags['update-baseline']) { + updateBaseline(BASELINE_PATH, results.filter(r => r.tier === 'ci')) + console.log(`\nupdated ${BASELINE_PATH}`) + return + } + + if (regressed) { + console.error('\n✗ perf regression vs baseline (see REGRESSED rows above)') + process.exit(1) + } + + console.log('\n✓ no perf regressions') +} + +main().catch(err => { + console.error('\nperf harness failed:', err.stack ?? err.message) + process.exit(1) +}) diff --git a/apps/desktop/scripts/perf/scenarios/index.mjs b/apps/desktop/scripts/perf/scenarios/index.mjs new file mode 100644 index 000000000000..02771b2f0d98 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/index.mjs @@ -0,0 +1,23 @@ +// Scenario registry. Add a scenario module here and it's automatically +// available to the runner, the default suite (tier 'ci'), and the baseline gate. + +import keystroke from './keystroke.mjs' +import profileSwitch from './profile-switch.mjs' +import sessionSwitch from './session-switch.mjs' +import stream from './stream.mjs' +import submit from './submit.mjs' +import transcript from './transcript.mjs' + +export const SCENARIOS = { + [stream.name]: stream, + [keystroke.name]: keystroke, + [transcript.name]: transcript, + [submit.name]: submit, + [sessionSwitch.name]: sessionSwitch, + [profileSwitch.name]: profileSwitch +} + +/** Scenarios safe to run with no LLM credits / no live backend — the default suite. */ +export const CI_SCENARIOS = Object.values(SCENARIOS) + .filter(s => s.tier === 'ci') + .map(s => s.name) diff --git a/apps/desktop/scripts/perf/scenarios/keystroke.mjs b/apps/desktop/scripts/perf/scenarios/keystroke.mjs new file mode 100644 index 000000000000..ad705e1614db --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/keystroke.mjs @@ -0,0 +1,99 @@ +// Composer input latency — keystroke → next paint. Subsumes measure-latency, +// profile-typing, and leak-typing. This is the most-felt latency in a chat app +// (users type constantly) and nothing measured it against a baseline before. +// +// Each synthetic char records the time from dispatch to the first rAF after the +// composer mutates (a paint proxy). Metrics are p50/p95/p99 and the count of +// keystrokes that missed a 16ms frame. + +import { SELECTORS, sleep } from '../lib/cdp.mjs' +import { percentile } from '../lib/stats.mjs' + +const INSTALL = ` + (() => { + const el = document.querySelector(${JSON.stringify(SELECTORS.composer)}) + if (!el) return false + el.focus() + const range = document.createRange() + range.selectNodeContents(el) + range.collapse(false) + const sel = window.getSelection() + sel.removeAllRanges() + sel.addRange(range) + window.__KEY__ = { samples: [], pending: null } + const obs = new MutationObserver(() => { + const start = window.__KEY__.pending + if (start === null) return + window.__KEY__.pending = null + requestAnimationFrame(() => window.__KEY__.samples.push(performance.now() - start)) + }) + obs.observe(el, { childList: true, subtree: true, characterData: true }) + window.__KEY__.obs = obs + return true + })() +` + +const CLEAR = ` + (() => { + const el = document.querySelector(${JSON.stringify(SELECTORS.composer)}) + if (el) { el.innerHTML = ''; el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContentBackward' })) } + window.__KEY__ && window.__KEY__.obs && window.__KEY__.obs.disconnect() + })() +` + +const SENTENCE = + 'the quick brown fox jumps over the lazy dog while typing into this composer, which should feel instant. ' + +export default { + name: 'keystroke', + tier: 'ci', + description: 'Composer keystroke → paint latency while idle.', + async run(cdp, opts = {}) { + const chars = Number(opts.chars ?? 120) + const cps = Number(opts.cps ?? 15) + + await cdp.send('Runtime.enable') + + const installed = await cdp.eval(INSTALL) + + if (!installed) { + throw new Error(`composer not found (${SELECTORS.composer}); is a chat view open?`) + } + + let text = '' + + while (text.length < chars) { + text += SENTENCE + } + + text = text.slice(0, chars) + const intervalMs = Math.max(1, Math.round(1000 / cps)) + const start = Date.now() + + for (let i = 0; i < text.length; i++) { + await cdp.eval('window.__KEY__.pending = performance.now()') + await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: text[i], unmodifiedText: text[i] }) + const wait = start + (i + 1) * intervalMs - Date.now() + + if (wait > 0) { + await sleep(wait) + } + } + + await sleep(300) + const samples = await cdp.eval('window.__KEY__.samples') + await cdp.eval(CLEAR) + + const round = n => Math.round(n * 10) / 10 + + return { + metrics: { + keystroke_p50_ms: round(percentile(samples, 0.5)), + keystroke_p95_ms: round(percentile(samples, 0.95)), + keystroke_p99_ms: round(percentile(samples, 0.99)), + keystroke_slow_16: samples.filter(s => s > 16).length + }, + detail: { n: samples.length, typed: text.length } + } + } +} diff --git a/apps/desktop/scripts/perf/scenarios/profile-switch.mjs b/apps/desktop/scripts/perf/scenarios/profile-switch.mjs new file mode 100644 index 000000000000..03ff104e3cf0 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/profile-switch.mjs @@ -0,0 +1,60 @@ +// Profile-switch latency. Subsumes measure-profile-switch. Backend tier: needs +// a configured profile in the rail and a live backend. Report-only. +// +// node scripts/perf/run.mjs profile-switch --profile + +import { SELECTORS, sleep } from '../lib/cdp.mjs' + +export default { + name: 'profile-switch', + tier: 'backend', + description: 'Click a profile in the rail and wait for its sidebar to settle.', + requiredOpts: ['profile'], + async run(cdp, opts = {}) { + const profile = opts.profile + const settleTimeoutMs = Number(opts.settleTimeoutMs ?? 60000) + + if (!profile) { + throw new Error('profile-switch needs --profile ') + } + + await cdp.send('Runtime.enable') + + const t0 = await cdp.eval(`(() => { + const rail = document.querySelector(${JSON.stringify(SELECTORS.profileRail)}) + if (!rail) return null + const target = [...rail.querySelectorAll('button, [role="tab"]')].find(b => + ((b.getAttribute('aria-label') || '') + ' ' + (b.title || '') + ' ' + (b.textContent || '')) + .toLowerCase().includes(${JSON.stringify(String(profile).toLowerCase())})) + if (!target) return null + target.click() + return performance.now() + })()`) + + if (t0 === null) { + throw new Error(`profile "${profile}" not found in the rail`) + } + + const deadline = Date.now() + settleTimeoutMs + let settledMs = null + + while (Date.now() < deadline) { + await sleep(100) + const s = await cdp.eval(`(() => { + const label = [...document.querySelectorAll('div[aria-hidden]')].find(el => /waking up/i.test(el.textContent || '')) + const overlayVisible = label ? Number(getComputedStyle(label).opacity) > 0.05 : false + return { t: performance.now(), overlayVisible, rows: document.querySelectorAll(${JSON.stringify(SELECTORS.rowButton)}).length } + })()`) + + if (!s.overlayVisible && s.rows > 0) { + settledMs = s.t - t0 + break + } + } + + return { + metrics: { profile_switch_settled_ms: settledMs === null ? -1 : Math.round(settledMs) }, + detail: { profile, timedOut: settledMs === null } + } + } +} diff --git a/apps/desktop/scripts/perf/scenarios/session-switch.mjs b/apps/desktop/scripts/perf/scenarios/session-switch.mjs new file mode 100644 index 000000000000..fb7fb383dcf7 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/session-switch.mjs @@ -0,0 +1,80 @@ +// Session-switch latency. Subsumes profile-session-switch. Backend tier: needs +// two real stored session ids and a live backend. Report-only. +// +// node scripts/perf/run.mjs session-switch --a --b [--rounds 2] + +import { SELECTORS, sleep } from '../lib/cdp.mjs' +import { summarize } from '../lib/stats.mjs' + +export default { + name: 'session-switch', + tier: 'backend', + description: 'Route to a session and wait for first-paint + settle of its transcript.', + requiredOpts: ['a', 'b'], + async run(cdp, opts = {}) { + const { a, b } = opts + const rounds = Number(opts.rounds ?? 2) + const settleTimeoutMs = Number(opts.settleTimeoutMs ?? 30000) + + if (!a || !b) { + throw new Error('session-switch needs --a --b ') + } + + await cdp.send('Runtime.enable') + + const switchTo = async sid => { + const t0 = await cdp.eval(`(() => { location.hash = '#/' + ${JSON.stringify(sid)}; return performance.now() })()`) + const deadline = Date.now() + settleTimeoutMs + let firstPaint = null + let stable = 0 + let lastCount = -1 + + while (Date.now() < deadline) { + await sleep(50) + const s = await cdp.eval(`({ + t: performance.now(), + route: location.hash, + msgs: document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length + })`) + + if (!String(s.route).includes(sid)) { + continue + } + + if (s.msgs > 0 && firstPaint === null) { + firstPaint = s.t - t0 + } + + stable = s.msgs === lastCount && s.msgs > 0 ? stable + 1 : 0 + lastCount = s.msgs + + if (stable >= 3) { + return { firstPaint, settled: s.t - t0 } + } + } + + return { firstPaint, settled: null } + } + + const firstPaints = [] + const settles = [] + + for (let round = 0; round < rounds; round++) { + for (const sid of [a, b]) { + const r = await switchTo(sid) + + if (typeof r.firstPaint === 'number') firstPaints.push(r.firstPaint) + if (typeof r.settled === 'number') settles.push(r.settled) + await sleep(800) + } + } + + return { + metrics: { + switch_first_paint_p95_ms: summarize(firstPaints).p95, + switch_settled_p95_ms: summarize(settles).p95 + }, + detail: { rounds, firstPaint: summarize(firstPaints), settled: summarize(settles) } + } + } +} diff --git a/apps/desktop/scripts/perf/scenarios/stream.mjs b/apps/desktop/scripts/perf/scenarios/stream.mjs new file mode 100644 index 000000000000..0f42339132c1 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/stream.mjs @@ -0,0 +1,178 @@ +// Streaming render cost. Subsumes measure-synthetic-stream, profile-synth-stream, +// profile-long-stream (synthetic) and measure-real-stream / profile-real-stream +// (via --real). CPU profiling is provided by the runner's --cpuprofile flag. +// +// Metrics (lower is better): longtask count + max, frame p95/p99, slow-frame +// count, inter-mutation p95. These are what "is streaming smooth?" reduces to. + +import { SELECTORS, sleep, typeIntoComposer } from '../lib/cdp.mjs' +import { frameHistogram, percentile } from '../lib/stats.mjs' + +const RECORDERS = ` + (() => { + window.__FT__ = { times: [], stop: false } + let last = performance.now() + const tick = () => { + if (window.__FT__.stop) return + const now = performance.now() + window.__FT__.times.push(now - last) + last = now + requestAnimationFrame(tick) + } + requestAnimationFrame(tick) + + window.__LT__ = { entries: [], stop: false } + try { + const po = new PerformanceObserver((list) => { + if (window.__LT__.stop) return + for (const e of list.getEntries()) window.__LT__.entries.push({ duration: e.duration, startTime: e.startTime }) + }) + po.observe({ entryTypes: ['longtask'] }) + window.__LT__.po = po + } catch {} + + window.__MO__ = { mutations: [], stop: false, current: null } + window.__MO__.arm = () => { + const all = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}) + const last = all[all.length - 1] + if (!last || last === window.__MO__.current) return + window.__MO__.current = last + window.__MO__.obs && window.__MO__.obs.disconnect() + const obs = new MutationObserver(() => { + if (window.__MO__.stop) return + window.__MO__.mutations.push({ t: performance.now(), len: last.textContent.length }) + }) + obs.observe(last, { childList: true, subtree: true, characterData: true }) + window.__MO__.obs = obs + } + return 'armed' + })() +` + +const COLLECT = ` + (() => { + window.__FT__.stop = true + window.__LT__.stop = true + window.__MO__.stop = true + try { window.__LT__.po && window.__LT__.po.disconnect() } catch {} + try { window.__MO__.obs && window.__MO__.obs.disconnect() } catch {} + return JSON.stringify({ + frames: window.__FT__.times, + longtasks: window.__LT__.entries, + mutations: window.__MO__.mutations + }) + })() +` + +function analyze(data, warmupMs) { + // Drop warm-up frames (recorder installs before the stream starts). + const frames = [] + let acc = 0 + + for (const f of data.frames) { + acc += f + + if (acc >= warmupMs) { + frames.push(f) + } + } + + const interMut = [] + + for (let i = 1; i < data.mutations.length; i++) { + interMut.push(data.mutations[i].t - data.mutations[i - 1].t) + } + + const ltDurations = data.longtasks.map(e => e.duration) + const windowS = frames.reduce((a, b) => a + b, 0) / 1000 + + return { + metrics: { + longtasks_n: data.longtasks.length, + longtask_max_ms: Math.round((ltDurations.length ? Math.max(...ltDurations) : 0) * 10) / 10, + frame_p95_ms: Math.round(percentile(frames, 0.95) * 10) / 10, + frame_p99_ms: Math.round(percentile(frames, 0.99) * 10) / 10, + slow_frames_33: frames.filter(f => f > 33).length, + intermut_p95_ms: Math.round(percentile(interMut, 0.95) * 10) / 10 + }, + detail: { + windowS: Math.round(windowS * 10) / 10, + avgFps: windowS ? Math.round((frames.length / windowS) * 10) / 10 : 0, + frameHistogram: frameHistogram(frames), + mutations: data.mutations.length, + finalLen: data.mutations.at(-1)?.len ?? 0 + } + } +} + +export default { + name: 'stream', + tier: 'ci', + description: 'Assistant-message streaming: longtasks, frame pacing, mutation cadence.', + async run(cdp, opts = {}) { + const tokens = Number(opts.tokens ?? 400) + const intervalMs = Number(opts.intervalMs ?? 16) + const flushMinMs = Number(opts.flushMinMs ?? 33) + const chunk = opts.chunk ?? '**word** in _italic_ with `code`, a [link](https://x.dev) and prose. ' + const real = Boolean(opts.real) + + await cdp.send('Runtime.enable') + await cdp.eval(RECORDERS) + + if (real) { + // Backend path: fire a real prompt and wait for the stream to appear. + const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`) + await typeIntoComposer(cdp, opts.prompt ?? 'count from 1 to 80, one number per line', { cps: 40 }) + await cdp.eval(`(() => { + const el = document.querySelector(${JSON.stringify(SELECTORS.composer)}) + el && el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true })) + })()`) + + const deadline = Date.now() + Number(opts.timeoutMs ?? 60000) + let started = false + + while (Date.now() < deadline) { + await sleep(50) + const n = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`) + + if (n > baseCount) { + started = true + break + } + } + + if (!started) { + throw new Error('real stream never started (no LLM credit / backend?)') + } + + await cdp.eval('window.__MO__.arm()') + // Let it run to completion or timeout. + const runDeadline = Date.now() + Number(opts.runMs ?? 30000) + + while (Date.now() < runDeadline) { + await sleep(250) + const busy = await cdp.eval(`!!document.querySelector('[data-status="running"], [data-busy="true"]')`) + + if (!busy) { + break + } + } + } else { + // Synthetic path: drive $messages directly. No LLM, no credits. + await cdp.eval( + `window.__PERF_DRIVE__.stream({ chunk: ${JSON.stringify(chunk)}, intervalMs: ${intervalMs}, totalTokens: ${tokens}, flushMinMs: ${flushMinMs} })` + ) + await sleep(200) + await cdp.eval('window.__MO__.arm()') + await sleep(tokens * intervalMs + 1500) + } + + const data = JSON.parse(await cdp.eval(COLLECT)) + + if (!real) { + await cdp.eval('window.__PERF_DRIVE__.reset()') + } + + return analyze(data, real ? 0 : 500) + } +} diff --git a/apps/desktop/scripts/perf/scenarios/submit.mjs b/apps/desktop/scripts/perf/scenarios/submit.mjs new file mode 100644 index 000000000000..9b5efc1934e2 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/submit.mjs @@ -0,0 +1,87 @@ +// Submit (Enter) latency + scroll stability. Subsumes measure-submit and +// measure-jump. Backend tier: fires a REAL prompt, so run it on a throwaway +// session with a live backend. Report-only (no committed baseline — real +// round-trips are too environment-dependent to gate). + +import { SELECTORS, sleep, typeIntoComposer } from '../lib/cdp.mjs' +import { summarize } from '../lib/stats.mjs' + +const MEASURE = ` + new Promise((resolve) => { + const composer = document.querySelector(${JSON.stringify(SELECTORS.composer)}) + const thread = document.querySelector(${JSON.stringify(SELECTORS.threadContent)}) || + document.querySelector(${JSON.stringify(SELECTORS.threadViewport)}) + const viewport = document.querySelector(${JSON.stringify(SELECTORS.threadViewport)}) + const startCount = thread ? thread.querySelectorAll(${JSON.stringify(SELECTORS.turnPair)}).length : 0 + const startScroll = viewport ? viewport.scrollTop : 0 + const m = { start: performance.now(), maxJumpPx: 0 } + let done = false + + const finish = (reason) => { + if (done) return + done = true + clearTimeout(timer); composerObs.disconnect(); threadObs && threadObs.disconnect() + m.reason = reason + resolve(m) + } + + const composerObs = new MutationObserver(() => { + if (!m.composerClearedMs && composer && composer.innerText.length === 0) { + m.composerClearedMs = performance.now() - m.start + } + }) + composer && composerObs.observe(composer, { childList: true, subtree: true, characterData: true }) + + let threadObs = null + if (thread) { + threadObs = new MutationObserver(() => { + if (viewport) m.maxJumpPx = Math.max(m.maxJumpPx, Math.abs(viewport.scrollTop - startScroll)) + const c = thread.querySelectorAll(${JSON.stringify(SELECTORS.turnPair)}).length + if (!m.userMsgRenderedMs && c > startCount) { + m.userMsgRenderedMs = performance.now() - m.start + requestAnimationFrame(() => { m.userMsgPaintMs = performance.now() - m.start; finish('paint') }) + } + }) + threadObs.observe(thread, { childList: true, subtree: true }) + } + + const timer = setTimeout(() => finish('timeout'), 5000) + composer && composer.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true })) + }) +` + +export default { + name: 'submit', + tier: 'backend', + description: 'Enter → composer cleared → user message painted, plus scroll jump.', + async run(cdp, opts = {}) { + const rounds = Number(opts.rounds ?? 3) + await cdp.send('Runtime.enable') + + const clears = [] + const paints = [] + const jumps = [] + + for (let i = 0; i < rounds; i++) { + await typeIntoComposer(cdp, `perf submit round ${i} ${'x'.repeat(30)}`, { cps: 60 }) + await sleep(250) + const m = await cdp.eval(MEASURE) + + if (typeof m.composerClearedMs === 'number') clears.push(m.composerClearedMs) + if (typeof m.userMsgPaintMs === 'number') paints.push(m.userMsgPaintMs) + jumps.push(m.maxJumpPx ?? 0) + + // Let the turn finish before the next round so they don't pile up. + await sleep(4000) + } + + return { + metrics: { + submit_clear_p95_ms: summarize(clears).p95, + submit_paint_p95_ms: summarize(paints).p95, + submit_scroll_jump_max_px: Math.max(0, ...jumps) + }, + detail: { rounds, clears: summarize(clears), paints: summarize(paints) } + } + } +} diff --git a/apps/desktop/scripts/perf/scenarios/transcript.mjs b/apps/desktop/scripts/perf/scenarios/transcript.mjs new file mode 100644 index 000000000000..75b86b7d8ec7 --- /dev/null +++ b/apps/desktop/scripts/perf/scenarios/transcript.mjs @@ -0,0 +1,50 @@ +// Large-transcript mount cost. New scenario (no prior script measured this): +// loads N synthetic turns of mixed markdown into $messages and records the +// mount→paint time plus any longtasks the mount blocks the main thread with. +// This is the "open a long session" path — a first-impression latency. + +import { sleep } from '../lib/cdp.mjs' + +const OBSERVE = ` + (() => { + window.__TM__ = { longtasks: [] } + try { + const po = new PerformanceObserver((l) => { + for (const e of l.getEntries()) window.__TM__.longtasks.push(e.duration) + }) + po.observe({ entryTypes: ['longtask'] }) + window.__TM__.po = po + } catch {} + return 'observing' + })() +` + +export default { + name: 'transcript', + tier: 'ci', + description: 'Mount + paint cost of loading a long transcript.', + async run(cdp, opts = {}) { + const turns = Number(opts.turns ?? 200) + + await cdp.send('Runtime.enable') + await cdp.eval(OBSERVE) + + const mountMs = await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${turns})`) + + // Let post-mount longtasks (content-visibility passes, virtualizer) settle. + await sleep(1500) + + const longtasks = await cdp.eval('window.__TM__.longtasks') + await cdp.eval('try { window.__TM__.po && window.__TM__.po.disconnect() } catch {}') + await cdp.eval('window.__PERF_DRIVE__.reset()') + + return { + metrics: { + transcript_mount_ms: Math.round(mountMs * 10) / 10, + transcript_longtask_ms: Math.round(longtasks.reduce((a, b) => a + b, 0) * 10) / 10, + transcript_longtask_max_ms: Math.round((longtasks.length ? Math.max(...longtasks) : 0) * 10) / 10 + }, + detail: { turns, messages: turns * 2, longtasks: longtasks.length } + } + } +} diff --git a/apps/desktop/scripts/perf/serve.mjs b/apps/desktop/scripts/perf/serve.mjs new file mode 100644 index 000000000000..adf9dbfb5e5e --- /dev/null +++ b/apps/desktop/scripts/perf/serve.mjs @@ -0,0 +1,40 @@ +// Launch a standalone, fully isolated perf instance and leave it running so you +// can attach the harness (`npm run perf`) or DevTools to it. Ctrl-C tears it +// down and removes its temp dirs. +// +// npm run perf:serve # :9222, temp HERMES_HOME + user-data-dir +// PERF_PORT=9333 npm run perf:serve # custom CDP port +// +// This is the isolation seam: because it uses its own --user-data-dir the +// Electron single-instance lock never collides with a running `hgui`. + +import { startIsolatedInstance } from './lib/launch.mjs' + +const port = Number(process.env.PERF_PORT ?? 9222) +const devPort = Number(process.env.PERF_DEV_PORT ?? 5174) + +console.log(`[perf:serve] starting isolated instance (CDP :${port}, dev :${devPort})…`) + +const instance = await startIsolatedInstance({ + port, + devPort, + hermesHome: process.env.PERF_HERMES_HOME, + userDataDir: process.env.PERF_USER_DATA +}) + +console.log(`[perf:serve] READY — attach with: npm run perf -- --port ${port}`) + +let closing = false +const shutdown = () => { + if (closing) { + return + } + + closing = true + console.log('\n[perf:serve] tearing down…') + instance.teardown() + process.exit(0) +} + +process.on('SIGINT', shutdown) +process.on('SIGTERM', shutdown) diff --git a/apps/desktop/scripts/profile-long-stream.mjs b/apps/desktop/scripts/profile-long-stream.mjs deleted file mode 100644 index b0ae7922173e..000000000000 --- a/apps/desktop/scripts/profile-long-stream.mjs +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env node -// Long-running stream profile + frame-rate timeline. Submits a prompt that -// asks for ~30 paragraphs of output, then captures both a CPU profile and -// a per-100ms frame counter so we can see if FPS sags as the message grows. - -import { writeFileSync } from 'node:fs' - -const args = Object.fromEntries( - process.argv.slice(2).flatMap(s => { - const m = s.match(/^--([^=]+)(?:=(.*))?$/) - return m ? [[m[1], m[2] ?? true]] : [] - }) -) -const PORT = Number(args.port ?? 9222) -const OUT = String(args.out ?? `/tmp/hermes-long-stream-${Date.now()}`) -const STREAM_SEC = Number(args.seconds ?? 25) - -async function pickRenderer() { - const list = await (await fetch(`http://127.0.0.1:${PORT}/json/list`)).json() - return list.find(t => t.type === 'page' && t.url.startsWith('http')) -} - -function connect(url) { - return new Promise((resolve, reject) => { - const ws = new WebSocket(url) - let id = 0 - const pending = new Map() - ws.addEventListener('open', () => - resolve({ - send(method, params = {}) { - const myId = ++id - ws.send(JSON.stringify({ id: myId, method, params })) - return new Promise((res, rej) => pending.set(myId, { res, rej })) - }, - close: () => ws.close() - }) - ) - ws.addEventListener('error', reject) - ws.addEventListener('message', ev => { - const m = JSON.parse(typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8')) - if (m.id != null) { - const p = pending.get(m.id) - if (!p) return - pending.delete(m.id) - m.error ? p.rej(new Error(m.error.message)) : p.res(m.result) - } - }) - }) -} - -async function evalP(cdp, expr) { - const r = await cdp.send('Runtime.evaluate', { expression: expr, returnByValue: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.text) - return r.result.value -} - -async function main() { - const tgt = await pickRenderer() - console.log('target', tgt.url) - const cdp = await connect(tgt.webSocketDebuggerUrl) - await cdp.send('Runtime.enable') - await cdp.send('Profiler.enable') - await cdp.send('Performance.enable') - - // Submit a long-form prompt - await evalP( - cdp, - `(() => { - const el = document.querySelector('[data-slot="composer-rich-input"]') - el.focus() - const r = document.createRange(); r.selectNodeContents(el); r.collapse(false) - window.getSelection().removeAllRanges(); window.getSelection().addRange(r) - })()` - ) - const prompt = 'write 15 paragraphs about gpu memory bandwidth, memory hierarchies, roofline model, and how modern transformer inference benefits from these. include diagrams in ascii where relevant. no code. fully detailed.' - for (const c of prompt) { - await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: c, unmodifiedText: c }) - await new Promise(r => setTimeout(r, 5)) - } - await new Promise(r => setTimeout(r, 200)) - await cdp.send('Input.dispatchKeyEvent', { - type: 'rawKeyDown', windowsVirtualKeyCode: 13, key: 'Enter', code: 'Enter', text: '\r', unmodifiedText: '\r' - }) - await cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', windowsVirtualKeyCode: 13, key: 'Enter', code: 'Enter' }) - - console.log('waiting for assistant…') - let streaming = false - for (let i = 0; i < 100; i++) { - const c = await evalP(cdp, `document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length`) - if (c > 0) { streaming = true; break } - await new Promise(r => setTimeout(r, 100)) - } - if (!streaming) { - console.error('no assistant message') - cdp.close() - return - } - - // Install a per-rAF frame counter - await evalP( - cdp, - `(() => { - window.__fpsSamples = [] - window.__fpsT0 = performance.now() - window.__fpsLast = performance.now() - window.__fpsFrameCount = 0 - window.__fpsHistogram = [] // {t, fps, contentLen} - const tick = () => { - const now = performance.now() - const dt = now - window.__fpsLast - window.__fpsLast = now - window.__fpsFrameCount++ - window.__fpsSamples.push({ t: now - window.__fpsT0, dt }) - if (performance.now() - window.__fpsT0 < ${STREAM_SEC * 1000}) { - requestAnimationFrame(tick) - } - } - requestAnimationFrame(tick) - // Bucket fps every 500ms - window.__fpsBucket = setInterval(() => { - const now = performance.now() - const recentCount = window.__fpsSamples.filter(s => now - window.__fpsT0 - s.t < 500).length - const root = document.querySelector('[data-slot="aui_thread-content"]') - const len = root ? root.innerText.length : 0 - const v = document.querySelector('[data-slot="aui_thread-viewport"]') - window.__fpsHistogram.push({ - t: now - window.__fpsT0, - frames500ms: recentCount, - fps: recentCount * 2, - contentLen: len, - scrollTop: v?.scrollTop ?? 0, - scrollHeight: v?.scrollHeight ?? 0 - }) - }, 500) - })()` - ) - - // Start CPU profile - await cdp.send('Profiler.setSamplingInterval', { interval: 1000 }) - await cdp.send('Profiler.start') - - await new Promise(r => setTimeout(r, STREAM_SEC * 1000)) - - const { profile } = await cdp.send('Profiler.stop') - await evalP(cdp, `clearInterval(window.__fpsBucket)`) - - writeFileSync(`${OUT}.cpuprofile`, JSON.stringify(profile)) - console.log(`cpu profile → ${OUT}.cpuprofile`) - - // Pull fps histogram - const hist = JSON.parse(await evalP(cdp, `JSON.stringify(window.__fpsHistogram || [])`)) - writeFileSync(`${OUT}.fps.json`, JSON.stringify(hist, null, 2)) - - console.log(`\n=== FPS over time ===`) - console.log(` t(s) fps contentLen scrollTop/scrollHeight`) - for (const h of hist) { - const bar = '█'.repeat(Math.min(40, Math.max(0, Math.round(h.fps / 2)))) - console.log(` ${(h.t / 1000).toFixed(1).padStart(5)} ${String(h.fps).padStart(3)} ${String(h.contentLen).padStart(10)} ${h.scrollTop}/${h.scrollHeight} ${bar}`) - } - - // Top self frames - const total = (profile.endTime - profile.startTime) / 1000 - const intMs = total / Math.max(1, profile.samples?.length ?? 1) - const counts = new Map() - for (const s of profile.samples ?? []) counts.set(s, (counts.get(s) ?? 0) + 1) - const rows = profile.nodes - .map(n => ({ id: n.id, fn: n.callFrame.functionName || '(anon)', url: n.callFrame.url || '', line: n.callFrame.lineNumber, self: counts.get(n.id) ?? 0 })) - .sort((a, b) => b.self - a.self) - .slice(0, 25) - console.log(`\n=== ${total.toFixed(0)}ms wall, ${profile.samples?.length ?? 0} samples (${intMs.toFixed(2)}ms each) ===`) - for (const r of rows) { - if (r.self === 0) break - const url = r.url.replace(/^.*\/src\//, 'src/').replace(/\?.*$/, '').slice(0, 70) - console.log(` ${(r.self * intMs).toFixed(1).padStart(7)}ms (${String(r.self).padStart(4)} samp) ${r.fn.padEnd(45)} ${url}:${r.line}`) - } - - await evalP(cdp, ` - (() => { - for (const b of document.querySelectorAll('button')) { - if ((b.getAttribute('aria-label') || '').toLowerCase().includes('stop')) { b.click(); return } - } - })() - `) - - cdp.close() -} - -main().catch(e => { - console.error('fatal:', e.stack ?? e.message) - process.exit(1) -}) diff --git a/apps/desktop/scripts/profile-real-stream.mjs b/apps/desktop/scripts/profile-real-stream.mjs deleted file mode 100644 index cb5da652b331..000000000000 --- a/apps/desktop/scripts/profile-real-stream.mjs +++ /dev/null @@ -1,137 +0,0 @@ -// CPU-profile during a real LLM stream — confirms or refutes whether the -// synthetic stream's hotspots (Streamdown markdown re-parse, FadeText) -// match real-world content. -// -// Run *after* model is set to something fast + cheap (gpt-4o-mini etc.). -// Sends a prompt likely to produce markdown + a numbered list. - -import { writeFileSync } from 'node:fs' - -const CDP_HTTP = 'http://127.0.0.1:9222' -const PROMPT = process.env.PROMPT || 'Give me a numbered list of 8 useful bash one-liners. For each: a brief description, then the command in a code block. No preamble.' -const OUT = process.env.OUT || `/tmp/real-stream-${Date.now()}.cpuprofile` -const START_TIMEOUT = Number(process.env.START_TIMEOUT || 45000) -const STREAM_TIMEOUT = Number(process.env.STREAM_TIMEOUT || 60000) - -class CDP { - constructor(ws) { this.ws = ws; this.id = 0; this.pending = new Map() } - static async open(url) { - const ws = new WebSocket(url) - await new Promise((r) => ws.addEventListener('open', r, { once: true })) - const cdp = new CDP(ws) - ws.addEventListener('message', (ev) => { - const m = JSON.parse(ev.data.toString()) - if (m.id != null && cdp.pending.has(m.id)) { - const { resolve, reject } = cdp.pending.get(m.id) - cdp.pending.delete(m.id) - if (m.error) reject(new Error(m.error.message)) - else resolve(m.result) - } - }) - return cdp - } - send(method, params) { - const id = ++this.id - return new Promise((res, rej) => { - this.pending.set(id, { resolve: res, reject: rej }) - this.ws.send(JSON.stringify({ id, method, params })) - }) - } - async eval(expr) { - const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'eval') - return r.result.value - } - close() { this.ws.close() } -} - -async function main() { - const list = await (await fetch(`${CDP_HTTP}/json`)).json() - const target = list.find((t) => t.type === 'page' && /5174/.test(t.url)) - const cdp = await CDP.open(target.webSocketDebuggerUrl) - - const baseCount = await cdp.eval('document.querySelectorAll("[data-slot=aui_assistant-message-root]").length') - - // Submit prompt - await cdp.eval(`(() => { - const ed = document.querySelector('[contenteditable="true"]') - ed.focus() - document.execCommand('insertText', false, ${JSON.stringify(PROMPT)}) - ed.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', which: 13, keyCode: 13, bubbles: true, cancelable: true })) - return 'submitted' - })()`) - - // Wait for real stream start (assistant count grows). - const submitT0 = Date.now() - let streamT = null - for (let i = 0; i < START_TIMEOUT / 50; i++) { - await new Promise((r) => setTimeout(r, 50)) - const n = await cdp.eval('document.querySelectorAll("[data-slot=aui_assistant-message-root]").length') - if (n > baseCount) { streamT = Date.now(); break } - } - if (!streamT) { - console.error('stream never started within', START_TIMEOUT, 'ms') - cdp.close() - process.exit(2) - } - console.log('REAL stream started after', streamT - submitT0, 'ms — starting CPU profile NOW') - - // Start CPU profile NOW, only during stream phase. - await cdp.send('Profiler.enable') - await cdp.send('Profiler.setSamplingInterval', { interval: 100 }) - await cdp.send('Profiler.start') - - // Wait until busy goes false + grace, or timeout. - const cutoff = Date.now() + STREAM_TIMEOUT - while (Date.now() < cutoff) { - await new Promise((r) => setTimeout(r, 500)) - const busy = await cdp.eval('!!document.querySelector("[data-status=running], [data-busy=true]")') - if (!busy) { - await new Promise((r) => setTimeout(r, 500)) - break - } - } - - const { profile } = await cdp.send('Profiler.stop') - writeFileSync(OUT, JSON.stringify(profile)) - console.log('wrote', OUT) - - const samples = profile.samples || [] - const timeDeltas = profile.timeDeltas || [] - const nodes = new Map(profile.nodes.map((n) => [n.id, n])) - const selfTime = new Map() - for (let i = 0; i < samples.length; i++) { - const id = samples[i] - const dt = timeDeltas[i] ?? 0 - selfTime.set(id, (selfTime.get(id) || 0) + dt) - } - const ranked = [...selfTime.entries()] - .map(([id, us]) => { - const n = nodes.get(id) - const cf = n?.callFrame || {} - return { - ms: us / 1000, - name: cf.functionName || '(anonymous)', - url: (cf.url || '').slice(-60), - line: cf.lineNumber - } - }) - .filter((x) => !/\(root\)|\(idle\)|\(garbage collector\)|\(program\)/.test(x.name)) - .sort((a, b) => b.ms - a.ms) - .slice(0, 25) - - const finalText = await cdp.eval(`(() => { - const all = document.querySelectorAll('[data-slot="aui_assistant-message-root"]') - return all.length ? all[all.length-1].textContent.length : 0 - })()`) - console.log('\nfinal assistant message length:', finalText, 'chars') - - console.log('\n=== TOP 25 SELF TIME (ms) DURING REAL STREAM ===') - for (const r of ranked) { - console.log(`${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(40)} ${r.url}:${r.line}`) - } - - cdp.close() -} - -main().catch((e) => { console.error(e); process.exit(1) }) diff --git a/apps/desktop/scripts/profile-session-switch.mjs b/apps/desktop/scripts/profile-session-switch.mjs deleted file mode 100644 index e706cbbbc2df..000000000000 --- a/apps/desktop/scripts/profile-session-switch.mjs +++ /dev/null @@ -1,163 +0,0 @@ -// CPU-profile a session switch — outputs a .cpuprofile, a top-self ranking, -// longtask timings, and paint milestones for cold + warm switches. -// -// Drives the real resume path by setting location.hash (same code path as a -// sidebar click: use-route-resume → resumeSession → prefetch + resume RPC). -// -// Usage: -// node apps/desktop/scripts/profile-session-switch.mjs [rounds] -// OUT=/tmp/switch.cpuprofile node scripts/profile-session-switch.mjs 2026.. 2026.. - -import { writeFileSync } from 'node:fs' - -const CDP_HTTP = process.env.CDP_HTTP || 'http://127.0.0.1:9222' -const A = process.argv[2] -const B = process.argv[3] -const ROUNDS = Number(process.argv[4] || 2) -const OUT = process.env.OUT || `/tmp/session-switch-${Date.now()}.cpuprofile` -const SETTLE_TIMEOUT = Number(process.env.SETTLE_TIMEOUT || 30000) - -if (!A || !B) { - console.error('usage: profile-session-switch.mjs [rounds]') - process.exit(1) -} - -class CDP { - constructor(ws) { this.ws = ws; this.id = 0; this.pending = new Map() } - static async open(url) { - const ws = new WebSocket(url) - await new Promise((r) => ws.addEventListener('open', r, { once: true })) - const cdp = new CDP(ws) - ws.addEventListener('message', (ev) => { - const m = JSON.parse(ev.data.toString()) - if (m.id != null && cdp.pending.has(m.id)) { - const { resolve, reject } = cdp.pending.get(m.id) - cdp.pending.delete(m.id) - if (m.error) reject(new Error(m.error.message)) - else resolve(m.result) - } - }) - return cdp - } - send(method, params) { - const id = ++this.id - return new Promise((res, rej) => { - this.pending.set(id, { resolve: res, reject: rej }) - this.ws.send(JSON.stringify({ id, method, params })) - }) - } - async eval(expr) { - const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'eval failed') - return r.result.value - } - close() { this.ws.close() } -} - -async function main() { - const list = await (await fetch(`${CDP_HTTP}/json`)).json() - const target = list.find((t) => t.type === 'page' && /5174/.test(t.url)) - if (!target) { console.error('renderer not found on 9222'); process.exit(1) } - const cdp = await CDP.open(target.webSocketDebuggerUrl) - - // Install observers once: longtasks + rAF frame gaps, tagged per switch. - await cdp.eval(`(() => { - if (window.__SWITCH_OBS__) return 'already' - const obs = { longtasks: [], marks: [] } - new PerformanceObserver((l) => { - for (const e of l.getEntries()) obs.longtasks.push({ t: e.startTime, dur: e.duration }) - }).observe({ entryTypes: ['longtask'] }) - window.__SWITCH_OBS__ = obs - return 'installed' - })()`) - - const switchTo = async (sid, label) => { - const t0 = await cdp.eval(`(() => { - const o = window.__SWITCH_OBS__ - o.marks.push({ label: ${JSON.stringify(label)}, sid: ${JSON.stringify(sid)}, t: performance.now() }) - location.hash = '#/' + ${JSON.stringify(sid)} - return performance.now() - })()`) - - // Poll until the transcript for this session has painted and settled: - // route matches, >0 message roots, and message count stable for 3 polls. - const deadline = Date.now() + SETTLE_TIMEOUT - let stable = 0 - let lastCount = -1 - let firstPaintT = null - while (Date.now() < deadline) { - await new Promise((r) => setTimeout(r, 50)) - const s = await cdp.eval(`({ - t: performance.now(), - route: location.hash, - msgs: document.querySelectorAll('[data-slot="aui_message"], [data-slot="aui_assistant-message-root"], [data-slot="aui_user-message-root"]').length, - parts: document.querySelectorAll('[data-slot="aui_thread-content"] *').length - })`) - if (!s.route.includes(sid)) continue - if (s.msgs > 0 && firstPaintT === null) firstPaintT = s.t - stable = s.msgs === lastCount && s.msgs > 0 ? stable + 1 : 0 - lastCount = s.msgs - if (stable >= 3) return { t0, firstPaintT, settledT: s.t, msgs: s.msgs, domNodes: s.parts } - } - return { t0, firstPaintT, settledT: null, msgs: lastCount, timedOut: true } - } - - console.log('starting CPU profile') - await cdp.send('Profiler.enable') - await cdp.send('Profiler.setSamplingInterval', { interval: 100 }) - await cdp.send('Profiler.start') - - const results = [] - for (let round = 0; round < ROUNDS; round++) { - for (const [sid, tag] of [[A, 'A'], [B, 'B']]) { - const label = `round${round}:${tag}:${round === 0 ? 'cold' : 'warm'}` - const r = await switchTo(sid, label) - results.push({ label, sid, ...r }) - const ftp = r.firstPaintT != null ? (r.firstPaintT - r.t0).toFixed(0) : 'n/a' - const st = r.settledT != null ? (r.settledT - r.t0).toFixed(0) : 'TIMEOUT' - console.log(`${label.padEnd(18)} first-paint ${String(ftp).padStart(6)} ms settled ${String(st).padStart(6)} ms msgs ${r.msgs} dom ${r.domNodes ?? '?'}`) - await new Promise((r2) => setTimeout(r2, 800)) - } - } - - const { profile } = await cdp.send('Profiler.stop') - writeFileSync(OUT, JSON.stringify(profile)) - console.log('\nwrote', OUT) - - // Longtasks per switch window. - const obs = await cdp.eval('window.__SWITCH_OBS__') - console.log('\n=== LONGTASKS (>=50ms main-thread blocks) ===') - for (let i = 0; i < obs.marks.length; i++) { - const m = obs.marks[i] - const end = obs.marks[i + 1]?.t ?? Infinity - const lts = obs.longtasks.filter((lt) => lt.t >= m.t && lt.t < end) - const total = lts.reduce((a, b) => a + b.dur, 0) - console.log(`${m.label.padEnd(18)} ${String(lts.length).padStart(2)} longtasks, ${total.toFixed(0).padStart(5)} ms total ${lts.map((l) => Math.round(l.dur)).join(', ')}`) - } - - // Self-time ranking. - const samples = profile.samples || [] - const timeDeltas = profile.timeDeltas || [] - const nodes = new Map(profile.nodes.map((n) => [n.id, n])) - const selfTime = new Map() - for (let i = 0; i < samples.length; i++) { - selfTime.set(samples[i], (selfTime.get(samples[i]) || 0) + (timeDeltas[i] ?? 0)) - } - const ranked = [...selfTime.entries()] - .map(([id, us]) => { - const cf = nodes.get(id)?.callFrame || {} - return { ms: us / 1000, name: cf.functionName || '(anonymous)', url: (cf.url || '').slice(-70), line: cf.lineNumber } - }) - .filter((x) => !/\(root\)|\(idle\)|\(garbage collector\)|\(program\)/.test(x.name)) - .sort((a, b) => b.ms - a.ms) - .slice(0, 30) - - console.log('\n=== TOP 30 SELF TIME (ms) ACROSS ALL SWITCHES ===') - for (const r of ranked) { - console.log(`${r.ms.toFixed(1).padStart(8)} ${r.name.padEnd(44)} ${r.url}:${r.line}`) - } - - cdp.close() -} - -main().catch((e) => { console.error(e); process.exit(1) }) diff --git a/apps/desktop/scripts/profile-synth-stream.mjs b/apps/desktop/scripts/profile-synth-stream.mjs deleted file mode 100644 index 1cc395c1bab5..000000000000 --- a/apps/desktop/scripts/profile-synth-stream.mjs +++ /dev/null @@ -1,103 +0,0 @@ -// CPU-profile a synthetic stream — outputs a .cpuprofile and a top-self ranking. -// Open the .cpuprofile in Chrome DevTools Performance panel for a flamegraph. - -import { writeFileSync } from 'node:fs' - -const CDP_HTTP = 'http://127.0.0.1:9222' -const TOKENS = Number(process.env.TOKENS || 400) -const INTERVAL_MS = Number(process.env.INTERVAL_MS || 8) -const CHUNK = process.env.CHUNK || '**word** in _italic_ with `code` ' -const LABEL = process.env.LABEL || 'profile' -const OUT = process.env.OUT || `synth-${LABEL}.cpuprofile` - -class CDP { - constructor(ws) { this.ws = ws; this.id = 0; this.pending = new Map() } - static async open(url) { - const ws = new WebSocket(url) - await new Promise((r) => ws.addEventListener('open', r, { once: true })) - const cdp = new CDP(ws) - ws.addEventListener('message', (ev) => { - const m = JSON.parse(ev.data.toString()) - if (m.id != null && cdp.pending.has(m.id)) { - const { resolve, reject } = cdp.pending.get(m.id) - cdp.pending.delete(m.id) - if (m.error) reject(new Error(m.error.message)) - else resolve(m.result) - } - }) - return cdp - } - send(method, params) { - const id = ++this.id - return new Promise((res, rej) => { - this.pending.set(id, { resolve: res, reject: rej }) - this.ws.send(JSON.stringify({ id, method, params })) - }) - } - async eval(expr) { - const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }) - if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'eval') - return r.result.value - } - close() { this.ws.close() } -} - -async function main() { - const list = await (await fetch(`${CDP_HTTP}/json`)).json() - const target = list.find((t) => t.type === 'page' && /5174/.test(t.url)) - const cdp = await CDP.open(target.webSocketDebuggerUrl) - - if (!await cdp.eval('!!window.__PERF_DRIVE__')) { - console.error('no __PERF_DRIVE__') - cdp.close() - process.exit(2) - } - - await cdp.send('Profiler.enable') - // High-resolution sampling: 100us - await cdp.send('Profiler.setSamplingInterval', { interval: 100 }) - await cdp.send('Profiler.start') - - await cdp.eval(`window.__PERF_DRIVE__.stream({ chunk: ${JSON.stringify(CHUNK)}, intervalMs: ${INTERVAL_MS}, totalTokens: ${TOKENS} })`) - await new Promise((r) => setTimeout(r, TOKENS * INTERVAL_MS + 1500)) - await cdp.eval('window.__PERF_DRIVE__.reset()') - - const { profile } = await cdp.send('Profiler.stop') - writeFileSync(OUT, JSON.stringify(profile)) - console.log('wrote', OUT) - - // Compute top self time per function. - const samples = profile.samples || [] - const timeDeltas = profile.timeDeltas || [] - const nodes = new Map(profile.nodes.map((n) => [n.id, n])) - const selfTime = new Map() // id -> microseconds - for (let i = 0; i < samples.length; i++) { - const id = samples[i] - const dt = timeDeltas[i] ?? 0 - selfTime.set(id, (selfTime.get(id) || 0) + dt) - } - const ranked = [...selfTime.entries()] - .map(([id, us]) => { - const n = nodes.get(id) - const cf = n?.callFrame || {} - return { - us, - ms: us / 1000, - name: cf.functionName || '(anonymous)', - url: (cf.url || '').slice(-60), - line: cf.lineNumber - } - }) - .filter((x) => !/\(root\)|\(idle\)|\(garbage collector\)|\(program\)/.test(x.name)) - .sort((a, b) => b.us - a.us) - .slice(0, 30) - - console.log('\n=== TOP 30 SELF TIME (ms) ===') - for (const r of ranked) { - console.log(`${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(40)} ${r.url}:${r.line}`) - } - - cdp.close() -} - -main().catch((e) => { console.error(e); process.exit(1) }) diff --git a/apps/desktop/scripts/profile-typing-lag.md b/apps/desktop/scripts/profile-typing-lag.md index a0b09b92ab53..85ee84bba633 100644 --- a/apps/desktop/scripts/profile-typing-lag.md +++ b/apps/desktop/scripts/profile-typing-lag.md @@ -3,6 +3,13 @@ Workflow for empirically measuring (and fixing) typing/submit lag in the desktop chat composer. +> **Note (Jul 2026):** the standalone `measure-*` / `profile-*` scripts this +> doc references have been consolidated into the systematized perf harness at +> `scripts/perf/` (`npm run perf`, `npm run perf:serve`). CPU profiling is now a +> `--cpuprofile` flag on any scenario. This doc is kept as an investigation log; +> for the current tooling and the scenario→old-script mapping see +> `scripts/perf/README.md`. + ## Quick boot for profiling Vite 8 + plugin-react 6 has a known issue where the React Fast Refresh diff --git a/apps/desktop/scripts/profile-typing.mjs b/apps/desktop/scripts/profile-typing.mjs deleted file mode 100644 index f57cb40adf65..000000000000 --- a/apps/desktop/scripts/profile-typing.mjs +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env node -// Profile typing lag in the Electron renderer by: -// 1. Connecting to a running renderer via CDP (--remote-debugging-port=9222) -// 2. Focusing the composer contentEditable -// 3. Starting CPU profile + heap snapshot -// 4. Synthesizing keystrokes via Input.dispatchKeyEvent (so the run is -// reproducible, no human-typing variance) -// 5. Stopping the profile + capturing a second heap snapshot -// 6. Saving .cpuprofile + .heapsnapshot -// -// Usage: -// node apps/desktop/scripts/profile-typing.mjs -// [--port=9222] [--out=/tmp/hermes-typing] -// [--chars=400] # how many characters to type -// [--cps=30] # keystrokes per second -// [--text="..."] # override generated text -// [--no-heap] # skip heap snapshots -// [--seconds=N] # idle-record for N seconds instead of typing -// -// Zero deps — uses Node 24's global WebSocket + fetch. - -import { writeFileSync } from 'node:fs' - -const args = Object.fromEntries( - process.argv.slice(2).flatMap(s => { - const m = s.match(/^--([^=]+)(?:=(.*))?$/) - return m ? [[m[1], m[2] ?? true]] : [] - }) -) - -const PORT = Number(args.port ?? 9222) -const OUT = String(args.out ?? `/tmp/hermes-typing-${Date.now()}`) -const CHARS = Number(args.chars ?? 400) -const CPS = Number(args.cps ?? 30) -const HEAP = args['no-heap'] ? false : true -const IDLE_SECONDS = args.seconds ? Number(args.seconds) : null -const CUSTOM_TEXT = args.text === undefined || args.text === true ? null : String(args.text) - -const log = (...m) => console.log('[profile]', ...m) -const banner = m => console.log(`\n========== ${m} ==========`) - -async function pickRenderer() { - const list = await (await fetch(`http://127.0.0.1:${PORT}/json/list`)).json() - const pages = list.filter(t => t.type === 'page' && t.url.startsWith('http')) - if (!pages.length) { - console.error('No renderer page. Targets:') - list.forEach(t => console.error(' ', t.type, t.url)) - process.exit(2) - } - return pages[0] -} - -function connect(url) { - return new Promise((resolve, reject) => { - const ws = new WebSocket(url) - let id = 0 - const pending = new Map() - const events = new Map() - ws.addEventListener('open', () => - resolve({ - send(method, params = {}) { - const myId = ++id - ws.send(JSON.stringify({ id: myId, method, params })) - return new Promise((res, rej) => pending.set(myId, { res, rej })) - }, - on(method, h) { - if (!events.has(method)) events.set(method, []) - events.get(method).push(h) - }, - close: () => ws.close() - }) - ) - ws.addEventListener('error', reject) - ws.addEventListener('message', ev => { - const txt = typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8') - const m = JSON.parse(txt) - if (m.id != null) { - const p = pending.get(m.id) - if (!p) return - pending.delete(m.id) - m.error ? p.rej(new Error(m.error.message)) : p.res(m.result) - } else if (m.method) { - ;(events.get(m.method) ?? []).forEach(h => h(m.params)) - } - }) - }) -} - -async function captureHeap(cdp, path) { - log(`heap snapshot → ${path}`) - const chunks = [] - cdp.on('HeapProfiler.addHeapSnapshotChunk', ({ chunk }) => chunks.push(chunk)) - await cdp.send('HeapProfiler.enable') - await cdp.send('HeapProfiler.takeHeapSnapshot', { reportProgress: false, captureNumericValue: true }) - writeFileSync(path, chunks.join('')) - log(` ${(Buffer.byteLength(chunks.join(''), 'utf8') / 1024 / 1024).toFixed(1)} MB`) -} - -async function focusComposer(cdp) { - // Focus the rich-input contentEditable. RICH_INPUT_SLOT is the data-slot - // value used by the composer's editable div. If focus fails (no composer - // mounted yet — disabled state, etc.) the script logs and continues; the - // profile will still show idle behavior. - const result = await cdp.send('Runtime.evaluate', { - expression: ` - (() => { - const el = document.querySelector('[data-slot="composer-rich-input"]') - if (!el) return { ok: false, reason: 'composer-rich-input not found' } - el.focus() - // place caret at end - const range = document.createRange() - range.selectNodeContents(el) - range.collapse(false) - const sel = window.getSelection() - sel.removeAllRanges() - sel.addRange(range) - return { ok: true, text: el.innerText.length } - })() - `, - returnByValue: true - }) - if (!result.result.value?.ok) { - log(`focus failed: ${result.result.value?.reason ?? 'unknown'}`) - return false - } - log(`composer focused (existing text length: ${result.result.value.text})`) - return true -} - -function genText(n) { - const lorem = - 'the quick brown fox jumps over the lazy dog while the agent thinks really hard about why typing into this composer feels like wading through molasses on a hot afternoon ' - let s = '' - while (s.length < n) s += lorem - return s.slice(0, n) -} - -async function dispatchChar(cdp, ch) { - // For printable chars, char + keypress is enough — Electron treats it as text input - // and the contentEditable input event fires. For Enter / Space we could add - // specials; this run is one long line. - await cdp.send('Input.dispatchKeyEvent', { - type: 'char', - text: ch, - unmodifiedText: ch - }) -} - -async function typeText(cdp, text, cps) { - const intervalMs = Math.max(1, Math.round(1000 / cps)) - const start = Date.now() - for (let i = 0; i < text.length; i++) { - await dispatchChar(cdp, text[i]) - // Pace evenly; account for dispatch latency so we don't drift much. - const expected = start + (i + 1) * intervalMs - const wait = expected - Date.now() - if (wait > 0) await new Promise(r => setTimeout(r, wait)) - } -} - -async function main() { - log(`CDP port ${PORT}, out ${OUT}`) - const target = await pickRenderer() - log(`target ${target.url}`) - const cdp = await connect(target.webSocketDebuggerUrl) - await cdp.send('Runtime.enable') - await cdp.send('Page.enable') - await cdp.send('Profiler.enable') - - // Pre-GC so the cpu profile + heap delta are clean. - try { - await cdp.send('HeapProfiler.collectGarbage') - } catch (e) { - log('GC skipped:', e.message) - } - - if (HEAP) await captureHeap(cdp, `${OUT}.before.heapsnapshot`) - - // 1ms sampling — fine enough for per-frame React work. - await cdp.send('Profiler.setSamplingInterval', { interval: 1000 }) - - let typedText = '' - if (!IDLE_SECONDS) { - const focused = await focusComposer(cdp) - if (!focused) { - log('aborting — composer not focusable. Make sure the app is past the boot screen.') - cdp.close() - process.exit(3) - } - typedText = CUSTOM_TEXT ?? genText(CHARS) - } - - await cdp.send('Profiler.start') - - if (IDLE_SECONDS) { - banner(`IDLE recording for ${IDLE_SECONDS}s — DO NOT TOUCH`) - await new Promise(r => setTimeout(r, IDLE_SECONDS * 1000)) - } else { - banner(`TYPING ${typedText.length} chars @ ${CPS} cps (≈${(typedText.length / CPS).toFixed(1)}s)`) - const t0 = Date.now() - await typeText(cdp, typedText, CPS) - log(`typing wall time: ${((Date.now() - t0) / 1000).toFixed(2)}s`) - // Settle frame for trailing React work. - await new Promise(r => setTimeout(r, 500)) - } - - banner('STOP — saving profile') - const { profile } = await cdp.send('Profiler.stop') - writeFileSync(`${OUT}.cpuprofile`, JSON.stringify(profile)) - log(`cpu profile → ${OUT}.cpuprofile (${(JSON.stringify(profile).length / 1024 / 1024).toFixed(1)} MB)`) - - if (HEAP) { - try { - await cdp.send('HeapProfiler.collectGarbage') - } catch {} - await captureHeap(cdp, `${OUT}.after.heapsnapshot`) - } - - // Quick triage: top-self-time frames from the profile. - const top = summarizeProfile(profile) - banner('TOP SELF-TIME FRAMES') - for (const row of top.slice(0, 20)) { - console.log( - ` ${row.selfMs.toFixed(1).padStart(7)}ms ${row.functionName || '(anonymous)'}` + - ` ${row.url ? '· ' + row.url.replace(/^.*\/src\//, 'src/').slice(0, 80) : ''}` - ) - } - console.log() - log(`total samples: ${top.totalSamples}, total time: ${(top.totalMs / 1000).toFixed(2)}s`) - - cdp.close() -} - -function summarizeProfile(profile) { - // Cumulative samples = how many sampling ticks landed on each node. - // selfMs = own time only, using sampling interval. - const intervalMs = (profile.endTime - profile.startTime) / 1000 / Math.max(1, profile.samples?.length ?? 1) - const counts = new Map() - for (const s of profile.samples ?? []) counts.set(s, (counts.get(s) ?? 0) + 1) - const rows = profile.nodes.map(n => { - const self = counts.get(n.id) ?? 0 - return { - id: n.id, - functionName: n.callFrame.functionName, - url: n.callFrame.url, - lineNumber: n.callFrame.lineNumber, - selfSamples: self, - selfMs: self * intervalMs - } - }) - rows.sort((a, b) => b.selfSamples - a.selfSamples) - rows.totalSamples = (profile.samples ?? []).length - rows.totalMs = ((profile.endTime - profile.startTime) / 1000) - return rows -} - -main().catch(e => { - console.error('[profile] fatal:', e.stack ?? e.message) - process.exit(1) -}) diff --git a/apps/desktop/src/app/chat/perf-probe.tsx b/apps/desktop/src/app/chat/perf-probe.tsx index 383e4bcba939..52af1cafc676 100644 --- a/apps/desktop/src/app/chat/perf-probe.tsx +++ b/apps/desktop/src/app/chat/perf-probe.tsx @@ -24,6 +24,13 @@ declare global { __PERF_DRIVE__?: { /** Inject an assistant message and grow it by `chunk` every `intervalMs`. Returns a stop handle. */ stream: (opts?: { chunk?: string; intervalMs?: number; totalTokens?: number }) => SyntheticDriverHandle + /** + * Replace the transcript with `turns` synthetic user/assistant pairs of + * realistic mixed markdown, then resolve with the ms elapsed from the + * `setMessages` commit to the second animation frame (a mount+paint + * proxy). Used by the `transcript` perf scenario. `reset()` restores. + */ + loadTranscript: (turns?: number) => Promise reset: () => void snapshotMsgs: () => number } @@ -84,7 +91,7 @@ const onRender: ProfilerOnRenderCallback = (id, phase, actualDuration, baseDurat if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) { // Synthetic stream driver — pushes tokens through the live $messages atom so the // assistant-ui runtime + react tree sees them exactly as a real LLM stream would. - // Used by scripts/measure-real-stream.mjs when no live LLM credit is available. + // Driven by the perf harness (scripts/perf/) when no live LLM credit is available. let baseline: ReturnType | null = null let activeHandle: SyntheticDriverHandle | null = null @@ -93,8 +100,78 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) { setBusy(false) } + // One synthetic turn's worth of mixed markdown — prose, a list, a fenced + // code block, inline code, a link, and a short table — so a loaded transcript + // exercises the same render cost (Streamdown blocks, code cards) a real one + // would. Kept deterministic (seeded by index) so runs are comparable. + const syntheticTurn = (i: number): ReturnType => { + const user = { + id: `perf-u-${i}`, + role: 'user' as const, + parts: [{ type: 'text' as const, text: `Question ${i}: how does the widget in module ${i} handle back-pressure?` }], + timestamp: Date.now() + } + + const assistant = { + id: `perf-a-${i}`, + role: 'assistant' as const, + parts: [ + { + type: 'text' as const, + text: [ + `## Answer ${i}`, + '', + `The widget buffers writes and applies a bounded queue. Key points for module \`${i}\`:`, + '', + '- It coalesces bursts into a single flush.', + '- Back-pressure propagates via a `Promise` that resolves on drain.', + '- See [the design note](https://example.com/design) for the state machine.', + '', + '```ts', + `function flush${i}(items: number[]) {`, + ' return items.reduce((a, b) => a + b, 0)', + '}', + '```', + '', + '| stage | cost |', + '|---|---|', + '| enqueue | O(1) |', + '| flush | O(n) |', + '' + ].join('\n') + } + ], + timestamp: Date.now(), + pending: false + } + + return [user, assistant] + } + window.__PERF_DRIVE__ = { snapshotMsgs: () => $messages.get().length, + loadTranscript: (turns = 200) => { + if (!baseline) { + baseline = $messages.get() + } + + const next: ReturnType = [] + + for (let i = 0; i < turns; i += 1) { + next.push(...syntheticTurn(i)) + } + + const t0 = performance.now() + setMessages(next) + + return new Promise(resolve => { + requestAnimationFrame(() => { + requestAnimationFrame(() => { + resolve(performance.now() - t0) + }) + }) + }) + }, reset: () => { activeHandle?.stop()