mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
bench(desktop): trustworthy --spawn stream numbers + real baseline (#67694)
Chased the "stream frame p95 = 60ms with ZERO longtasks" mystery to its actual cause: the default stream chunk had no paragraph breaks, so it grew into one giant ~22KB block that re-rendered fully every flush — defeating the block memoization real streaming relies on. Plain text = 21ms; realistic chunk with `\n\n` breaks (blocks settle, only the tail re-renders) = 23ms. Fixed the default chunk to model real LLM output; a break-less `--chunk` remains available as a single-block worst-case stress. Also hardened the isolated instance so measurements reflect real cost: - Wait for the gateway socket to actually connect before measuring (a booting/ absent backend's reconnect backoff churns the main thread). Exposed via a new __PERF_DRIVE__.connected() probe reading $gateway.connectionState. - Focus emulation + anti-throttle/occlusion flags so a backgrounded perf window isn't frame-throttled (no OS focus stealing). - Generation-guarded the rAF frame recorder so repeated runs don't leave overlapping recorders polluting frame intervals. Baseline re-captured as the median of 5 --spawn runs (darwin-arm64); all three CI scenarios now green and stable. Absolute values are dev-build (noted in _meta) — regression guards, not shipped numbers.
This commit is contained in:
brooklyn!
GitHub
parent
0d2ad3993e
commit
dd418284db
4 changed files with 123 additions and 27 deletions
|
|
@ -1,37 +1,46 @@
|
|||
{
|
||||
"_meta": {
|
||||
"note": "Captured on darwin-arm64 via `npm run perf -- --spawn`. keystroke + transcript are clean medians. stream = a clean single-run capture; under --spawn the isolated backend may not connect, and its reconnect churn inflates frame-pacing, so re-capture stream on a backend-CONNECTED instance (perf:serve against a live profile, or attach) for tighter tolerances. Re-baseline with `npm run perf -- --update-baseline`.",
|
||||
"note": "Median of 5 runs, darwin-arm64, `npm run perf -- --spawn` (dev renderer, gateway waited-for-connected, realistic block-settling stream chunk). Absolute values are dev-build (React dev mode is ~3x prod) so treat them as regression guards, not shipped numbers; re-baseline per device with `npm run perf -- --update-baseline`. Tolerances are loose to absorb cross-machine variance.",
|
||||
"platform": "darwin-arm64",
|
||||
"node": null,
|
||||
"updated": "2026-07-19"
|
||||
"node": "v24.11.0",
|
||||
"updated": "2026-07-19T21:24:19.047Z"
|
||||
},
|
||||
"scenarios": {
|
||||
"stream": {
|
||||
"tolerance": { "tolFrac": 0.6, "tolAbs": 5 },
|
||||
"tolerance": {
|
||||
"tolFrac": 0.6,
|
||||
"tolAbs": 5
|
||||
},
|
||||
"metrics": {
|
||||
"longtasks_n": 2,
|
||||
"longtask_max_ms": 97,
|
||||
"frame_p95_ms": 18.4,
|
||||
"frame_p99_ms": 28.5,
|
||||
"slow_frames_33": 4,
|
||||
"intermut_p95_ms": 37.1
|
||||
"longtasks_n": 1,
|
||||
"longtask_max_ms": 145,
|
||||
"frame_p95_ms": 23.5,
|
||||
"frame_p99_ms": 26.9,
|
||||
"slow_frames_33": 1,
|
||||
"intermut_p95_ms": 48.2
|
||||
}
|
||||
},
|
||||
"keystroke": {
|
||||
"tolerance": { "tolFrac": 0.6, "tolAbs": 4 },
|
||||
"tolerance": {
|
||||
"tolFrac": 0.6,
|
||||
"tolAbs": 4
|
||||
},
|
||||
"metrics": {
|
||||
"keystroke_p50_ms": 2.7,
|
||||
"keystroke_p95_ms": 9.9,
|
||||
"keystroke_p99_ms": 17.7,
|
||||
"keystroke_p50_ms": 2,
|
||||
"keystroke_p95_ms": 8.2,
|
||||
"keystroke_p99_ms": 17.4,
|
||||
"keystroke_slow_16": 2
|
||||
}
|
||||
},
|
||||
"transcript": {
|
||||
"tolerance": { "tolFrac": 0.75, "tolAbs": 40 },
|
||||
"tolerance": {
|
||||
"tolFrac": 0.75,
|
||||
"tolAbs": 40
|
||||
},
|
||||
"metrics": {
|
||||
"transcript_mount_ms": 287.3,
|
||||
"transcript_longtask_ms": 559,
|
||||
"transcript_longtask_max_ms": 295
|
||||
"transcript_mount_ms": 280.1,
|
||||
"transcript_longtask_ms": 431,
|
||||
"transcript_longtask_max_ms": 221
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,6 +83,28 @@ function resolveViteBin() {
|
|||
return join(dirname(pkgPath), rel)
|
||||
}
|
||||
|
||||
// Poll the perf driver's `connected()` until the gateway socket is open.
|
||||
// Returns false if the probe predates this helper or the timeout elapses.
|
||||
async function waitForConnected(cdp, timeoutMs) {
|
||||
const hasProbe = await cdp.eval('typeof window.__PERF_DRIVE__.connected === "function"')
|
||||
|
||||
if (!hasProbe) {
|
||||
return false
|
||||
}
|
||||
|
||||
const deadline = Date.now() + timeoutMs
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (await cdp.eval('window.__PERF_DRIVE__.connected()')) {
|
||||
return true
|
||||
}
|
||||
|
||||
await sleep(500)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
function runNode(scriptRelPath, args = []) {
|
||||
return new Promise((resolveRun, reject) => {
|
||||
const child = spawn(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args], {
|
||||
|
|
@ -114,7 +136,8 @@ export async function startIsolatedInstance({
|
|||
userDataDir,
|
||||
seedConfig = true,
|
||||
bootFakeStepMs = 120,
|
||||
settleMs = 2500
|
||||
settleMs = 2500,
|
||||
connectTimeoutMs = 90000
|
||||
} = {}) {
|
||||
const children = []
|
||||
const tempDirs = []
|
||||
|
|
@ -173,7 +196,21 @@ export async function startIsolatedInstance({
|
|||
const electronBin = require('electron')
|
||||
const electron = spawn(
|
||||
electronBin,
|
||||
['.', `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`],
|
||||
[
|
||||
'.',
|
||||
`--user-data-dir=${userData}`,
|
||||
`--remote-debugging-port=${port}`,
|
||||
// The perf window usually opens behind the user's other windows, and
|
||||
// Chromium throttles frame production for backgrounded/occluded windows
|
||||
// (~17fps), which shows up as choppy frames with ZERO longtasks and
|
||||
// wrecks the stream frame-pacing metric. Disable every throttle path so
|
||||
// measurements reflect real render cost regardless of window state
|
||||
// (CalculateNativeWinOcclusion is the macOS/Windows occlusion detector).
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
'--disable-features=CalculateNativeWinOcclusion'
|
||||
],
|
||||
{
|
||||
cwd: DESKTOP_DIR,
|
||||
stdio: ['ignore', 'inherit', 'inherit'],
|
||||
|
|
@ -209,12 +246,37 @@ export async function startIsolatedInstance({
|
|||
{ timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' }
|
||||
)
|
||||
|
||||
// Let cold-start contention (vite dep pre-bundling, first backend-connect
|
||||
// attempts, initial paint) drain before scenarios measure, so numbers
|
||||
// reflect steady state rather than launch noise.
|
||||
// Electron throttles rAF/timers for a window that isn't foregrounded
|
||||
// (per-window backgroundThrottling, which the Chromium CLI flags above don't
|
||||
// override). Focus emulation makes the renderer behave as if focused so
|
||||
// frame-pacing measurements are real even though the perf window sits behind
|
||||
// the user's other windows — WITHOUT actually stealing OS focus.
|
||||
try {
|
||||
// Behave as if focused so frame-pacing isn't throttled while the perf
|
||||
// window sits behind the user's IDE/terminal — WITHOUT stealing OS focus.
|
||||
await cdp.send('Emulation.setFocusEmulationEnabled', { enabled: true })
|
||||
} catch {
|
||||
// Older CDP / not supported — fall back to the anti-throttle flags above.
|
||||
}
|
||||
|
||||
// Wait for the gateway socket to actually open. A booting/absent backend
|
||||
// retries on a 1–15s backoff, and that churn contaminates frame-pacing
|
||||
// (the `stream` scenario). Best-effort: proceed after the timeout so the
|
||||
// backend-independent scenarios (keystroke, transcript) still run.
|
||||
const connected = await waitForConnected(cdp, connectTimeoutMs)
|
||||
|
||||
if (!connected) {
|
||||
console.warn(
|
||||
`[perf] gateway did not connect within ${connectTimeoutMs}ms — ` +
|
||||
'stream/frame numbers may be inflated by reconnect churn.'
|
||||
)
|
||||
}
|
||||
|
||||
// Let residual cold-start work (vite dep pre-bundling, initial paint) drain.
|
||||
await sleep(settleMs)
|
||||
|
||||
return {
|
||||
connected,
|
||||
cdp,
|
||||
devUrl,
|
||||
port,
|
||||
|
|
|
|||
|
|
@ -10,10 +10,16 @@ import { frameHistogram, percentile } from '../lib/stats.mjs'
|
|||
|
||||
const RECORDERS = `
|
||||
(() => {
|
||||
// Generation guard: a prior run's rAF loop re-reads window.__FT__ each frame,
|
||||
// so simply reassigning it would leave the old loop running and pushing into
|
||||
// the new array (overlapping recorders inflate frame intervals on run 2+).
|
||||
// Bumping the generation makes every stale loop exit on its next tick.
|
||||
window.__FT_GEN__ = (window.__FT_GEN__ || 0) + 1
|
||||
const ftGen = window.__FT_GEN__
|
||||
window.__FT__ = { times: [], stop: false }
|
||||
let last = performance.now()
|
||||
const tick = () => {
|
||||
if (window.__FT__.stop) return
|
||||
if (window.__FT_GEN__ !== ftGen || window.__FT__.stop) return
|
||||
const now = performance.now()
|
||||
window.__FT__.times.push(now - last)
|
||||
last = now
|
||||
|
|
@ -113,9 +119,14 @@ export default {
|
|||
const tokens = Number(opts.tokens ?? 400)
|
||||
const intervalMs = Number(opts.intervalMs ?? 16)
|
||||
const flushMinMs = Number(opts.flushMinMs ?? 33)
|
||||
// No raw autolink in the default chunk — a resolvable URL triggers link
|
||||
// embeds / DNS lookups that add noise unrelated to render cost.
|
||||
const chunk = opts.chunk ?? '**word** in _italic_ with `code` and a bit of ordinary prose. '
|
||||
// Realistic default: a short markdown paragraph ending in a blank line, so
|
||||
// blocks SETTLE as they stream — exactly how real LLM output behaves, and
|
||||
// what block-memoization is designed for (only the growing tail re-renders).
|
||||
// A chunk with NO paragraph break (e.g. `--chunk 'word '`) instead grows one
|
||||
// ever-larger block that re-renders fully every flush — a useful worst-case
|
||||
// stress, but not the typical number. No raw autolink (avoids DNS/link-embed
|
||||
// noise unrelated to render cost).
|
||||
const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n'
|
||||
const real = Boolean(opts.real)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { Profiler, type ProfilerOnRenderCallback, type ReactNode } from 'react'
|
||||
|
||||
import { $gateway } from '@/store/gateway'
|
||||
import { $messages, setBusy, setMessages } from '@/store/session'
|
||||
|
||||
type Sample = {
|
||||
|
|
@ -31,6 +32,12 @@ declare global {
|
|||
* proxy). Used by the `transcript` perf scenario. `reset()` restores.
|
||||
*/
|
||||
loadTranscript: (turns?: number) => Promise<number>
|
||||
/**
|
||||
* Whether the active gateway socket is open. The perf harness waits on
|
||||
* this before measuring so background reconnect churn (a booting/absent
|
||||
* backend) doesn't contaminate frame-pacing numbers.
|
||||
*/
|
||||
connected: () => boolean
|
||||
reset: () => void
|
||||
snapshotMsgs: () => number
|
||||
}
|
||||
|
|
@ -152,6 +159,13 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) {
|
|||
|
||||
window.__PERF_DRIVE__ = {
|
||||
snapshotMsgs: () => $messages.get().length,
|
||||
connected: () => {
|
||||
try {
|
||||
return $gateway.get()?.connectionState === 'open'
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
},
|
||||
loadTranscript: (turns = 200) => {
|
||||
if (!baseline) {
|
||||
baseline = $messages.get()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue