mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
bench(desktop): measure the full picture — prod build, cold-start, first-token (#67697)
Stop drip-feeding scenarios: extend the harness to cover the latencies that actually dominate perceived speed, and measure them on a REAL production build. - --prod: build a production renderer with the probe included (VITE_PERF_PROBE=1, off in normal builds) and launch it from dist/. Measures minified React, so numbers are representative shipped figures instead of ~3x-inflated dev ones. - cold-start scenario (tier "cold"): launch → CDP → driver → first paint, via a fresh isolated spawn per run. Captures spawn_to_cdp_ms, spawn_to_driver_ms, fcp_ms. - first-token scenario (backend tier): Enter → first assistant token painted — the TTFT latency an agent app is uniquely judged on. - run.mjs gained --prod (build once), cold-start fresh-spawn loop, and gates ci+cold tiers against the baseline. Baseline re-captured on a PRODUCTION build (median of 5), darwin-arm64 — all green. Representative numbers: cold-start spawn→interactive ~1.6s, FCP ~0.5s stream frame p95 22ms, 1 longtask keystroke p50 2ms, p95 8.7ms transcript mount 145ms, 82ms longtask (400-msg open) The prod build also settled the open question from the dev numbers: the transcript-mount "lead" (221ms longtask in dev) is only ~72-82ms in prod — not actionable. Measurement did its job.
This commit is contained in:
brooklyn!
GitHub
parent
dd418284db
commit
b1fb3c5285
8 changed files with 358 additions and 126 deletions
|
|
@ -19,10 +19,21 @@ npm run perf # attaches, runs the CI suite, gates on baseline
|
|||
# One scenario, with a CPU profile:
|
||||
npm run perf -- stream --cpuprofile --tokens 800
|
||||
|
||||
# Representative PRODUCTION numbers (minified React, not the ~3x-slower dev build):
|
||||
npm run perf -- cold-start stream keystroke transcript --spawn --prod
|
||||
|
||||
# Re-capture the baseline on your reference device, then commit baseline.json:
|
||||
npm run perf -- --update-baseline
|
||||
npm run perf -- cold-start stream keystroke transcript --spawn --prod --update-baseline
|
||||
```
|
||||
|
||||
## Dev vs prod
|
||||
|
||||
By default the harness measures the **dev** renderer (fast to spin up, good for
|
||||
relative regression checks). Pass `--prod` (with `--spawn`) to build a
|
||||
production renderer *with the probe included* (`VITE_PERF_PROBE=1`) and measure
|
||||
minified React — the representative shipped numbers. The committed baseline is
|
||||
captured with `--prod`.
|
||||
|
||||
## Why isolation matters
|
||||
|
||||
The measurement this harness exists to run was historically blocked: a running
|
||||
|
|
@ -40,13 +51,16 @@ directly via `window.__PERF_DRIVE__`, so no LLM credits are spent.
|
|||
| `stream --real` | backend | same, from a real LLM stream | measure-real-stream, profile-real-stream |
|
||||
| `keystroke` | ci | composer keystroke → paint latency | measure-latency, profile-typing, leak-typing |
|
||||
| `transcript` | ci | large-transcript mount + paint cost | (new) |
|
||||
| `cold-start` | cold | launch → CDP → driver → first paint (fresh spawn/run) | (new) |
|
||||
| `first-token` | backend | Enter → first assistant token painted (TTFT) | (new) |
|
||||
| `submit` | backend | Enter → cleared → user msg painted, scroll jump | measure-submit, measure-jump |
|
||||
| `session-switch` | backend | route → first-paint → settle | profile-session-switch |
|
||||
| `profile-switch` | backend | rail click → sidebar settled | measure-profile-switch |
|
||||
|
||||
`ci` scenarios need no backend/credits and are gated against `baseline.json`.
|
||||
`backend` scenarios need a live backend (and `--spawn` or a real session) and
|
||||
are report-only.
|
||||
`ci` + `cold` scenarios need no backend/credits and are gated against
|
||||
`baseline.json` (`cold-start` requires `--spawn` since it measures a fresh
|
||||
launch, and must be run in its own invocation). `backend` scenarios need a live
|
||||
backend (and `--spawn` or a real session/credits) and are report-only.
|
||||
|
||||
CPU profiling is a cross-cutting `--cpuprofile` flag on any scenario (it wraps
|
||||
the run in `Profiler.start/stop` and prints a top-self-time table), replacing
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"_meta": {
|
||||
"note": "Median of 5 runs, darwin-arm64, `npm run perf -- --spawn` (dev renderer, gateway waited-for-connected, realistic block-settling stream chunk). Absolute values are dev-build (React dev mode is ~3x prod) so treat them as regression guards, not shipped numbers; re-baseline per device with `npm run perf -- --update-baseline`. Tolerances are loose to absorb cross-machine variance.",
|
||||
"note": "Median of 5 runs, darwin-arm64, `npm run perf -- cold-start stream keystroke transcript --spawn --prod` — a PRODUCTION renderer (minified React), so these are representative shipped numbers, not dev-inflated. Re-baseline per device with the same command + `--update-baseline`. Tolerances are loose to absorb cross-machine variance; cold-start especially varies with disk/OS state.",
|
||||
"platform": "darwin-arm64",
|
||||
"node": "v24.11.0",
|
||||
"updated": "2026-07-19T21:24:19.047Z"
|
||||
"updated": "2026-07-19T21:38:19.701Z"
|
||||
},
|
||||
"scenarios": {
|
||||
"stream": {
|
||||
|
|
@ -13,11 +13,11 @@
|
|||
},
|
||||
"metrics": {
|
||||
"longtasks_n": 1,
|
||||
"longtask_max_ms": 145,
|
||||
"frame_p95_ms": 23.5,
|
||||
"frame_p99_ms": 26.9,
|
||||
"longtask_max_ms": 67,
|
||||
"frame_p95_ms": 22,
|
||||
"frame_p99_ms": 23.7,
|
||||
"slow_frames_33": 1,
|
||||
"intermut_p95_ms": 48.2
|
||||
"intermut_p95_ms": 36.1
|
||||
}
|
||||
},
|
||||
"keystroke": {
|
||||
|
|
@ -26,9 +26,9 @@
|
|||
"tolAbs": 4
|
||||
},
|
||||
"metrics": {
|
||||
"keystroke_p50_ms": 2,
|
||||
"keystroke_p95_ms": 8.2,
|
||||
"keystroke_p99_ms": 17.4,
|
||||
"keystroke_p50_ms": 2.1,
|
||||
"keystroke_p95_ms": 8.7,
|
||||
"keystroke_p99_ms": 16.9,
|
||||
"keystroke_slow_16": 2
|
||||
}
|
||||
},
|
||||
|
|
@ -38,9 +38,20 @@
|
|||
"tolAbs": 40
|
||||
},
|
||||
"metrics": {
|
||||
"transcript_mount_ms": 280.1,
|
||||
"transcript_longtask_ms": 431,
|
||||
"transcript_longtask_max_ms": 221
|
||||
"transcript_mount_ms": 145,
|
||||
"transcript_longtask_ms": 82,
|
||||
"transcript_longtask_max_ms": 82
|
||||
}
|
||||
},
|
||||
"cold-start": {
|
||||
"tolerance": {
|
||||
"tolFrac": 0.6,
|
||||
"tolAbs": 150
|
||||
},
|
||||
"metrics": {
|
||||
"spawn_to_cdp_ms": 326,
|
||||
"spawn_to_driver_ms": 1647,
|
||||
"fcp_ms": 500
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,17 +105,32 @@ async function waitForConnected(cdp, timeoutMs) {
|
|||
return false
|
||||
}
|
||||
|
||||
function runNode(scriptRelPath, args = []) {
|
||||
function runProcess(command, args, { env } = {}) {
|
||||
return new Promise((resolveRun, reject) => {
|
||||
const child = spawn(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args], {
|
||||
const child = spawn(command, args, {
|
||||
cwd: DESKTOP_DIR,
|
||||
stdio: 'inherit'
|
||||
stdio: 'inherit',
|
||||
env: env ? { ...process.env, ...env } : process.env
|
||||
})
|
||||
child.on('error', reject)
|
||||
child.on('exit', code => (code === 0 ? resolveRun() : reject(new Error(`${scriptRelPath} exited ${code}`))))
|
||||
child.on('exit', code => (code === 0 ? resolveRun() : reject(new Error(`${command} ${args[0]} exited ${code}`))))
|
||||
})
|
||||
}
|
||||
|
||||
function runNode(scriptRelPath, args = []) {
|
||||
return runProcess(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args])
|
||||
}
|
||||
|
||||
// Build a production renderer WITH the perf probe included (VITE_PERF_PROBE=1),
|
||||
// plus the prod electron-main bundle, so the harness can measure a real,
|
||||
// minified React build instead of the ~3x-slower dev build. Slow (a full vite
|
||||
// build); do it once, then run/attach many times.
|
||||
export async function buildProdRenderer() {
|
||||
const viteBin = resolveViteBin()
|
||||
await runProcess(process.execPath, [viteBin, 'build'], { env: { VITE_PERF_PROBE: '1' } })
|
||||
await runNode('scripts/bundle-electron-main.mjs')
|
||||
}
|
||||
|
||||
/** Attach to a renderer already listening on `port` (launched via perf:serve or with --remote-debugging-port). */
|
||||
export async function attach({ port = 9222, match } = {}) {
|
||||
const cdp = await CDP.connect({ port, match })
|
||||
|
|
@ -129,9 +144,29 @@ export async function attach({ port = 9222, match } = {}) {
|
|||
* and return `{ cdp, teardown, devUrl, port }`. `teardown` kills both children
|
||||
* and removes any temp dirs it created.
|
||||
*/
|
||||
// Chromium switches that stop frame-production throttling for a window that
|
||||
// isn't foregrounded (the perf window usually sits behind the IDE/terminal).
|
||||
const ANTI_THROTTLE_FLAGS = [
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
'--disable-features=CalculateNativeWinOcclusion'
|
||||
]
|
||||
|
||||
/**
|
||||
* Spawn an isolated instance and connect the perf driver. Two render modes:
|
||||
* · dev (default): vite dev server + dev electron-main bundle.
|
||||
* · prod (`prod: true`): a production build (call buildProdRenderer first);
|
||||
* electron loads dist/index.html — representative, minified React.
|
||||
* `coldStart: true` skips the gateway-connect wait and settle (for launch-time
|
||||
* measurement) and returns `timings` (spawn→CDP, spawn→driver) plus renderer
|
||||
* boot marks (FCP, time-to-composer).
|
||||
*/
|
||||
export async function startIsolatedInstance({
|
||||
port = 9222,
|
||||
devPort = 5174,
|
||||
prod = false,
|
||||
coldStart = false,
|
||||
hermesHome,
|
||||
userDataDir,
|
||||
seedConfig = true,
|
||||
|
|
@ -151,9 +186,8 @@ export async function startIsolatedInstance({
|
|||
|
||||
const home = hermesHome ?? mkTemp('hermes-perf-home-')
|
||||
const userData = userDataDir ?? mkTemp('hermes-perf-ud-')
|
||||
const devUrl = `http://127.0.0.1:${devPort}`
|
||||
const devUrl = prod ? null : `http://127.0.0.1:${devPort}`
|
||||
|
||||
// Only seed a temp home we created — never scribble into a user-provided one.
|
||||
if (seedConfig && !hermesHome) {
|
||||
seedConfigFrom(join(homedir(), '.hermes'), home)
|
||||
}
|
||||
|
|
@ -177,61 +211,56 @@ export async function startIsolatedInstance({
|
|||
}
|
||||
|
||||
try {
|
||||
// 1. Renderer: reuse an already-running dev server, else start one.
|
||||
if (!(await reachable(devUrl))) {
|
||||
const viteBin = resolveViteBin()
|
||||
const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], {
|
||||
cwd: DESKTOP_DIR,
|
||||
stdio: ['ignore', 'inherit', 'inherit']
|
||||
})
|
||||
children.push(vite)
|
||||
await waitFor(() => reachable(devUrl), { timeoutMs: 60000, label: `vite dev server on :${devPort}` })
|
||||
if (prod) {
|
||||
// Renderer + main are expected pre-built (buildProdRenderer). Cheap to
|
||||
// re-bundle main so an isolated run always matches current source.
|
||||
await runNode('scripts/bundle-electron-main.mjs')
|
||||
} else {
|
||||
if (!(await reachable(devUrl))) {
|
||||
const viteBin = resolveViteBin()
|
||||
const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], {
|
||||
cwd: DESKTOP_DIR,
|
||||
stdio: ['ignore', 'inherit', 'inherit']
|
||||
})
|
||||
children.push(vite)
|
||||
await waitFor(() => reachable(devUrl), { timeoutMs: 60000, label: `vite dev server on :${devPort}` })
|
||||
}
|
||||
|
||||
await runNode('scripts/bundle-electron-main.mjs', ['--dev'])
|
||||
}
|
||||
|
||||
// 2. Electron main bundle (dev variant) — same step the dev script runs.
|
||||
await runNode('scripts/bundle-electron-main.mjs', ['--dev'])
|
||||
|
||||
// 3. Isolated Electron. --user-data-dir gives it its own single-instance
|
||||
// lock scope; HERMES_HOME gives it its own backend + sessions.
|
||||
// Isolated Electron: own --user-data-dir (single-instance lock scope) + own
|
||||
// HERMES_HOME (backend + sessions). No DEV_SERVER env in prod → dist load.
|
||||
const electronBin = require('electron')
|
||||
const env = {
|
||||
...process.env,
|
||||
HERMES_HOME: home,
|
||||
HERMES_DESKTOP_BOOT_FAKE: '1',
|
||||
HERMES_DESKTOP_BOOT_FAKE_STEP_MS: String(bootFakeStepMs),
|
||||
XCURSOR_SIZE: '24'
|
||||
}
|
||||
|
||||
if (devUrl) {
|
||||
env.HERMES_DESKTOP_DEV_SERVER = devUrl
|
||||
}
|
||||
|
||||
const spawnAt = Date.now()
|
||||
const electron = spawn(
|
||||
electronBin,
|
||||
[
|
||||
'.',
|
||||
`--user-data-dir=${userData}`,
|
||||
`--remote-debugging-port=${port}`,
|
||||
// The perf window usually opens behind the user's other windows, and
|
||||
// Chromium throttles frame production for backgrounded/occluded windows
|
||||
// (~17fps), which shows up as choppy frames with ZERO longtasks and
|
||||
// wrecks the stream frame-pacing metric. Disable every throttle path so
|
||||
// measurements reflect real render cost regardless of window state
|
||||
// (CalculateNativeWinOcclusion is the macOS/Windows occlusion detector).
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
'--disable-features=CalculateNativeWinOcclusion'
|
||||
],
|
||||
{
|
||||
cwd: DESKTOP_DIR,
|
||||
stdio: ['ignore', 'inherit', 'inherit'],
|
||||
env: {
|
||||
...process.env,
|
||||
HERMES_HOME: home,
|
||||
HERMES_DESKTOP_DEV_SERVER: devUrl,
|
||||
HERMES_DESKTOP_BOOT_FAKE: '1',
|
||||
HERMES_DESKTOP_BOOT_FAKE_STEP_MS: String(bootFakeStepMs),
|
||||
XCURSOR_SIZE: '24'
|
||||
}
|
||||
}
|
||||
['.', `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`, ...ANTI_THROTTLE_FLAGS],
|
||||
{ cwd: DESKTOP_DIR, stdio: ['ignore', 'inherit', 'inherit'], env }
|
||||
)
|
||||
children.push(electron)
|
||||
|
||||
// 4. Wait for the renderer + the perf driver to be live.
|
||||
// Wait for the renderer + perf driver. In prod the target URL is file://,
|
||||
// so don't match on the dev port.
|
||||
let cdp = null
|
||||
let cdpAt = 0
|
||||
await waitFor(
|
||||
async () => {
|
||||
try {
|
||||
cdp = await CDP.connect({ port, match: String(devPort), timeoutMs: 2000 })
|
||||
cdp = await CDP.connect({ port, match: devUrl ? String(devPort) : undefined, timeoutMs: 2000 })
|
||||
cdpAt = cdpAt || Date.now()
|
||||
|
||||
return await cdp.eval('!!(window.__PERF_DRIVE__ && window.__PERF_DRIVE__.stream)')
|
||||
} catch {
|
||||
|
|
@ -245,41 +274,46 @@ export async function startIsolatedInstance({
|
|||
},
|
||||
{ timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' }
|
||||
)
|
||||
const driverAt = Date.now()
|
||||
|
||||
// Electron throttles rAF/timers for a window that isn't foregrounded
|
||||
// (per-window backgroundThrottling, which the Chromium CLI flags above don't
|
||||
// override). Focus emulation makes the renderer behave as if focused so
|
||||
// frame-pacing measurements are real even though the perf window sits behind
|
||||
// the user's other windows — WITHOUT actually stealing OS focus.
|
||||
try {
|
||||
// Behave as if focused so frame-pacing isn't throttled while the perf
|
||||
// window sits behind the user's IDE/terminal — WITHOUT stealing OS focus.
|
||||
await cdp.send('Emulation.setFocusEmulationEnabled', { enabled: true })
|
||||
} catch {
|
||||
// Older CDP / not supported — fall back to the anti-throttle flags above.
|
||||
// Older CDP / not supported — fall back to the anti-throttle flags.
|
||||
}
|
||||
|
||||
// Wait for the gateway socket to actually open. A booting/absent backend
|
||||
// retries on a 1–15s backoff, and that churn contaminates frame-pacing
|
||||
// (the `stream` scenario). Best-effort: proceed after the timeout so the
|
||||
// backend-independent scenarios (keystroke, transcript) still run.
|
||||
const connected = await waitForConnected(cdp, connectTimeoutMs)
|
||||
|
||||
if (!connected) {
|
||||
console.warn(
|
||||
`[perf] gateway did not connect within ${connectTimeoutMs}ms — ` +
|
||||
'stream/frame numbers may be inflated by reconnect churn.'
|
||||
)
|
||||
// Renderer-side boot marks (relative to its own navigation start).
|
||||
const bootMarks = await readBootMarks(cdp)
|
||||
const timings = {
|
||||
spawn_to_cdp_ms: cdpAt ? cdpAt - spawnAt : null,
|
||||
spawn_to_driver_ms: driverAt - spawnAt,
|
||||
...bootMarks
|
||||
}
|
||||
|
||||
// Let residual cold-start work (vite dep pre-bundling, initial paint) drain.
|
||||
await sleep(settleMs)
|
||||
let connected = true
|
||||
|
||||
if (!coldStart) {
|
||||
// Steady-state scenarios: wait for the gateway to connect (reconnect churn
|
||||
// contaminates frame pacing) and let residual cold-start work drain.
|
||||
connected = await waitForConnected(cdp, connectTimeoutMs)
|
||||
|
||||
if (!connected) {
|
||||
console.warn(
|
||||
`[perf] gateway did not connect within ${connectTimeoutMs}ms — ` +
|
||||
'stream/frame numbers may be inflated by reconnect churn.'
|
||||
)
|
||||
}
|
||||
|
||||
await sleep(settleMs)
|
||||
}
|
||||
|
||||
return {
|
||||
connected,
|
||||
cdp,
|
||||
devUrl,
|
||||
port,
|
||||
prod,
|
||||
timings,
|
||||
teardown: () => {
|
||||
cdp?.close()
|
||||
teardown()
|
||||
|
|
@ -291,4 +325,25 @@ export async function startIsolatedInstance({
|
|||
}
|
||||
}
|
||||
|
||||
// Read First Contentful Paint + time-to-composer from the renderer, relative to
|
||||
// its navigation start (the process-spawn deltas live in `timings`).
|
||||
async function readBootMarks(cdp) {
|
||||
try {
|
||||
return await cdp.eval(`(() => {
|
||||
const paints = performance.getEntriesByType('paint')
|
||||
const fcp = paints.find(p => p.name === 'first-contentful-paint')
|
||||
const composer = document.querySelector('[data-slot="composer-rich-input"]')
|
||||
return {
|
||||
fcp_ms: fcp ? Math.round(fcp.startTime) : null,
|
||||
// performance.now() at read time ≈ time since nav start; only meaningful
|
||||
// right after boot (cold-start reads it immediately).
|
||||
nav_to_read_ms: Math.round(performance.now()),
|
||||
composer_present: !!composer
|
||||
}
|
||||
})()`)
|
||||
} catch {
|
||||
return { fcp_ms: null, nav_to_read_ms: null, composer_present: false }
|
||||
}
|
||||
}
|
||||
|
||||
export { DESKTOP_DIR }
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import { fileURLToPath } from 'node:url'
|
|||
|
||||
import { withCpuProfile } from './lib/cdp.mjs'
|
||||
import { compareScenario, loadBaseline, updateBaseline } from './lib/baseline.mjs'
|
||||
import { attach, startIsolatedInstance } from './lib/launch.mjs'
|
||||
import { attach, buildProdRenderer, startIsolatedInstance } from './lib/launch.mjs'
|
||||
import { cpuProfileTopSelf, median } from './lib/stats.mjs'
|
||||
import { CI_SCENARIOS, SCENARIOS } from './scenarios/index.mjs'
|
||||
|
||||
|
|
@ -111,52 +111,94 @@ async function main() {
|
|||
const runs = Number(flags.runs ?? 1)
|
||||
const port = Number(flags.port ?? 9222)
|
||||
const devPort = Number(flags['dev-port'] ?? 5174)
|
||||
const prod = 'prod' in flags
|
||||
const cpuProfile = 'cpuprofile' in flags
|
||||
const cpuProfileDir = typeof flags.cpuprofile === 'string' ? flags.cpuprofile : HERE
|
||||
|
||||
const connection = flags.spawn
|
||||
? await startIsolatedInstance({ port, devPort })
|
||||
: await attach({ port, match: String(devPort) })
|
||||
const coldNames = names.filter(n => SCENARIOS[n].tier === 'cold')
|
||||
const liveNames = names.filter(n => SCENARIOS[n].tier !== 'cold')
|
||||
|
||||
const { cdp, teardown } = connection
|
||||
// ci + cold metrics are stable enough to gate against the baseline; backend
|
||||
// scenarios vary too much with the live environment, so they're report-only.
|
||||
const GATED = new Set(['ci', 'cold'])
|
||||
const baseline = loadBaseline(BASELINE_PATH)
|
||||
const results = []
|
||||
let regressed = false
|
||||
|
||||
try {
|
||||
const baseline = loadBaseline(BASELINE_PATH)
|
||||
const record = (name, tier, metrics, detail) => {
|
||||
const comparison = GATED.has(tier) ? compareScenario(name, metrics, baseline) : null
|
||||
regressed = regressed || Boolean(comparison?.regressed)
|
||||
results.push({ name, tier, metrics, detail })
|
||||
printMetrics(name, metrics, comparison)
|
||||
}
|
||||
|
||||
for (const name of names) {
|
||||
const scenario = SCENARIOS[name]
|
||||
const perRun = []
|
||||
let detail = null
|
||||
|
||||
for (let i = 0; i < runs; i++) {
|
||||
if (cpuProfile && i === 0) {
|
||||
const { result, profile } = await withCpuProfile(cdp, () => scenario.run(cdp, flags))
|
||||
const out = join(cpuProfileDir, `${name}-${Date.now()}.cpuprofile`)
|
||||
writeFileSync(out, JSON.stringify(profile))
|
||||
console.log(`\n[cpuprofile] wrote ${out}`)
|
||||
console.log('[cpuprofile] top self-time (ms):')
|
||||
for (const r of cpuProfileTopSelf(profile, 15)) {
|
||||
console.log(` ${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(38)} ${r.url}:${r.line}`)
|
||||
}
|
||||
perRun.push(result.metrics)
|
||||
detail = result.detail
|
||||
} else {
|
||||
const result = await scenario.run(cdp, flags)
|
||||
perRun.push(result.metrics)
|
||||
detail = result.detail
|
||||
}
|
||||
}
|
||||
|
||||
const metrics = medianMetrics(perRun)
|
||||
const comparison = scenario.tier === 'ci' ? compareScenario(name, metrics, baseline) : null
|
||||
regressed = regressed || Boolean(comparison?.regressed)
|
||||
results.push({ name, tier: scenario.tier, metrics, detail })
|
||||
printMetrics(name, metrics, comparison)
|
||||
if (prod) {
|
||||
if (!flags.spawn) {
|
||||
console.error('--prod requires --spawn (it builds and launches an isolated production renderer)')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
console.log('[perf] building production renderer with the probe (VITE_PERF_PROBE=1)…')
|
||||
await buildProdRenderer()
|
||||
}
|
||||
|
||||
// Cold start measures the launch itself → a fresh spawn per run.
|
||||
if (coldNames.length) {
|
||||
if (!flags.spawn) {
|
||||
console.error('cold-start requires --spawn (it measures a fresh launch)')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
const perRun = []
|
||||
|
||||
for (let i = 0; i < runs; i++) {
|
||||
const inst = await startIsolatedInstance({ port, devPort, prod, coldStart: true })
|
||||
const t = inst.timings
|
||||
perRun.push({ spawn_to_cdp_ms: t.spawn_to_cdp_ms, spawn_to_driver_ms: t.spawn_to_driver_ms, fcp_ms: t.fcp_ms })
|
||||
inst.teardown()
|
||||
}
|
||||
|
||||
record('cold-start', 'cold', medianMetrics(perRun), { runs })
|
||||
}
|
||||
|
||||
// Steady-state scenarios share one persistent connection.
|
||||
if (liveNames.length) {
|
||||
const connection = flags.spawn
|
||||
? await startIsolatedInstance({ port, devPort, prod })
|
||||
: await attach({ port, match: prod ? undefined : String(devPort) })
|
||||
|
||||
const { cdp, teardown } = connection
|
||||
|
||||
try {
|
||||
for (const name of liveNames) {
|
||||
const scenario = SCENARIOS[name]
|
||||
const perRun = []
|
||||
let detail = null
|
||||
|
||||
for (let i = 0; i < runs; i++) {
|
||||
if (cpuProfile && i === 0) {
|
||||
const { result, profile } = await withCpuProfile(cdp, () => scenario.run(cdp, flags))
|
||||
const out = join(cpuProfileDir, `${name}-${Date.now()}.cpuprofile`)
|
||||
writeFileSync(out, JSON.stringify(profile))
|
||||
console.log(`\n[cpuprofile] wrote ${out}`)
|
||||
console.log('[cpuprofile] top self-time (ms):')
|
||||
for (const r of cpuProfileTopSelf(profile, 15)) {
|
||||
console.log(` ${r.ms.toFixed(1).padStart(7)} ${r.name.padEnd(38)} ${r.url}:${r.line}`)
|
||||
}
|
||||
perRun.push(result.metrics)
|
||||
detail = result.detail
|
||||
} else {
|
||||
const result = await scenario.run(cdp, flags)
|
||||
perRun.push(result.metrics)
|
||||
detail = result.detail
|
||||
}
|
||||
}
|
||||
|
||||
record(name, scenario.tier, medianMetrics(perRun), detail)
|
||||
}
|
||||
} finally {
|
||||
teardown()
|
||||
}
|
||||
} finally {
|
||||
teardown()
|
||||
}
|
||||
|
||||
if (flags.json) {
|
||||
|
|
@ -165,7 +207,7 @@ async function main() {
|
|||
}
|
||||
|
||||
if (flags['update-baseline']) {
|
||||
updateBaseline(BASELINE_PATH, results.filter(r => r.tier === 'ci'))
|
||||
updateBaseline(BASELINE_PATH, results.filter(r => GATED.has(r.tier)))
|
||||
console.log(`\nupdated ${BASELINE_PATH}`)
|
||||
return
|
||||
}
|
||||
|
|
|
|||
18
apps/desktop/scripts/perf/scenarios/cold-start.mjs
Normal file
18
apps/desktop/scripts/perf/scenarios/cold-start.mjs
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
// Cold start — launch → renderer → interactive. Unlike the other scenarios this
|
||||
// measures the LAUNCH itself, so it can't run against an already-up instance:
|
||||
// the runner spawns a fresh isolated instance per run (requires --spawn) and
|
||||
// reads the timings/boot-marks the launcher captures. Registered here so it's a
|
||||
// known name with a baseline entry; the actual measurement lives in run.mjs.
|
||||
//
|
||||
// Metrics (lower is better):
|
||||
// spawn_to_cdp_ms process spawn → CDP page target reachable (electron/V8 up)
|
||||
// spawn_to_driver_ms process spawn → renderer mounted + perf driver present
|
||||
// fcp_ms renderer nav start → first contentful paint
|
||||
export default {
|
||||
name: 'cold-start',
|
||||
tier: 'cold',
|
||||
description: 'Launch → first paint → interactive (fresh spawn per run).',
|
||||
run() {
|
||||
throw new Error('cold-start is measured by the runner via fresh spawns; use `--spawn`.')
|
||||
}
|
||||
}
|
||||
84
apps/desktop/scripts/perf/scenarios/first-token.mjs
Normal file
84
apps/desktop/scripts/perf/scenarios/first-token.mjs
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
// Time-to-first-token — Enter → first assistant token painted. The latency an
|
||||
// agent app is uniquely judged on, spanning the desktop submit path AND the
|
||||
// backend/agent-loop first-token time. Backend tier: fires a REAL prompt, needs
|
||||
// a live backend (and credits). Report-only.
|
||||
//
|
||||
// node scripts/perf/run.mjs first-token --spawn --prompt "hi"
|
||||
|
||||
import { SELECTORS, sleep, typeIntoComposer } from '../lib/cdp.mjs'
|
||||
import { summarize } from '../lib/stats.mjs'
|
||||
|
||||
export default {
|
||||
name: 'first-token',
|
||||
tier: 'backend',
|
||||
description: 'Enter → first assistant token painted (real backend).',
|
||||
async run(cdp, opts = {}) {
|
||||
const rounds = Number(opts.rounds ?? 3)
|
||||
const prompt = opts.prompt ?? 'reply with a single short sentence'
|
||||
const timeoutMs = Number(opts.timeoutMs ?? 60000)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
const firstTokens = []
|
||||
|
||||
for (let i = 0; i < rounds; i++) {
|
||||
const baseText = await cdp.eval(`(() => {
|
||||
const a = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)})
|
||||
return a.length ? a[a.length - 1].textContent.length : 0
|
||||
})()`)
|
||||
const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`)
|
||||
|
||||
await typeIntoComposer(cdp, `${prompt} (${i})`, { cps: 60 })
|
||||
const submitAt = Date.now()
|
||||
await cdp.eval(`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
el && el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', bubbles: true, cancelable: true }))
|
||||
})()`)
|
||||
|
||||
const deadline = Date.now() + timeoutMs
|
||||
let firstTokenMs = null
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
await sleep(25)
|
||||
const grown = await cdp.eval(`(() => {
|
||||
const a = document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)})
|
||||
if (a.length > ${baseCount}) return true
|
||||
return a.length ? a[a.length - 1].textContent.length > ${baseText} : false
|
||||
})()`)
|
||||
|
||||
if (grown) {
|
||||
firstTokenMs = Date.now() - submitAt
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (firstTokenMs !== null) {
|
||||
firstTokens.push(firstTokenMs)
|
||||
}
|
||||
|
||||
// Let the turn finish before the next round.
|
||||
const turnDeadline = Date.now() + timeoutMs
|
||||
while (Date.now() < turnDeadline) {
|
||||
await sleep(250)
|
||||
const busy = await cdp.eval(`!!document.querySelector('[data-status="running"], [data-busy="true"]')`)
|
||||
|
||||
if (!busy) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(500)
|
||||
}
|
||||
|
||||
if (!firstTokens.length) {
|
||||
throw new Error('no first token observed — is a backend with credits connected?')
|
||||
}
|
||||
|
||||
const s = summarize(firstTokens)
|
||||
|
||||
return {
|
||||
metrics: { first_token_p50_ms: s.p50, first_token_p95_ms: s.p95 },
|
||||
detail: { rounds, samples: firstTokens, summary: s }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
// Scenario registry. Add a scenario module here and it's automatically
|
||||
// available to the runner, the default suite (tier 'ci'), and the baseline gate.
|
||||
|
||||
import coldStart from './cold-start.mjs'
|
||||
import firstToken from './first-token.mjs'
|
||||
import keystroke from './keystroke.mjs'
|
||||
import profileSwitch from './profile-switch.mjs'
|
||||
import sessionSwitch from './session-switch.mjs'
|
||||
|
|
@ -12,6 +14,8 @@ export const SCENARIOS = {
|
|||
[stream.name]: stream,
|
||||
[keystroke.name]: keystroke,
|
||||
[transcript.name]: transcript,
|
||||
[coldStart.name]: coldStart,
|
||||
[firstToken.name]: firstToken,
|
||||
[submit.name]: submit,
|
||||
[sessionSwitch.name]: sessionSwitch,
|
||||
[profileSwitch.name]: profileSwitch
|
||||
|
|
|
|||
|
|
@ -17,7 +17,11 @@ import { ThemeProvider } from './themes/context'
|
|||
|
||||
installClipboardShim()
|
||||
|
||||
if (import.meta.env.MODE !== 'production') {
|
||||
// The perf probe ships in dev, and in a production build ONLY when explicitly
|
||||
// opted in (VITE_PERF_PROBE=1) — this lets the perf harness measure a real,
|
||||
// minified production renderer for representative absolute numbers. Normal
|
||||
// `npm run build` leaves the flag unset, so the probe never reaches users.
|
||||
if (import.meta.env.MODE !== 'production' || import.meta.env.VITE_PERF_PROBE === '1') {
|
||||
import('./app/chat/perf-probe')
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue