bench(desktop): make --spawn work + capture a real baseline (#67670)

- Resolve the vite CLI via vite/package.json `bin` (Vite 8's exports block
  importing vite/bin/vite.js directly — --spawn failed with ERR_PACKAGE_PATH_NOT_EXPORTED).
- Add a post-launch settle so cold-start contention (vite dep pre-bundling,
  first backend-connect attempts) doesn't contaminate the first scenario.
- Drop the raw autolink from the default stream chunk (resolvable URLs trigger
  link-embed DNS lookups unrelated to render cost).
- Replace seed baseline with real numbers from a darwin-arm64 --spawn run.
  keystroke + transcript are clean; stream is a clean single-run capture (the
  isolated backend may not connect, and its reconnect churn inflates frame
  pacing — re-capture on a connected instance for tighter tolerances).
This commit is contained in:
brooklyn! 2026-07-19 16:37:35 -04:00 committed by GitHub
parent e361c5e204
commit 3345b3cdfd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 44 additions and 22 deletions

View file

@ -1,37 +1,37 @@
{
"_meta": {
"note": "SEED baseline only. Values are approximations from apps/desktop/scripts/profile-typing-lag.md (May 2026, 34MB session, PRE incremental-lex). Run `npm run perf -- --update-baseline` on YOUR reference device to capture real numbers, then commit. Tolerances are intentionally loose until then.",
"platform": null,
"note": "Captured on darwin-arm64 via `npm run perf -- --spawn`. keystroke + transcript are clean medians. stream = a clean single-run capture; under --spawn the isolated backend may not connect, and its reconnect churn inflates frame-pacing, so re-capture stream on a backend-CONNECTED instance (perf:serve against a live profile, or attach) for tighter tolerances. Re-baseline with `npm run perf -- --update-baseline`.",
"platform": "darwin-arm64",
"node": null,
"updated": null
"updated": "2026-07-19"
},
"scenarios": {
"stream": {
"tolerance": { "tolFrac": 0.5, "tolAbs": 2 },
"tolerance": { "tolFrac": 0.6, "tolAbs": 5 },
"metrics": {
"longtasks_n": 2,
"longtask_max_ms": 127,
"frame_p95_ms": 25.6,
"frame_p99_ms": 31.4,
"slow_frames_33": 6,
"intermut_p95_ms": 45
"longtask_max_ms": 97,
"frame_p95_ms": 18.4,
"frame_p99_ms": 28.5,
"slow_frames_33": 4,
"intermut_p95_ms": 37.1
}
},
"keystroke": {
"tolerance": { "tolFrac": 0.5, "tolAbs": 3 },
"tolerance": { "tolFrac": 0.6, "tolAbs": 4 },
"metrics": {
"keystroke_p50_ms": 8,
"keystroke_p95_ms": 17,
"keystroke_p99_ms": 28,
"keystroke_slow_16": 6
"keystroke_p50_ms": 2.7,
"keystroke_p95_ms": 9.9,
"keystroke_p99_ms": 17.7,
"keystroke_slow_16": 2
}
},
"transcript": {
"tolerance": { "tolFrac": 0.5, "tolAbs": 20 },
"tolerance": { "tolFrac": 0.75, "tolAbs": 40 },
"metrics": {
"transcript_mount_ms": 450,
"transcript_longtask_ms": 350,
"transcript_longtask_max_ms": 160
"transcript_mount_ms": 287.3,
"transcript_longtask_ms": 559,
"transcript_longtask_max_ms": 295
}
}
}

View file

@ -12,7 +12,7 @@
// spent regardless of the isolated backend.
import { spawn } from 'node:child_process'
import { copyFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs'
import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
import { createRequire } from 'node:module'
import { homedir, tmpdir } from 'node:os'
import { dirname, join, resolve } from 'node:path'
@ -69,6 +69,20 @@ function seedConfigFrom(sourceHome, targetHome) {
}
}
// Resolve the vite CLI entry via its package.json `bin` (Vite 8's `exports`
// blocks importing `vite/bin/vite.js` directly).
function resolveViteBin() {
const pkgPath = require.resolve('vite/package.json')
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'))
const rel = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin?.vite
if (!rel) {
throw new Error('could not resolve the vite CLI from vite/package.json')
}
return join(dirname(pkgPath), rel)
}
function runNode(scriptRelPath, args = []) {
return new Promise((resolveRun, reject) => {
const child = spawn(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args], {
@ -99,7 +113,8 @@ export async function startIsolatedInstance({
hermesHome,
userDataDir,
seedConfig = true,
bootFakeStepMs = 120
bootFakeStepMs = 120,
settleMs = 2500
} = {}) {
const children = []
const tempDirs = []
@ -141,7 +156,7 @@ export async function startIsolatedInstance({
try {
// 1. Renderer: reuse an already-running dev server, else start one.
if (!(await reachable(devUrl))) {
const viteBin = require.resolve('vite/bin/vite.js')
const viteBin = resolveViteBin()
const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], {
cwd: DESKTOP_DIR,
stdio: ['ignore', 'inherit', 'inherit']
@ -194,6 +209,11 @@ export async function startIsolatedInstance({
{ timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' }
)
// Let cold-start contention (vite dep pre-bundling, first backend-connect
// attempts, initial paint) drain before scenarios measure, so numbers
// reflect steady state rather than launch noise.
await sleep(settleMs)
return {
cdp,
devUrl,

View file

@ -113,7 +113,9 @@ export default {
const tokens = Number(opts.tokens ?? 400)
const intervalMs = Number(opts.intervalMs ?? 16)
const flushMinMs = Number(opts.flushMinMs ?? 33)
const chunk = opts.chunk ?? '**word** in _italic_ with `code`, a [link](https://x.dev) and prose. '
// No raw autolink in the default chunk — a resolvable URL triggers link
// embeds / DNS lookups that add noise unrelated to render cost.
const chunk = opts.chunk ?? '**word** in _italic_ with `code` and a bit of ordinary prose. '
const real = Boolean(opts.real)
await cdp.send('Runtime.enable')