mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
bench(desktop): make --spawn work + capture a real baseline (#67670)
- Resolve the vite CLI via vite/package.json `bin` (Vite 8's exports block importing vite/bin/vite.js directly — --spawn failed with ERR_PACKAGE_PATH_NOT_EXPORTED). - Add a post-launch settle so cold-start contention (vite dep pre-bundling, first backend-connect attempts) doesn't contaminate the first scenario. - Drop the raw autolink from the default stream chunk (resolvable URLs trigger link-embed DNS lookups unrelated to render cost). - Replace seed baseline with real numbers from a darwin-arm64 --spawn run. keystroke + transcript are clean; stream is a clean single-run capture (the isolated backend may not connect, and its reconnect churn inflates frame pacing — re-capture on a connected instance for tighter tolerances).
This commit is contained in:
parent
e361c5e204
commit
3345b3cdfd
3 changed files with 44 additions and 22 deletions
|
|
@ -1,37 +1,37 @@
|
|||
{
|
||||
"_meta": {
|
||||
"note": "SEED baseline only. Values are approximations from apps/desktop/scripts/profile-typing-lag.md (May 2026, 34MB session, PRE incremental-lex). Run `npm run perf -- --update-baseline` on YOUR reference device to capture real numbers, then commit. Tolerances are intentionally loose until then.",
|
||||
"platform": null,
|
||||
"note": "Captured on darwin-arm64 via `npm run perf -- --spawn`. keystroke + transcript are clean medians. stream = a clean single-run capture; under --spawn the isolated backend may not connect, and its reconnect churn inflates frame-pacing, so re-capture stream on a backend-CONNECTED instance (perf:serve against a live profile, or attach) for tighter tolerances. Re-baseline with `npm run perf -- --update-baseline`.",
|
||||
"platform": "darwin-arm64",
|
||||
"node": null,
|
||||
"updated": null
|
||||
"updated": "2026-07-19"
|
||||
},
|
||||
"scenarios": {
|
||||
"stream": {
|
||||
"tolerance": { "tolFrac": 0.5, "tolAbs": 2 },
|
||||
"tolerance": { "tolFrac": 0.6, "tolAbs": 5 },
|
||||
"metrics": {
|
||||
"longtasks_n": 2,
|
||||
"longtask_max_ms": 127,
|
||||
"frame_p95_ms": 25.6,
|
||||
"frame_p99_ms": 31.4,
|
||||
"slow_frames_33": 6,
|
||||
"intermut_p95_ms": 45
|
||||
"longtask_max_ms": 97,
|
||||
"frame_p95_ms": 18.4,
|
||||
"frame_p99_ms": 28.5,
|
||||
"slow_frames_33": 4,
|
||||
"intermut_p95_ms": 37.1
|
||||
}
|
||||
},
|
||||
"keystroke": {
|
||||
"tolerance": { "tolFrac": 0.5, "tolAbs": 3 },
|
||||
"tolerance": { "tolFrac": 0.6, "tolAbs": 4 },
|
||||
"metrics": {
|
||||
"keystroke_p50_ms": 8,
|
||||
"keystroke_p95_ms": 17,
|
||||
"keystroke_p99_ms": 28,
|
||||
"keystroke_slow_16": 6
|
||||
"keystroke_p50_ms": 2.7,
|
||||
"keystroke_p95_ms": 9.9,
|
||||
"keystroke_p99_ms": 17.7,
|
||||
"keystroke_slow_16": 2
|
||||
}
|
||||
},
|
||||
"transcript": {
|
||||
"tolerance": { "tolFrac": 0.5, "tolAbs": 20 },
|
||||
"tolerance": { "tolFrac": 0.75, "tolAbs": 40 },
|
||||
"metrics": {
|
||||
"transcript_mount_ms": 450,
|
||||
"transcript_longtask_ms": 350,
|
||||
"transcript_longtask_max_ms": 160
|
||||
"transcript_mount_ms": 287.3,
|
||||
"transcript_longtask_ms": 559,
|
||||
"transcript_longtask_max_ms": 295
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
// spent regardless of the isolated backend.
|
||||
|
||||
import { spawn } from 'node:child_process'
|
||||
import { copyFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs'
|
||||
import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { homedir, tmpdir } from 'node:os'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
|
|
@ -69,6 +69,20 @@ function seedConfigFrom(sourceHome, targetHome) {
|
|||
}
|
||||
}
|
||||
|
||||
// Resolve the vite CLI entry via its package.json `bin` (Vite 8's `exports`
|
||||
// blocks importing `vite/bin/vite.js` directly).
|
||||
function resolveViteBin() {
|
||||
const pkgPath = require.resolve('vite/package.json')
|
||||
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'))
|
||||
const rel = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin?.vite
|
||||
|
||||
if (!rel) {
|
||||
throw new Error('could not resolve the vite CLI from vite/package.json')
|
||||
}
|
||||
|
||||
return join(dirname(pkgPath), rel)
|
||||
}
|
||||
|
||||
function runNode(scriptRelPath, args = []) {
|
||||
return new Promise((resolveRun, reject) => {
|
||||
const child = spawn(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args], {
|
||||
|
|
@ -99,7 +113,8 @@ export async function startIsolatedInstance({
|
|||
hermesHome,
|
||||
userDataDir,
|
||||
seedConfig = true,
|
||||
bootFakeStepMs = 120
|
||||
bootFakeStepMs = 120,
|
||||
settleMs = 2500
|
||||
} = {}) {
|
||||
const children = []
|
||||
const tempDirs = []
|
||||
|
|
@ -141,7 +156,7 @@ export async function startIsolatedInstance({
|
|||
try {
|
||||
// 1. Renderer: reuse an already-running dev server, else start one.
|
||||
if (!(await reachable(devUrl))) {
|
||||
const viteBin = require.resolve('vite/bin/vite.js')
|
||||
const viteBin = resolveViteBin()
|
||||
const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], {
|
||||
cwd: DESKTOP_DIR,
|
||||
stdio: ['ignore', 'inherit', 'inherit']
|
||||
|
|
@ -194,6 +209,11 @@ export async function startIsolatedInstance({
|
|||
{ timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' }
|
||||
)
|
||||
|
||||
// Let cold-start contention (vite dep pre-bundling, first backend-connect
|
||||
// attempts, initial paint) drain before scenarios measure, so numbers
|
||||
// reflect steady state rather than launch noise.
|
||||
await sleep(settleMs)
|
||||
|
||||
return {
|
||||
cdp,
|
||||
devUrl,
|
||||
|
|
|
|||
|
|
@ -113,7 +113,9 @@ export default {
|
|||
const tokens = Number(opts.tokens ?? 400)
|
||||
const intervalMs = Number(opts.intervalMs ?? 16)
|
||||
const flushMinMs = Number(opts.flushMinMs ?? 33)
|
||||
const chunk = opts.chunk ?? '**word** in _italic_ with `code`, a [link](https://x.dev) and prose. '
|
||||
// No raw autolink in the default chunk — a resolvable URL triggers link
|
||||
// embeds / DNS lookups that add noise unrelated to render cost.
|
||||
const chunk = opts.chunk ?? '**word** in _italic_ with `code` and a bit of ordinary prose. '
|
||||
const real = Boolean(opts.real)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue