mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-22 16:25:58 +00:00
Replaces the dozen ad-hoc measure-*/profile-* scripts (each reinventing the CDP client — 4 different copies — plus its own arg parsing, stats, output path, and none with a baseline) with one framework under scripts/perf/: - lib/cdp.mjs one CDP client + target discovery + typing + CPU-profile wrapper + DOM selectors - lib/stats.mjs percentiles, histograms, CPU-profile self-time ranking - lib/baseline.mjs load/compare/update baseline + regression gate (new capability) - lib/launch.mjs attach, OR spawn a fully ISOLATED instance - scenarios/* one module per measurement, registered in scenarios/index.mjs - run.mjs / serve.mjs, baseline.json, README.md Isolation solves the long-standing measurement blocker: a running `hgui` held the Electron single-instance lock, so a second instance quit. `--spawn` / `perf:serve` launch with their own --user-data-dir (separate lock scope), their own HERMES_HOME (separate backend/sessions, config seeded from ~/.hermes so it reaches a chat view without onboarding), and their own --remote-debugging-port. Synthetic scenarios drive $messages via window.__PERF_DRIVE__, so no LLM credits. Scenario -> sunset script mapping: stream <- measure-synthetic-stream, profile-synth-stream, profile-long-stream stream --real <- measure-real-stream, profile-real-stream keystroke <- measure-latency, profile-typing, leak-typing transcript <- (new: long-transcript mount cost) submit <- measure-submit, measure-jump session-switch <- profile-session-switch profile-switch <- measure-profile-switch CPU profiling is now a cross-cutting --cpuprofile flag, not 5 separate scripts. CI-tier scenarios (stream, keystroke, transcript) need no backend/credits and are gated against baseline.json (seed values; re-capture with --update-baseline on a reference device). Backend-tier scenarios are report-only. perf-probe.tsx gains loadTranscript() for the transcript scenario. No core files touched; isolation is via CLI args, not env-gated app changes. Verified: node --check all modules, tsc, eslint, and a unit smoke of the stats + regression-gate logic. The end-to-end GUI run (which opens a window) is left to run interactively via `npm run perf -- --spawn`.
212 lines
6.2 KiB
JavaScript
212 lines
6.2 KiB
JavaScript
// Connect the harness to a renderer — either an already-running debug instance
|
|
// (`attach`) or a freshly spawned, fully isolated one (`startIsolatedInstance`).
|
|
//
|
|
// The isolated instance is what makes the harness self-contained and unblocks
|
|
// the measurement that the single-instance lock used to prevent:
|
|
// · its own --user-data-dir → its own Electron single-instance lock, so it
|
|
// never collides with (or steals focus from) the user's running `hgui`.
|
|
// · its own HERMES_HOME → its own backend + sessions, no shared state.
|
|
// · its own --remote-debugging-port → a private CDP endpoint.
|
|
// · HERMES_DESKTOP_BOOT_FAKE=1 → deterministic boot overlay.
|
|
// The synthetic scenarios drive `$messages` directly, so no LLM credits are
|
|
// spent regardless of the isolated backend.
|
|
|
|
import { spawn } from 'node:child_process'
|
|
import { copyFileSync, existsSync, mkdtempSync, rmSync } from 'node:fs'
|
|
import { createRequire } from 'node:module'
|
|
import { homedir, tmpdir } from 'node:os'
|
|
import { dirname, join, resolve } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
import { CDP, requireDriver, sleep } from './cdp.mjs'
|
|
|
|
const require = createRequire(import.meta.url)
|
|
const DESKTOP_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..', '..')
|
|
|
|
async function reachable(url) {
|
|
try {
|
|
await fetch(url)
|
|
|
|
return true
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
async function waitFor(fn, { timeoutMs, label }) {
|
|
const deadline = Date.now() + timeoutMs
|
|
|
|
while (Date.now() < deadline) {
|
|
if (await fn()) {
|
|
return
|
|
}
|
|
|
|
await sleep(300)
|
|
}
|
|
|
|
throw new Error(`timed out after ${timeoutMs}ms waiting for ${label}`)
|
|
}
|
|
|
|
// Seed an isolated HERMES_HOME with just enough config (NOT sessions) so the
|
|
// spawned instance reaches an empty chat view instead of the onboarding wizard.
|
|
// A separate HERMES_HOME dir means a separate gateway lock — no collision with
|
|
// the user's running app, which keeps its own sessions DB and state.
|
|
function seedConfigFrom(sourceHome, targetHome) {
|
|
if (!existsSync(sourceHome)) {
|
|
return
|
|
}
|
|
|
|
for (const name of ['config.yaml', '.env', 'auth.json']) {
|
|
const from = join(sourceHome, name)
|
|
|
|
if (existsSync(from)) {
|
|
try {
|
|
copyFileSync(from, join(targetHome, name))
|
|
} catch {
|
|
// best-effort — a missing file just means onboarding may appear.
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function runNode(scriptRelPath, args = []) {
|
|
return new Promise((resolveRun, reject) => {
|
|
const child = spawn(process.execPath, [join(DESKTOP_DIR, scriptRelPath), ...args], {
|
|
cwd: DESKTOP_DIR,
|
|
stdio: 'inherit'
|
|
})
|
|
child.on('error', reject)
|
|
child.on('exit', code => (code === 0 ? resolveRun() : reject(new Error(`${scriptRelPath} exited ${code}`))))
|
|
})
|
|
}
|
|
|
|
/** Attach to a renderer already listening on `port` (launched via perf:serve or with --remote-debugging-port). */
|
|
export async function attach({ port = 9222, match } = {}) {
|
|
const cdp = await CDP.connect({ port, match })
|
|
await requireDriver(cdp)
|
|
|
|
return { cdp, teardown: () => cdp.close() }
|
|
}
|
|
|
|
/**
|
|
* Spawn an isolated dev instance (vite + electron), wait for the perf driver,
|
|
* and return `{ cdp, teardown, devUrl, port }`. `teardown` kills both children
|
|
* and removes any temp dirs it created.
|
|
*/
|
|
export async function startIsolatedInstance({
|
|
port = 9222,
|
|
devPort = 5174,
|
|
hermesHome,
|
|
userDataDir,
|
|
seedConfig = true,
|
|
bootFakeStepMs = 120
|
|
} = {}) {
|
|
const children = []
|
|
const tempDirs = []
|
|
|
|
const mkTemp = prefix => {
|
|
const dir = mkdtempSync(join(tmpdir(), prefix))
|
|
tempDirs.push(dir)
|
|
|
|
return dir
|
|
}
|
|
|
|
const home = hermesHome ?? mkTemp('hermes-perf-home-')
|
|
const userData = userDataDir ?? mkTemp('hermes-perf-ud-')
|
|
const devUrl = `http://127.0.0.1:${devPort}`
|
|
|
|
// Only seed a temp home we created — never scribble into a user-provided one.
|
|
if (seedConfig && !hermesHome) {
|
|
seedConfigFrom(join(homedir(), '.hermes'), home)
|
|
}
|
|
|
|
const teardown = () => {
|
|
for (const child of children) {
|
|
try {
|
|
child.kill('SIGTERM')
|
|
} catch {
|
|
// already gone
|
|
}
|
|
}
|
|
|
|
for (const dir of tempDirs) {
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true })
|
|
} catch {
|
|
// best-effort
|
|
}
|
|
}
|
|
}
|
|
|
|
try {
|
|
// 1. Renderer: reuse an already-running dev server, else start one.
|
|
if (!(await reachable(devUrl))) {
|
|
const viteBin = require.resolve('vite/bin/vite.js')
|
|
const vite = spawn(process.execPath, [viteBin, '--host', '127.0.0.1', '--port', String(devPort)], {
|
|
cwd: DESKTOP_DIR,
|
|
stdio: ['ignore', 'inherit', 'inherit']
|
|
})
|
|
children.push(vite)
|
|
await waitFor(() => reachable(devUrl), { timeoutMs: 60000, label: `vite dev server on :${devPort}` })
|
|
}
|
|
|
|
// 2. Electron main bundle (dev variant) — same step the dev script runs.
|
|
await runNode('scripts/bundle-electron-main.mjs', ['--dev'])
|
|
|
|
// 3. Isolated Electron. --user-data-dir gives it its own single-instance
|
|
// lock scope; HERMES_HOME gives it its own backend + sessions.
|
|
const electronBin = require('electron')
|
|
const electron = spawn(
|
|
electronBin,
|
|
['.', `--user-data-dir=${userData}`, `--remote-debugging-port=${port}`],
|
|
{
|
|
cwd: DESKTOP_DIR,
|
|
stdio: ['ignore', 'inherit', 'inherit'],
|
|
env: {
|
|
...process.env,
|
|
HERMES_HOME: home,
|
|
HERMES_DESKTOP_DEV_SERVER: devUrl,
|
|
HERMES_DESKTOP_BOOT_FAKE: '1',
|
|
HERMES_DESKTOP_BOOT_FAKE_STEP_MS: String(bootFakeStepMs),
|
|
XCURSOR_SIZE: '24'
|
|
}
|
|
}
|
|
)
|
|
children.push(electron)
|
|
|
|
// 4. Wait for the renderer + the perf driver to be live.
|
|
let cdp = null
|
|
await waitFor(
|
|
async () => {
|
|
try {
|
|
cdp = await CDP.connect({ port, match: String(devPort), timeoutMs: 2000 })
|
|
|
|
return await cdp.eval('!!(window.__PERF_DRIVE__ && window.__PERF_DRIVE__.stream)')
|
|
} catch {
|
|
if (cdp) {
|
|
cdp.close()
|
|
cdp = null
|
|
}
|
|
|
|
return false
|
|
}
|
|
},
|
|
{ timeoutMs: 120000, label: 'isolated renderer + __PERF_DRIVE__' }
|
|
)
|
|
|
|
return {
|
|
cdp,
|
|
devUrl,
|
|
port,
|
|
teardown: () => {
|
|
cdp?.close()
|
|
teardown()
|
|
}
|
|
}
|
|
} catch (err) {
|
|
teardown()
|
|
throw err
|
|
}
|
|
}
|
|
|
|
export { DESKTOP_DIR }
|