mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
Replaces the dozen ad-hoc measure-*/profile-* scripts (each reinventing the CDP client — 4 different copies — plus its own arg parsing, stats, output path, and none with a baseline) with one framework under scripts/perf/: - lib/cdp.mjs one CDP client + target discovery + typing + CPU-profile wrapper + DOM selectors - lib/stats.mjs percentiles, histograms, CPU-profile self-time ranking - lib/baseline.mjs load/compare/update baseline + regression gate (new capability) - lib/launch.mjs attach, OR spawn a fully ISOLATED instance - scenarios/* one module per measurement, registered in scenarios/index.mjs - run.mjs / serve.mjs, baseline.json, README.md Isolation solves the long-standing measurement blocker: a running `hgui` held the Electron single-instance lock, so a second instance quit. `--spawn` / `perf:serve` launch with their own --user-data-dir (separate lock scope), their own HERMES_HOME (separate backend/sessions, config seeded from ~/.hermes so it reaches a chat view without onboarding), and their own --remote-debugging-port. Synthetic scenarios drive $messages via window.__PERF_DRIVE__, so no LLM credits. Scenario -> sunset script mapping: stream <- measure-synthetic-stream, profile-synth-stream, profile-long-stream stream --real <- measure-real-stream, profile-real-stream keystroke <- measure-latency, profile-typing, leak-typing transcript <- (new: long-transcript mount cost) submit <- measure-submit, measure-jump session-switch <- profile-session-switch profile-switch <- measure-profile-switch CPU profiling is now a cross-cutting --cpuprofile flag, not 5 separate scripts. CI-tier scenarios (stream, keystroke, transcript) need no backend/credits and are gated against baseline.json (seed values; re-capture with --update-baseline on a reference device). Backend-tier scenarios are report-only. perf-probe.tsx gains loadTranscript() for the transcript scenario. No core files touched; isolation is via CLI args, not env-gated app changes. Verified: node --check all modules, tsc, eslint, and a unit smoke of the stats + regression-gate logic. The end-to-end GUI run (which opens a window) is left to run interactively via `npm run perf -- --spawn`.
202 lines
6 KiB
JavaScript
202 lines
6 KiB
JavaScript
// The one Chrome DevTools Protocol client for the desktop perf harness.
|
|
//
|
|
// Before this, every measure-*/profile-* script shipped its own copy-pasted
|
|
// `CDP` class (four subtly different implementations), its own `/json` vs
|
|
// `/json/list` target discovery, and its own Profiler ranking. Scenarios now
|
|
// import from here so there is a single place to fix a protocol bug.
|
|
|
|
const DEFAULT_PORT = 9222
|
|
|
|
// Stable DOM hooks the renderer exposes. Centralised so a component refactor
|
|
// updates one constant instead of a dozen scattered querySelector strings.
|
|
export const SELECTORS = {
|
|
composer: '[data-slot="composer-rich-input"]',
|
|
threadViewport: '[data-slot="aui_thread-viewport"]',
|
|
threadContent: '[data-slot="aui_thread-content"]',
|
|
assistantMessage: '[data-slot="aui_assistant-message-root"]',
|
|
turnPair: '[data-slot="aui_turn-pair"]',
|
|
profileRail: '[data-slot="profile-rail"]',
|
|
rowButton: '[data-slot="row-button"]'
|
|
}
|
|
|
|
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
|
|
|
|
/**
|
|
* Poll the CDP HTTP endpoint until a page target is available.
|
|
* @param {object} [opts]
|
|
* @param {number} [opts.port] remote-debugging-port (default 9222).
|
|
* @param {string} [opts.match] substring the target URL must contain (e.g. a dev-server port).
|
|
* @param {number} [opts.timeoutMs] how long to wait for a target.
|
|
*/
|
|
export async function discoverTarget({ port = DEFAULT_PORT, match, timeoutMs = 30000 } = {}) {
|
|
const deadline = Date.now() + timeoutMs
|
|
|
|
for (;;) {
|
|
try {
|
|
const list = await (await fetch(`http://127.0.0.1:${port}/json/list`)).json()
|
|
const pages = list.filter(t => t.type === 'page' && typeof t.webSocketDebuggerUrl === 'string')
|
|
const target = match
|
|
? pages.find(t => String(t.url).includes(match))
|
|
: pages.find(t => String(t.url).startsWith('http')) ?? pages[0]
|
|
|
|
if (target) {
|
|
return target
|
|
}
|
|
} catch {
|
|
// debug port not up yet — keep polling until the deadline.
|
|
}
|
|
|
|
if (Date.now() >= deadline) {
|
|
throw new Error(`no CDP page target on :${port}${match ? ` matching "${match}"` : ''} within ${timeoutMs}ms`)
|
|
}
|
|
|
|
await sleep(250)
|
|
}
|
|
}
|
|
|
|
export class CDP {
|
|
constructor(ws) {
|
|
this.ws = ws
|
|
this.id = 0
|
|
this.pending = new Map()
|
|
this.listeners = new Map()
|
|
}
|
|
|
|
static async open(url) {
|
|
const ws = new WebSocket(url)
|
|
|
|
await new Promise((resolve, reject) => {
|
|
ws.addEventListener('open', resolve, { once: true })
|
|
ws.addEventListener('error', reject, { once: true })
|
|
})
|
|
|
|
const cdp = new CDP(ws)
|
|
|
|
ws.addEventListener('message', ev => {
|
|
const m = JSON.parse(typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8'))
|
|
|
|
if (m.id != null && cdp.pending.has(m.id)) {
|
|
const { resolve, reject } = cdp.pending.get(m.id)
|
|
cdp.pending.delete(m.id)
|
|
|
|
if (m.error) {
|
|
reject(new Error(m.error.message))
|
|
} else {
|
|
resolve(m.result)
|
|
}
|
|
} else if (m.method) {
|
|
for (const handler of cdp.listeners.get(m.method) ?? []) {
|
|
handler(m.params)
|
|
}
|
|
}
|
|
})
|
|
|
|
ws.addEventListener('close', () => {
|
|
for (const { reject } of cdp.pending.values()) {
|
|
reject(new Error('CDP socket closed'))
|
|
}
|
|
|
|
cdp.pending.clear()
|
|
})
|
|
|
|
return cdp
|
|
}
|
|
|
|
/** Connect straight to a discovered target. */
|
|
static async connect(opts) {
|
|
const target = await discoverTarget(opts)
|
|
|
|
return CDP.open(target.webSocketDebuggerUrl)
|
|
}
|
|
|
|
send(method, params = {}) {
|
|
const id = ++this.id
|
|
|
|
return new Promise((resolve, reject) => {
|
|
this.pending.set(id, { resolve, reject })
|
|
this.ws.send(JSON.stringify({ id, method, params }))
|
|
})
|
|
}
|
|
|
|
on(method, handler) {
|
|
if (!this.listeners.has(method)) {
|
|
this.listeners.set(method, [])
|
|
}
|
|
|
|
this.listeners.get(method).push(handler)
|
|
}
|
|
|
|
/** Evaluate an expression in the page and return its value (awaits promises). */
|
|
async eval(expression) {
|
|
const r = await this.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true })
|
|
|
|
if (r.exceptionDetails) {
|
|
throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text || 'eval failed')
|
|
}
|
|
|
|
return r.result.value
|
|
}
|
|
|
|
close() {
|
|
this.ws.close()
|
|
}
|
|
}
|
|
|
|
/** Assert the renderer has the dev-only `__PERF_DRIVE__` harness attached. */
|
|
export async function requireDriver(cdp) {
|
|
const ok = await cdp.eval('!!(window.__PERF_DRIVE__ && window.__PERF_DRIVE__.stream)')
|
|
|
|
if (!ok) {
|
|
throw new Error(
|
|
'__PERF_DRIVE__ not on window. The perf harness needs a DEV renderer ' +
|
|
'(perf-probe.tsx is excluded from production builds). Launch with `npm run perf:serve`.'
|
|
)
|
|
}
|
|
}
|
|
|
|
/** Type real key events into the composer, one char at a time, at `cps` chars/sec. */
|
|
export async function typeIntoComposer(cdp, text, { cps = 15 } = {}) {
|
|
await cdp.eval(`(() => {
|
|
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
|
if (!el) return false
|
|
el.focus()
|
|
const range = document.createRange()
|
|
range.selectNodeContents(el)
|
|
range.collapse(false)
|
|
const sel = window.getSelection()
|
|
sel.removeAllRanges()
|
|
sel.addRange(range)
|
|
return true
|
|
})()`)
|
|
|
|
const intervalMs = Math.max(1, Math.round(1000 / cps))
|
|
|
|
for (const ch of text) {
|
|
await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: ch, unmodifiedText: ch })
|
|
await sleep(intervalMs)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run `body()` while a V8 CPU profile is recording. Returns
|
|
* `{ result, profile }`; the caller decides whether to write the .cpuprofile.
|
|
*/
|
|
export async function withCpuProfile(cdp, body, { samplingIntervalUs = 100 } = {}) {
|
|
await cdp.send('Profiler.enable')
|
|
await cdp.send('Profiler.setSamplingInterval', { interval: samplingIntervalUs })
|
|
await cdp.send('Profiler.start')
|
|
|
|
let result
|
|
let stopped
|
|
|
|
try {
|
|
result = await body()
|
|
} finally {
|
|
// Always stop so a scenario error can't leave the profiler running.
|
|
stopped = await cdp.send('Profiler.stop')
|
|
}
|
|
|
|
return { result, profile: stopped.profile }
|
|
}
|
|
|
|
export { sleep }
|