mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-24 16:54:43 +00:00
d1c455acf7
Replaces the dozen ad-hoc measure-*/profile-* scripts (each reinventing the CDP client — 4 different copies — plus its own arg parsing, stats, output path, and none with a baseline) with one framework under scripts/perf/: - lib/cdp.mjs one CDP client + target discovery + typing + CPU-profile wrapper + DOM selectors - lib/stats.mjs percentiles, histograms, CPU-profile self-time ranking - lib/baseline.mjs load/compare/update baseline + regression gate (new capability) - lib/launch.mjs attach, OR spawn a fully ISOLATED instance - scenarios/* one module per measurement, registered in scenarios/index.mjs - run.mjs / serve.mjs, baseline.json, README.md Isolation solves the long-standing measurement blocker: a running `hgui` held the Electron single-instance lock, so a second instance quit. `--spawn` / `perf:serve` launch with their own --user-data-dir (separate lock scope), their own HERMES_HOME (separate backend/sessions, config seeded from ~/.hermes so it reaches a chat view without onboarding), and their own --remote-debugging-port. Synthetic scenarios drive $messages via window.__PERF_DRIVE__, so no LLM credits. Scenario -> sunset script mapping: stream <- measure-synthetic-stream, profile-synth-stream, profile-long-stream stream --real <- measure-real-stream, profile-real-stream keystroke <- measure-latency, profile-typing, leak-typing transcript <- (new: long-transcript mount cost) submit <- measure-submit, measure-jump session-switch <- profile-session-switch profile-switch <- measure-profile-switch CPU profiling is now a cross-cutting --cpuprofile flag, not 5 separate scripts. CI-tier scenarios (stream, keystroke, transcript) need no backend/credits and are gated against baseline.json (seed values; re-capture with --update-baseline on a reference device). Backend-tier scenarios are report-only. perf-probe.tsx gains loadTranscript() for the transcript scenario. No core files touched; isolation is via CLI args, not env-gated app changes. Verified: node --check all modules, tsc, eslint, and a unit smoke of the stats + regression-gate logic. The end-to-end GUI run (which opens a window) is left to run interactively via `npm run perf -- --spawn`.
99 lines
3.2 KiB
JavaScript
99 lines
3.2 KiB
JavaScript
// Composer input latency — keystroke → next paint. Subsumes measure-latency,
|
|
// profile-typing, and leak-typing. This is the most-felt latency in a chat app
|
|
// (users type constantly) and nothing measured it against a baseline before.
|
|
//
|
|
// Each synthetic char records the time from dispatch to the first rAF after the
|
|
// composer mutates (a paint proxy). Metrics are p50/p95/p99 and the count of
|
|
// keystrokes that missed a 16ms frame.
|
|
|
|
import { SELECTORS, sleep } from '../lib/cdp.mjs'
|
|
import { percentile } from '../lib/stats.mjs'
|
|
|
|
const INSTALL = `
|
|
(() => {
|
|
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
|
if (!el) return false
|
|
el.focus()
|
|
const range = document.createRange()
|
|
range.selectNodeContents(el)
|
|
range.collapse(false)
|
|
const sel = window.getSelection()
|
|
sel.removeAllRanges()
|
|
sel.addRange(range)
|
|
window.__KEY__ = { samples: [], pending: null }
|
|
const obs = new MutationObserver(() => {
|
|
const start = window.__KEY__.pending
|
|
if (start === null) return
|
|
window.__KEY__.pending = null
|
|
requestAnimationFrame(() => window.__KEY__.samples.push(performance.now() - start))
|
|
})
|
|
obs.observe(el, { childList: true, subtree: true, characterData: true })
|
|
window.__KEY__.obs = obs
|
|
return true
|
|
})()
|
|
`
|
|
|
|
const CLEAR = `
|
|
(() => {
|
|
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
|
if (el) { el.innerHTML = ''; el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContentBackward' })) }
|
|
window.__KEY__ && window.__KEY__.obs && window.__KEY__.obs.disconnect()
|
|
})()
|
|
`
|
|
|
|
const SENTENCE =
|
|
'the quick brown fox jumps over the lazy dog while typing into this composer, which should feel instant. '
|
|
|
|
export default {
|
|
name: 'keystroke',
|
|
tier: 'ci',
|
|
description: 'Composer keystroke → paint latency while idle.',
|
|
async run(cdp, opts = {}) {
|
|
const chars = Number(opts.chars ?? 120)
|
|
const cps = Number(opts.cps ?? 15)
|
|
|
|
await cdp.send('Runtime.enable')
|
|
|
|
const installed = await cdp.eval(INSTALL)
|
|
|
|
if (!installed) {
|
|
throw new Error(`composer not found (${SELECTORS.composer}); is a chat view open?`)
|
|
}
|
|
|
|
let text = ''
|
|
|
|
while (text.length < chars) {
|
|
text += SENTENCE
|
|
}
|
|
|
|
text = text.slice(0, chars)
|
|
const intervalMs = Math.max(1, Math.round(1000 / cps))
|
|
const start = Date.now()
|
|
|
|
for (let i = 0; i < text.length; i++) {
|
|
await cdp.eval('window.__KEY__.pending = performance.now()')
|
|
await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: text[i], unmodifiedText: text[i] })
|
|
const wait = start + (i + 1) * intervalMs - Date.now()
|
|
|
|
if (wait > 0) {
|
|
await sleep(wait)
|
|
}
|
|
}
|
|
|
|
await sleep(300)
|
|
const samples = await cdp.eval('window.__KEY__.samples')
|
|
await cdp.eval(CLEAR)
|
|
|
|
const round = n => Math.round(n * 10) / 10
|
|
|
|
return {
|
|
metrics: {
|
|
keystroke_p50_ms: round(percentile(samples, 0.5)),
|
|
keystroke_p95_ms: round(percentile(samples, 0.95)),
|
|
keystroke_p99_ms: round(percentile(samples, 0.99)),
|
|
keystroke_slow_16: samples.filter(s => s > 16).length
|
|
},
|
|
detail: { n: samples.length, typed: text.length }
|
|
}
|
|
}
|
|
}
|