mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-24 16:54:43 +00:00
d1c455acf7
Replaces the dozen ad-hoc measure-*/profile-* scripts (each reinventing the CDP client — 4 different copies — plus its own arg parsing, stats, output path, and none with a baseline) with one framework under scripts/perf/: - lib/cdp.mjs one CDP client + target discovery + typing + CPU-profile wrapper + DOM selectors - lib/stats.mjs percentiles, histograms, CPU-profile self-time ranking - lib/baseline.mjs load/compare/update baseline + regression gate (new capability) - lib/launch.mjs attach, OR spawn a fully ISOLATED instance - scenarios/* one module per measurement, registered in scenarios/index.mjs - run.mjs / serve.mjs, baseline.json, README.md Isolation solves the long-standing measurement blocker: a running `hgui` held the Electron single-instance lock, so a second instance quit. `--spawn` / `perf:serve` launch with their own --user-data-dir (separate lock scope), their own HERMES_HOME (separate backend/sessions, config seeded from ~/.hermes so it reaches a chat view without onboarding), and their own --remote-debugging-port. Synthetic scenarios drive $messages via window.__PERF_DRIVE__, so no LLM credits. Scenario -> sunset script mapping: stream <- measure-synthetic-stream, profile-synth-stream, profile-long-stream stream --real <- measure-real-stream, profile-real-stream keystroke <- measure-latency, profile-typing, leak-typing transcript <- (new: long-transcript mount cost) submit <- measure-submit, measure-jump session-switch <- profile-session-switch profile-switch <- measure-profile-switch CPU profiling is now a cross-cutting --cpuprofile flag, not 5 separate scripts. CI-tier scenarios (stream, keystroke, transcript) need no backend/credits and are gated against baseline.json (seed values; re-capture with --update-baseline on a reference device). Backend-tier scenarios are report-only. perf-probe.tsx gains loadTranscript() for the transcript scenario. No core files touched; isolation is via CLI args, not env-gated app changes. Verified: node --check all modules, tsc, eslint, and a unit smoke of the stats + regression-gate logic. The end-to-end GUI run (which opens a window) is left to run interactively via `npm run perf -- --spawn`.
60 lines
2.1 KiB
JavaScript
60 lines
2.1 KiB
JavaScript
// Profile-switch latency. Subsumes measure-profile-switch. Backend tier: needs
|
|
// a configured profile in the rail and a live backend. Report-only.
|
|
//
|
|
// node scripts/perf/run.mjs profile-switch --profile <name>
|
|
|
|
import { SELECTORS, sleep } from '../lib/cdp.mjs'
|
|
|
|
export default {
|
|
name: 'profile-switch',
|
|
tier: 'backend',
|
|
description: 'Click a profile in the rail and wait for its sidebar to settle.',
|
|
requiredOpts: ['profile'],
|
|
async run(cdp, opts = {}) {
|
|
const profile = opts.profile
|
|
const settleTimeoutMs = Number(opts.settleTimeoutMs ?? 60000)
|
|
|
|
if (!profile) {
|
|
throw new Error('profile-switch needs --profile <name>')
|
|
}
|
|
|
|
await cdp.send('Runtime.enable')
|
|
|
|
const t0 = await cdp.eval(`(() => {
|
|
const rail = document.querySelector(${JSON.stringify(SELECTORS.profileRail)})
|
|
if (!rail) return null
|
|
const target = [...rail.querySelectorAll('button, [role="tab"]')].find(b =>
|
|
((b.getAttribute('aria-label') || '') + ' ' + (b.title || '') + ' ' + (b.textContent || ''))
|
|
.toLowerCase().includes(${JSON.stringify(String(profile).toLowerCase())}))
|
|
if (!target) return null
|
|
target.click()
|
|
return performance.now()
|
|
})()`)
|
|
|
|
if (t0 === null) {
|
|
throw new Error(`profile "${profile}" not found in the rail`)
|
|
}
|
|
|
|
const deadline = Date.now() + settleTimeoutMs
|
|
let settledMs = null
|
|
|
|
while (Date.now() < deadline) {
|
|
await sleep(100)
|
|
const s = await cdp.eval(`(() => {
|
|
const label = [...document.querySelectorAll('div[aria-hidden]')].find(el => /waking up/i.test(el.textContent || ''))
|
|
const overlayVisible = label ? Number(getComputedStyle(label).opacity) > 0.05 : false
|
|
return { t: performance.now(), overlayVisible, rows: document.querySelectorAll(${JSON.stringify(SELECTORS.rowButton)}).length }
|
|
})()`)
|
|
|
|
if (!s.overlayVisible && s.rows > 0) {
|
|
settledMs = s.t - t0
|
|
break
|
|
}
|
|
}
|
|
|
|
return {
|
|
metrics: { profile_switch_settled_ms: settledMs === null ? -1 : Math.round(settledMs) },
|
|
detail: { profile, timedOut: settledMs === null }
|
|
}
|
|
}
|
|
}
|