mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
perf(desktop): add a stream-history scenario to the perf harness
The existing `stream` scenario measures streaming into an empty thread, which is exactly the case where per-delta transcript work does not show up. This adds `--historyTurns`, which mounts a settled transcript and lets it drain BEFORE the recorders start, so the measurement window contains streaming work only — and `stream-history`, a report-only preset that turns it on. Report-only (tier: manual) rather than gated: how much history a host can mount varies, so the absolute number is not comparable across machines. It is meant for same-machine before/after runs. Co-authored-by: Jakub Wolniewicz <frizikk@users.noreply.github.com>
This commit is contained in:
parent
ad09bf3872
commit
20fcef5207
3 changed files with 47 additions and 3 deletions
|
|
@ -8,11 +8,13 @@ import multitab from './multitab.mjs'
|
|||
import profileSwitch from './profile-switch.mjs'
|
||||
import sessionSwitch from './session-switch.mjs'
|
||||
import stream from './stream.mjs'
|
||||
import streamHistory from './stream-history.mjs'
|
||||
import submit from './submit.mjs'
|
||||
import transcript from './transcript.mjs'
|
||||
|
||||
export const SCENARIOS = {
|
||||
[stream.name]: stream,
|
||||
[streamHistory.name]: streamHistory,
|
||||
[keystroke.name]: keystroke,
|
||||
[transcript.name]: transcript,
|
||||
[multitab.name]: multitab,
|
||||
|
|
|
|||
18
apps/desktop/scripts/perf/scenarios/stream-history.mjs
Normal file
18
apps/desktop/scripts/perf/scenarios/stream-history.mjs
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
// Streaming into an ALREADY-LONG transcript. Same measurement as `stream`, but
|
||||
// the history is mounted and allowed to settle before the recorders start, so
|
||||
// what it captures is the per-delta cost that scales with transcript length —
|
||||
// the regression reported in #69120.
|
||||
//
|
||||
// Report-only (tier: manual): the number depends on how much history the host
|
||||
// can mount, so it is not gated against the committed baseline.
|
||||
|
||||
import stream from './stream.mjs'
|
||||
|
||||
export default {
|
||||
name: 'stream-history',
|
||||
tier: 'manual',
|
||||
description: 'Streaming cost with a long settled transcript already mounted.',
|
||||
run(cdp, opts = {}) {
|
||||
return stream.run(cdp, { ...opts, historyTurns: Number(opts.historyTurns ?? 200) })
|
||||
}
|
||||
}
|
||||
|
|
@ -70,7 +70,7 @@ const COLLECT = `
|
|||
})()
|
||||
`
|
||||
|
||||
function analyze(data, warmupMs) {
|
||||
function analyze(data, warmupMs, extra = {}) {
|
||||
// Drop warm-up frames (recorder installs before the stream starts).
|
||||
const frames = []
|
||||
let acc = 0
|
||||
|
|
@ -102,6 +102,7 @@ function analyze(data, warmupMs) {
|
|||
intermut_p95_ms: Math.round(percentile(interMut, 0.95) * 10) / 10
|
||||
},
|
||||
detail: {
|
||||
...extra,
|
||||
windowS: Math.round(windowS * 10) / 10,
|
||||
avgFps: windowS ? Math.round((frames.length / windowS) * 10) / 10 : 0,
|
||||
frameHistogram: frameHistogram(frames),
|
||||
|
|
@ -128,13 +129,36 @@ export default {
|
|||
// noise unrelated to render cost).
|
||||
const chunk = opts.chunk ?? 'A streamed sentence with **bold**, `code`, and ordinary prose like a normal reply.\n\n'
|
||||
const real = Boolean(opts.real)
|
||||
const historyTurns = Number(opts.historyTurns ?? 0)
|
||||
const historySettleMs = Number(opts.historySettleMs ?? 1500)
|
||||
|
||||
await cdp.send('Runtime.enable')
|
||||
|
||||
// Mount the settled history BEFORE the recorders start, so the measurement
|
||||
// window contains only streaming work — not the one-off mount cost.
|
||||
if (historyTurns > 0) {
|
||||
if (real) {
|
||||
throw new Error('--historyTurns is only supported by the synthetic stream path')
|
||||
}
|
||||
|
||||
await cdp.eval(`window.__PERF_DRIVE__.loadTranscript(${historyTurns})`)
|
||||
await sleep(historySettleMs)
|
||||
|
||||
const mounted = Number(await cdp.eval('window.__PERF_DRIVE__.snapshotMsgs()'))
|
||||
const expected = historyTurns * 2
|
||||
|
||||
if (mounted !== expected) {
|
||||
throw new Error(`expected ${expected} preloaded history messages, got ${mounted}`)
|
||||
}
|
||||
}
|
||||
|
||||
await cdp.eval(RECORDERS)
|
||||
|
||||
if (real) {
|
||||
// Backend path: fire a real prompt and wait for the stream to appear.
|
||||
const baseCount = await cdp.eval(`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`)
|
||||
const baseCount = await cdp.eval(
|
||||
`document.querySelectorAll(${JSON.stringify(SELECTORS.assistantMessage)}).length`
|
||||
)
|
||||
await typeIntoComposer(cdp, opts.prompt ?? 'count from 1 to 80, one number per line', { cps: 40 })
|
||||
await cdp.eval(`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(SELECTORS.composer)})
|
||||
|
|
@ -186,6 +210,6 @@ export default {
|
|||
await cdp.eval('window.__PERF_DRIVE__.reset()')
|
||||
}
|
||||
|
||||
return analyze(data, real ? 0 : 500)
|
||||
return analyze(data, real ? 0 : 500, { historyTurns })
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue