hermes-agent/apps/desktop/scripts/measure-synthetic-stream.mjs

// Measure render cost of a synthetic stream driven through the live $messages atom.
//
// Why synthetic: the user's LLM credits are depleted; we can't fire a real stream.
// The synthetic stream exercises the exact same React pipeline (assistant-ui runtime →
// repository.addOrUpdateMessage → MessagePrimitive re-render → markdown reflow) as a
// real stream. The only thing it does NOT exercise is the gateway → SSE → optimistic-
// merge path, which is orthogonal to the rendering question.
//
// What we record:
//   1) rAF frame intervals (long-frame histogram; >33ms = perceived jank, >100ms = bad)
//   2) PerformanceObserver `longtask` entries (task >50ms blocks input)
//   3) MutationObserver: per-message mutation count & inter-mutation latency
//   4) Optional: typing latency overlay — typing into composer while streaming
//
// Output is plain text suitable for terminal + a JSON sidecar for diffing across runs.

import { writeFileSync } from 'node:fs'

const CDP_HTTP = 'http://127.0.0.1:9222'
const TOKENS = Number(process.env.TOKENS || 300)
const INTERVAL_MS = Number(process.env.INTERVAL_MS || 16)
// Upstream flush throttle to apply in the synthetic driver. Mirrors what the
// real gateway path does in `use-message-stream.scheduleDeltaFlush`. 0
// disables (worst-case, every token = one React commit).
const FLUSH_MIN_MS = Number(process.env.FLUSH_MIN_MS || 0)
const CHUNK = process.env.CHUNK || 'lorem ipsum '
const TYPE_WHILE_STREAMING = process.env.TYPE_WHILE_STREAMING === '1'
const LABEL = process.env.LABEL || 'baseline'
const OUT = process.env.OUT || `frame-times-${LABEL}.json`

async function getTarget() {
  const list = await (await fetch(`${CDP_HTTP}/json`)).json()
  const t = list.find((t) => t.type === 'page' && /5174/.test(t.url))
  if (!t) throw new Error('renderer not found')
  return t
}

class CDP {
  constructor(ws) { this.ws = ws; this.id = 0; this.pending = new Map() }
  static async open(url) {
    const ws = new WebSocket(url)
    await new Promise((r, j) => {
      ws.addEventListener('open', r, { once: true })
      ws.addEventListener('error', (e) => j(e), { once: true })
    })
    const cdp = new CDP(ws)
    ws.addEventListener('message', (ev) => {
      const m = JSON.parse(ev.data.toString())
      if (m.id != null && cdp.pending.has(m.id)) {
        const { resolve, reject } = cdp.pending.get(m.id)
        cdp.pending.delete(m.id)
        if (m.error) reject(new Error(m.error.message))
        else resolve(m.result)
      }
    })
    return cdp
  }
  send(method, params) {
    const id = ++this.id
    return new Promise((res, rej) => {
      this.pending.set(id, { resolve: res, reject: rej })
      this.ws.send(JSON.stringify({ id, method, params }))
    })
  }
  async eval(expr) {
    const r = await this.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true })
    if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'eval')
    return r.result.value
  }
  close() { this.ws.close() }
}

function pct(arr, p) {
  if (!arr.length) return 0
  const i = Math.min(arr.length - 1, Math.floor(arr.length * p))
  return arr[i]
}

async function main() {
  const target = await getTarget()
  const cdp = await CDP.open(target.webSocketDebuggerUrl)

  // Sanity check driver is loaded.
  const probeOk = await cdp.eval('!!window.__PERF_DRIVE__ && !!window.__PERF_DRIVE__.stream')
  if (!probeOk) {
    console.error('__PERF_DRIVE__ not on window — did you reload the renderer after editing perf-probe.tsx?')
    cdp.close()
    process.exit(2)
  }

  // Install recorders.
  await cdp.eval(`
    (() => {
      window.__FT__ = { times: [], stop: false }
      let last = performance.now()
      const tick = () => {
        if (window.__FT__.stop) return
        const now = performance.now()
        window.__FT__.times.push(now - last)
        last = now
        requestAnimationFrame(tick)
      }
      requestAnimationFrame(tick)

      window.__LT__ = { entries: [], stop: false }
      try {
        const po = new PerformanceObserver((list) => {
          if (window.__LT__.stop) return
          for (const e of list.getEntries()) {
            window.__LT__.entries.push({ name: e.name, duration: e.duration, startTime: e.startTime })
          }
        })
        po.observe({ entryTypes: ['longtask'] })
        window.__LT__.po = po
      } catch {}

      window.__MO__ = { mutations: [], stop: false, currentMsg: null }
      const arm = () => {
        const all = document.querySelectorAll('[data-slot="aui_assistant-message-root"]')
        const last = all[all.length - 1]
        if (!last || last === window.__MO__.currentMsg) return
        window.__MO__.currentMsg = last
        if (window.__MO__.obs) window.__MO__.obs.disconnect()
        const obs = new MutationObserver((muts) => {
          if (window.__MO__.stop) return
          const t = performance.now()
          window.__MO__.mutations.push({ t, count: muts.length, len: last.textContent.length })
        })
        obs.observe(last, { childList: true, subtree: true, characterData: true })
        window.__MO__.obs = obs
      }
      window.__MO__.arm = arm

      // Optional: typing observer — fires keystroke timings if asked.
      window.__TYP__ = { times: [], stop: false, lastKey: 0 }
      return 'recorders armed'
    })()
  `)

  // Baseline state.
  const base = JSON.parse(await cdp.eval(`
    JSON.stringify({
      assistantCount: document.querySelectorAll('[data-slot="aui_assistant-message-root"]').length,
      atomCount: window.__PERF_DRIVE__.snapshotMsgs()
    })
  `))
  console.log('baseline:', base)

  // Drive a synthetic stream.
  const streamStart = Date.now()
  await cdp.eval(`window.__PERF_DRIVE__.stream({ chunk: ${JSON.stringify(CHUNK)}, intervalMs: ${INTERVAL_MS}, totalTokens: ${TOKENS}, flushMinMs: ${FLUSH_MIN_MS} })`)

  // After the first paint, arm MO on the new message.
  await new Promise((r) => setTimeout(r, 200))
  await cdp.eval('window.__MO__.arm()')

  // Optional: type while streaming.
  if (TYPE_WHILE_STREAMING) {
    await new Promise((r) => setTimeout(r, 400))
    await cdp.eval(`(() => {
      const ed = document.querySelector('[contenteditable="true"]')
      ed.focus()
      window.__TYP__.startedAt = performance.now()
      const text = 'the quick brown fox jumps over the lazy dog '
      let i = 0
      const tick = () => {
        if (i >= text.length) return
        const t0 = performance.now()
        document.execCommand('insertText', false, text[i])
        // requestAnimationFrame to wait for next paint
        requestAnimationFrame(() => {
          window.__TYP__.times.push(performance.now() - t0)
        })
        i++
        setTimeout(tick, 60)
      }
      tick()
      return 'typing'
    })()`)
  }

  // Wait for stream to complete + small grace.
  const expectedMs = TOKENS * INTERVAL_MS + 1500
  await new Promise((r) => setTimeout(r, expectedMs))

  // Pull recordings.
  const data = JSON.parse(await cdp.eval(`
    (() => {
      window.__FT__.stop = true
      window.__LT__.stop = true
      window.__MO__.stop = true
      window.__TYP__.stop = true
      try { window.__LT__.po && window.__LT__.po.disconnect() } catch {}
      try { window.__MO__.obs && window.__MO__.obs.disconnect() } catch {}
      return JSON.stringify({
        frames: window.__FT__.times,
        longtasks: window.__LT__.entries,
        mutations: window.__MO__.mutations,
        typing: window.__TYP__.times,
        finalText: (() => { const a = document.querySelectorAll('[data-slot="aui_assistant-message-root"]'); return a.length ? a[a.length-1].textContent.length : 0 })()
      })
    })()
  `))

  // Reset DOM back to baseline so we don't accumulate fake messages.
  await cdp.eval('window.__PERF_DRIVE__.reset()')

  // Analysis (trim warm-up: drop frames before first mutation timestamp).
  const firstMut = data.mutations[0]?.t
  const frames = data.frames

  // Sum durations to figure out when each frame happened (relative to recorder start).
  const frameTimeline = []
  let acc = 0
  for (const f of frames) { acc += f; frameTimeline.push(acc) }

  // Mutations are in performance.now() ms; frames started recording when we installed
  // the recorder (before stream). To align: compute total stream window from frames
  // after mutation activity began. Simpler heuristic: drop first 500ms of frames as warm-up.
  const WARMUP_MS = 500
  let dropIdx = 0
  for (let i = 0; i < frames.length; i++) {
    if (frameTimeline[i] >= WARMUP_MS) { dropIdx = i; break }
  }
  const streamFrames = frames.slice(dropIdx)

  const buckets = { '<=16.7': 0, '16.7-33': 0, '33-50': 0, '50-100': 0, '100-200': 0, '>200': 0 }
  let frameTotal = 0
  let maxFrame = 0
  for (const f of streamFrames) {
    frameTotal += f
    if (f > maxFrame) maxFrame = f
    if (f <= 16.7) buckets['<=16.7']++
    else if (f <= 33) buckets['16.7-33']++
    else if (f <= 50) buckets['33-50']++
    else if (f <= 100) buckets['50-100']++
    else if (f <= 200) buckets['100-200']++
    else buckets['>200']++
  }
  const sortedFrames = [...streamFrames].sort((a, b) => a - b)
  const fAvgFps = streamFrames.length ? (streamFrames.length / (frameTotal / 1000)).toFixed(1) : 'n/a'
  const fP50 = pct(sortedFrames, 0.5).toFixed(1)
  const fP95 = pct(sortedFrames, 0.95).toFixed(1)
  const fP99 = pct(sortedFrames, 0.99).toFixed(1)
  const slowFrames = streamFrames.filter((f) => f > 33).length
  const veryslowFrames = streamFrames.filter((f) => f > 100).length

  const ltDur = data.longtasks.map((e) => e.duration).sort((a, b) => a - b)
  const ltMs = ltDur.reduce((a, b) => a + b, 0)
  const ltMax = ltDur.length ? ltDur[ltDur.length - 1] : 0
  const ltP95 = pct(ltDur, 0.95)

  // Mutation cadence.
  const mutDurs = []
  for (let i = 1; i < data.mutations.length; i++) mutDurs.push(data.mutations[i].t - data.mutations[i - 1].t)
  mutDurs.sort((a, b) => a - b)
  const mutP50 = pct(mutDurs, 0.5)
  const mutP95 = pct(mutDurs, 0.95)
  const mutMax = mutDurs.length ? mutDurs[mutDurs.length - 1] : 0

  // Typing latency (optional).
  let typingSummary = null
  if (TYPE_WHILE_STREAMING && data.typing.length) {
    const t = [...data.typing].sort((a, b) => a - b)
    typingSummary = {
      n: t.length,
      p50: pct(t, 0.5).toFixed(1),
      p95: pct(t, 0.95).toFixed(1),
      max: t[t.length - 1].toFixed(1)
    }
  }

  const result = {
    label: LABEL,
    timestamp: new Date().toISOString(),
    config: { TOKENS, INTERVAL_MS, CHUNK, TYPE_WHILE_STREAMING, FLUSH_MIN_MS },
    streamWallMs: Date.now() - streamStart,
    frames: {
      total: streamFrames.length,
      avgFps: fAvgFps,
      windowS: (frameTotal / 1000).toFixed(1),
      p50: fP50,
      p95: fP95,
      p99: fP99,
      max: maxFrame.toFixed(1),
      slow33: slowFrames,
      veryslow100: veryslowFrames,
      histogram: buckets
    },
    longtasks: {
      n: data.longtasks.length,
      totalMs: ltMs.toFixed(0),
      maxMs: ltMax.toFixed(1),
      p95Ms: ltP95.toFixed(1)
    },
    mutations: {
      n: data.mutations.length,
      finalTextLen: data.finalText,
      interMutP50ms: mutP50.toFixed(1),
      interMutP95ms: mutP95.toFixed(1),
      interMutMaxMs: mutMax.toFixed(1)
    },
    typing: typingSummary
  }

  writeFileSync(OUT, JSON.stringify(result, null, 2))

  console.log('\n=== SYNTHETIC STREAM RESULTS ===')
  console.log('label:', LABEL, '| tokens:', TOKENS, '@', INTERVAL_MS, 'ms')
  console.log('streamWallMs:', result.streamWallMs)
  console.log('FRAMES: avgFps', fAvgFps, '| p50', fP50, 'ms | p95', fP95, 'ms | p99', fP99, 'ms | max', maxFrame.toFixed(1), 'ms')
  console.log('FRAMES histogram:', buckets)
  console.log('FRAMES slow(>33):', slowFrames, '/ veryslow(>100):', veryslowFrames, 'of', streamFrames.length)
  console.log('LONGTASKS:', data.longtasks.length, '| total', ltMs.toFixed(0), 'ms | max', ltMax.toFixed(1), 'ms | p95', ltP95.toFixed(1), 'ms')
  console.log('MUTATIONS:', data.mutations.length, '| finalLen', data.finalText, 'chars | inter p50', mutP50.toFixed(1), 'ms | p95', mutP95.toFixed(1), 'ms')
  if (typingSummary) console.log('TYPING-WHILE-STREAMING latency: p50', typingSummary.p50, 'ms | p95', typingSummary.p95, 'ms | n=', typingSummary.n)
  console.log('written to', OUT)

  cdp.close()
}

main().catch((e) => { console.error(e); process.exit(1) })