hermes-agent/apps/desktop/scripts/perf/run.mjs

// Desktop perf harness entrypoint.
//
//   node scripts/perf/run.mjs [scenarios...] [flags]
//
// Default (no scenarios): runs the CI suite (stream, keystroke, transcript)
// against a renderer on :9222 and diffs the committed baseline.
//
// Flags:
//   --spawn                launch a fully isolated instance (own user-data-dir +
//                          HERMES_HOME + debug port) instead of attaching
//   --port <n>             CDP port to attach to (default 9222)
//   --dev-port <n>         vite dev-server port to match / spawn (default 5174)
//   --runs <n>             repeat each scenario n times, report the median (default 1)
//   --cpuprofile [dir]     also record a V8 CPU profile per scenario (top-30 self time)
//   --update-baseline      overwrite baseline.json with this run's numbers
//   --json <path>          write the full results JSON here
//   --tier <ci|backend>    run all scenarios of a tier
//   ...scenario opts       e.g. --tokens 600, --turns 400, --real, --a <sid> --b <sid>, --profile <name>
//
// Examples:
//   npm run perf                       # attach to :9222, run CI suite, gate on baseline
//   npm run perf -- --spawn            # isolated instance, no running app needed
//   npm run perf -- stream --cpuprofile --tokens 800
//   npm run perf -- --update-baseline

import { writeFileSync } from 'node:fs'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'

import { withCpuProfile } from './lib/cdp.mjs'
import { compareScenario, loadBaseline, updateBaseline } from './lib/baseline.mjs'
import { attach, startIsolatedInstance } from './lib/launch.mjs'
import { cpuProfileTopSelf, median } from './lib/stats.mjs'
import { CI_SCENARIOS, SCENARIOS } from './scenarios/index.mjs'

const HERE = dirname(fileURLToPath(import.meta.url))
const BASELINE_PATH = join(HERE, 'baseline.json')

function parseArgs(argv) {
  const positional = []
  const flags = {}

  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i]

    if (arg.startsWith('--')) {
      const [key, inlineValue] = arg.slice(2).split(/=(.*)/s)
      const next = argv[i + 1]

      if (inlineValue !== undefined) {
        flags[key] = inlineValue
      } else if (next === undefined || next.startsWith('--')) {
        flags[key] = true
      } else {
        flags[key] = next
        i++
      }
    } else {
      positional.push(arg)
    }
  }

  return { positional, flags }
}

function medianMetrics(runs) {
  const keys = new Set(runs.flatMap(r => Object.keys(r)))
  const out = {}

  for (const key of keys) {
    const values = runs.map(r => r[key]).filter(v => typeof v === 'number')
    out[key] = values.length ? Math.round(median(values) * 10) / 10 : runs[0][key]
  }

  return out
}

function printMetrics(name, metrics, comparison) {
  console.log(`\n● ${name}`)
  const byMetric = new Map((comparison?.rows ?? []).map(r => [r.metric, r]))

  for (const [metric, value] of Object.entries(metrics)) {
    const row = byMetric.get(metric)

    if (!row || row.baseline === null) {
      console.log(`   ${metric.padEnd(26)} ${String(value).padStart(9)}${row ? '   (new)' : ''}`)
    } else {
      const tag = row.status === 'REGRESSED' ? '  ✗ REGRESSED' : '  ✓'
      const delta = row.deltaPct === null ? '' : ` (${row.deltaPct > 0 ? '+' : ''}${row.deltaPct}%)`
      console.log(
        `   ${metric.padEnd(26)} ${String(value).padStart(9)}  vs ${String(row.baseline).padStart(9)}${delta}${tag}`
      )
    }
  }
}

async function main() {
  const { positional, flags } = parseArgs(process.argv.slice(2))

  let names = positional
  if (!names.length) {
    names = flags.tier ? Object.values(SCENARIOS).filter(s => s.tier === flags.tier).map(s => s.name) : CI_SCENARIOS
  }

  const unknown = names.filter(n => !SCENARIOS[n])
  if (unknown.length) {
    console.error(`unknown scenario(s): ${unknown.join(', ')}\nknown: ${Object.keys(SCENARIOS).join(', ')}`)
    process.exit(2)
  }

  const runs = Number(flags.runs ?? 1)
  const port = Number(flags.port ?? 9222)
  const devPort = Number(flags['dev-port'] ?? 5174)
  const cpuProfile = 'cpuprofile' in flags
  const cpuProfileDir = typeof flags.cpuprofile === 'string' ? flags.cpuprofile : HERE

  const connection = flags.spawn
    ? await startIsolatedInstance({ port, devPort })
    : await attach({ port, match: String(devPort) })

  const { cdp, teardown } = connection
  const results = []
  let regressed = false

  try {
    const baseline = loadBaseline(BASELINE_PATH)

    for (const name of names) {
      const scenario = SCENARIOS[name]
      const perRun = []
      let detail = null

      for (let i = 0; i < runs; i++) {
        if (cpuProfile && i === 0) {
          const { result, profile } = await withCpuProfile(cdp, () => scenario.run(cdp, flags))
          const out = join(cpuProfileDir, `${name}-${Date.now()}.cpuprofile`)
          writeFileSync(out, JSON.stringify(profile))
          console.log(`\n[cpuprofile] wrote ${out}`)
          console.log('[cpuprofile] top self-time (ms):')
          for (const r of cpuProfileTopSelf(profile, 15)) {
            console.log(`   ${r.ms.toFixed(1).padStart(7)}  ${r.name.padEnd(38)}  ${r.url}:${r.line}`)
          }
          perRun.push(result.metrics)
          detail = result.detail
        } else {
          const result = await scenario.run(cdp, flags)
          perRun.push(result.metrics)
          detail = result.detail
        }
      }

      const metrics = medianMetrics(perRun)
      const comparison = scenario.tier === 'ci' ? compareScenario(name, metrics, baseline) : null
      regressed = regressed || Boolean(comparison?.regressed)
      results.push({ name, tier: scenario.tier, metrics, detail })
      printMetrics(name, metrics, comparison)
    }
  } finally {
    teardown()
  }

  if (flags.json) {
    writeFileSync(resolve(String(flags.json)), `${JSON.stringify({ timestamp: new Date().toISOString(), results }, null, 2)}\n`)
    console.log(`\nwrote ${flags.json}`)
  }

  if (flags['update-baseline']) {
    updateBaseline(BASELINE_PATH, results.filter(r => r.tier === 'ci'))
    console.log(`\nupdated ${BASELINE_PATH}`)
    return
  }

  if (regressed) {
    console.error('\n✗ perf regression vs baseline (see REGRESSED rows above)')
    process.exit(1)
  }

  console.log('\n✓ no perf regressions')
}

main().catch(err => {
  console.error('\nperf harness failed:', err.stack ?? err.message)
  process.exit(1)
})