hermes-agent/apps/desktop/scripts/perf/lib/baseline.mjs

// Baseline + regression gate. This is the capability the old one-off scripts
// never had: measured numbers are compared against a committed baseline so a
// PR that regresses streaming/typing/mount cost fails loudly instead of
// silently drifting.
//
// Every tracked metric is "lower is better" (longtask counts, frame/keystroke
// percentiles, mount ms). A metric regresses when it exceeds
// `baseline * (1 + tolFrac) + tolAbs`. tolAbs absorbs sub-millisecond jitter on
// already-fast metrics so they don't false-positive.

import { readFileSync, writeFileSync } from 'node:fs'

const DEFAULT_TOLERANCE = { tolFrac: 0.25, tolAbs: 1 }

export function loadBaseline(path) {
  try {
    return JSON.parse(readFileSync(path, 'utf8'))
  } catch {
    return { _meta: {}, scenarios: {} }
  }
}

/**
 * Compare a scenario's measured metrics against the baseline.
 * @returns {{ rows: Array, regressed: boolean }}
 */
export function compareScenario(name, measured, baseline) {
  const base = baseline.scenarios?.[name]
  const tol = { ...DEFAULT_TOLERANCE, ...(base?.tolerance ?? {}) }
  const rows = []
  let regressed = false

  for (const [metric, value] of Object.entries(measured)) {
    if (typeof value !== 'number') {
      continue
    }

    const baseValue = base?.metrics?.[metric]

    if (typeof baseValue !== 'number') {
      rows.push({ metric, measured: value, baseline: null, limit: null, status: 'new' })

      continue
    }

    const limit = baseValue * (1 + tol.tolFrac) + tol.tolAbs
    const over = value > limit
    regressed = regressed || over

    rows.push({
      metric,
      measured: value,
      baseline: baseValue,
      limit: Math.round(limit * 100) / 100,
      deltaPct: baseValue ? Math.round(((value - baseValue) / baseValue) * 1000) / 10 : null,
      status: over ? 'REGRESSED' : 'ok'
    })
  }

  return { rows, regressed }
}

/** Write measured metrics back as the new baseline for the given scenarios. */
export function updateBaseline(path, results) {
  const baseline = loadBaseline(path)
  baseline.scenarios ??= {}

  for (const { name, metrics } of results) {
    const numeric = Object.fromEntries(Object.entries(metrics).filter(([, v]) => typeof v === 'number'))
    const prev = baseline.scenarios[name] ?? {}
    baseline.scenarios[name] = { ...prev, metrics: numeric }
  }

  baseline._meta = {
    ...baseline._meta,
    updated: new Date().toISOString(),
    platform: `${process.platform}-${process.arch}`,
    node: process.version
  }

  writeFileSync(path, `${JSON.stringify(baseline, null, 2)}\n`)
}

export { DEFAULT_TOLERANCE }