mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-09 13:21:42 +00:00
1102 lines
51 KiB
JavaScript
1102 lines
51 KiB
JavaScript
#!/usr/bin/env node
|
||
// Report renderer — reads bench/results/*.json + bench/session-distribution.json
|
||
// and emits ONE self-contained bench/report.html (inline SVG, zero CDN/network)
|
||
// plus PNG exports of each chart to bench/report-assets/ (rasterized with the
|
||
// resvg-js available at ~/.claude/skills/tmux-pane-screenshot/scripts/node_modules).
|
||
// Real data only: cells with no results render as "not run".
|
||
//
|
||
// Audience note: the report is written for a 60-second skim — verdicts first,
|
||
// plain language everywhere, precise stats terms kept in parentheses.
|
||
|
||
import { createRequire } from 'node:module'
|
||
import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
|
||
import { homedir } from 'node:os'
|
||
import { dirname, join } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
|
||
const here = dirname(fileURLToPath(import.meta.url))
|
||
const RESULTS_DIR = join(here, 'results')
|
||
const ASSETS_DIR = join(here, 'report-assets')
|
||
const OUT_HTML = join(here, 'report.html')
|
||
const DIST_FILE = join(here, 'session-distribution.json')
|
||
|
||
const CAP_MB = 2048
|
||
|
||
// ── data load ───────────────────────────────────────────────────────────
|
||
function loadResults() {
|
||
let files = []
|
||
try {
|
||
files = readdirSync(RESULTS_DIR).filter(f => f.endsWith('.json'))
|
||
} catch {
|
||
return []
|
||
}
|
||
const out = []
|
||
for (const f of files.sort()) {
|
||
try {
|
||
const r = JSON.parse(readFileSync(join(RESULTS_DIR, f), 'utf8'))
|
||
r._file = f
|
||
out.push(r)
|
||
} catch {
|
||
/* skip unparseable */
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
function loadDistribution() {
|
||
try {
|
||
return JSON.parse(readFileSync(DIST_FILE, 'utf8'))
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
// ── stats ───────────────────────────────────────────────────────────────
|
||
const quantile = (sorted, q) => {
|
||
if (!sorted.length) return null
|
||
const pos = (sorted.length - 1) * q
|
||
const lo = Math.floor(pos)
|
||
const hi = Math.ceil(pos)
|
||
return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo)
|
||
}
|
||
const pq = (xs, q) => quantile(xs.slice().sort((a, b) => a - b), q)
|
||
const median = xs => pq(xs, 0.5)
|
||
const iqr = xs => {
|
||
const s = xs.slice().sort((a, b) => a - b)
|
||
return [quantile(s, 0.25), quantile(s, 0.75)]
|
||
}
|
||
const fmt = (x, d = 1) => (x === null || x === undefined || Number.isNaN(x) ? '—' : Number(x).toFixed(d))
|
||
const fmtMedIqr = (xs, d = 1) => {
|
||
if (!xs.length) return 'not run'
|
||
const [lo, hi] = iqr(xs)
|
||
return `${fmt(median(xs), d)} <span class="iqr">[middle half: ${fmt(lo, d)}–${fmt(hi, d)}]</span>`
|
||
}
|
||
|
||
// least-squares slope of rss_mb vs msgs over points
|
||
function lsSlope(points) {
|
||
if (points.length < 3) return null
|
||
const n = points.length
|
||
let sx = 0
|
||
let sy = 0
|
||
let sxx = 0
|
||
let sxy = 0
|
||
for (const [x, y] of points) {
|
||
sx += x
|
||
sy += y
|
||
sxx += x * x
|
||
sxy += x * y
|
||
}
|
||
const denom = n * sxx - sx * sx
|
||
if (denom === 0) return null
|
||
return (n * sxy - sx * sy) / denom // MB per message
|
||
}
|
||
|
||
// Per-run back-half slope (MB/1k msgs): fit over msgs >= max(500, maxMsgs/2)
|
||
// — warmup (first 500 msgs) always excluded per protocol.
|
||
function runSlope(run) {
|
||
const pts = run.samples
|
||
.filter(s => s.kind === 'boundary' && s.msgs != null && s.rss_kb != null)
|
||
.map(s => [s.msgs, s.rss_kb / 1024])
|
||
if (pts.length < 4) return null
|
||
const maxMsgs = pts[pts.length - 1][0]
|
||
const cut = Math.max(500, maxMsgs / 2)
|
||
const back = pts.filter(([x]) => x >= cut)
|
||
const slope = lsSlope(back)
|
||
return slope === null ? null : slope * 1000
|
||
}
|
||
|
||
// plateau: median RSS over the final quartile of boundary samples
|
||
function runPlateau(run) {
|
||
const pts = run.samples.filter(s => s.kind === 'boundary' && s.rss_kb != null).map(s => s.rss_kb / 1024)
|
||
if (pts.length < 4) return null
|
||
return median(pts.slice(Math.floor(pts.length * 0.75)))
|
||
}
|
||
|
||
// ── SVG primitives ──────────────────────────────────────────────────────
|
||
const COLORS = { ink: '#e06c75', 'otui-capped': '#61afef', 'otui-uncapped': '#56b6c2', other: '#c678dd' }
|
||
const NICE = { ink: 'Ink', 'otui-capped': 'OpenTUI', 'otui-uncapped': 'OpenTUI (no cap)' }
|
||
const esc = s => String(s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||
|
||
function chart({ title, w = 1040, h = 500, xLabel, yLabel, xMax, yMax, series, capLine, markers = [], note }) {
|
||
const padL = 72
|
||
const padR = 18
|
||
const padT = 48
|
||
const padB = 70
|
||
const pw = w - padL - padR
|
||
const ph = h - padT - padB
|
||
const X = x => padL + (x / xMax) * pw
|
||
const Y = y => padT + ph - (y / yMax) * ph
|
||
const parts = []
|
||
parts.push(`<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}" viewBox="0 0 ${w} ${h}" font-family="ui-monospace,monospace">`)
|
||
parts.push(`<rect width="${w}" height="${h}" fill="#11151c"/>`)
|
||
parts.push(`<text x="${padL}" y="28" fill="#e8eaf0" font-size="18" font-weight="bold">${esc(title)}</text>`)
|
||
// grid + axes
|
||
const xticks = 6
|
||
const yticks = 5
|
||
for (let i = 0; i <= xticks; i++) {
|
||
const xv = (xMax / xticks) * i
|
||
const x = X(xv)
|
||
parts.push(`<line x1="${x}" y1="${padT}" x2="${x}" y2="${padT + ph}" stroke="#262c38" stroke-width="1"/>`)
|
||
parts.push(`<text x="${x}" y="${padT + ph + 20}" fill="#8b93a7" font-size="12" text-anchor="middle">${Math.round(xv)}</text>`)
|
||
}
|
||
for (let i = 0; i <= yticks; i++) {
|
||
const yv = (yMax / yticks) * i
|
||
const y = Y(yv)
|
||
parts.push(`<line x1="${padL}" y1="${y}" x2="${padL + pw}" y2="${y}" stroke="#262c38" stroke-width="1"/>`)
|
||
parts.push(`<text x="${padL - 8}" y="${y + 4}" fill="#8b93a7" font-size="12" text-anchor="end">${Math.round(yv)}</text>`)
|
||
}
|
||
parts.push(`<text x="${padL + pw / 2}" y="${h - 26}" fill="#aab2c5" font-size="13" text-anchor="middle">${esc(xLabel)}</text>`)
|
||
parts.push(
|
||
`<text x="18" y="${padT + ph / 2}" fill="#aab2c5" font-size="13" text-anchor="middle" transform="rotate(-90 18 ${padT + ph / 2})">${esc(yLabel)}</text>`
|
||
)
|
||
if (capLine != null && capLine <= yMax) {
|
||
parts.push(
|
||
`<line x1="${padL}" y1="${Y(capLine)}" x2="${padL + pw}" y2="${Y(capLine)}" stroke="#e5c07b" stroke-width="1.5" stroke-dasharray="7 5"/>`
|
||
)
|
||
parts.push(`<text x="${padL + pw - 4}" y="${Y(capLine) - 6}" fill="#e5c07b" font-size="12" text-anchor="end">hard memory cap ${capLine} MB</text>`)
|
||
}
|
||
let legendY = padT + 10
|
||
for (const s of series) {
|
||
if (!s.points.length) continue
|
||
const d = s.points.map(([x, y], i) => `${i === 0 ? 'M' : 'L'}${X(Math.min(x, xMax)).toFixed(1)},${Y(Math.min(y, yMax)).toFixed(1)}`).join(' ')
|
||
parts.push(`<path d="${d}" fill="none" stroke="${s.color}" stroke-width="${s.width ?? 2}" opacity="${s.opacity ?? 1}"/>`)
|
||
if (s.label) {
|
||
parts.push(`<rect x="${padL + pw - 230}" y="${legendY - 9}" width="14" height="3" fill="${s.color}"/>`)
|
||
parts.push(`<text x="${padL + pw - 210}" y="${legendY - 2}" fill="#cdd3e0" font-size="12">${esc(s.label)}</text>`)
|
||
legendY += 18
|
||
}
|
||
}
|
||
for (const m of markers) {
|
||
const x = X(Math.min(m.x, xMax))
|
||
const y = Y(Math.min(m.y, yMax))
|
||
parts.push(`<text x="${x}" y="${y + 5}" fill="${m.color ?? '#e5c07b'}" font-size="17" text-anchor="middle" font-weight="bold">×</text>`)
|
||
if (m.label) parts.push(`<text x="${x}" y="${y - 10}" fill="${m.color ?? '#e5c07b'}" font-size="11" text-anchor="middle">${esc(m.label)}</text>`)
|
||
}
|
||
if (note) parts.push(`<text x="${padL}" y="${h - 8}" fill="#6f7689" font-size="11">${esc(note)}</text>`)
|
||
parts.push('</svg>')
|
||
return parts.join('\n')
|
||
}
|
||
|
||
function barChart({ title, w = 1040, h = 420, groups, yLabel, note, barWidth = 56 }) {
|
||
// groups: [{label, bars: [{name, value, lo, hi, color}]}]
|
||
const padL = 72
|
||
const padR = 18
|
||
const padT = 48
|
||
const padB = 74
|
||
const pw = w - padL - padR
|
||
const ph = h - padT - padB
|
||
const vals = groups.flatMap(g => g.bars.map(b => b.hi ?? b.value)).filter(v => v != null)
|
||
if (!vals.length) return null
|
||
const yMax = Math.max(...vals) * 1.18
|
||
const Y = y => padT + ph - (y / yMax) * ph
|
||
const parts = []
|
||
parts.push(`<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}" viewBox="0 0 ${w} ${h}" font-family="ui-monospace,monospace">`)
|
||
parts.push(`<rect width="${w}" height="${h}" fill="#11151c"/>`)
|
||
parts.push(`<text x="${padL}" y="28" fill="#e8eaf0" font-size="18" font-weight="bold">${esc(title)}</text>`)
|
||
for (let i = 0; i <= 5; i++) {
|
||
const yv = (yMax / 5) * i
|
||
parts.push(`<line x1="${padL}" y1="${Y(yv)}" x2="${padL + pw}" y2="${Y(yv)}" stroke="#262c38"/>`)
|
||
parts.push(`<text x="${padL - 8}" y="${Y(yv) + 4}" fill="#8b93a7" font-size="12" text-anchor="end">${yv >= 100 ? Math.round(yv) : yv.toFixed(1)}</text>`)
|
||
}
|
||
parts.push(
|
||
`<text x="18" y="${padT + ph / 2}" fill="#aab2c5" font-size="13" text-anchor="middle" transform="rotate(-90 18 ${padT + ph / 2})">${esc(yLabel)}</text>`
|
||
)
|
||
const gw = pw / groups.length
|
||
groups.forEach((g, gi) => {
|
||
const bw = Math.min(barWidth, (gw - 28) / Math.max(1, g.bars.length))
|
||
g.bars.forEach((b, bi) => {
|
||
if (b.value == null) return
|
||
const x = padL + gi * gw + gw / 2 - (g.bars.length * bw) / 2 + bi * bw
|
||
parts.push(`<rect x="${x + 3}" y="${Y(b.value)}" width="${bw - 6}" height="${padT + ph - Y(b.value)}" fill="${b.color}" opacity="0.85"/>`)
|
||
if (b.lo != null && b.hi != null) {
|
||
const cx = x + bw / 2
|
||
parts.push(`<line x1="${cx}" y1="${Y(b.lo)}" x2="${cx}" y2="${Y(b.hi)}" stroke="#e8eaf0" stroke-width="1.5"/>`)
|
||
}
|
||
parts.push(`<text x="${x + bw / 2}" y="${Y(b.value) - 6}" fill="#e8eaf0" font-size="12" text-anchor="middle">${b.value >= 100 ? Math.round(b.value) : b.value.toFixed(1)}</text>`)
|
||
parts.push(`<text x="${x + bw / 2}" y="${padT + ph + 17}" fill="#8b93a7" font-size="11" text-anchor="middle">${esc(b.name)}</text>`)
|
||
})
|
||
parts.push(`<text x="${padL + gi * gw + gw / 2}" y="${padT + ph + 38}" fill="#aab2c5" font-size="13" text-anchor="middle">${esc(g.label)}</text>`)
|
||
})
|
||
if (note) parts.push(`<text x="${padL}" y="${h - 8}" fill="#6f7689" font-size="11">${esc(note)}</text>`)
|
||
parts.push('</svg>')
|
||
return parts.join('\n')
|
||
}
|
||
|
||
// categorical histogram with percentile markers (uneven buckets, equal-width bars)
|
||
function histChart({ title, buckets, percentiles, w = 1040, h = 460, yLabel, note }) {
|
||
const bs = buckets.filter(b => b.count > 0 || b.hi != null)
|
||
while (bs.length && bs[bs.length - 1].count === 0) bs.pop()
|
||
if (!bs.length) return null
|
||
const padL = 72
|
||
const padR = 18
|
||
const padT = 96 // room for staggered percentile labels
|
||
const padB = 70
|
||
const pw = w - padL - padR
|
||
const ph = h - padT - padB
|
||
const maxC = Math.max(...bs.map(b => b.count))
|
||
const yMax = maxC * 1.1
|
||
const Y = y => padT + ph - (y / yMax) * ph
|
||
const bw = pw / bs.length
|
||
const parts = []
|
||
parts.push(`<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}" viewBox="0 0 ${w} ${h}" font-family="ui-monospace,monospace">`)
|
||
parts.push(`<rect width="${w}" height="${h}" fill="#11151c"/>`)
|
||
parts.push(`<text x="${padL}" y="28" fill="#e8eaf0" font-size="18" font-weight="bold">${esc(title)}</text>`)
|
||
for (let i = 0; i <= 5; i++) {
|
||
const yv = (yMax / 5) * i
|
||
parts.push(`<line x1="${padL}" y1="${Y(yv)}" x2="${padL + pw}" y2="${Y(yv)}" stroke="#262c38"/>`)
|
||
parts.push(`<text x="${padL - 8}" y="${Y(yv) + 4}" fill="#8b93a7" font-size="12" text-anchor="end">${Math.round(yv)}</text>`)
|
||
}
|
||
parts.push(
|
||
`<text x="18" y="${padT + ph / 2}" fill="#aab2c5" font-size="13" text-anchor="middle" transform="rotate(-90 18 ${padT + ph / 2})">${esc(yLabel)}</text>`
|
||
)
|
||
bs.forEach((b, i) => {
|
||
const x = padL + i * bw
|
||
parts.push(`<rect x="${x + 4}" y="${Y(b.count)}" width="${bw - 8}" height="${padT + ph - Y(b.count)}" fill="#c678dd" opacity="0.75"/>`)
|
||
parts.push(`<text x="${x + bw / 2}" y="${Y(b.count) - 6}" fill="#e8eaf0" font-size="12" text-anchor="middle">${b.count}</text>`)
|
||
const lbl = b.hi == null ? `${b.lo}+` : `${b.lo}–${b.hi}`
|
||
parts.push(`<text x="${x + bw / 2}" y="${padT + ph + 20}" fill="#8b93a7" font-size="12" text-anchor="middle">${esc(lbl)}</text>`)
|
||
})
|
||
parts.push(`<text x="${padL + pw / 2}" y="${h - 26}" fill="#aab2c5" font-size="13" text-anchor="middle">messages in the session</text>`)
|
||
// percentile markers: position within the containing bucket (linear within bucket)
|
||
percentiles.forEach((p, i) => {
|
||
let bi = bs.findIndex(b => p.v >= b.lo && (b.hi == null || p.v < b.hi))
|
||
if (bi < 0) bi = bs.length - 1
|
||
const b = bs[bi]
|
||
const frac = b.hi == null ? 0.5 : (p.v - b.lo) / (b.hi - b.lo)
|
||
const x = padL + (bi + frac) * bw
|
||
const labelY = 46 + (i % 3) * 16
|
||
const flip = x > w - 160
|
||
parts.push(`<line x1="${x}" y1="${labelY + 4}" x2="${x}" y2="${padT + ph}" stroke="#e5c07b" stroke-width="1.5" stroke-dasharray="5 4" opacity="0.85"/>`)
|
||
parts.push(`<text x="${flip ? x - 4 : x + 4}" y="${labelY}" fill="#e5c07b" font-size="12" text-anchor="${flip ? 'end' : 'start'}">${esc(p.label)}</text>`)
|
||
})
|
||
if (note) parts.push(`<text x="${padL}" y="${h - 8}" fill="#6f7689" font-size="11">${esc(note)}</text>`)
|
||
parts.push('</svg>')
|
||
return parts.join('\n')
|
||
}
|
||
|
||
// ── aggregate stats used by verdicts + sections ─────────────────────────
|
||
function aggregate(results) {
|
||
const A = {}
|
||
const cell = (name, cfg) =>
|
||
results.filter(r => r.meta.cell === name && (cfg == null || r.meta.config === cfg) && !r.meta.instrumented)
|
||
|
||
// memory peaks (VmHWM = process peak RSS) per cell/config
|
||
A.memPeak = {}
|
||
for (const c of ['mem100', 'mem300', 'mem2000', 'mem3000']) {
|
||
A.memPeak[c] = {}
|
||
for (const cfg of ['ink', 'otui-capped', 'otui-uncapped']) {
|
||
const v = cell(c, cfg).filter(r => r.meta.mode === 'mem').map(r => r.summary.vmhwm_kb).filter(Boolean).map(k => k / 1024)
|
||
if (v.length) A.memPeak[c][cfg] = { med: median(v), n: v.length, all: v }
|
||
}
|
||
}
|
||
// mem3000 plateaus
|
||
A.plateau3000 = {}
|
||
for (const cfg of ['ink', 'otui-capped', 'otui-uncapped']) {
|
||
const v = cell('mem3000', cfg).filter(r => r.meta.mode === 'mem').map(runPlateau).filter(x => x != null)
|
||
if (v.length) A.plateau3000[cfg] = { med: median(v), all: v }
|
||
}
|
||
|
||
// scroll latencies pooled per config
|
||
A.scroll = {}
|
||
for (const r of results.filter(r => r.meta.cell.startsWith('scroll') && r.summary.scroll_latencies_ms?.length)) {
|
||
;(A.scroll[r.meta.config] ??= []).push(...r.summary.scroll_latencies_ms)
|
||
}
|
||
|
||
// echo
|
||
A.echo = {}
|
||
for (const r of results.filter(r => r.meta.cell === 'echo' && r.summary.echo)) A.echo[r.meta.config] = r.summary.echo
|
||
|
||
// pipeline (cpu + frame pacing)
|
||
A.pipeline = {}
|
||
for (const r of results.filter(r => r.meta.cell === 'pipeline' && r.summary.pipeline)) {
|
||
A.pipeline[r.meta.config] = { cpu: r.summary.pipeline.cpu_s, fp: r.summary.frame_pacing, bytes: r.summary.pipeline.bytes_total, msgs: r.summary.msgs_streamed }
|
||
}
|
||
|
||
// chaos: scenario × config
|
||
A.chaos = {}
|
||
for (const r of results.filter(r => r.meta.cell === 'chaos' && r.summary.chaos)) {
|
||
const sc = r.summary.chaos.scenario
|
||
;(A.chaos[sc] ??= {})[r.meta.config] = r.summary.chaos
|
||
}
|
||
|
||
// startup
|
||
A.startup = {}
|
||
for (const r of results.filter(r => r.meta.cell === 'startup')) {
|
||
const c = (A.startup[r.meta.config] ??= { fb: [], sc: [] })
|
||
if (r.summary.first_byte_ms != null) c.fb.push(r.summary.first_byte_ms)
|
||
if (r.summary.session_create_ms != null) c.sc.push(r.summary.session_create_ms)
|
||
}
|
||
return A
|
||
}
|
||
|
||
// ── chart builders ──────────────────────────────────────────────────────
|
||
function rssChart(results) {
|
||
const runs = results.filter(
|
||
r => (r.meta.cell.startsWith('mem') || r.meta.cell.startsWith('slope')) && !r.meta.instrumented && r.meta.mode === 'mem'
|
||
)
|
||
if (!runs.length) return null
|
||
let xMax = 0
|
||
let yMax = CAP_MB * 1.05
|
||
const series = []
|
||
const markers = []
|
||
const seen = new Set()
|
||
for (const r of runs) {
|
||
const pts = r.samples.filter(s => s.kind === 'boundary' && s.msgs != null && s.rss_kb != null).map(s => [s.msgs, s.rss_kb / 1024])
|
||
if (!pts.length) continue
|
||
xMax = Math.max(xMax, pts[pts.length - 1][0])
|
||
yMax = Math.max(yMax, ...pts.map(p => p[1]))
|
||
const color = COLORS[r.meta.config] ?? COLORS.other
|
||
const key = r.meta.config
|
||
series.push({
|
||
points: pts,
|
||
color,
|
||
width: r.meta.cell.startsWith('slope') ? 2.5 : 1.5,
|
||
opacity: 0.8,
|
||
label: seen.has(key) ? null : `${NICE[key] ?? key}`
|
||
})
|
||
seen.add(key)
|
||
if (r.summary.cap_hit) {
|
||
const last = pts[pts.length - 1]
|
||
markers.push({ x: last[0], y: last[1], label: `out of memory @${last[0]}`, color: '#e5c07b' })
|
||
} else if (r.summary.result === 'died' || r.summary.result === 'crashed_after_stream') {
|
||
const last = pts[pts.length - 1]
|
||
markers.push({ x: last[0], y: last[1], label: `crash @${last[0]}`, color: '#e06c75' })
|
||
}
|
||
}
|
||
// de-duplicate stacked markers (several repeats crash at the same boundary)
|
||
const seenMarks = new Set()
|
||
const dedup = markers.filter(m => {
|
||
const k = `${m.label}:${Math.round(m.x / 100)}`
|
||
if (seenMarks.has(k)) return false
|
||
seenMarks.add(k)
|
||
return true
|
||
})
|
||
markers.length = 0
|
||
markers.push(...dedup)
|
||
return chart({
|
||
title: 'Memory used as the conversation grows (stress runs, every repeat shown)',
|
||
xLabel: 'messages streamed into the session',
|
||
yLabel: 'memory (MB)',
|
||
xMax: Math.max(xMax, 1000),
|
||
yMax: yMax * 1.08,
|
||
series,
|
||
capLine: CAP_MB,
|
||
markers,
|
||
note: '2GB hard cap (systemd cgroup). × = killed for running out of memory. Crash marks on older OpenTUI runs are the exit-7 bug, fixed in the latest runs.'
|
||
})
|
||
}
|
||
|
||
function nodesChart(results) {
|
||
const runs = results.filter(r => r.meta.cell.startsWith('nodes'))
|
||
if (!runs.length) return null
|
||
const series = []
|
||
let xMax = 0
|
||
let yMax = 0
|
||
for (const r of runs) {
|
||
let pts = []
|
||
if (r.node_samples?.length) {
|
||
const t0 = Date.parse(r.meta.utc)
|
||
const bounds = r.samples.filter(s => s.kind === 'boundary' && s.msgs != null)
|
||
pts = r.node_samples.map(ns => {
|
||
const el = ns.t - t0
|
||
let msgs = 0
|
||
for (const b of bounds) if (b.t_ms <= el) msgs = b.msgs
|
||
return [msgs, ns.yoga]
|
||
})
|
||
const byMsg = new Map()
|
||
for (const [m, y] of pts) byMsg.set(m, y)
|
||
pts = [...byMsg.entries()].sort((a, b) => a[0] - b[0])
|
||
series.push({ points: pts, color: COLORS.ink, label: 'Ink live layout nodes' })
|
||
} else if (r.samples.some(s => s.renderables != null)) {
|
||
pts = r.samples.filter(s => s.renderables != null).map(s => [s.msgs, s.renderables])
|
||
series.push({
|
||
points: pts,
|
||
color: COLORS[r.meta.config] ?? COLORS.other,
|
||
label: `${NICE[r.meta.config] ?? r.meta.config} renderables`
|
||
})
|
||
}
|
||
for (const [x, y] of pts) {
|
||
xMax = Math.max(xMax, x)
|
||
yMax = Math.max(yMax, y)
|
||
}
|
||
}
|
||
if (!series.length) return null
|
||
return chart({
|
||
title: 'Live UI nodes during the 3,000-msg marathon (diagnostic run)',
|
||
xLabel: 'messages streamed into the session',
|
||
yLabel: 'live nodes',
|
||
xMax: Math.max(xMax, 100),
|
||
yMax: yMax * 1.1,
|
||
series,
|
||
note: 'Diagnostic instrumented runs only — never used for the headline numbers.'
|
||
})
|
||
}
|
||
|
||
function scrollCdfChart(results) {
|
||
const runs = results.filter(r => r.meta.cell.startsWith('scroll') && r.summary.scroll_latencies_ms?.length)
|
||
if (!runs.length) return null
|
||
const transcriptMsgs = runs[0].meta.fixture?.msgs ?? '?'
|
||
const byConfig = {}
|
||
for (const r of runs) {
|
||
;(byConfig[r.meta.config] ??= []).push(...r.summary.scroll_latencies_ms)
|
||
}
|
||
const series = []
|
||
let xMax = 0
|
||
for (const [config, lats] of Object.entries(byConfig)) {
|
||
const s = lats.slice().sort((a, b) => a - b)
|
||
xMax = Math.max(xMax, quantile(s, 0.995))
|
||
const pts = s.map((v, i) => [v, ((i + 1) / s.length) * 100])
|
||
series.push({ points: pts, color: COLORS[config] ?? COLORS.other, label: `${NICE[config] ?? config} (${s.length} scrolls)` })
|
||
}
|
||
return chart({
|
||
title: `What fraction of scroll responses finished within X ms (${transcriptMsgs}-msg transcript)`,
|
||
xLabel: 'time from scroll input to first screen response (ms)',
|
||
yLabel: '% of scroll responses at least this fast',
|
||
xMax: Math.max(1, xMax),
|
||
yMax: 100,
|
||
series,
|
||
note: 'Mouse wheel fired 30×/s for 15s, three repeats pooled. Higher curve further left = more responses answered fast.'
|
||
})
|
||
}
|
||
|
||
function startupChart(results) {
|
||
const runs = results.filter(r => r.meta.cell === 'startup')
|
||
if (!runs.length) return null
|
||
const byConfig = {}
|
||
for (const r of runs) {
|
||
const c = (byConfig[r.meta.config] ??= { fb: [], sc: [] })
|
||
if (r.summary.first_byte_ms != null) c.fb.push(r.summary.first_byte_ms)
|
||
if (r.summary.session_create_ms != null) c.sc.push(r.summary.session_create_ms)
|
||
}
|
||
const groups = Object.entries(byConfig).map(([config, v]) => ({
|
||
label: NICE[config] ?? config,
|
||
bars: [
|
||
{ name: 'first paint', value: median(v.fb), lo: iqr(v.fb)[0], hi: iqr(v.fb)[1], color: COLORS[config] ?? COLORS.other },
|
||
{ name: 'session ready', value: median(v.sc), lo: iqr(v.sc)[0], hi: iqr(v.sc)[1], color: '#98c379' }
|
||
]
|
||
}))
|
||
return barChart({
|
||
title: 'Startup: time until something is on screen, and until the session is ready',
|
||
yLabel: 'ms after launch (lower = faster)',
|
||
groups,
|
||
barWidth: 110,
|
||
note: 'Typical of 10 launches (median); whisker = middle half of runs. "first paint" = first byte drawn to the terminal.'
|
||
})
|
||
}
|
||
|
||
function ptyRateChart(results) {
|
||
const runs = results.filter(r => r.meta.cell.startsWith('cpu') && r.summary.stream_done)
|
||
if (!runs.length) return null
|
||
const byConfig = {}
|
||
for (const r of runs) {
|
||
const done = r.samples.filter(s => s.kind === 'done')[0]
|
||
const start = r.summary.stream_start_ms
|
||
if (!done || start == null) continue
|
||
const secs = (done.t_ms - start) / 1000
|
||
const c = (byConfig[r.meta.config] ??= { rate: [], cpu: [] })
|
||
c.rate.push(done.pty_bytes / secs / 1024)
|
||
const sb = r.samples.filter(s => s.kind === 'boundary')
|
||
if (sb.length >= 2) {
|
||
const first = sb[0]
|
||
const last = sb[sb.length - 1]
|
||
const ticks = last.utime_ticks + last.stime_ticks - first.utime_ticks - first.stime_ticks
|
||
const events = (last.events ?? 0) - (first.events ?? 0)
|
||
if (events > 0) c.cpu.push((ticks * 10) / events)
|
||
}
|
||
}
|
||
const groups = Object.entries(byConfig).map(([config, v]) => ({
|
||
label: NICE[config] ?? config,
|
||
bars: [
|
||
{ name: 'KiB/s', value: median(v.rate), lo: iqr(v.rate)[0], hi: iqr(v.rate)[1], color: COLORS[config] ?? COLORS.other },
|
||
{ name: 'ms/event', value: median(v.cpu), lo: iqr(v.cpu)[0], hi: iqr(v.cpu)[1], color: '#d19a66' }
|
||
]
|
||
}))
|
||
return barChart({
|
||
title: 'Streaming at 30 events/s: terminal output volume and CPU cost per event',
|
||
yLabel: 'output KiB/s · CPU ms per event',
|
||
groups,
|
||
barWidth: 100,
|
||
note: 'Typical of 3 runs (median); whisker = middle half. CPU is the UI process only, measured over the stream.'
|
||
})
|
||
}
|
||
|
||
function sessionHistChart(dist) {
|
||
if (!dist?.tui_cli?.histogram) return null
|
||
const p = dist.tui_cli
|
||
return histChart({
|
||
title: `How long the user's real terminal sessions actually are (${p.n} sessions)`,
|
||
buckets: p.histogram,
|
||
yLabel: 'number of sessions',
|
||
percentiles: [
|
||
{ v: p.p50, label: `half are ≤${p.p50} (p50)` },
|
||
{ v: p.p75, label: `75% ≤${p.p75}` },
|
||
{ v: p.p90, label: `90% ≤${p.p90}` },
|
||
{ v: p.p95, label: `95% ≤${p.p95}` },
|
||
{ v: p.p99, label: `99% ≤${p.p99} (p99)` }
|
||
],
|
||
note: `Every TUI/CLI session in the real session DB (${dist.db ?? 'state.db'}); message counts per session.`
|
||
})
|
||
}
|
||
|
||
function memRealChart(A) {
|
||
const groups = []
|
||
for (const [cellName, label] of [
|
||
['mem100', '100 msgs (heavy-ish day)'],
|
||
['mem300', '300 msgs (top 5% of sessions)'],
|
||
['mem2000', '2,000 msgs (longest real sessions)']
|
||
]) {
|
||
const m = A.memPeak[cellName]
|
||
if (!m) continue
|
||
const bars = []
|
||
if (m.ink) bars.push({ name: 'Ink', value: m.ink.med, color: COLORS.ink })
|
||
if (m['otui-capped']) bars.push({ name: 'OpenTUI', value: m['otui-capped'].med, color: COLORS['otui-capped'] })
|
||
if (bars.length) groups.push({ label, bars })
|
||
}
|
||
if (!groups.length) return null
|
||
return barChart({
|
||
title: 'Peak memory at real session sizes — Ink vs OpenTUI',
|
||
yLabel: 'peak memory (MB)',
|
||
groups,
|
||
note: 'Peak resident memory of the UI process (VmHWM), typical of 2 repeats (median).'
|
||
})
|
||
}
|
||
|
||
function frameRateChart(A) {
|
||
const groups = []
|
||
const fpsBars = []
|
||
const gapBars = []
|
||
for (const cfg of ['ink', 'otui-capped']) {
|
||
const fp = A.pipeline[cfg]?.fp
|
||
if (!fp) continue
|
||
fpsBars.push({ name: NICE[cfg], value: fp.fps_avg, color: COLORS[cfg] })
|
||
gapBars.push({ name: NICE[cfg], value: fp.interframe_ms_p95, color: COLORS[cfg] })
|
||
}
|
||
if (!fpsBars.length) return null
|
||
groups.push({ label: 'screen updates per second (higher = smoother)', bars: fpsBars })
|
||
return barChart({
|
||
title: 'Frame smoothness while text streams in',
|
||
yLabel: 'frames per second',
|
||
groups,
|
||
barWidth: 90,
|
||
note: '800-message stream at 30 events/s; a frame = a burst of terminal output separated by a ≥4ms gap.'
|
||
})
|
||
}
|
||
|
||
function frameGapChart(A) {
|
||
const mk = key => {
|
||
const bars = []
|
||
for (const cfg of ['ink', 'otui-capped']) {
|
||
const fp = A.pipeline[cfg]?.fp
|
||
if (fp) bars.push({ name: NICE[cfg], value: fp[key], color: COLORS[cfg] })
|
||
}
|
||
return bars
|
||
}
|
||
const typical = mk('interframe_ms_p50')
|
||
const worst = mk('interframe_ms_p95')
|
||
if (!worst.length) return null
|
||
return barChart({
|
||
title: 'Pauses between screen updates while streaming (lower = steadier)',
|
||
yLabel: 'gap between frames (ms)',
|
||
groups: [
|
||
{ label: 'typical gap (p50)', bars: typical },
|
||
{ label: 'slowest 1 in 20 gaps (p95)', bars: worst }
|
||
],
|
||
barWidth: 90,
|
||
note: 'Same 800-message stream. The right-hand pair is the stutter you actually notice.'
|
||
})
|
||
}
|
||
|
||
function pipelineCpuChart(A) {
|
||
const groups = []
|
||
for (const cfg of ['ink', 'otui-capped']) {
|
||
const c = A.pipeline[cfg]?.cpu
|
||
if (!c) continue
|
||
groups.push({
|
||
label: NICE[cfg],
|
||
bars: [
|
||
{ name: 'UI', value: c.ui, color: COLORS[cfg] },
|
||
{ name: 'gateway', value: c.gateway, color: '#98c379' },
|
||
{ name: 'tmux', value: c.tmux_server, color: '#d19a66' }
|
||
]
|
||
})
|
||
}
|
||
if (!groups.length) return null
|
||
return barChart({
|
||
title: 'Total CPU burned streaming the same 800-message conversation',
|
||
yLabel: 'CPU seconds',
|
||
groups,
|
||
note: 'Whole pipeline measured inside tmux: UI process + gateway + the tmux server that has to parse the UI’s output.'
|
||
})
|
||
}
|
||
|
||
// ── verdicts ────────────────────────────────────────────────────────────
|
||
function buildVerdicts(A, dist, results) {
|
||
const r1 = x => (x == null ? null : Math.round(x))
|
||
const rows = []
|
||
const add = (dim, winner, headline, detail) => rows.push({ dim, winner, headline, detail })
|
||
|
||
// memory typical
|
||
{
|
||
const d100 = A.memPeak.mem100?.['otui-capped'] && A.memPeak.mem100?.ink ? A.memPeak.mem100['otui-capped'].med - A.memPeak.mem100.ink.med : null
|
||
const d300 = A.memPeak.mem300?.['otui-capped'] && A.memPeak.mem300?.ink ? A.memPeak.mem300['otui-capped'].med - A.memPeak.mem300.ink.med : null
|
||
if (d100 != null || d300 != null) {
|
||
add(
|
||
'Memory — typical real sessions (20–300 msgs)',
|
||
'ink',
|
||
'Ink wins, modestly',
|
||
`OpenTUI uses ~${r1(d100)}–${r1(d300)}MB more (${r1(A.memPeak.mem100.ink.med)} vs ${r1(A.memPeak.mem100['otui-capped'].med)}MB at 100 msgs; ${r1(A.memPeak.mem300.ink.med)} vs ${r1(A.memPeak.mem300['otui-capped'].med)}MB at 300).`
|
||
)
|
||
}
|
||
}
|
||
// memory p99 tail
|
||
{
|
||
const i = A.memPeak.mem2000?.ink?.med
|
||
const o = A.memPeak.mem2000?.['otui-capped']?.med
|
||
if (i != null && o != null) {
|
||
add(
|
||
'Memory — longest real sessions (~2,000 msgs, the longest 1 in 100 = p99)',
|
||
'ink',
|
||
'Ink wins big',
|
||
`${r1(i)}MB vs ${r1(o)}MB peak — ${(o / i).toFixed(1)}× more. Sessions this long really happen (6 of them in the DB).`
|
||
)
|
||
}
|
||
}
|
||
// memory stress
|
||
{
|
||
const ip = A.plateau3000?.ink?.med
|
||
const op = A.plateau3000?.['otui-capped']?.med
|
||
const opk = A.memPeak.mem3000?.['otui-capped']?.med
|
||
if (ip != null && op != null) {
|
||
add(
|
||
'Memory — 3,000-msg stress marathon (beyond any real session)',
|
||
'ink',
|
||
'Ink wins',
|
||
`Ink levels off near ${r1(ip)}MB; OpenTUI climbs to ~${r1(op)}MB (peak ~${r1(opk)}MB), and its syntax styling degrades past ~1,400 rows. Stress test only — past the longest real session.`
|
||
)
|
||
}
|
||
}
|
||
// scroll
|
||
{
|
||
const i = A.scroll.ink
|
||
const oc = A.scroll['otui-capped']
|
||
const ou = A.scroll['otui-uncapped']
|
||
if (i?.length && oc?.length) {
|
||
const perRep = cfg =>
|
||
results
|
||
.filter(r => r.meta.cell.startsWith('scroll') && r.meta.config === cfg && r.summary.scroll_latencies_ms?.length)
|
||
.map(r => pq(r.summary.scroll_latencies_ms, 0.99))
|
||
const iReps = perRep('ink')
|
||
const oReps = [...perRep('otui-capped'), ...perRep('otui-uncapped')]
|
||
const op99s = [pq(oc, 0.99), ou?.length ? pq(ou, 0.99) : null].filter(x => x != null)
|
||
add(
|
||
'Scroll responsiveness on a long transcript',
|
||
'otui',
|
||
'OpenTUI wins decisively',
|
||
`Slowest 1-in-100 scroll responses (p99): ${r1(Math.min(...op99s))}–${r1(Math.max(...op99s))}ms vs ${r1(Math.min(...iReps))}–${r1(Math.max(...iReps))}ms across repeats. Typical scrolls are ~2ms on both — the difference is the stutters.`
|
||
)
|
||
}
|
||
}
|
||
// frame smoothness
|
||
{
|
||
const fi = A.pipeline.ink?.fp
|
||
const fo = A.pipeline['otui-capped']?.fp
|
||
if (fi && fo) {
|
||
add(
|
||
'Frame smoothness while streaming',
|
||
'otui',
|
||
'OpenTUI wins',
|
||
`${fo.fps_avg.toFixed(0)} vs ${fi.fps_avg.toFixed(0)} screen updates/s, and its worst pauses between updates are half as long (${r1(fo.interframe_ms_p95)}ms vs ${r1(fi.interframe_ms_p95)}ms, slowest 1 in 20).`
|
||
)
|
||
}
|
||
}
|
||
// echo
|
||
{
|
||
const ei = A.echo.ink
|
||
const eo = A.echo['otui-capped']
|
||
if (ei && eo) {
|
||
add('Typing echo (keystroke → it appears)', 'tie', 'Tie', `Both answer a keystroke in ${r1(ei.echo_ms.p50)}–${r1(eo.echo_ms.p50)}ms — under any human threshold.`)
|
||
add(
|
||
'Submit → first reply paint',
|
||
'ink',
|
||
'Ink wins',
|
||
`${r1(ei.submit_first_token_paint_ms)}ms vs ${r1(eo.submit_first_token_paint_ms)}ms from pressing Enter to the first reply text on screen.`
|
||
)
|
||
}
|
||
}
|
||
// CPU
|
||
{
|
||
const ci = A.pipeline.ink?.cpu
|
||
const co = A.pipeline['otui-capped']?.cpu
|
||
if (ci && co) {
|
||
add(
|
||
'CPU, including the terminal-emulator (tmux) side',
|
||
'tie',
|
||
'Tie',
|
||
`~80 CPU-seconds either way for the same 800-message stream (${ci.total.toFixed(0)} vs ${co.total.toFixed(0)}s total); the tmux leg is ~0.4s for both.`
|
||
)
|
||
}
|
||
}
|
||
// chaos
|
||
{
|
||
const scen = ['gw-kill-stream', 'gw-kill-tool', 'gw-stop']
|
||
const iT = scen.map(s => A.chaos[s]?.ink?.time_to_respawn_ms).filter(x => x != null)
|
||
const oT = scen.map(s => A.chaos[s]?.['otui-capped']?.time_to_respawn_ms).filter(x => x != null)
|
||
if (iT.length && oT.length) {
|
||
add(
|
||
'Crash recovery (gateway shot mid-stream)',
|
||
'tie',
|
||
'Tie',
|
||
`Both auto-respawn the killed gateway and end with the full transcript intact and zero orphan processes. Ink respawns in ~${r1(median(iT))}ms, OpenTUI in ~${(median(oT) / 1000).toFixed(1)}s.`
|
||
)
|
||
}
|
||
}
|
||
// startup
|
||
{
|
||
const si = A.startup.ink
|
||
const so = A.startup['otui-capped']
|
||
if (si?.fb.length && so?.fb.length) {
|
||
add(
|
||
'Startup',
|
||
'ink',
|
||
'Ink wins, modestly',
|
||
`First paint ${r1(median(si.fb))}ms vs ${r1(median(so.fb))}ms. Both feel instant; OpenTUI actually reaches “session ready” slightly sooner (${r1(median(so.sc))} vs ${r1(median(si.sc))}ms).`
|
||
)
|
||
}
|
||
}
|
||
return rows
|
||
}
|
||
|
||
function verdictTable(rows) {
|
||
if (!rows.length) return '<p class="notrun">No results yet.</p>'
|
||
const cellFor = r => {
|
||
const cls = r.winner === 'ink' ? 'win-ink' : r.winner === 'otui' ? 'win-otui' : 'win-tie'
|
||
return `<td class="${cls}">${esc(r.headline)}</td>`
|
||
}
|
||
return `<table class="verdict">
|
||
<tr><th style="width:32%">dimension</th><th style="width:18%">winner</th><th>the numbers</th></tr>
|
||
${rows.map(r => `<tr><td>${r.dim}</td>${cellFor(r)}<td>${r.detail}</td></tr>`).join('\n')}
|
||
</table>
|
||
<p class="legend"><span class="sw win-ink"> </span> red = Ink (current UI) wins · <span class="sw win-otui"> </span> green = OpenTUI (new engine) wins · <span class="sw win-tie"> </span> grey = tie</p>`
|
||
}
|
||
|
||
// ── tables ──────────────────────────────────────────────────────────────
|
||
function memMediansTable(A, dist) {
|
||
const rowsDef = [
|
||
['mem100', '100 msgs', 'a heavier-than-usual day (typical session is ~20 msgs)'],
|
||
['mem300', '300 msgs', 'top ~5% of real sessions'],
|
||
['mem2000', '2,000 msgs', 'the longest sessions that actually occur (~1 in 100, p99)']
|
||
]
|
||
const rows = []
|
||
for (const [c, label, gloss] of rowsDef) {
|
||
const m = A.memPeak[c]
|
||
if (!m?.ink || !m['otui-capped']) continue
|
||
const i = m.ink.med
|
||
const o = m['otui-capped'].med
|
||
rows.push(
|
||
`<tr><td>${label}</td><td class="dim">${gloss}</td><td><b style="color:${COLORS.ink}">${Math.round(i)} MB</b></td><td><b style="color:${COLORS['otui-capped']}">${Math.round(o)} MB</b></td><td>OpenTUI +${Math.round(o - i)} MB (${(o / i).toFixed(1)}×)</td></tr>`
|
||
)
|
||
}
|
||
if (!rows.length) return '<p class="notrun">memory-at-size cells: not run</p>'
|
||
return `<table><tr><th>session size</th><th>what that means in practice</th><th>Ink peak</th><th>OpenTUI peak</th><th>difference</th></tr>${rows.join('\n')}</table>`
|
||
}
|
||
|
||
function chaosTable(A) {
|
||
const scen = Object.keys(A.chaos)
|
||
if (!scen.length) return '<p class="notrun">chaos cells: not run</p>'
|
||
const DESC = {
|
||
'gw-kill-stream': 'shot the gateway (kill -9) while reply text was streaming',
|
||
'gw-kill-tool': 'shot the gateway (kill -9) in the middle of a tool call',
|
||
'gw-stop': 'froze the gateway for 30 seconds mid-session, then let the UI recover',
|
||
'pty-eof': 'closed the terminal out from under the UI (should exit cleanly, leave nothing behind)',
|
||
'resize-storm': 'resized the window 30 times in 3 seconds'
|
||
}
|
||
const ORDER = ['gw-kill-stream', 'gw-kill-tool', 'gw-stop', 'resize-storm', 'pty-eof']
|
||
const ok = '<span class="ok">yes</span>'
|
||
const no = '<span class="bad">no</span>'
|
||
const cellFor = c => {
|
||
if (!c) return '<td class="notrun">not run</td>'
|
||
const bits = []
|
||
if (c.scenario === 'pty-eof') {
|
||
bits.push(`exited cleanly: ${c.ui_exited_after_eof ? ok : no}`)
|
||
bits.push(`gateway cleaned up: ${c.gateway_reaped ? ok : no} (${c.gateway_reaped_ms}ms)`)
|
||
} else if (c.scenario === 'resize-storm') {
|
||
bits.push(`survived: ${c.ui_survived ? ok : no}`)
|
||
bits.push(`transcript intact: ${c.transcript_preserved ? ok : no}`)
|
||
} else {
|
||
bits.push(`UI survived: ${c.ui_survived ? ok : no}`)
|
||
if (c.gateway_respawned != null)
|
||
bits.push(`gateway respawned: ${c.gateway_respawned ? ok : no} (${c.time_to_respawn_ms >= 1000 ? (c.time_to_respawn_ms / 1000).toFixed(1) + 's' : c.time_to_respawn_ms + 'ms'})`)
|
||
if (c.stream_resumed != null) bits.push(`stream resumed: ${c.stream_resumed ? ok : no}`)
|
||
bits.push(`transcript intact: ${c.transcript_preserved ? ok : no}`)
|
||
}
|
||
bits.push(`orphan processes: ${(c.orphans ?? []).length === 0 ? '<span class="ok">none</span>' : `<span class="bad">${c.orphans.length}</span>`}`)
|
||
return `<td>${bits.join('<br>')}</td>`
|
||
}
|
||
const rows = ORDER.filter(s => A.chaos[s]).map(
|
||
s => `<tr><td><b>${esc(s)}</b><br><span class="dim">${esc(DESC[s] ?? '')}</span></td>${cellFor(A.chaos[s].ink)}${cellFor(A.chaos[s]['otui-capped'])}</tr>`
|
||
)
|
||
return `<table><tr><th style="width:34%">what we did</th><th>Ink</th><th>OpenTUI</th></tr>${rows.join('\n')}</table>`
|
||
}
|
||
|
||
function echoTable(A) {
|
||
const ei = A.echo.ink
|
||
const eo = A.echo['otui-capped']
|
||
if (!ei && !eo) return '<p class="notrun">echo cells: not run</p>'
|
||
const row = (cfg, e) =>
|
||
e
|
||
? `<tr><td><b style="color:${COLORS[cfg]}">${NICE[cfg]}</b></td>
|
||
<td>${fmt(e.echo_ms.p50, 0)} ms</td><td>${fmt(e.echo_ms.p95, 1)} ms</td>
|
||
<td>${fmt(e.submit_first_token_paint_ms, 0)} ms</td><td>${e.keystrokes_matched}/${e.keystrokes_sent}</td></tr>`
|
||
: ''
|
||
return `<table>
|
||
<tr><th>UI</th><th>keystroke echo, typical (p50)</th><th>keystroke echo, slowest 1 in 20 (p95)</th><th>Enter → first reply text on screen</th><th>keystrokes verified</th></tr>
|
||
${row('ink', ei)}${row('otui-capped', eo)}
|
||
</table>`
|
||
}
|
||
|
||
function matrixTable(results) {
|
||
const memRuns = results.filter(r => r.meta.cell === 'mem3000' && r.meta.mode === 'mem' && !r.meta.instrumented)
|
||
const slopeRuns = results.filter(r => r.meta.cell.startsWith('slope'))
|
||
const scrollRuns = results.filter(r => r.meta.cell.startsWith('scroll'))
|
||
const cpuRuns = results.filter(r => r.meta.cell.startsWith('cpu'))
|
||
const configs = ['ink', 'otui-capped', 'otui-uncapped']
|
||
const rows = []
|
||
for (const config of configs) {
|
||
const mem = memRuns.filter(r => r.meta.config === config)
|
||
const slopes = mem.map(runSlope).filter(s => s != null)
|
||
const plateaus = mem.map(runPlateau).filter(s => s != null)
|
||
const vmhwm = mem.map(r => r.summary.vmhwm_kb).filter(Boolean).map(k => k / 1024)
|
||
const slope10k = slopeRuns.filter(r => r.meta.config === config).map(runSlope).filter(s => s != null)
|
||
const lat = scrollRuns.filter(r => r.meta.config === config).flatMap(r => r.summary.scroll_latencies_ms ?? [])
|
||
const latS = lat.slice().sort((a, b) => a - b)
|
||
const cpu = []
|
||
for (const r of cpuRuns.filter(x => x.meta.config === config)) {
|
||
const sb = r.samples.filter(s => s.kind === 'boundary')
|
||
if (sb.length >= 2) {
|
||
const f = sb[0]
|
||
const l = sb[sb.length - 1]
|
||
const ev = (l.events ?? 0) - (f.events ?? 0)
|
||
if (ev > 0) cpu.push(((l.utime_ticks + l.stime_ticks - f.utime_ticks - f.stime_ticks) * 10) / ev)
|
||
}
|
||
}
|
||
const capHits = [...mem, ...slopeRuns.filter(r => r.meta.config === config)].filter(r => r.summary.cap_hit)
|
||
rows.push(`<tr>
|
||
<td><b style="color:${COLORS[config]}">${NICE[config] ?? config}</b></td>
|
||
<td>${fmtMedIqr(slopes, 2)}</td>
|
||
<td>${slope10k.length ? fmt(median(slope10k), 2) : 'not run'}</td>
|
||
<td>${fmtMedIqr(plateaus, 0)}</td>
|
||
<td>${fmtMedIqr(vmhwm, 0)}</td>
|
||
<td>${latS.length ? `${fmt(quantile(latS, 0.5), 1)} / ${fmt(quantile(latS, 0.9), 1)} / ${fmt(quantile(latS, 0.99), 1)}` : 'not run'}</td>
|
||
<td>${fmtMedIqr(cpu, 2)}</td>
|
||
<td>${capHits.length ? capHits.map(r => `${r.meta.cell}@${r.summary.at_messages ?? '?'}msgs`).join('<br>') : mem.length ? 'none' : 'not run'}</td>
|
||
</tr>`)
|
||
}
|
||
return `<table>
|
||
<tr><th>config</th><th>memory growth<br>MB per 1k msgs<br>(3,000-msg runs)</th><th>memory growth<br>MB per 1k<br>(10,000-msg run)</th><th>settled memory MB<br>(plateau, end of run)</th><th>peak memory MB</th><th>scroll ms — typical /<br>slowest 1 in 10 /<br>slowest 1 in 100<br>(p50/p90/p99)</th><th>CPU ms per event<br>(paced stream)</th><th>killed by 2GB cap?</th></tr>
|
||
${rows.join('\n')}
|
||
</table>`
|
||
}
|
||
|
||
function survivalTable(results) {
|
||
const runs = results.filter(r => r.meta.cell.startsWith('e3'))
|
||
if (!runs.length) return '<p class="notrun">Low-memory survival (Docker): not run.</p>'
|
||
const fmtLimit = v => {
|
||
const n = Number(v)
|
||
if (Number.isFinite(n) && n > 1e6) return `${Math.round(n / 1073741824 * 10) / 10} GB`
|
||
return String(v ?? '?')
|
||
}
|
||
const rows = runs.map(
|
||
r => `<tr><td>${esc(r.meta.cell)}</td><td>${esc(NICE[r.meta.config] ?? r.meta.config)}</td><td>${esc(fmtLimit(r.meta.memory_max ?? r.meta.container_memory))}</td>
|
||
<td>${r.summary.result}</td><td>${r.summary.at_messages ?? r.summary.msgs_streamed ?? '—'}</td>
|
||
<td>${fmt((r.summary.vmhwm_kb ?? 0) / 1024, 0)} MB</td><td>${esc(r.summary.cap_hit_basis ?? '—')}</td></tr>`
|
||
)
|
||
return `<table><tr><th>cell</th><th>UI</th><th>memory limit</th><th>result</th><th>msgs survived</th><th>peak memory</th><th>basis</th></tr>${rows.join('')}</table>`
|
||
}
|
||
|
||
function gateTable(results) {
|
||
const runs = results.filter(r => r.meta.cell === 'gate')
|
||
if (!runs.length) return '<p class="notrun">Determinism gate: not run.</p>'
|
||
const byConfig = {}
|
||
for (const r of runs) (byConfig[r.meta.config] ??= []).push(r.summary.digest)
|
||
const rows = Object.entries(byConfig).map(([c, ds]) => {
|
||
const ok = ds.length >= 2 && ds.every(d => d && d === ds[0])
|
||
return `<tr><td>${esc(NICE[c] ?? c)}</td><td>${ds.map(d => (d ?? '∅').slice(0, 16)).join(' · ')}</td><td style="color:${ok ? '#98c379' : '#e06c75'}">${ok ? 'PASS' : 'FAIL'}</td></tr>`
|
||
})
|
||
return `<table><tr><th>UI</th><th>replay fingerprints (same input must give same screen)</th><th>gate</th></tr>${rows.join('')}</table>`
|
||
}
|
||
|
||
function drainTable(results) {
|
||
const bad = results.filter(r => r.summary && r.summary.drain_ok === false)
|
||
if (!bad.length) return '<p>Harness self-check: no run was distorted by the test rig itself (event loop never starved >10ms).</p>'
|
||
const rows = bad.map(
|
||
r => `<tr><td>${esc(r._file)}</td><td>${fmt(r.summary.drain_max_loop_lag_ms, 0)} ms</td><td>${r.summary.drain_lag_violations}</td></tr>`
|
||
)
|
||
return `<p style="color:#e5c07b">⚠ some runs were flagged: the test rig itself lagged (results kept, but read with care):</p>
|
||
<table><tr><th>run</th><th>max rig lag</th><th>lags >10ms</th></tr>${rows.join('')}</table>`
|
||
}
|
||
|
||
// ── PNG export ──────────────────────────────────────────────────────────
|
||
function exportPng(name, svg) {
|
||
try {
|
||
const require2 = createRequire(import.meta.url)
|
||
const resvgPath = join(homedir(), '.claude/skills/tmux-pane-screenshot/scripts/node_modules/@resvg/resvg-js')
|
||
const { Resvg } = require2(resvgPath)
|
||
const png = new Resvg(svg, { fitTo: { mode: 'width', value: 1280 } }).render().asPng()
|
||
writeFileSync(join(ASSETS_DIR, `${name}.png`), png)
|
||
return true
|
||
} catch (e) {
|
||
process.stderr.write(`png export failed for ${name}: ${e.message}\n`)
|
||
return false
|
||
}
|
||
}
|
||
|
||
// ── main ────────────────────────────────────────────────────────────────
|
||
const results = loadResults()
|
||
const dist = loadDistribution()
|
||
const A = aggregate(results)
|
||
mkdirSync(ASSETS_DIR, { recursive: true })
|
||
|
||
const charts = {
|
||
'session-histogram': sessionHistChart(dist),
|
||
'mem-real-workloads': memRealChart(A),
|
||
'scroll-cdf': scrollCdfChart(results),
|
||
'frame-rate': frameRateChart(A),
|
||
'frame-gaps': frameGapChart(A),
|
||
'pipeline-cpu': pipelineCpuChart(A),
|
||
'pty-rate': ptyRateChart(results),
|
||
startup: startupChart(results),
|
||
'rss-vs-msgs': rssChart(results),
|
||
'node-count': nodesChart(results)
|
||
}
|
||
|
||
const pngs = []
|
||
for (const [name, svg] of Object.entries(charts)) {
|
||
if (svg && exportPng(name, svg)) pngs.push(`${name}.png`)
|
||
}
|
||
|
||
const fig = (name, caption) =>
|
||
charts[name]
|
||
? `<figure>${charts[name]}<figcaption>${caption}</figcaption></figure>`
|
||
: `<p class="notrun">${esc(name)}: not run</p>`
|
||
|
||
const verdicts = buildVerdicts(A, dist, results)
|
||
|
||
const metaRuns = results.length
|
||
? `${results.length} result files · sha ${esc(results[0].meta.sha ?? '?')} · node ${esc(results.find(r => r.meta.node_version)?.meta.node_version ?? '?')}`
|
||
: 'no results yet'
|
||
|
||
const p99real = dist?.tui_cli?.p99 ?? '~2,000'
|
||
|
||
const html = `<!doctype html>
|
||
<html lang="en"><head><meta charset="utf-8">
|
||
<title>Hermes TUI bench — Ink vs OpenTUI</title>
|
||
<style>
|
||
body { background:#0b0e14; color:#cdd3e0; font-family:ui-monospace,SFMono-Regular,monospace; margin:0; padding:32px; max-width:1100px; margin-inline:auto; }
|
||
h1 { color:#e8eaf0; font-size:24px; }
|
||
h2 { color:#e8eaf0; font-size:19px; margin-top:48px; border-bottom:1px solid #262c38; padding-bottom:6px;}
|
||
table { border-collapse:collapse; margin:12px 0; font-size:13px; width:100%; }
|
||
th,td { border:1px solid #262c38; padding:7px 10px; text-align:left; vertical-align:top;}
|
||
th { background:#161b24; color:#aab2c5; }
|
||
.verdict td { font-size:13.5px; }
|
||
.win-ink { background:#3a2126; color:#e8b4ba; font-weight:bold; }
|
||
.win-otui { background:#1f3a28; color:#a7d8b4; font-weight:bold; }
|
||
.win-tie { background:#252b36; color:#aab2c5; font-weight:bold; }
|
||
.sw { display:inline-block; width:12px; height:12px; border-radius:2px; vertical-align:-1px; }
|
||
.legend { color:#8b93a7; font-size:12px; }
|
||
.iqr { color:#6f7689; font-size:11px; }
|
||
.dim { color:#8b93a7; }
|
||
.ok { color:#98c379; } .bad { color:#e06c75; }
|
||
.notrun { color:#6f7689; font-style:italic; }
|
||
svg { margin:12px 0 4px; border-radius:6px; max-width:100%; height:auto; }
|
||
figure { margin:18px 0; }
|
||
figcaption { color:#9aa3b8; font-size:13px; margin:2px 0 0 4px; font-style:italic; }
|
||
p, li { font-size:14px; line-height:1.55; }
|
||
code { color:#98c379; }
|
||
.callout { background:#161b24; border-left:3px solid #61afef; padding:10px 14px; margin:14px 0; font-size:14px; }
|
||
.stress { border-left-color:#e5c07b; }
|
||
</style></head><body>
|
||
<h1>Hermes TUI benchmark — Ink (current UI) vs OpenTUI (new engine)</h1>
|
||
<p>Both UIs were run as real binaries in a real terminal, fed the exact same scripted conversations by a fake
|
||
gateway, and measured from outside the process. Every number below is the typical of repeated runs (median)
|
||
unless said otherwise. ${metaRuns} · generated ${new Date().toISOString()}</p>
|
||
|
||
<h2>The verdict — who won what</h2>
|
||
${verdictTable(verdicts)}
|
||
<p class="callout">One-line summary: <b>OpenTUI is the smoother UI</b> (scrolling, streaming) and
|
||
<b>Ink is the lighter one</b> (memory, first paint). Everything else is a wash — including reliability,
|
||
where both recover from a killed gateway with the transcript intact.</p>
|
||
|
||
<h2>Memory at real workloads — what sessions actually look like</h2>
|
||
<p>The memory debate was framed around 200–300-message sessions. The real session database says that band is
|
||
the top 5–10%: <b>the typical session is ~${dist?.tui_cli?.p50 ?? 20} messages</b>, 90% stay under
|
||
${dist?.tui_cli?.p90 ?? 182}, and the longest real sessions reach ~${p99real} messages (the longest
|
||
1 in 100 — p99) — and at that tail the memory gap widens to ${A.memPeak.mem2000?.ink && A.memPeak.mem2000?.['otui-capped'] ? (A.memPeak.mem2000['otui-capped'].med / A.memPeak.mem2000.ink.med).toFixed(1) : '~2.9'}×.</p>
|
||
${fig('session-histogram', 'Takeaway: real sessions are short — half end within ~20 messages; the 200–300-msg sizes the debate assumed are actually the top 5–10% of sessions.')}
|
||
${fig('mem-real-workloads', 'Takeaway: at everyday sizes the gap is a modest 60–90MB; at the rare-but-real 2,000-msg session it becomes 234MB vs 671MB — 2.9× — which is where Ink genuinely wins.')}
|
||
${memMediansTable(A, dist)}
|
||
|
||
<h2>Scroll responsiveness — where OpenTUI wins</h2>
|
||
${fig('scroll-cdf', 'Takeaway: both feel identical on a typical scroll (~2ms), but Ink’s slowest 1-in-100 responses (p99) take 82–101ms — visible hitches — while OpenTUI stays under ~17ms.')}
|
||
|
||
<h2>Frame smoothness while streaming — where OpenTUI wins</h2>
|
||
${fig('frame-rate', 'Takeaway: while a long reply streams in, OpenTUI repaints ~22×/s vs Ink’s ~16×/s — text appears noticeably more fluid.')}
|
||
${fig('frame-gaps', 'Takeaway: typical pauses are similar, but Ink’s worst stutters between repaints (slowest 1 in 20, p95) are twice as long — 209ms vs 103ms.')}
|
||
|
||
<h2>Typing echo & first reply paint</h2>
|
||
${echoTable(A)}
|
||
<p>Keystroke echo is a tie — 1–2ms on both, far below anything a human can perceive. The one real difference:
|
||
after pressing Enter, Ink paints the first reply text in ${A.echo.ink?.submit_first_token_paint_ms ?? '—'}ms
|
||
vs OpenTUI's ${A.echo['otui-capped']?.submit_first_token_paint_ms ?? '—'}ms. (Single run per UI.)</p>
|
||
|
||
<h2>CPU cost of streaming — including the terminal's side of the work</h2>
|
||
${fig('pipeline-cpu', 'Takeaway: a tie — same 800-message stream costs ~80 CPU-seconds on either UI, and the terminal emulator’s share (tmux) is a rounding error (~0.4s) for both.')}
|
||
<p>The hypothesis that Ink's bigger output stream costs meaningfully more CPU in the terminal emulator
|
||
<b>did not hold at this workload</b>: Ink did push more bytes (${A.pipeline.ink ? (A.pipeline.ink.bytes / 1048576).toFixed(1) : '—'}MB
|
||
vs ${A.pipeline['otui-capped'] ? (A.pipeline['otui-capped'].bytes / 1048576).toFixed(1) : '—'}MB), but the tmux
|
||
server burned ~0.4 CPU-seconds either way.</p>
|
||
${fig('pty-rate', 'Takeaway: per streamed event the CPU cost is in the same ballpark for all configs — neither UI is the cheap one on CPU.')}
|
||
|
||
<h2>Crash recovery — we shot the gateway mid-stream and watched what happened</h2>
|
||
<p>Each scenario kills, freezes, or yanks something out from under the UI on a live session, then checks:
|
||
did the UI survive, did the gateway come back, did the stream resume, is the final transcript identical
|
||
to an undisturbed run, and is anything left running afterwards?</p>
|
||
${chaosTable(A)}
|
||
<p><b>Result: a tie.</b> Both UIs auto-respawn a killed gateway and finish with a byte-identical final
|
||
transcript and zero orphan processes. Ink respawns faster (~35–87ms vs ~1.0s), but both are well within
|
||
"didn't lose anything".</p>
|
||
|
||
<h2>Startup</h2>
|
||
${fig('startup', 'Takeaway: Ink gets pixels on screen first (~67ms vs ~127ms); OpenTUI finishes its session bootstrap slightly sooner (~176ms vs ~202ms). Both feel instant.')}
|
||
|
||
<h2 id="stress">Stress appendix — beyond real usage</h2>
|
||
<p class="callout stress">Everything below streams 3,000–10,000 messages into one session — past the longest
|
||
session ever recorded in the real database (~${p99real} msgs). It shows where the engines break, not what
|
||
daily use feels like.</p>
|
||
${fig('rss-vs-msgs', 'Takeaway: Ink stays flat (~250MB) no matter how long the marathon runs; OpenTUI climbs toward ~870MB and only levels off thanks to its rolling row cap. Neither hits the 2GB kill line.')}
|
||
|
||
<h3>Stress-run numbers (3,000-msg marathons + one 10,000-msg run)</h3>
|
||
${matrixTable(results)}
|
||
<p class="dim">Reading guide: "typical" = median; the bracketed range is the middle half of runs
|
||
(interquartile range). "slowest 1 in 100" = p99. OpenTUI's syntax styling visibly degrades past ~1,400
|
||
rendered rows in these marathons (style-handle exhaustion fallback); older OpenTUI runs also show a
|
||
post-stream crash that was fixed before the latest runs.</p>
|
||
|
||
${fig('node-count', 'Takeaway: Ink keeps a bounded few hundred nodes mounted no matter how long the transcript gets — consistent with its flat memory line above. (No OpenTUI node-walk run in this result set.)')}
|
||
|
||
<h3>Low-memory survival (1GB Docker container)</h3>
|
||
${survivalTable(results)}
|
||
|
||
<h2>Run health — can you trust these numbers?</h2>
|
||
<p>Determinism gate: each UI replayed the same input twice and must produce a pixel-identical final screen
|
||
(same fingerprint). If this fails, none of the comparisons above are meaningful.</p>
|
||
${gateTable(results)}
|
||
${drainTable(results)}
|
||
<p class="dim">Methodology: <code>docs/plans/opentui-bench-suite.md</code>. Real binaries on a real PTY
|
||
(120×40), fake gateway via <code>HERMES_PYTHON</code> (zero UI changes), outside-the-process /proc sampling,
|
||
2GB cgroup caps via systemd. Instrumented diagnostic runs are flagged and never headlined.</p>
|
||
|
||
</body></html>
|
||
`
|
||
|
||
writeFileSync(OUT_HTML, html)
|
||
process.stdout.write(`report → ${OUT_HTML}\npngs → ${pngs.join(', ') || '(none)'}\n`)
|