#!/usr/bin/env node // Report renderer — reads bench/results/*.json and emits ONE self-contained // bench/report.html (inline SVG, zero CDN/network) plus PNG exports of each // chart to bench/report-assets/ (rasterized with the resvg-js available at // ~/.claude/skills/tmux-pane-screenshot/scripts/node_modules — see README). // Real data only: cells with no results render as "not run". import { createRequire } from 'node:module' import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' const here = dirname(fileURLToPath(import.meta.url)) const RESULTS_DIR = join(here, 'results') const ASSETS_DIR = join(here, 'report-assets') const OUT_HTML = join(here, 'report.html') const CAP_MB = 2048 // ── data load ─────────────────────────────────────────────────────────── function loadResults() { let files = [] try { files = readdirSync(RESULTS_DIR).filter(f => f.endsWith('.json')) } catch { return [] } const out = [] for (const f of files.sort()) { try { const r = JSON.parse(readFileSync(join(RESULTS_DIR, f), 'utf8')) r._file = f out.push(r) } catch { /* skip unparseable */ } } return out } // ── stats ─────────────────────────────────────────────────────────────── const quantile = (sorted, q) => { if (!sorted.length) return null const pos = (sorted.length - 1) * q const lo = Math.floor(pos) const hi = Math.ceil(pos) return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo) } const median = xs => quantile(xs.slice().sort((a, b) => a - b), 0.5) const iqr = xs => { const s = xs.slice().sort((a, b) => a - b) return [quantile(s, 0.25), quantile(s, 0.75)] } const fmt = (x, d = 1) => (x === null || x === undefined || Number.isNaN(x) ? '—' : Number(x).toFixed(d)) const fmtMedIqr = (xs, d = 1) => { if (!xs.length) return 'not run' const [lo, hi] = iqr(xs) return `${fmt(median(xs), d)} [${fmt(lo, d)}–${fmt(hi, d)}]` } // least-squares slope of rss_mb vs msgs over points function lsSlope(points) { if (points.length < 3) return null const n = points.length let sx = 0 let sy = 0 let sxx = 0 let sxy = 0 for (const [x, y] of points) { sx += x sy += y sxx += x * x sxy += x * y } const denom = n * sxx - sx * sx if (denom === 0) return null return (n * sxy - sx * sy) / denom // MB per message } // Per-run back-half slope (MB/1k msgs): fit over msgs >= max(500, maxMsgs/2) // — warmup (first 500 msgs) always excluded per protocol. function runSlope(run) { const pts = run.samples .filter(s => s.kind === 'boundary' && s.msgs != null && s.rss_kb != null) .map(s => [s.msgs, s.rss_kb / 1024]) if (pts.length < 4) return null const maxMsgs = pts[pts.length - 1][0] const cut = Math.max(500, maxMsgs / 2) const back = pts.filter(([x]) => x >= cut) const slope = lsSlope(back) return slope === null ? null : slope * 1000 } // plateau: median RSS over the final quartile of boundary samples function runPlateau(run) { const pts = run.samples.filter(s => s.kind === 'boundary' && s.rss_kb != null).map(s => s.rss_kb / 1024) if (pts.length < 4) return null return median(pts.slice(Math.floor(pts.length * 0.75))) } // ── SVG primitives ────────────────────────────────────────────────────── const COLORS = { ink: '#e06c75', 'otui-capped': '#61afef', 'otui-uncapped': '#56b6c2', other: '#c678dd' } const esc = s => String(s).replace(/&/g, '&').replace(//g, '>') function chart({ title, w = 860, h = 440, xLabel, yLabel, xMax, yMax, series, capLine, markers = [], note }) { const padL = 64 const padR = 16 const padT = 40 const padB = 64 const pw = w - padL - padR const ph = h - padT - padB const X = x => padL + (x / xMax) * pw const Y = y => padT + ph - (y / yMax) * ph const parts = [] parts.push(`') return parts.join('\n') } function barChart({ title, w = 860, h = 360, groups, yLabel, note }) { // groups: [{label, bars: [{name, value, lo, hi, color}]}] const padL = 64 const padR = 16 const padT = 40 const padB = 64 const pw = w - padL - padR const ph = h - padT - padB const vals = groups.flatMap(g => g.bars.map(b => b.hi ?? b.value)).filter(v => v != null) if (!vals.length) return null const yMax = Math.max(...vals) * 1.15 const Y = y => padT + ph - (y / yMax) * ph const parts = [] parts.push(`') return parts.join('\n') } // ── chart builders ────────────────────────────────────────────────────── function rssChart(results) { const runs = results.filter( r => (r.meta.cell.startsWith('mem') || r.meta.cell.startsWith('slope')) && !r.meta.instrumented && r.meta.mode === 'mem' ) if (!runs.length) return null let xMax = 0 let yMax = CAP_MB * 1.05 const series = [] const markers = [] const seen = new Set() for (const r of runs) { const pts = r.samples.filter(s => s.kind === 'boundary' && s.msgs != null && s.rss_kb != null).map(s => [s.msgs, s.rss_kb / 1024]) if (!pts.length) continue xMax = Math.max(xMax, pts[pts.length - 1][0]) yMax = Math.max(yMax, ...pts.map(p => p[1])) const color = COLORS[r.meta.config] ?? COLORS.other const key = r.meta.config series.push({ points: pts, color, width: r.meta.cell.startsWith('slope') ? 2.5 : 1.5, opacity: 0.8, label: seen.has(key) ? null : `${key}` }) seen.add(key) if (r.summary.cap_hit) { const last = pts[pts.length - 1] markers.push({ x: last[0], y: last[1], label: `OOM @${last[0]}`, color: '#e5c07b' }) } else if (r.summary.result === 'died' || r.summary.result === 'crashed_after_stream') { const last = pts[pts.length - 1] markers.push({ x: last[0], y: last[1], label: `crash @${last[0]}`, color: '#e06c75' }) } } return chart({ title: 'RSS vs fixture messages (clean memory runs, all reps + slope runs)', xLabel: 'fixture messages (rowsPerTurn accounting)', yLabel: 'RSS (MB)', xMax: Math.max(xMax, 1000), yMax: yMax * 1.08, series, capLine: CAP_MB, markers, note: '2GB cgroup cap (systemd-run, MemorySwapMax=0). × = cgroup OOM kill. Samples every 100 msgs from /proc/PID/smaps_rollup.' }) } function nodesChart(results) { const runs = results.filter(r => r.meta.cell.startsWith('nodes')) if (!runs.length) return null const series = [] let xMax = 0 let yMax = 0 for (const r of runs) { let pts = [] if (r.node_samples?.length) { // Ink fd-3 sampler: align wall-clock node samples to msg boundaries. const t0 = Date.parse(r.meta.utc) const bounds = r.samples.filter(s => s.kind === 'boundary' && s.msgs != null) pts = r.node_samples.map(ns => { const el = ns.t - t0 let msgs = 0 for (const b of bounds) if (b.t_ms <= el) msgs = b.msgs return [msgs, ns.yoga] }) // collapse to last sample per msg count const byMsg = new Map() for (const [m, y] of pts) byMsg.set(m, y) pts = [...byMsg.entries()].sort((a, b) => a[0] - b[0]) series.push({ points: pts, color: COLORS.ink, label: 'ink live yoga nodes (fd-3 sampler)' }) } else if (r.samples.some(s => s.renderables != null)) { pts = r.samples.filter(s => s.renderables != null).map(s => [s.msgs, s.renderables]) series.push({ points: pts, color: COLORS[r.meta.config] ?? COLORS.other, label: `${r.meta.config} renderables (headless walk)` }) } for (const [x, y] of pts) { xMax = Math.max(xMax, x) yMax = Math.max(yMax, y) } } if (!series.length) return null return chart({ title: 'Mounted node count vs messages (instrumented runs — mechanism witness, never headlined)', xLabel: 'fixture messages', yLabel: 'live nodes', xMax: Math.max(xMax, 100), yMax: yMax * 1.1, series, note: 'instrumented:true. Ink: HERMES_TUI_MEMSAMPLE_FD walk of the forked reconciler root. OpenTUI: scripts/mem-bench.tsx renderable walk (headless, diagnostic-only).' }) } function scrollCdfChart(results) { const runs = results.filter(r => r.meta.cell.startsWith('scroll') && r.summary.scroll_latencies_ms?.length) if (!runs.length) return null const transcriptMsgs = runs[0].meta.fixture?.msgs ?? '?' const byConfig = {} for (const r of runs) { ;(byConfig[r.meta.config] ??= []).push(...r.summary.scroll_latencies_ms) } const series = [] let xMax = 0 for (const [config, lats] of Object.entries(byConfig)) { const s = lats.slice().sort((a, b) => a - b) xMax = Math.max(xMax, quantile(s, 0.995)) const pts = s.map((v, i) => [v, ((i + 1) / s.length) * 100]) series.push({ points: pts, color: COLORS[config] ?? COLORS.other, label: `${config} (n=${s.length})` }) } return chart({ title: `Scroll latency CDF — SGR wheel 30Hz×15s on ${transcriptMsgs}-msg transcript (all reps pooled)`, xLabel: 'write → first output byte (ms)', yLabel: 'percentile (%)', xMax: Math.max(1, xMax), yMax: 100, series, note: 'Latency = PTY write of the wheel event to the next PTY output chunk.' }) } function startupChart(results) { const runs = results.filter(r => r.meta.cell === 'startup') if (!runs.length) return null const byConfig = {} for (const r of runs) { const c = (byConfig[r.meta.config] ??= { fb: [], sc: [] }) if (r.summary.first_byte_ms != null) c.fb.push(r.summary.first_byte_ms) if (r.summary.session_create_ms != null) c.sc.push(r.summary.session_create_ms) } const groups = Object.entries(byConfig).map(([config, v]) => ({ label: config, bars: [ { name: 'first byte', value: median(v.fb), lo: iqr(v.fb)[0], hi: iqr(v.fb)[1], color: COLORS[config] ?? COLORS.other }, { name: 'session.create', value: median(v.sc), lo: iqr(v.sc)[0], hi: iqr(v.sc)[1], color: '#98c379' } ] })) return barChart({ title: 'Startup (fake gateway) — median of 10 reps, whiskers = IQR', yLabel: 'ms from spawn', groups, note: 'first byte = first PTY output; session.create = the UI reaches its session bootstrap RPC.' }) } function ptyRateChart(results) { const runs = results.filter(r => r.meta.cell.startsWith('cpu') && r.summary.stream_done) if (!runs.length) return null const byConfig = {} for (const r of runs) { const done = r.samples.filter(s => s.kind === 'done')[0] const start = r.summary.stream_start_ms if (!done || start == null) continue const secs = (done.t_ms - start) / 1000 const c = (byConfig[r.meta.config] ??= { rate: [], cpu: [] }) c.rate.push(done.pty_bytes / secs / 1024) // CPU ms per event over the streaming window const sb = r.samples.filter(s => s.kind === 'boundary') if (sb.length >= 2) { const first = sb[0] const last = sb[sb.length - 1] const ticks = last.utime_ticks + last.stime_ticks - first.utime_ticks - first.stime_ticks const events = (last.events ?? 0) - (first.events ?? 0) if (events > 0) c.cpu.push((ticks * 10) / events) // 100Hz ticks → ms } } const groups = Object.entries(byConfig).map(([config, v]) => ({ label: config, bars: [ { name: 'PTY KiB/s', value: median(v.rate), lo: iqr(v.rate)[0], hi: iqr(v.rate)[1], color: COLORS[config] ?? COLORS.other }, { name: 'CPU ms/event', value: median(v.cpu), lo: iqr(v.cpu)[0], hi: iqr(v.cpu)[1], color: '#d19a66' } ] })) return barChart({ title: 'Paced streaming (30 ev/s): PTY output rate + CPU per event — median±IQR over reps', yLabel: 'KiB/s · ms/event', groups, note: 'CPU from /proc/PID/stat utime+stime deltas across the stream window (UI process only).' }) } // ── tables ────────────────────────────────────────────────────────────── function matrixTable(results) { const memRuns = results.filter(r => r.meta.cell.startsWith('mem') && r.meta.mode === 'mem' && !r.meta.instrumented) const slopeRuns = results.filter(r => r.meta.cell.startsWith('slope')) const scrollRuns = results.filter(r => r.meta.cell.startsWith('scroll')) const cpuRuns = results.filter(r => r.meta.cell.startsWith('cpu')) const configs = ['ink', 'otui-capped', 'otui-uncapped'] const rows = [] for (const config of configs) { const mem = memRuns.filter(r => r.meta.config === config) const slopes = mem.map(runSlope).filter(s => s != null) const plateaus = mem.map(runPlateau).filter(s => s != null) const vmhwm = mem.map(r => r.summary.vmhwm_kb).filter(Boolean).map(k => k / 1024) const slope10k = slopeRuns.filter(r => r.meta.config === config).map(runSlope).filter(s => s != null) const lat = scrollRuns.filter(r => r.meta.config === config).flatMap(r => r.summary.scroll_latencies_ms ?? []) const latS = lat.slice().sort((a, b) => a - b) const cpu = [] for (const r of cpuRuns.filter(x => x.meta.config === config)) { const sb = r.samples.filter(s => s.kind === 'boundary') if (sb.length >= 2) { const f = sb[0] const l = sb[sb.length - 1] const ev = (l.events ?? 0) - (f.events ?? 0) if (ev > 0) cpu.push(((l.utime_ticks + l.stime_ticks - f.utime_ticks - f.stime_ticks) * 10) / ev) } } const capHits = [...mem, ...slopeRuns.filter(r => r.meta.config === config)].filter(r => r.summary.cap_hit) rows.push(`
| config | slope MB/1k msgs (3000-msg runs) | slope MB/1k (10k run) | plateau RSS MB (final quartile) | VmHWM MB | scroll p50/p90/p99 ms | CPU ms/event (paced) | cap hits (2GB) |
|---|
E3 (memory-constrained Docker survival): not run.
' const rows = runs.map( r => `| cell | config | limit | result | msgs survived | VmHWM | basis |
|---|
Determinism gate: not run.
' const byConfig = {} for (const r of runs) (byConfig[r.meta.config] ??= []).push(r.summary.digest) const rows = Object.entries(byConfig).map(([c, ds]) => { const ok = ds.length >= 2 && ds.every(d => d && d === ds[0]) return `| config | replay digests | gate |
|---|
PTY drain assertion: no run exceeded the 10ms event-loop starvation budget.
' const rows = bad.map( r => `⚠ drain assertion violations (runs kept, flagged):
| run | max loop lag | violations >10ms |
|---|
${metaRuns} · generated ${new Date().toISOString()}
Methodology: docs/plans/opentui-bench-suite.md. Real binaries over a real node-pty PTY (120×40,
xterm-256color), fake gateway via HERMES_PYTHON (zero UI changes), external /proc sampling on
100-msg boundaries, 2GB cgroup-v2 caps via systemd-run --user --scope. Median±IQR throughout;
instrumented node-count runs are flagged and never headlined.
not run
'}not run
'}not run
'}not run
'}not run
'}