From 14ee1a52c0480d49b7a2c50257860f9c4186b2a2 Mon Sep 17 00:00:00 2001 From: alt-glitch Date: Thu, 11 Jun 2026 02:29:37 +0530 Subject: [PATCH] bench: fake-gateway + PTY harness + matrix runner (methodology in docs/plans) --- bench/.gitignore | 3 + bench/README.md | 66 ++++ bench/fake-gateway.mjs | 170 +++++++++ bench/fixture-stream.mjs | 86 +++++ bench/harness.mjs | 720 +++++++++++++++++++++++++++++++++++++++ bench/package-lock.json | 31 ++ bench/package.json | 13 + bench/render.mjs | 549 +++++++++++++++++++++++++++++ bench/run-e3-inner.mjs | 71 ++++ bench/run-e3.sh | 33 ++ bench/run.mjs | 360 ++++++++++++++++++++ 11 files changed, 2102 insertions(+) create mode 100644 bench/.gitignore create mode 100644 bench/README.md create mode 100755 bench/fake-gateway.mjs create mode 100644 bench/fixture-stream.mjs create mode 100644 bench/harness.mjs create mode 100644 bench/package-lock.json create mode 100644 bench/package.json create mode 100644 bench/render.mjs create mode 100644 bench/run-e3-inner.mjs create mode 100644 bench/run-e3.sh create mode 100644 bench/run.mjs diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 00000000000..1af43aec4d1 --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +.cache/ +*.cpuprofile diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000000..551aeaf1c24 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,66 @@ +# TUI benchmark suite — Ink (`ui-tui`) vs OpenTUI (`ui-opentui`) + +Methodology (settled, binding): `docs/plans/opentui-bench-suite.md`. This +directory is the implementation: real binaries over a real node-pty PTY +(120×40, xterm-256color), a fake gateway substituted via `HERMES_PYTHON` +(ZERO changes to either UI), external `/proc` sampling, cgroup-v2 memory caps. +No tmux anywhere in measurement. + +## Pieces + +| file | role | +|---|---| +| `fake-gateway.mjs` | NDJSON JSON-RPC gateway stand-in. Both UIs spawn it as `$HERMES_PYTHON -m tui_gateway.entry`. Answers every startup RPC with canned results, then streams the fixture (burst / paced / load-then-idle). Never writes stderr (the UIs render gateway stderr). | +| `fixture-stream.mjs` | Serializes the deterministic lumpy-turn fixture (`ui-opentui/scripts/fixture.ts`, imported directly via Node ≥26 type stripping — no port) to NDJSON. Cached under `.cache/`, sha256-stamped. | +| `harness.mjs` | One scenario = one UI boot: node-pty PTY, tight drain loop (event-loop starvation probe, 10ms budget asserted), `/proc/PID/{smaps_rollup,status,stat}` samples on 100-msg boundaries (UI PID only), `systemd-run --user --scope -p MemoryMax=… -p MemorySwapMax=0` caps, SGR wheel injection, resize-jiggle digest capture. | +| `run.mjs` | The matrix runner (protocol: determinism gate first, sequential SUTs, randomized per-rep config order, 10s cooldowns, load gate). | +| `render.mjs` | `results/*.json` → self-contained `report.html` (inline SVG, no CDN) + PNGs in `report-assets/`. | + +## Running cells + +Node 26 is required (`BENCH_NODE_BIN` overrides the default fnm path). Build +both UIs first; results land in `results/-----r.json`. + +```sh +cd ui-opentui && node scripts/build.mjs && cd ../ui-tui && node scripts/build.mjs && cd ../bench +npm install # node-pty (bench-local devDep) + +node run.mjs --cell gate # determinism gate (digest replay ×2 per UI) — run FIRST +node run.mjs --cell mem3000 # clean memory runs, 3 reps × 3 configs, 2GB cap +node run.mjs --cell slope10k # one 10k-msg slope run: ink + otui-uncapped (cap-hit IS a datapoint) +node run.mjs --cell nodes # instrumented node counts (ink fd-3 sampler; opentui headless walk) +node run.mjs --cell cpu # paced 30 ev/s streaming ×3 +node run.mjs --cell scroll # SGR wheel 30Hz×15s on a 3000-msg transcript ×3 +node run.mjs --cell startup # ×10, fake gateway +node render.mjs # report.html + report-assets/*.png +``` + +Configs: `ink` · `otui-capped` (`HERMES_TUI_MAX_MESSAGES=3000`, the default) · +`otui-uncapped` (`=100000`). Launch parity with `hermes_cli/main.py`: +Ink = `node --expose-gc ui-tui/dist/entry.js`, OpenTUI = +`node --experimental-ffi --no-warnings ui-opentui/dist/main.js`, both with +`NODE_OPTIONS=--max-old-space-size=` (8192 on the unconstrained host — +what the launcher picks outside a container). + +## E3 (constrained Docker survival) + +`E3-lite` runs the same harness inside a generic `node:26` container (NOT the +shipped image) with the worktree bind-mounted read-only and `--memory=1g +--memory-swap=1g`; the whole container (UI + fake gateway + harness) shares the +limit. See `run-e3.sh` if present, or the report's survival table for the exact +invocation used. + +## Accounting + known deviations + +- **"messages" = fixture rows** (`rowsPerTurn` accounting, identical to + `ui-opentui/scripts/mem-bench.tsx`), so numbers are comparable with the + pre-registered expectations. ~46% of fixture rows are user/system rows. +- **User/system rows are not streamed**: they are composer-local in both UIs + (no wire event exists), so PTY runs mount only the assistant/tool rows — + the renderable-heavy part that carries the memory claim. Consequence: the + OpenTUI store cap (3000 rows) binds at ≈6.6k fixture-msgs in PTY runs. +- **Digest gate**: final-screen digest after a resize-forced repaint, ANSI + stripped, cut at the composer hint, `up: Ns` normalized (the OpenTUI status + bar has a 1Hz uptime clock; the transcript region itself is deterministic). +- The headless `scripts/mem-bench.tsx` numbers are diagnostic-only and flagged + `instrumented`/`diagnostic_only` — never headlined. diff --git a/bench/fake-gateway.mjs b/bench/fake-gateway.mjs new file mode 100755 index 00000000000..a3703f48b0a --- /dev/null +++ b/bench/fake-gateway.mjs @@ -0,0 +1,170 @@ +#!/usr/bin/env node +// Fake tui_gateway — substituted via HERMES_PYTHON so BOTH UIs spawn THIS +// executable as `$HERMES_PYTHON -m tui_gateway.entry` (argv ignored) and speak +// the identical NDJSON JSON-RPC wire over stdio. ZERO changes to either UI. +// +// Wire contract (mirrors tui_gateway/entry.py + both UI clients): +// - unsolicited {jsonrpc:"2.0",method:"event",params:{type:"gateway.ready",payload:{skin:{}}}} +// - events: {jsonrpc:"2.0",method:"event",params:{type,payload?}} (no id) +// - responses: {jsonrpc:"2.0",id,result} for every request, canned per method. +// +// NEVER writes to stderr (both UIs surface gateway stderr lines INTO the UI as +// activity rows / gateway.stderr events, which would perturb the rendered +// transcript). Progress/telemetry goes to HERMES_FAKE_PROGRESS (append-only +// NDJSON file the harness tails). +// +// Env config: +// HERMES_FAKE_FIXTURE NDJSON fixture path (from fixture-stream.mjs). Optional. +// HERMES_FAKE_MODE burst | paced | load-then-idle (default burst) +// HERMES_FAKE_RATE events/sec for paced mode (default 30) +// HERMES_FAKE_START_DELAY_MS delay after session.create reply before streaming (default 1500) +// HERMES_FAKE_SAMPLE_EVERY fixture-msg boundary cadence for progress lines (default 100) +// HERMES_FAKE_PROGRESS progress NDJSON file path (required for harness runs) +// +// Modes: burst = write as fast as the pipe accepts (await 'drain' on +// backpressure, so emission tracks UI ingestion within the ~64KB pipe buffer); +// paced = HERMES_FAKE_RATE events/sec; load-then-idle = burst, then sit idle +// (scroll-latency runs drive input afterwards). Exits on stdin EOF (the UIs +// close stdin to stop the gateway) — same lifecycle as the real child. + +import { appendFileSync, readFileSync } from 'node:fs' +import { createInterface } from 'node:readline' + +const FIXTURE = process.env.HERMES_FAKE_FIXTURE || '' +const MODE = process.env.HERMES_FAKE_MODE || 'burst' +const RATE = Math.max(1, Number.parseInt(process.env.HERMES_FAKE_RATE ?? '30', 10) || 30) +const START_DELAY_MS = Number.parseInt(process.env.HERMES_FAKE_START_DELAY_MS ?? '1500', 10) || 1500 +const SAMPLE_EVERY = Math.max(1, Number.parseInt(process.env.HERMES_FAKE_SAMPLE_EVERY ?? '100', 10) || 100) +const PROGRESS = process.env.HERMES_FAKE_PROGRESS || '' + +const t0 = Date.now() +const progress = obj => { + if (!PROGRESS) return + try { + appendFileSync(PROGRESS, JSON.stringify({ ...obj, t: Date.now() - t0, wall: Date.now() }) + '\n') + } catch { + /* progress is best-effort; never crash the wire */ + } +} + +// UI gone (pipe closed) → exit quietly like the real child on stdin EOF. +process.stdout.on('error', () => process.exit(0)) + +const writeFrame = obj => { + const ok = process.stdout.write(JSON.stringify(obj) + '\n') + return ok ? null : new Promise(r => process.stdout.once('drain', r)) +} +const emitEvent = params => writeFrame({ jsonrpc: '2.0', method: 'event', params }) + +// ── Canned RPC results (recon'd from both UIs' startup sequences) ────── +const SESSION_ID = 'bench-session-0001' +const INFO = { + model: 'bench/fake-model', + version: '0.0.0-bench', + cwd: process.env.HERMES_CWD || process.cwd(), + skills: {}, + tools: { core: ['terminal', 'read_file'] }, + usage: { calls: 0, input: 0, output: 0, total: 0 } +} + +function resultFor(method, params) { + switch (method) { + case 'setup.status': + return { provider_configured: true } + case 'session.create': + return { session_id: SESSION_ID, info: INFO } + case 'session.resume': + case 'session.activate': + return { session_id: SESSION_ID, messages: [], info: INFO } + case 'session.most_recent': + return {} + case 'session.list': + case 'session.active_list': + return { sessions: [] } + case 'config.get': + if (params && params.key === 'mtime') return { mtime: 1 } + if (params && params.key === 'full') return { config: { display: {} } } + return { value: '' } + case 'commands.catalog': + return { pairs: [['help', 'show help']], canon: {}, categories: [], sub: {}, skill_count: 0 } + case 'startup.catalog': + return { tools: {}, skills: {}, mcp_servers: [] } + case 'model.options': + return { providers: [] } + case 'session.title': + return { title: 'bench' } + case 'prompt.submit': + return { ok: true } + case 'session.interrupt': + return { ok: true } + case 'complete.slash': + case 'complete.path': + return { items: [] } + default: + return {} + } +} + +// ── Fixture streaming ────────────────────────────────────────────────── +let streaming = false +async function streamFixture() { + if (streaming || !FIXTURE) return + streaming = true + const lines = readFileSync(FIXTURE, 'utf8').split('\n') + let msgs = 0 + let events = 0 + let nextBoundary = SAMPLE_EVERY + const paced = MODE === 'paced' + const interval = paced ? 1000 / RATE : 0 + let nextAt = Date.now() + progress({ k: 'stream_start', mode: MODE }) + for (const raw of lines) { + if (!raw) continue + const item = JSON.parse(raw) + if (item.k === 'e') { + if (paced) { + const wait = nextAt - Date.now() + if (wait > 0) await new Promise(r => setTimeout(r, wait)) + nextAt += interval + } + const drained = emitEvent(item.v) + if (drained) await drained + events++ + } else if (item.k === 't') { + msgs = item.msgs + if (msgs >= nextBoundary) { + progress({ k: 'boundary', msgs, events }) + while (nextBoundary <= msgs) nextBoundary += SAMPLE_EVERY + } + } + // {"k":"r"} row markers: composer-local rows, nothing on the wire. + } + progress({ k: 'done', msgs, events }) +} + +// ── Main: handshake + request loop ───────────────────────────────────── +progress({ k: 'start', pid: process.pid, mode: MODE, fixture: FIXTURE }) +emitEvent({ type: 'gateway.ready', payload: { skin: {} } }) + +const rl = createInterface({ input: process.stdin }) +rl.on('line', line => { + let msg + try { + msg = JSON.parse(line) + } catch { + return + } + if (!msg || typeof msg !== 'object' || msg.id === undefined) return + const method = String(msg.method ?? '') + progress({ k: 'req', method }) + void writeFrame({ jsonrpc: '2.0', id: msg.id, result: resultFor(method, msg.params) }) + if (method === 'session.create' || method === 'session.resume') { + setTimeout(() => { + streamFixture().catch(() => process.exit(1)) + }, START_DELAY_MS) + } +}) +rl.on('close', () => { + progress({ k: 'eof' }) + process.exit(0) +}) diff --git a/bench/fixture-stream.mjs b/bench/fixture-stream.mjs new file mode 100644 index 00000000000..3c53f59080f --- /dev/null +++ b/bench/fixture-stream.mjs @@ -0,0 +1,86 @@ +#!/usr/bin/env node +// Serialize the deterministic lumpy-turn fixture (ui-opentui/scripts/fixture.ts) +// to NDJSON for the fake gateway. We check in THIS generator invocation, not the +// generated file (it is megabytes); the stream is byte-reproducible for a given +// message count because the fixture is seeded by turn index. +// +// The generator is imported DIRECTLY from ui-opentui/scripts/fixture.ts via +// Node >=26 type stripping — no port, no drift. `applyTurn(store, turn)` only +// calls store.pushUser/pushSystem/apply, so a recorder stub extracts the exact +// per-turn action stream the OpenTUI mem-bench drives. +// +// Line format (one JSON object per line): +// {"k":"e","v":{...GatewayEvent...}} → sent on the wire as +// {jsonrpc:"2.0",method:"event",params:v} +// {"k":"r","role":"user"|"system"} → row marker, NOT sent (composer-local +// rows have no wire representation — +// see README "deviation: user rows") +// {"k":"t","msgs":N} → end-of-turn marker with the CUMULATIVE +// fixture-message count (rowsPerTurn +// accounting, same as scripts/mem-bench.tsx) +// +// Usage: node fixture-stream.mjs --msgs 3000 [--out path] +// Default out: bench/.cache/fixture-.ndjson (prints path + sha256) + +import { createHash } from 'node:crypto' +import { createWriteStream, mkdirSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +const here = dirname(fileURLToPath(import.meta.url)) +const fixtureTs = resolve(here, '../ui-opentui/scripts/fixture.ts') + +function parseArgs(argv) { + const args = { msgs: 3000, out: null } + for (let i = 2; i < argv.length; i++) { + if (argv[i] === '--msgs') args.msgs = Number.parseInt(argv[++i], 10) + else if (argv[i] === '--out') args.out = argv[++i] + } + if (!Number.isFinite(args.msgs) || args.msgs <= 0) throw new Error('--msgs must be a positive integer') + return args +} + +export async function generate(msgs, outPath) { + const { applyTurn, rowsPerTurn } = await import(pathToFileURL(fixtureTs).href) + mkdirSync(dirname(outPath), { recursive: true }) + const out = createWriteStream(outPath) + const hash = createHash('sha256') + const write = line => { + const data = line + '\n' + hash.update(data) + if (!out.write(data)) return new Promise(r => out.once('drain', r)) + return null + } + + let pushed = 0 + let events = 0 + let turn = 0 + while (pushed < msgs) { + const lines = [] + const recorder = { + pushUser: () => lines.push('{"k":"r","role":"user"}'), + pushSystem: () => lines.push('{"k":"r","role":"system"}'), + apply: ev => { + lines.push(JSON.stringify({ k: 'e', v: ev })) + events++ + } + } + applyTurn(recorder, turn) + pushed += rowsPerTurn(turn) + lines.push(JSON.stringify({ k: 't', msgs: pushed })) + for (const line of lines) { + const wait = write(line) + if (wait) await wait + } + turn++ + } + await new Promise((res, rej) => out.end(err => (err ? rej(err) : res()))) + return { path: outPath, msgs: pushed, events, turns: turn, sha256: hash.digest('hex') } +} + +if (import.meta.main) { + const args = parseArgs(process.argv) + const outPath = args.out ?? resolve(here, `.cache/fixture-${args.msgs}.ndjson`) + const info = await generate(args.msgs, outPath) + process.stdout.write(JSON.stringify(info) + '\n') +} diff --git a/bench/harness.mjs b/bench/harness.mjs new file mode 100644 index 00000000000..9b2c76c8f0d --- /dev/null +++ b/bench/harness.mjs @@ -0,0 +1,720 @@ +// PTY harness — boots ONE UI (the real binary) over a real node-pty PTY at +// 120×40 with the fake gateway substituted via HERMES_PYTHON, drains the master +// side tightly, samples /proc/PID externally on fixture-message boundaries, and +// (optionally) wraps the UI in a cgroup-v2 scope via systemd-run. +// Methodology: docs/plans/opentui-bench-suite.md. No tmux anywhere. + +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, statSync, unlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import pty from 'node-pty' + +const here = dirname(fileURLToPath(import.meta.url)) +export const REPO_ROOT = resolve(here, '..') +const FAKE_GATEWAY = join(here, 'fake-gateway.mjs') + +export const NODE26_BIN = process.env.BENCH_NODE_BIN + || join(process.env.HOME ?? '', '.local/share/fnm/node-versions/v26.3.0/installation/bin/node') + +const sleep = ms => new Promise(r => setTimeout(r, ms)) +const now = () => Date.now() + +// ── /proc readers (UI PID only — never the gateway child) ────────────── +function readProcSample(pid) { + try { + const rollup = readFileSync(`/proc/${pid}/smaps_rollup`, 'utf8') + const status = readFileSync(`/proc/${pid}/status`, 'utf8') + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8') + const kb = (text, key) => { + const m = text.match(new RegExp(`^${key}:\\s+(\\d+) kB`, 'm')) + return m ? Number(m[1]) : null + } + // stat: fields after the parenthesized comm; utime=14 stime=15 (1-indexed). + const afterComm = stat.slice(stat.lastIndexOf(')') + 2).split(' ') + return { + rss_kb: kb(rollup, 'Rss'), + pss_kb: kb(rollup, 'Pss'), + private_dirty_kb: kb(rollup, 'Private_Dirty'), + vmhwm_kb: kb(status, 'VmHWM'), + utime_ticks: Number(afterComm[11]), + stime_ticks: Number(afterComm[12]) + } + } catch { + return null // process gone + } +} + +function readCgroup(pid) { + try { + const line = readFileSync(`/proc/${pid}/cgroup`, 'utf8').trim() + const path = line.split('::')[1] + if (!path) return null + return `/sys/fs/cgroup${path}` + } catch { + return null + } +} + +function readCgroupStats(cgPath) { + if (!cgPath) return null + try { + const read = f => { + try { + return readFileSync(join(cgPath, f), 'utf8').trim() + } catch { + return null + } + } + const events = read('memory.events') + const oomKill = events ? Number(events.match(/^oom_kill (\d+)$/m)?.[1] ?? 0) : null + return { + current: Number(read('memory.current') ?? NaN) || null, + peak: Number(read('memory.peak') ?? NaN) || null, + oom_kill: oomKill + } + } catch { + return null + } +} + +function childrenOf(pid) { + try { + return readFileSync(`/proc/${pid}/task/${pid}/children`, 'utf8').trim().split(/\s+/).filter(Boolean).map(Number) + } catch { + return [] + } +} + +function commOf(pid) { + try { + return readFileSync(`/proc/${pid}/comm`, 'utf8').trim() + } catch { + return '' + } +} + +// ── ANSI strip for the determinism digest ────────────────────────────── +// Removes CSI/OSC/DCS/SOS/PM/APC sequences, single ESC sequences, and control +// chars, then normalizes whitespace. Good enough to compare final rendered +// transcript text across replays of the SAME UI. +export function stripAnsi(text) { + return text + .replace(/\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)/g, '') // OSC + .replace(/\x1b[PX^_][^\x1b]*\x1b\\/g, '') // DCS/SOS/PM/APC + .replace(/\x1b\[[0-9;:<=>?]*[ -/]*[@-~]/g, '') // CSI + .replace(/\x1b[@-Z\\-_]/g, '') // single ESC + .replace(/[\x00-\x08\x0b-\x1f\x7f]/g, '') + .replace(/[ \t]+/g, ' ') + .replace(/\n{2,}/g, '\n') + .trim() +} + +// Digest normalization: the final screen contains a 1Hz uptime clock (OpenTUI +// status bar `up: Ns`) whose incremental repaints trail the full post-resize +// frame. The transcript region paints deterministically; the clock does not. +// Cut everything after the composer hint (the last stable screen region) and +// normalize the uptime token inside the kept prefix. +export function normalizeForDigest(text) { + const marker = 'Type your message' + const idx = text.indexOf(marker) + const head = idx >= 0 ? text.slice(0, idx + marker.length) : text + return head.replace(/up: \d+s/g, 'up: Ns') +} + +// ── env / argv composition (mirrors hermes_cli/main.py _launch_tui) ──── +function composeEnv({ ui, opentuiCap, heapMb, fakeEnv, activeSessionFile }) { + const keep = ['HOME', 'USER', 'LANG', 'LC_ALL', 'XDG_RUNTIME_DIR', 'DBUS_SESSION_BUS_ADDRESS', 'SHELL'] + const env = {} + for (const k of keep) if (process.env[k]) env[k] = process.env[k] + env.PATH = `${dirname(NODE26_BIN)}:/usr/bin:/bin:/usr/local/bin` + env.TERM = 'xterm-256color' + env.NODE_ENV = 'production' + env.HERMES_PYTHON = FAKE_GATEWAY + env.HERMES_PYTHON_SRC_ROOT = REPO_ROOT + env.HERMES_CWD = REPO_ROOT + env.HERMES_TUI_ACTIVE_SESSION_FILE = activeSessionFile + // Launcher parity: NODE_OPTIONS carries the V8 heap cap (8192 on an + // unconstrained host; _resolve_tui_heap_mb sizes it under a cgroup limit). + env.NODE_OPTIONS = `--max-old-space-size=${heapMb}` + if (ui === 'opentui') { + env.HERMES_TUI_MOUSE = '1' + if (opentuiCap != null) env.HERMES_TUI_MAX_MESSAGES = String(opentuiCap) + } + Object.assign(env, fakeEnv) + return env +} + +function uiArgv(ui) { + if (ui === 'ink') { + return { file: NODE26_BIN, args: ['--expose-gc', join(REPO_ROOT, 'ui-tui/dist/entry.js')], cwd: join(REPO_ROOT, 'ui-tui') } + } + return { + file: NODE26_BIN, + args: ['--experimental-ffi', '--no-warnings', join(REPO_ROOT, 'ui-opentui/dist/main.js')], + cwd: join(REPO_ROOT, 'ui-opentui') + } +} + +// ── the scenario runner ──────────────────────────────────────────────── +/** + * opts: + * ui: 'ink' | 'opentui' + * configName: 'ink' | 'otui-capped' | 'otui-uncapped' + * opentuiCap: number|null (HERMES_TUI_MAX_MESSAGES) + * mode: 'mem' | 'cpu-paced' | 'scroll' | 'startup' | 'digest' + * fixturePath, fixtureMsgs, fixtureSha + * memoryMax: string|null ('2G' → systemd-run --user --scope) + * heapMb: number (--max-old-space-size) + * sampleEvery: number (default 100) + * scroll: { hz, seconds } (scroll mode) + * pacedRate: number (cpu-paced mode, events/s) + * cell, rep, outFile + * startDelayMs, quiesceMs, runTimeoutMs + */ +export async function runScenario(opts) { + const { + ui, + configName, + opentuiCap = null, + mode, + fixturePath = '', + fixtureMsgs = 0, + fixtureSha = '', + memoryMax = null, + heapMb = 8192, + sampleEvery = 100, + scroll = { hz: 30, seconds: 15 }, + pacedRate = 30, + cell = 'E1', + rep = 0, + outFile = null, + startDelayMs = 1500, + quiesceMs = 800, + runTimeoutMs = 30 * 60 * 1000 + } = opts + + const runId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}` + const progressFile = join(tmpdir(), `hermes-bench-progress-${runId}.ndjson`) + const activeSessionFile = join(tmpdir(), `hermes-bench-session-${runId}.json`) + writeFileSync(progressFile, '') + + const fakeEnv = { + HERMES_FAKE_FIXTURE: fixturePath, + HERMES_FAKE_MODE: mode === 'cpu-paced' ? 'paced' : mode === 'scroll' ? 'load-then-idle' : 'burst', + HERMES_FAKE_RATE: String(pacedRate), + HERMES_FAKE_START_DELAY_MS: String(startDelayMs), + HERMES_FAKE_SAMPLE_EVERY: String(sampleEvery), + HERMES_FAKE_PROGRESS: progressFile + } + const env = composeEnv({ ui, opentuiCap, heapMb, fakeEnv, activeSessionFile }) + const { file, args, cwd } = uiArgv(ui) + + // Instrumented node-count runs (Ink): open fd 3 onto an NDJSON file via a + // shell wrapper (node-pty cannot pass extra fds) and gate the in-process + // sampler with HERMES_TUI_MEMSAMPLE_FD=3. NEVER combined with headline + // memory runs — results carry instrumented:true. + let nodeSampleFile = null + let spawnFile = file + let spawnArgs = args + if (opts.inkNodeSampler) { + nodeSampleFile = join(tmpdir(), `hermes-bench-nodes-${runId}.ndjson`) + writeFileSync(nodeSampleFile, '') + env.HERMES_TUI_MEMSAMPLE_FD = '3' + const quoted = [file, ...args].map(a => `'${a.replace(/'/g, `'\\''`)}'`).join(' ') + spawnFile = '/bin/sh' + spawnArgs = ['-c', `exec 3>>'${nodeSampleFile}'; exec ${quoted}`] + } + + const unitName = `hermes-bench-${runId}.scope` + if (memoryMax) { + const innerFile = spawnFile + const innerArgs = spawnArgs + spawnFile = 'systemd-run' + spawnArgs = [ + '--user', + '--scope', + '--quiet', + '--collect', + `--unit=${unitName.replace(/\.scope$/, '')}`, + '-p', + `MemoryMax=${memoryMax}`, + '-p', + 'MemorySwapMax=0', + '--', + innerFile, + ...innerArgs + ] + } + + const t0 = now() + const term = pty.spawn(spawnFile, spawnArgs, { + name: 'xterm-256color', + cols: 120, + rows: 40, + cwd, + env + }) + term.resize(120, 40) // explicit TIOCSWINSZ per protocol + + // ── tight drain loop ──────────────────────────────────────────────── + let bytesOut = 0 + let dataWrites = 0 + let firstByteAt = null + let lastDataAt = null + const dataTimestamps = [] // for scroll latency (epoch ms of each data chunk) + let recordDataTimestamps = false + let tailBuf = [] + let tailLen = 0 + const TAIL_MAX = 4 * 1024 * 1024 + term.onData(d => { + const t = now() + bytesOut += Buffer.byteLength(d) + dataWrites++ + if (firstByteAt === null) firstByteAt = t + lastDataAt = t + if (recordDataTimestamps) dataTimestamps.push(t) + tailBuf.push(d) + tailLen += d.length + while (tailLen > TAIL_MAX && tailBuf.length > 1) tailLen -= tailBuf.shift().length + }) + const resetTail = () => { + tailBuf = [] + tailLen = 0 + } + + // Drain-starvation probe: if OUR event loop stalls, the PTY master isn't + // being drained. 5ms cadence; any observed gap >10ms is recorded (assert). + let maxLoopLagMs = 0 + let lagViolations = 0 + let lastTick = now() + const lagTimer = setInterval(() => { + const t = now() + const lag = t - lastTick - 5 + if (lag > maxLoopLagMs) maxLoopLagMs = lag + if (lag > 10) lagViolations++ + lastTick = t + }, 5) + + // ── exit tracking ─────────────────────────────────────────────────── + let exited = null + const exitPromise = new Promise(res => { + term.onExit(({ exitCode, signal }) => { + exited = { exitCode, signal, t: now() - t0 } + res(exited) + }) + }) + + // ── UI PID discovery ──────────────────────────────────────────────── + // `systemd-run --scope` (and the /bin/sh sampler wrapper) EXEC the target in + // place, so the pty child PID *is* the UI once its comm flips to 'node'. + // Wait for that flip (scope registration / sh exec take a moment); fall back + // to a child walk in case a future wrapper forks instead. + let uiPid = term.pid + if (memoryMax || opts.inkNodeSampler) { + uiPid = null + // Node 26 names its main thread: comm is 'node-MainThread'. + const isNode = pid => commOf(pid).startsWith('node') + for (let i = 0; i < 200 && !exited; i++) { + if (isNode(term.pid)) { + uiPid = term.pid + break + } + const nodeKid = childrenOf(term.pid).find(k => isNode(k)) + if (nodeKid) { + uiPid = nodeKid + break + } + await sleep(25) + } + } + // containerCap: the harness itself runs INSIDE a memory-capped container + // (E3) — the container cgroup is the cap, no systemd-run involved. + const cgPath = opts.containerCap ? '/sys/fs/cgroup' : memoryMax && uiPid ? readCgroup(uiPid) : null + + // ── sampling state ────────────────────────────────────────────────── + const samples = [] + const events = [] + let lastCg = null + let streamDone = false + let streamStartT = null + let doneInfo = null + let sessionCreateAt = null + let gwPid = null + + const takeSample = (kind, msgs, evCount) => { + if (!uiPid) return + const proc = readProcSample(uiPid) + if (!proc) return + const cg = readCgroupStats(cgPath) + if (cg) lastCg = cg + samples.push({ + kind, + t_ms: now() - t0, + msgs: msgs ?? null, + events: evCount ?? null, + pty_bytes: bytesOut, + pty_writes: dataWrites, + ...proc, + ...(cg ? { cg_current: cg.current, cg_peak: cg.peak, cg_oom_kill: cg.oom_kill } : {}) + }) + } + + const tailProgress = (() => { + let offset = 0 + return () => { + let size + try { + size = statSync(progressFile).size + } catch { + return [] + } + if (size <= offset) return [] + const fd = openSync(progressFile, 'r') + try { + const buf = Buffer.alloc(size - offset) + readSync(fd, buf, 0, buf.length, offset) + offset = size + const out = [] + let text = buf.toString('utf8') + const lastNl = text.lastIndexOf('\n') + if (lastNl < text.length - 1) { + offset -= Buffer.byteLength(text.slice(lastNl + 1), 'utf8') + text = text.slice(0, lastNl + 1) + } + for (const line of text.split('\n')) { + if (!line.trim()) continue + try { + out.push(JSON.parse(line)) + } catch { + /* skip malformed */ + } + } + return out + } finally { + closeSync(fd) + } + } + })() + + const handleProgress = item => { + if (item.k === 'start') gwPid = item.pid + if (item.k === 'req') { + events.push({ kind: 'rpc', method: item.method, t_ms: now() - t0 }) + if (item.method === 'session.create' && sessionCreateAt === null) sessionCreateAt = now() + } + if (item.k === 'stream_start') streamStartT = now() + if (item.k === 'boundary') takeSample('boundary', item.msgs, item.events) + if (item.k === 'done') { + streamDone = true + doneInfo = { msgs: item.msgs, events: item.events } + takeSample('done', item.msgs, item.events) + } + } + + // main poll loop driver + let pollTimer = null + const startPolling = () => { + let lastPeriodic = 0 + pollTimer = setInterval(() => { + for (const item of tailProgress()) handleProgress(item) + const t = now() + if (t - lastPeriodic >= 1000) { + lastPeriodic = t + takeSample('periodic', doneInfo?.msgs ?? null, null) + } + }, 25) + } + startPolling() + + const waitFor = async (cond, timeoutMs, pollMs = 50) => { + const start = now() + while (!cond()) { + if (exited) return false + if (now() - start > timeoutMs) return false + await sleep(pollMs) + } + return true + } + + // Wait until no PTY output for `ms`, bounded: idle-frame UIs still repaint + // periodically (1Hz status clock), so a quiesce can never be unbounded. + const quiesce = async (ms, maxWaitMs = 15_000) => { + const deadline = now() + maxWaitMs + for (;;) { + if (exited) return + const last = lastDataAt ?? t0 + const idle = now() - last + if (idle >= ms) return + if (now() > deadline) return + await sleep(Math.min(ms - idle + 10, 200)) + } + } + + let quitRequested = false + const gracefulQuit = async () => { + if (exited) return + quitRequested = true + try { + term.write('\x03') + await sleep(150) + term.write('\x03') + } catch { + /* already gone */ + } + await Promise.race([exitPromise, sleep(3000)]) + if (!exited) { + try { + term.kill('SIGTERM') + } catch { + /* ignore */ + } + await Promise.race([exitPromise, sleep(2000)]) + } + if (!exited) { + try { + term.kill('SIGKILL') + } catch { + /* ignore */ + } + await Promise.race([exitPromise, sleep(2000)]) + } + } + + // ── mode flows ────────────────────────────────────────────────────── + const result = {} + const scrollLatencies = [] + let digest = null + let digestText = null + + const sessionStarted = await waitFor(() => sessionCreateAt !== null, 30_000) + if (!sessionStarted && !exited) { + events.push({ kind: 'error', message: 'no session.create within 30s', t_ms: now() - t0 }) + } + + if (mode === 'startup') { + // settle: boot RPCs done + paint quiet + await quiesce(quiesceMs) + takeSample('final', 0, 0) + } else { + // wait for the stream to finish (or the UI to die — cap-hit IS a result) + const ok = await waitFor(() => streamDone, runTimeoutMs, 100) + if (ok) { + await quiesce(quiesceMs) + takeSample('final', doneInfo?.msgs ?? null, doneInfo?.events ?? null) + } + } + + if (mode === 'scroll' && !exited && streamDone) { + // SGR wheel bursts at scroll.hz for scroll.seconds: first half UP, second half DOWN. + const totalEvents = Math.round(scroll.hz * scroll.seconds) + const interval = 1000 / scroll.hz + const writeTimes = [] + recordDataTimestamps = true + const cpuBefore = readProcSample(uiPid) + const tScroll0 = now() + for (let i = 0; i < totalEvents && !exited; i++) { + const target = tScroll0 + i * interval + const wait = target - now() + if (wait > 0) await sleep(wait) + const btn = i < totalEvents / 2 ? 64 : 65 + term.write(`\x1b[<${btn};60;20M`) + writeTimes.push(now()) + } + await quiesce(500) + recordDataTimestamps = false + const cpuAfter = readProcSample(uiPid) + // latency: for each write, first data timestamp >= write time + let j = 0 + for (const wt of writeTimes) { + while (j < dataTimestamps.length && dataTimestamps[j] < wt) j++ + if (j < dataTimestamps.length) scrollLatencies.push(dataTimestamps[j] - wt) + } + result.scroll = { + events_sent: writeTimes.length, + responses: scrollLatencies.length, + cpu_ticks: cpuBefore && cpuAfter ? cpuAfter.utime_ticks + cpuAfter.stime_ticks - cpuBefore.utime_ticks - cpuBefore.stime_ticks : null + } + } + + if (mode === 'digest' && !exited && streamDone) { + // Force a full repaint via resize-jiggle, then digest the post-resize text. + resetTail() + term.resize(120, 39) + await sleep(400) + resetTail() + term.resize(120, 40) + await sleep(1200) // fixed window — a 1Hz status clock means true silence never comes + digestText = normalizeForDigest(stripAnsi(tailBuf.join(''))) + digest = createHash('sha256').update(digestText).digest('hex') + } + + await gracefulQuit() + clearInterval(pollTimer) + clearInterval(lagTimer) + + // cap-hit determination + const finalCg = lastCg + let capHit = false + let capHitBasis = null + if ((memoryMax || opts.containerCap) && exited) { + if ((finalCg?.oom_kill ?? 0) > 0) { + capHit = true + capHitBasis = 'memory.events oom_kill' + } else { + // journal fallback: systemd logs OOM kills on the scope + try { + const log = execFileSync( + 'journalctl', + ['--user', '-q', '--no-pager', '-u', unitName, '--since', '-30min'], + { encoding: 'utf8' } + ) + if (/oom|OOM/i.test(log)) { + capHit = true + capHitBasis = 'journalctl scope oom record' + } + } catch { + /* journal unavailable */ + } + if (!capHit && exited.signal === 9 && !streamDone) { + capHit = true + capHitBasis = 'SIGKILL before stream completion (inferred)' + } + } + } + + const lastSample = samples[samples.length - 1] ?? null + const summary = { + result: capHit + ? 'cap_hit' + : exited && !quitRequested && mode !== 'startup' + ? streamDone + ? 'crashed_after_stream' + : 'died' + : 'completed', + cap_hit: capHit, + cap_hit_basis: capHitBasis, + at_messages: capHit ? (samples.filter(s => s.kind === 'boundary').at(-1)?.msgs ?? null) : null, + exit: exited, + stream_done: streamDone, + msgs_streamed: doneInfo?.msgs ?? samples.filter(s => s.kind === 'boundary').at(-1)?.msgs ?? 0, + events_streamed: doneInfo?.events ?? null, + pty_bytes_total: bytesOut, + pty_data_callbacks: dataWrites, + first_byte_ms: firstByteAt ? firstByteAt - t0 : null, + session_create_ms: sessionCreateAt ? sessionCreateAt - t0 : null, + stream_start_ms: streamStartT ? streamStartT - t0 : null, + vmhwm_kb: lastSample?.vmhwm_kb ?? null, + cg_peak: finalCg?.peak ?? null, + drain_max_loop_lag_ms: maxLoopLagMs, + drain_lag_violations: lagViolations, + drain_ok: lagViolations === 0, + digest, + scroll_latencies_ms: scrollLatencies.length ? scrollLatencies : undefined, + ...result + } + + const out = { + meta: { + cell, + ui, + config: configName, + mode, + rep, + run_id: runId, + utc: new Date(t0).toISOString(), + sha: gitSha(), + node: NODE26_BIN, + node_version: nodeVersion(), + pty: { cols: 120, rows: 40, term: 'xterm-256color' }, + heap_mb: heapMb, + memory_max: memoryMax, + container_cap: Boolean(opts.containerCap), + container_memory: opts.containerMemory ?? null, + opentui_cap: opentuiCap, + fixture: { path: fixturePath, msgs: fixtureMsgs, sha256: fixtureSha }, + sample_every: sampleEvery, + mode_params: mode === 'cpu-paced' ? { rate: pacedRate } : mode === 'scroll' ? scroll : {}, + ui_pid: uiPid, + gw_pid: gwPid, + cgroup: cgPath, + load_avg_at_start: loadAvg(), + instrumented: Boolean(opts.inkNodeSampler) + }, + samples, + events, + summary + } + if (digestText !== null) out.digest_text = digestText + // Postmortem: keep the stripped tail of the PTY stream for any run that + // didn't complete cleanly (crash diagnostics — small, bounded). + if (summary.result !== 'completed') out.pty_tail = stripAnsi(tailBuf.join('')).slice(-4000) + if (nodeSampleFile) { + try { + out.node_samples = readFileSync(nodeSampleFile, 'utf8') + .split('\n') + .filter(Boolean) + .map(l => JSON.parse(l)) + } catch { + out.node_samples = [] + } + try { + unlinkSync(nodeSampleFile) + } catch { + /* ignore */ + } + } + + try { + unlinkSync(progressFile) + } catch { + /* ignore */ + } + try { + unlinkSync(activeSessionFile) + } catch { + /* ignore */ + } + + if (outFile) { + mkdirSync(dirname(outFile), { recursive: true }) + writeFileSync(outFile, JSON.stringify(out, null, 1)) + } + return out +} + +let _sha = null +function gitSha() { + if (_sha) return _sha + try { + _sha = execFileSync('git', ['-C', REPO_ROOT, 'rev-parse', '--short', 'HEAD'], { encoding: 'utf8' }).trim() + } catch { + _sha = 'unknown' + } + return _sha +} + +function nodeVersion() { + try { + return execFileSync(NODE26_BIN, ['--version'], { encoding: 'utf8' }).trim() + } catch { + return 'unknown' + } +} + +export function loadAvg() { + try { + return readFileSync('/proc/loadavg', 'utf8').split(' ').slice(0, 3).map(Number) + } catch { + return null + } +} + +export function fixtureCacheDir() { + const dir = join(here, '.cache') + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }) + return dir +} diff --git a/bench/package-lock.json b/bench/package-lock.json new file mode 100644 index 00000000000..2c9a1978022 --- /dev/null +++ b/bench/package-lock.json @@ -0,0 +1,31 @@ +{ + "name": "@hermes/bench", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@hermes/bench", + "version": "0.0.0", + "dependencies": { + "node-pty": "^1.1.0" + } + }, + "node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "license": "MIT" + }, + "node_modules/node-pty": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/node-pty/-/node-pty-1.1.0.tgz", + "integrity": "sha512-20JqtutY6JPXTUnL0ij1uad7Qe1baT46lyolh2sSENDd4sTzKZ4nmAFkeAARDKwmlLjPx6XKRlwRUxwjOy+lUg==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^7.1.0" + } + } + } +} diff --git a/bench/package.json b/bench/package.json new file mode 100644 index 00000000000..011d4432e76 --- /dev/null +++ b/bench/package.json @@ -0,0 +1,13 @@ +{ + "name": "@hermes/bench", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "TUI benchmark suite: Ink (ui-tui) vs OpenTUI (ui-opentui) over a real PTY with a fake gateway. Methodology: docs/plans/opentui-bench-suite.md.", + "scripts": { + "check": "node --check fake-gateway.mjs && node --check fixture-stream.mjs && node --check harness.mjs && node --check run.mjs && node --check render.mjs" + }, + "dependencies": { + "node-pty": "^1.1.0" + } +} diff --git a/bench/render.mjs b/bench/render.mjs new file mode 100644 index 00000000000..789657d561e --- /dev/null +++ b/bench/render.mjs @@ -0,0 +1,549 @@ +#!/usr/bin/env node +// Report renderer — reads bench/results/*.json and emits ONE self-contained +// bench/report.html (inline SVG, zero CDN/network) plus PNG exports of each +// chart to bench/report-assets/ (rasterized with the resvg-js available at +// ~/.claude/skills/tmux-pane-screenshot/scripts/node_modules — see README). +// Real data only: cells with no results render as "not run". + +import { createRequire } from 'node:module' +import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const here = dirname(fileURLToPath(import.meta.url)) +const RESULTS_DIR = join(here, 'results') +const ASSETS_DIR = join(here, 'report-assets') +const OUT_HTML = join(here, 'report.html') + +const CAP_MB = 2048 + +// ── data load ─────────────────────────────────────────────────────────── +function loadResults() { + let files = [] + try { + files = readdirSync(RESULTS_DIR).filter(f => f.endsWith('.json')) + } catch { + return [] + } + const out = [] + for (const f of files.sort()) { + try { + const r = JSON.parse(readFileSync(join(RESULTS_DIR, f), 'utf8')) + r._file = f + out.push(r) + } catch { + /* skip unparseable */ + } + } + return out +} + +// ── stats ─────────────────────────────────────────────────────────────── +const quantile = (sorted, q) => { + if (!sorted.length) return null + const pos = (sorted.length - 1) * q + const lo = Math.floor(pos) + const hi = Math.ceil(pos) + return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo) +} +const median = xs => quantile(xs.slice().sort((a, b) => a - b), 0.5) +const iqr = xs => { + const s = xs.slice().sort((a, b) => a - b) + return [quantile(s, 0.25), quantile(s, 0.75)] +} +const fmt = (x, d = 1) => (x === null || x === undefined || Number.isNaN(x) ? '—' : Number(x).toFixed(d)) +const fmtMedIqr = (xs, d = 1) => { + if (!xs.length) return 'not run' + const [lo, hi] = iqr(xs) + return `${fmt(median(xs), d)} [${fmt(lo, d)}–${fmt(hi, d)}]` +} + +// least-squares slope of rss_mb vs msgs over points +function lsSlope(points) { + if (points.length < 3) return null + const n = points.length + let sx = 0 + let sy = 0 + let sxx = 0 + let sxy = 0 + for (const [x, y] of points) { + sx += x + sy += y + sxx += x * x + sxy += x * y + } + const denom = n * sxx - sx * sx + if (denom === 0) return null + return (n * sxy - sx * sy) / denom // MB per message +} + +// Per-run back-half slope (MB/1k msgs): fit over msgs >= max(500, maxMsgs/2) +// — warmup (first 500 msgs) always excluded per protocol. +function runSlope(run) { + const pts = run.samples + .filter(s => s.kind === 'boundary' && s.msgs != null && s.rss_kb != null) + .map(s => [s.msgs, s.rss_kb / 1024]) + if (pts.length < 4) return null + const maxMsgs = pts[pts.length - 1][0] + const cut = Math.max(500, maxMsgs / 2) + const back = pts.filter(([x]) => x >= cut) + const slope = lsSlope(back) + return slope === null ? null : slope * 1000 +} + +// plateau: median RSS over the final quartile of boundary samples +function runPlateau(run) { + const pts = run.samples.filter(s => s.kind === 'boundary' && s.rss_kb != null).map(s => s.rss_kb / 1024) + if (pts.length < 4) return null + return median(pts.slice(Math.floor(pts.length * 0.75))) +} + +// ── SVG primitives ────────────────────────────────────────────────────── +const COLORS = { ink: '#e06c75', 'otui-capped': '#61afef', 'otui-uncapped': '#56b6c2', other: '#c678dd' } +const esc = s => String(s).replace(/&/g, '&').replace(//g, '>') + +function chart({ title, w = 860, h = 420, xLabel, yLabel, xMax, yMax, series, capLine, markers = [], note }) { + const padL = 64 + const padR = 16 + const padT = 40 + const padB = 48 + const pw = w - padL - padR + const ph = h - padT - padB + const X = x => padL + (x / xMax) * pw + const Y = y => padT + ph - (y / yMax) * ph + const parts = [] + parts.push(``) + parts.push(``) + parts.push(`${esc(title)}`) + // grid + axes + const xticks = 6 + const yticks = 5 + for (let i = 0; i <= xticks; i++) { + const xv = (xMax / xticks) * i + const x = X(xv) + parts.push(``) + parts.push(`${Math.round(xv)}`) + } + for (let i = 0; i <= yticks; i++) { + const yv = (yMax / yticks) * i + const y = Y(yv) + parts.push(``) + parts.push(`${Math.round(yv)}`) + } + parts.push(`${esc(xLabel)}`) + parts.push( + `${esc(yLabel)}` + ) + if (capLine != null && capLine <= yMax) { + parts.push( + `` + ) + parts.push(`cap ${capLine} MB`) + } + let legendY = padT + 8 + for (const s of series) { + if (!s.points.length) continue + const d = s.points.map(([x, y], i) => `${i === 0 ? 'M' : 'L'}${X(Math.min(x, xMax)).toFixed(1)},${Y(Math.min(y, yMax)).toFixed(1)}`).join(' ') + parts.push(``) + if (s.label) { + parts.push(``) + parts.push(`${esc(s.label)}`) + legendY += 16 + } + } + for (const m of markers) { + const x = X(Math.min(m.x, xMax)) + const y = Y(Math.min(m.y, yMax)) + parts.push(`×`) + if (m.label) parts.push(`${esc(m.label)}`) + } + if (note) parts.push(`${esc(note)}`) + parts.push('') + return parts.join('\n') +} + +function barChart({ title, w = 860, h = 360, groups, yLabel, note }) { + // groups: [{label, bars: [{name, value, lo, hi, color}]}] + const padL = 64 + const padR = 16 + const padT = 40 + const padB = 64 + const pw = w - padL - padR + const ph = h - padT - padB + const vals = groups.flatMap(g => g.bars.map(b => b.hi ?? b.value)).filter(v => v != null) + if (!vals.length) return null + const yMax = Math.max(...vals) * 1.15 + const Y = y => padT + ph - (y / yMax) * ph + const parts = [] + parts.push(``) + parts.push(``) + parts.push(`${esc(title)}`) + for (let i = 0; i <= 5; i++) { + const yv = (yMax / 5) * i + parts.push(``) + parts.push(`${yv >= 100 ? Math.round(yv) : yv.toFixed(1)}`) + } + parts.push( + `${esc(yLabel)}` + ) + const gw = pw / groups.length + groups.forEach((g, gi) => { + const bw = Math.min(46, (gw - 24) / Math.max(1, g.bars.length)) + g.bars.forEach((b, bi) => { + if (b.value == null) return + const x = padL + gi * gw + gw / 2 - (g.bars.length * bw) / 2 + bi * bw + parts.push(``) + if (b.lo != null && b.hi != null) { + const cx = x + bw / 2 + parts.push(``) + } + parts.push(`${b.value >= 100 ? Math.round(b.value) : b.value.toFixed(1)}`) + parts.push(`${esc(b.name)}`) + }) + parts.push(`${esc(g.label)}`) + }) + if (note) parts.push(`${esc(note)}`) + parts.push('') + return parts.join('\n') +} + +// ── chart builders ────────────────────────────────────────────────────── +function rssChart(results) { + const runs = results.filter( + r => (r.meta.cell.startsWith('mem') || r.meta.cell.startsWith('slope')) && !r.meta.instrumented && r.meta.mode === 'mem' + ) + if (!runs.length) return null + let xMax = 0 + let yMax = CAP_MB * 1.05 + const series = [] + const markers = [] + const seen = new Set() + for (const r of runs) { + const pts = r.samples.filter(s => s.kind === 'boundary' && s.msgs != null && s.rss_kb != null).map(s => [s.msgs, s.rss_kb / 1024]) + if (!pts.length) continue + xMax = Math.max(xMax, pts[pts.length - 1][0]) + yMax = Math.max(yMax, ...pts.map(p => p[1])) + const color = COLORS[r.meta.config] ?? COLORS.other + const key = r.meta.config + series.push({ points: pts, color, width: r.meta.cell.startsWith('slope') ? 2.5 : 1.5, opacity: 0.8, label: seen.has(key) ? null : `${key}` }) + seen.add(key) + if (r.summary.cap_hit) { + const last = pts[pts.length - 1] + markers.push({ x: last[0], y: last[1], label: `OOM @${last[0]}`, color: '#e5c07b' }) + } + } + return chart({ + title: 'RSS vs fixture messages (clean memory runs, all reps + slope runs)', + xLabel: 'fixture messages (rowsPerTurn accounting)', + yLabel: 'RSS (MB)', + xMax: Math.max(xMax, 1000), + yMax: yMax * 1.08, + series, + capLine: CAP_MB, + markers, + note: '2GB cgroup cap (systemd-run, MemorySwapMax=0). × = cgroup OOM kill. Samples every 100 msgs from /proc/PID/smaps_rollup.' + }) +} + +function nodesChart(results) { + const runs = results.filter(r => r.meta.cell.startsWith('nodes')) + if (!runs.length) return null + const series = [] + let xMax = 0 + let yMax = 0 + for (const r of runs) { + let pts = [] + if (r.node_samples?.length) { + // Ink fd-3 sampler: align wall-clock node samples to msg boundaries. + const t0 = Date.parse(r.meta.utc) + const bounds = r.samples.filter(s => s.kind === 'boundary' && s.msgs != null) + pts = r.node_samples.map(ns => { + const el = ns.t - t0 + let msgs = 0 + for (const b of bounds) if (b.t_ms <= el) msgs = b.msgs + return [msgs, ns.yoga] + }) + // collapse to last sample per msg count + const byMsg = new Map() + for (const [m, y] of pts) byMsg.set(m, y) + pts = [...byMsg.entries()].sort((a, b) => a[0] - b[0]) + series.push({ points: pts, color: COLORS.ink, label: 'ink live yoga nodes (fd-3 sampler)' }) + } else if (r.samples.some(s => s.renderables != null)) { + pts = r.samples.filter(s => s.renderables != null).map(s => [s.msgs, s.renderables]) + series.push({ + points: pts, + color: COLORS[r.meta.config] ?? COLORS.other, + label: `${r.meta.config} renderables (headless walk)` + }) + } + for (const [x, y] of pts) { + xMax = Math.max(xMax, x) + yMax = Math.max(yMax, y) + } + } + if (!series.length) return null + return chart({ + title: 'Mounted node count vs messages (instrumented runs — mechanism witness, never headlined)', + xLabel: 'fixture messages', + yLabel: 'live nodes', + xMax: Math.max(xMax, 100), + yMax: yMax * 1.1, + series, + note: 'instrumented:true. Ink: HERMES_TUI_MEMSAMPLE_FD walk of the forked reconciler root. OpenTUI: scripts/mem-bench.tsx renderable walk (headless, diagnostic-only).' + }) +} + +function scrollCdfChart(results) { + const runs = results.filter(r => r.meta.cell.startsWith('scroll') && r.summary.scroll_latencies_ms?.length) + if (!runs.length) return null + const byConfig = {} + for (const r of runs) { + ;(byConfig[r.meta.config] ??= []).push(...r.summary.scroll_latencies_ms) + } + const series = [] + let xMax = 0 + for (const [config, lats] of Object.entries(byConfig)) { + const s = lats.slice().sort((a, b) => a - b) + xMax = Math.max(xMax, quantile(s, 0.995)) + const pts = s.map((v, i) => [v, ((i + 1) / s.length) * 100]) + series.push({ points: pts, color: COLORS[config] ?? COLORS.other, label: `${config} (n=${s.length})` }) + } + return chart({ + title: 'Scroll latency CDF — SGR wheel 30Hz×15s on 3000-msg transcript (all reps pooled)', + xLabel: 'write → first output byte (ms)', + yLabel: 'percentile (%)', + xMax: Math.max(1, xMax), + yMax: 100, + series, + note: 'Latency = PTY write of the wheel event to the next PTY output chunk.' + }) +} + +function startupChart(results) { + const runs = results.filter(r => r.meta.cell === 'startup') + if (!runs.length) return null + const byConfig = {} + for (const r of runs) { + const c = (byConfig[r.meta.config] ??= { fb: [], sc: [] }) + if (r.summary.first_byte_ms != null) c.fb.push(r.summary.first_byte_ms) + if (r.summary.session_create_ms != null) c.sc.push(r.summary.session_create_ms) + } + const groups = Object.entries(byConfig).map(([config, v]) => ({ + label: config, + bars: [ + { name: 'first byte', value: median(v.fb), lo: iqr(v.fb)[0], hi: iqr(v.fb)[1], color: COLORS[config] ?? COLORS.other }, + { name: 'session.create', value: median(v.sc), lo: iqr(v.sc)[0], hi: iqr(v.sc)[1], color: '#98c379' } + ] + })) + return barChart({ + title: 'Startup (fake gateway) — median of 10 reps, whiskers = IQR', + yLabel: 'ms from spawn', + groups, + note: 'first byte = first PTY output; session.create = the UI reaches its session bootstrap RPC.' + }) +} + +function ptyRateChart(results) { + const runs = results.filter(r => r.meta.cell.startsWith('cpu') && r.summary.stream_done) + if (!runs.length) return null + const byConfig = {} + for (const r of runs) { + const done = r.samples.filter(s => s.kind === 'done')[0] + const start = r.summary.stream_start_ms + if (!done || start == null) continue + const secs = (done.t_ms - start) / 1000 + const c = (byConfig[r.meta.config] ??= { rate: [], cpu: [] }) + c.rate.push(done.pty_bytes / secs / 1024) + // CPU ms per event over the streaming window + const sb = r.samples.filter(s => s.kind === 'boundary') + if (sb.length >= 2) { + const first = sb[0] + const last = sb[sb.length - 1] + const ticks = last.utime_ticks + last.stime_ticks - first.utime_ticks - first.stime_ticks + const events = (last.events ?? 0) - (first.events ?? 0) + if (events > 0) c.cpu.push((ticks * 10) / events) // 100Hz ticks → ms + } + } + const groups = Object.entries(byConfig).map(([config, v]) => ({ + label: config, + bars: [ + { name: 'PTY KiB/s', value: median(v.rate), lo: iqr(v.rate)[0], hi: iqr(v.rate)[1], color: COLORS[config] ?? COLORS.other }, + { name: 'CPU ms/event', value: median(v.cpu), lo: iqr(v.cpu)[0], hi: iqr(v.cpu)[1], color: '#d19a66' } + ] + })) + return barChart({ + title: 'Paced streaming (30 ev/s): PTY output rate + CPU per event — median±IQR over reps', + yLabel: 'KiB/s · ms/event', + groups, + note: 'CPU from /proc/PID/stat utime+stime deltas across the stream window (UI process only).' + }) +} + +// ── tables ────────────────────────────────────────────────────────────── +function matrixTable(results) { + const memRuns = results.filter(r => r.meta.cell.startsWith('mem') && r.meta.mode === 'mem' && !r.meta.instrumented) + const slopeRuns = results.filter(r => r.meta.cell.startsWith('slope')) + const scrollRuns = results.filter(r => r.meta.cell.startsWith('scroll')) + const cpuRuns = results.filter(r => r.meta.cell.startsWith('cpu')) + const configs = ['ink', 'otui-capped', 'otui-uncapped'] + const rows = [] + for (const config of configs) { + const mem = memRuns.filter(r => r.meta.config === config) + const slopes = mem.map(runSlope).filter(s => s != null) + const plateaus = mem.map(runPlateau).filter(s => s != null) + const vmhwm = mem.map(r => r.summary.vmhwm_kb).filter(Boolean).map(k => k / 1024) + const slope10k = slopeRuns.filter(r => r.meta.config === config).map(runSlope).filter(s => s != null) + const lat = scrollRuns.filter(r => r.meta.config === config).flatMap(r => r.summary.scroll_latencies_ms ?? []) + const latS = lat.slice().sort((a, b) => a - b) + const cpu = [] + for (const r of cpuRuns.filter(x => x.meta.config === config)) { + const sb = r.samples.filter(s => s.kind === 'boundary') + if (sb.length >= 2) { + const f = sb[0] + const l = sb[sb.length - 1] + const ev = (l.events ?? 0) - (f.events ?? 0) + if (ev > 0) cpu.push(((l.utime_ticks + l.stime_ticks - f.utime_ticks - f.stime_ticks) * 10) / ev) + } + } + const capHits = [...mem, ...slopeRuns.filter(r => r.meta.config === config)].filter(r => r.summary.cap_hit) + rows.push(` + ${config} + ${fmtMedIqr(slopes, 2)} + ${slope10k.length ? fmt(median(slope10k), 2) : 'not run'} + ${fmtMedIqr(plateaus, 0)} + ${fmtMedIqr(vmhwm, 0)} + ${latS.length ? `${fmt(quantile(latS, 0.5), 1)} / ${fmt(quantile(latS, 0.9), 1)} / ${fmt(quantile(latS, 0.99), 1)}` : 'not run'} + ${fmtMedIqr(cpu, 2)} + ${capHits.length ? capHits.map(r => `${r.meta.cell}@${r.summary.at_messages ?? '?'}msgs`).join('
') : mem.length ? 'none' : 'not run'} + `) + } + return ` + + ${rows.join('\n')} +
configslope MB/1k msgs
(3000-msg runs)		slope MB/1k
(10k run)		plateau RSS MB
(final quartile)		VmHWM MBscroll p50/p90/p99 msCPU ms/event
(paced)		cap hits (2GB)
` +} + +function survivalTable(results) { + const runs = results.filter(r => r.meta.cell.startsWith('e3')) + if (!runs.length) return '

E3 (memory-constrained Docker survival): not run.

' + const rows = runs.map( + r => `${esc(r.meta.cell)}${esc(r.meta.config)}${esc(String(r.meta.memory_max ?? r.meta.container_memory ?? '?'))} + ${r.summary.result}${r.summary.at_messages ?? r.summary.msgs_streamed ?? '—'} + ${fmt((r.summary.vmhwm_kb ?? 0) / 1024, 0)} MB${esc(r.summary.cap_hit_basis ?? '—')}` + ) + return `${rows.join('')}
cellconfiglimitresultmsgs survivedVmHWMbasis
` +} + +function gateTable(results) { + const runs = results.filter(r => r.meta.cell === 'gate') + if (!runs.length) return '

Determinism gate: not run.

' + const byConfig = {} + for (const r of runs) (byConfig[r.meta.config] ??= []).push(r.summary.digest) + const rows = Object.entries(byConfig).map(([c, ds]) => { + const ok = ds.length >= 2 && ds.every(d => d && d === ds[0]) + return `${esc(c)}${ds.map(d => (d ?? '∅').slice(0, 16)).join(' · ')}${ok ? 'PASS' : 'FAIL'}` + }) + return `${rows.join('')}
configreplay digestsgate
` +} + +function drainTable(results) { + const bad = results.filter(r => r.summary && r.summary.drain_ok === false) + if (!bad.length) return '

PTY drain assertion: no run exceeded the 10ms event-loop starvation budget.

' + const rows = bad.map( + r => `${esc(r._file)}${fmt(r.summary.drain_max_loop_lag_ms, 0)} ms${r.summary.drain_lag_violations}` + ) + return `

⚠ drain assertion violations (runs kept, flagged):

+ ${rows.join('')}
runmax loop lagviolations >10ms
` +} + +// ── PNG export ────────────────────────────────────────────────────────── +function exportPng(name, svg) { + try { + const require2 = createRequire(import.meta.url) + const resvgPath = join(homedir(), '.claude/skills/tmux-pane-screenshot/scripts/node_modules/@resvg/resvg-js') + const { Resvg } = require2(resvgPath) + const png = new Resvg(svg, { fitTo: { mode: 'width', value: 1280 } }).render().asPng() + writeFileSync(join(ASSETS_DIR, `${name}.png`), png) + return true + } catch (e) { + process.stderr.write(`png export failed for ${name}: ${e.message}\n`) + return false + } +} + +// ── main ──────────────────────────────────────────────────────────────── +const results = loadResults() +mkdirSync(ASSETS_DIR, { recursive: true }) + +const charts = [ + ['rss-vs-msgs', rssChart(results)], + ['node-count', nodesChart(results)], + ['scroll-cdf', scrollCdfChart(results)], + ['startup', startupChart(results)], + ['pty-rate', ptyRateChart(results)] +] + +const pngs = [] +for (const [name, svg] of charts) { + if (svg && exportPng(name, svg)) pngs.push(`${name}.png`) +} + +const metaRuns = results.length + ? `${results.length} result files · sha ${esc(results[0].meta.sha ?? '?')} · node ${esc(results.find(r => r.meta.node_version)?.meta.node_version ?? '?')}` + : 'no results yet' + +const html = ` + +Hermes TUI bench — Ink vs OpenTUI + +

Hermes TUI benchmark — Ink (ui-tui) vs OpenTUI (ui-opentui)

+

${metaRuns} · generated ${new Date().toISOString()}
+Methodology: docs/plans/opentui-bench-suite.md. Real binaries over a real node-pty PTY (120×40, +xterm-256color), fake gateway via HERMES_PYTHON (zero UI changes), external /proc sampling on +100-msg boundaries, 2GB cgroup-v2 caps via systemd-run --user --scope. Median±IQR throughout; +instrumented node-count runs are flagged and never headlined.

+ +

Determinism gate

+${gateTable(results)} + +

Headline: RSS vs messages

+${charts[0][1] ?? '

not run

'} + +

Result matrix (median ± IQR)

+${matrixTable(results)} + +

Mechanism: mounted node count (instrumented)

+${charts[1][1] ?? '

not run

'} + +

Scroll latency

+${charts[2][1] ?? '

not run

'} + +

Startup

+${charts[3][1] ?? '

not run

'} + +

Streaming CPU / PTY throughput

+${charts[4][1] ?? '

not run

'} + +

E3 survival (memory-constrained Docker)

+${survivalTable(results)} + +

Run health

+${drainTable(results)} + + +` + +writeFileSync(OUT_HTML, html) +process.stdout.write(`report → ${OUT_HTML}\npngs → ${pngs.join(', ') || '(none)'}\n`) diff --git a/bench/run-e3-inner.mjs b/bench/run-e3-inner.mjs new file mode 100644 index 00000000000..0e8bc1a4339 --- /dev/null +++ b/bench/run-e3-inner.mjs @@ -0,0 +1,71 @@ +#!/usr/bin/env node +// E3-lite inner driver — runs INSIDE the memory-capped container (see +// run-e3.sh). Survival-to-OOM runs: Ink vs OpenTUI-capped on a long fixture; +// the CONTAINER cgroup is the cap (UI + fake gateway + this harness share it — +// labeled E3-lite, generic node image, not the shipped one). +// +// Heap sizing mirrors the production launcher inside a container: +// _resolve_tui_heap_mb reads /sys/fs/cgroup/memory.max → 75% of the limit +// (no floor at ≤2048MB limits): 1g → 768MB --max-old-space-size. + +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +import { runScenario } from './harness.mjs' + +const RESULTS = process.env.E3_RESULTS_DIR || '/results' +const MSGS = Number.parseInt(process.env.E3_MSGS ?? '10000', 10) +const FIXTURE = process.env.E3_FIXTURE || '' +const CELL = process.env.E3_CELL || 'e3lite-1g' + +function launcherHeapMb() { + try { + const raw = readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim() + if (raw === 'max') return 8192 + const limitMb = Math.floor(Number(raw) / (1024 * 1024)) + const sized = Math.floor(limitMb * 0.75) + if (sized >= 8192) return 8192 + return limitMb > 2048 ? Math.max(1536, sized) : sized + } catch { + return 8192 + } +} + +const heapMb = launcherHeapMb() +const memTotal = (() => { + try { + return readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim() + } catch { + return 'unknown' + } +})() + +process.stdout.write(`E3-lite: container memory.max=${memTotal} heapMb=${heapMb} msgs=${MSGS}\n`) + +for (const [config, ui, cap] of [ + ['ink', 'ink', null], + ['otui-capped', 'opentui', 3000] +]) { + const utc = new Date().toISOString().replace(/[:.]/g, '').slice(0, 15) + const outFile = join(RESULTS, `${utc}-e3-${CELL}-${ui}-${config}-r0.json`) + process.stdout.write(`▶ ${CELL} ${config}\n`) + const r = await runScenario({ + ui, + configName: config, + opentuiCap: cap, + mode: 'mem', + fixturePath: FIXTURE, + fixtureMsgs: MSGS, + fixtureSha: process.env.E3_FIXTURE_SHA ?? '', + memoryMax: null, + containerCap: true, + containerMemory: memTotal, + heapMb, + cell: CELL, + rep: 0, + outFile, + runTimeoutMs: 60 * 60 * 1000 + }) + process.stdout.write(` ✔ ${r.summary.result} msgs=${r.summary.msgs_streamed} vmhwm=${((r.summary.vmhwm_kb ?? 0) / 1024).toFixed(0)}MB basis=${r.summary.cap_hit_basis ?? '—'}\n`) + await new Promise(res => setTimeout(res, 5000)) +} diff --git a/bench/run-e3.sh b/bench/run-e3.sh new file mode 100644 index 00000000000..eaf4d4ef231 --- /dev/null +++ b/bench/run-e3.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# E3-lite — survival-to-OOM runs in a memory-constrained generic node:26 +# container (NOT the shipped image; labeled honestly in the report). The +# worktree is bind-mounted read-only; results land in bench/results/ via a +# writable mount. The whole container (UI + fake gateway + harness) shares the +# --memory limit, mirroring how the shipped container would be capped. +# +# Usage: bash run-e3.sh [memory] [msgs] (defaults: 1g, 10000) +set -euo pipefail + +MEM="${1:-1g}" +MSGS="${2:-10000}" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(dirname "$HERE")" +CELL="e3lite-${MEM}" + +# Fixture is generated on the HOST (the .cache mount below is read-only). +FIX_INFO=$(node "$HERE/fixture-stream.mjs" --msgs "$MSGS") +FIX_PATH=$(echo "$FIX_INFO" | sed -E 's/.*"path":"([^"]+)".*/\1/') +FIX_SHA=$(echo "$FIX_INFO" | sed -E 's/.*"sha256":"([^"]+)".*/\1/') +echo "fixture: $FIX_PATH ($FIX_SHA)" + +docker run --rm \ + --memory="$MEM" --memory-swap="$MEM" \ + -v "$REPO":/repo:ro \ + -v "$HERE/results":/results \ + -e BENCH_NODE_BIN=/usr/local/bin/node \ + -e E3_RESULTS_DIR=/results \ + -e E3_MSGS="$MSGS" \ + -e E3_FIXTURE="/repo/bench/.cache/$(basename "$FIX_PATH")" \ + -e E3_FIXTURE_SHA="$FIX_SHA" \ + -e E3_CELL="$CELL" \ + node:26 node /repo/bench/run-e3-inner.mjs diff --git a/bench/run.mjs b/bench/run.mjs new file mode 100644 index 00000000000..1d6f4017b34 --- /dev/null +++ b/bench/run.mjs @@ -0,0 +1,360 @@ +#!/usr/bin/env node +// Matrix runner — executes the protocol from docs/plans/opentui-bench-suite.md: +// golden-digest determinism gate first, then strictly SEQUENTIAL runs (one SUT +// at a time — this host has ~4.4GB free), A/B interleaved with randomized +// per-rep config order, 10s cooldowns, load-avg gate recorded, results to +// bench/results/-----r.json. +// +// Usage: +// node run.mjs --cell gate|mem3000|slope10k|nodes|cpu|scroll|startup +// node run.mjs --all (the full E1 host sequence, gate first) +// Knobs: --reps N, --msgs N, --cap 2G|none, --seed N + +import { execFileSync } from 'node:child_process' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { generate } from './fixture-stream.mjs' +import { loadAvg, NODE26_BIN, REPO_ROOT, runScenario } from './harness.mjs' + +const here = dirname(fileURLToPath(import.meta.url)) +const RESULTS_DIR = join(here, 'results') +const CACHE_DIR = join(here, '.cache') + +const CONFIGS = { + ink: { ui: 'ink', opentuiCap: null }, + 'otui-capped': { ui: 'opentui', opentuiCap: 3000 }, + 'otui-uncapped': { ui: 'opentui', opentuiCap: 100000 } +} + +const sleep = ms => new Promise(r => setTimeout(r, ms)) + +// Deterministic shuffle (mulberry32) so the randomized pair order is recorded +// and reproducible from the seed in each result's meta. +function rng(seed) { + let a = seed >>> 0 + return () => { + a |= 0 + a = (a + 0x6d2b79f5) | 0 + let t = Math.imul(a ^ (a >>> 15), 1 | a) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} +function shuffled(arr, rand) { + const a = arr.slice() + for (let i = a.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)) + ;[a[i], a[j]] = [a[j], a[i]] + } + return a +} + +function sha7() { + try { + return execFileSync('git', ['-C', REPO_ROOT, 'rev-parse', '--short=7', 'HEAD'], { encoding: 'utf8' }).trim() + } catch { + return 'unknown' + } +} + +function outFileFor(cell, config, rep) { + const utc = new Date().toISOString().replace(/[:.]/g, '').slice(0, 15) + return join(RESULTS_DIR, `${utc}-${sha7()}-${cell}-${CONFIGS[config].ui}-${config}-r${rep}.json`) +} + +async function ensureFixture(msgs) { + const path = join(CACHE_DIR, `fixture-${msgs}.ndjson`) + const metaPath = `${path}.meta.json` + if (existsSync(path) && existsSync(metaPath)) return JSON.parse(readFileSync(metaPath, 'utf8')) + const info = await generate(msgs, path) + writeFileSync(metaPath, JSON.stringify(info)) + return info +} + +// Load gate: record load average; wait (bounded) for load1 < 1 per protocol. +async function loadGate() { + for (let i = 0; i < 24; i++) { + const la = loadAvg() + if (!la || la[0] < 1) return la + process.stdout.write(` load gate: load1=${la[0]} — waiting\n`) + await sleep(5000) + } + return loadAvg() +} + +async function doRun(cell, config, rep, scenario) { + const la = await loadGate() + const outFile = outFileFor(cell, config, rep) + process.stdout.write(`▶ ${cell} ${config} r${rep} (load1=${la?.[0]})\n`) + const t0 = Date.now() + const result = await runScenario({ + ...CONFIGS[config], + configName: config, + cell, + rep, + outFile, + ...scenario + }) + process.stdout.write( + ` ✔ ${result.summary.result} in ${((Date.now() - t0) / 1000).toFixed(1)}s — rss=${( + (result.samples.at(-1)?.rss_kb ?? 0) / 1024 + ).toFixed(0)}MB vmhwm=${((result.summary.vmhwm_kb ?? 0) / 1024).toFixed(0)}MB → ${outFile.split('/').pop()}\n` + ) + await sleep(10_000) // cooldown + return result +} + +// ── cells ─────────────────────────────────────────────────────────────── +async function cellGate(opts) { + // Determinism gate: 2 digest replays per UI config; digests must match. + const fx = await ensureFixture(opts.gateMsgs ?? 300) + const digests = {} + for (const config of ['ink', 'otui-capped']) { + digests[config] = [] + for (let rep = 0; rep < 2; rep++) { + const r = await doRun('gate', config, rep, { + mode: 'digest', + fixturePath: fx.path, + fixtureMsgs: fx.msgs, + fixtureSha: fx.sha256, + memoryMax: null, + heapMb: 8192, + startDelayMs: 1200, + quiesceMs: 700 + }) + digests[config].push(r.summary.digest) + } + const [a, b] = digests[config] + if (!a || a !== b) { + throw new Error(`DETERMINISM GATE FAILED for ${config}: ${a} != ${b}`) + } + process.stdout.write(` gate OK ${config}: ${a.slice(0, 16)}…\n`) + } + return digests +} + +async function cellMem(opts) { + const msgs = opts.msgs ?? 3000 + const reps = opts.reps ?? 3 + const fx = await ensureFixture(msgs) + const rand = rng(opts.seed ?? 20260611) + for (let rep = 0; rep < reps; rep++) { + for (const config of shuffled(Object.keys(CONFIGS), rand)) { + await doRun(`mem${msgs}`, config, rep, { + mode: 'mem', + fixturePath: fx.path, + fixtureMsgs: fx.msgs, + fixtureSha: fx.sha256, + memoryMax: opts.cap === 'none' ? null : '2G', + heapMb: 8192, + runTimeoutMs: 45 * 60 * 1000 + }) + } + } +} + +async function cellSlope(opts) { + const msgs = opts.msgs ?? 10000 + const fx = await ensureFixture(msgs) + const rand = rng(opts.seed ?? 7) + for (const config of shuffled(['ink', 'otui-uncapped'], rand)) { + await doRun(`slope${msgs}`, config, 0, { + mode: 'mem', + fixturePath: fx.path, + fixtureMsgs: fx.msgs, + fixtureSha: fx.sha256, + memoryMax: opts.cap === 'none' ? null : '2G', + heapMb: 8192, + runTimeoutMs: 90 * 60 * 1000 + }) + } +} + +async function cellNodes(opts) { + // Instrumented node-count runs — NEVER headlined. Ink: env-gated fd-3 + // sampler in the real binary over the PTY. OpenTUI: the existing headless + // renderer-walk (scripts/mem-bench.tsx), labeled diagnostic. + const msgs = opts.msgs ?? 3000 + const fx = await ensureFixture(msgs) + await doRun(`nodes${msgs}`, 'ink', 0, { + mode: 'mem', + fixturePath: fx.path, + fixtureMsgs: fx.msgs, + fixtureSha: fx.sha256, + memoryMax: '2G', + heapMb: 8192, + inkNodeSampler: true, + runTimeoutMs: 45 * 60 * 1000 + }) + + // OpenTUI headless renderer-walk (diagnostic-only by methodology). + const benchDir = join(REPO_ROOT, 'ui-opentui/.bench') + process.stdout.write('▶ nodes opentui headless mem-bench (diagnostic)\n') + execFileSync(NODE26_BIN, ['scripts/build.mjs', 'scripts/mem-bench.tsx', '.bench'], { + cwd: join(REPO_ROOT, 'ui-opentui'), + stdio: 'inherit' + }) + for (const [config, cap] of [ + ['otui-capped', '3000'], + ['otui-uncapped', '100000'] + ]) { + const stdout = execFileSync( + NODE26_BIN, + ['--experimental-ffi', '--expose-gc', '--no-warnings', join(benchDir, 'mem-bench.js')], + { + cwd: join(REPO_ROOT, 'ui-opentui'), + encoding: 'utf8', + env: { ...process.env, MEM_BENCH_TOTAL: String(msgs), MEM_BENCH_SAMPLE: '250', HERMES_TUI_MAX_MESSAGES: cap }, + maxBuffer: 64 * 1024 * 1024 + } + ) + const outFile = outFileFor(`nodes${msgs}`, config, 0) + // parse the table: pushes | msgs | rss | heapUsed | external | arrayBuf | activeAllocs | renderables + const samples = [] + for (const line of stdout.split('\n')) { + const m = line.match(/^\s*(\d+) \|\s*(\d+) \|\s*([\d.]+) \|\s*([\d.]+) \|\s*([\d.]+) \|\s*([\d.]+) \|\s*(\d+) \|\s*(\d+)/) + if (m) { + samples.push({ + kind: 'boundary', + msgs: Number(m[1]), + mounted_msgs: Number(m[2]), + rss_kb: Math.round(Number(m[3]) * 1024), + heap_mb: Number(m[4]), + active_allocs: Number(m[7]), + renderables: Number(m[8]) + }) + } + } + writeFileSync( + outFile, + JSON.stringify( + { + meta: { + cell: `nodes${msgs}`, + ui: 'opentui', + config, + mode: 'headless-membench', + rep: 0, + utc: new Date().toISOString(), + sha: sha7(), + instrumented: true, + diagnostic_only: true, + opentui_cap: Number(cap), + fixture: { msgs } + }, + samples, + events: [], + summary: { result: 'completed', headless: true }, + raw_stdout: stdout + }, + null, + 1 + ) + ) + process.stdout.write(` ✔ headless ${config} → ${outFile.split('/').pop()}\n`) + } +} + +async function cellCpu(opts) { + const msgs = opts.msgs ?? 800 + const reps = opts.reps ?? 3 + const fx = await ensureFixture(msgs) + const rand = rng(opts.seed ?? 99) + for (let rep = 0; rep < reps; rep++) { + for (const config of shuffled(Object.keys(CONFIGS), rand)) { + await doRun(`cpu${msgs}`, config, rep, { + mode: 'cpu-paced', + pacedRate: 30, + fixturePath: fx.path, + fixtureMsgs: fx.msgs, + fixtureSha: fx.sha256, + memoryMax: '2G', + heapMb: 8192, + runTimeoutMs: 30 * 60 * 1000 + }) + } + } +} + +async function cellScroll(opts) { + const msgs = opts.msgs ?? 3000 + const reps = opts.reps ?? 3 + const fx = await ensureFixture(msgs) + const rand = rng(opts.seed ?? 31337) + for (let rep = 0; rep < reps; rep++) { + for (const config of shuffled(Object.keys(CONFIGS), rand)) { + await doRun(`scroll${msgs}`, config, rep, { + mode: 'scroll', + scroll: { hz: 30, seconds: 15 }, + fixturePath: fx.path, + fixtureMsgs: fx.msgs, + fixtureSha: fx.sha256, + memoryMax: '2G', + heapMb: 8192, + runTimeoutMs: 45 * 60 * 1000 + }) + } + } +} + +async function cellStartup(opts) { + const reps = opts.reps ?? 10 + const rand = rng(opts.seed ?? 4242) + for (let rep = 0; rep < reps; rep++) { + for (const config of shuffled(['ink', 'otui-capped'], rand)) { + await doRun('startup', config, rep, { + mode: 'startup', + fixturePath: '', + fixtureMsgs: 0, + fixtureSha: '', + memoryMax: null, + heapMb: 8192, + startDelayMs: 999999, + quiesceMs: 700, + runTimeoutMs: 60 * 1000 + }) + } + } +} + +// ── main ──────────────────────────────────────────────────────────────── +const args = process.argv.slice(2) +const opt = name => { + const i = args.indexOf(`--${name}`) + return i >= 0 ? args[i + 1] : undefined +} +const opts = { + reps: opt('reps') ? Number(opt('reps')) : undefined, + msgs: opt('msgs') ? Number(opt('msgs')) : undefined, + cap: opt('cap'), + seed: opt('seed') ? Number(opt('seed')) : undefined +} +const cell = opt('cell') +mkdirSync(RESULTS_DIR, { recursive: true }) + +const CELLS = { + gate: cellGate, + mem3000: cellMem, + slope10k: cellSlope, + nodes: cellNodes, + cpu: cellCpu, + scroll: cellScroll, + startup: cellStartup +} + +if (args.includes('--all')) { + await cellGate(opts) + await cellStartup(opts) + await cellMem(opts) + await cellCpu(opts) + await cellScroll(opts) + await cellNodes(opts) + await cellSlope(opts) +} else if (cell && CELLS[cell]) { + await CELLS[cell](opts) +} else { + process.stdout.write(`usage: node run.mjs --cell ${Object.keys(CELLS).join('|')} [--reps N --msgs N --cap none --seed N]\n`) + process.exit(cell ? 1 : 0) +}