diff --git a/bench/README.md b/bench/README.md index 4c7dafb71fe..8d5ae2f4cdb 100644 --- a/bench/README.md +++ b/bench/README.md @@ -32,9 +32,26 @@ node run.mjs --cell nodes # instrumented node counts (ink fd-3 sampler; node run.mjs --cell cpu # paced 30 ev/s streaming ×3 node run.mjs --cell scroll # SGR wheel 30Hz×15s on a 3000-msg transcript ×3 node run.mjs --cell startup # ×10, fake gateway +node run.mjs --cell chaos # stability: gw SIGKILL mid-stream/mid-tool, SIGSTOP 30s, resize storm, PTY EOF — 5 scenarios × {ink, otui-capped} node render.mjs # report.html + report-assets/*.png ``` +### Chaos cell notes + +- **chaos** (5 scenarios × ink/otui-capped, one JSON each, `summary.chaos`): + gateway death is SELF-inflicted (`HERMES_FAKE_DIE_AT=:` + → SIGKILL at fixture msg N, or at the first `tool.*` event after N) because + self-termination is deterministic vs racy external timing; a die-once flag + file keeps the auto-heal respawn from dying again. SIGSTOP (gw-stop) is + external via `HERMES_FAKE_PIDFILE`. Respawn detection = the respawned + gateway REWRITING that pidfile. Both UIs auto-heal (budget 3 respawns/60s): + OpenTUI with exponential backoff (`ui-opentui/src/boundary/gateway/liveGateway.ts` + `onExit`), Ink immediately (`ui-tui/src/app/useMainApp.ts` `exitHandler`). + `transcript_preserved` = after a forced full repaint (resize jiggle), the + screen still shows a recent pre-kill turn (`const xN` fixture markers). + `summary.result` keeps its usual semantics — for pty-eof the UI *should* + die, so read `summary.chaos`, not `summary.result`, for the verdicts. + Configs: `ink` · `otui-capped` (`HERMES_TUI_MAX_MESSAGES=3000`, the default) · `otui-uncapped` (`=100000`). Launch parity with `hermes_cli/main.py`: Ink = `node --expose-gc ui-tui/dist/entry.js`, OpenTUI = diff --git a/bench/fake-gateway.mjs b/bench/fake-gateway.mjs index a3703f48b0a..8bc68dc7893 100755 --- a/bench/fake-gateway.mjs +++ b/bench/fake-gateway.mjs @@ -20,6 +20,16 @@ // HERMES_FAKE_START_DELAY_MS delay after session.create reply before streaming (default 1500) // HERMES_FAKE_SAMPLE_EVERY fixture-msg boundary cadence for progress lines (default 100) // HERMES_FAKE_PROGRESS progress NDJSON file path (required for harness runs) +// HERMES_FAKE_PIDFILE write own pid here at startup (harness discovers the +// gateway pid; a REWRITE by a respawned instance is the +// harness's auto-heal detection signal) +// HERMES_FAKE_DIE_AT ":" — chaos cells: self-SIGKILL +// at fixture msg N (kill), or at the first tool.* event +// after msg N (tool-kill). Self-termination is deterministic +// vs racy external timing. SIGSTOP stays external (a stopped +// process can't stop itself usefully). +// HERMES_FAKE_DIE_FLAG die-once flag file: created just before the self-kill so +// the UI's auto-heal RESPAWN (same env) does not die again // // Modes: burst = write as fast as the pipe accepts (await 'drain' on // backpressure, so emission tracks UI ingestion within the ~64KB pipe buffer); @@ -27,7 +37,7 @@ // (scroll-latency runs drive input afterwards). Exits on stdin EOF (the UIs // close stdin to stop the gateway) — same lifecycle as the real child. -import { appendFileSync, readFileSync } from 'node:fs' +import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs' import { createInterface } from 'node:readline' const FIXTURE = process.env.HERMES_FAKE_FIXTURE || '' @@ -36,6 +46,30 @@ const RATE = Math.max(1, Number.parseInt(process.env.HERMES_FAKE_RATE ?? '30', 1 const START_DELAY_MS = Number.parseInt(process.env.HERMES_FAKE_START_DELAY_MS ?? '1500', 10) || 1500 const SAMPLE_EVERY = Math.max(1, Number.parseInt(process.env.HERMES_FAKE_SAMPLE_EVERY ?? '100', 10) || 100) const PROGRESS = process.env.HERMES_FAKE_PROGRESS || '' +const PIDFILE = process.env.HERMES_FAKE_PIDFILE || '' +const DIE_FLAG = process.env.HERMES_FAKE_DIE_FLAG || '' + +// Chaos self-termination (deterministic, no external kill races). Die-once: +// if the flag file exists a previous instance already died here — this is the +// auto-heal respawn, which must stream to completion. +let dieAtMsgs = null +let dieKind = 'kill' +{ + const m = (process.env.HERMES_FAKE_DIE_AT || '').match(/^(\d+):(kill|tool-kill)$/) + if (m) { + dieAtMsgs = Number(m[1]) + dieKind = m[2] + } + if (dieAtMsgs !== null && DIE_FLAG && existsSync(DIE_FLAG)) dieAtMsgs = null +} + +if (PIDFILE) { + try { + writeFileSync(PIDFILE, String(process.pid)) + } catch { + /* best-effort */ + } +} const t0 = Date.now() const progress = obj => { @@ -105,6 +139,21 @@ function resultFor(method, params) { } } +// ── Chaos self-kill ──────────────────────────────────────────────────── +// Flag first (sync — survives SIGKILL), then a 'dying' progress line (gives +// the harness the precise kill wall-clock), then SIGKILL self. +function dieNow(msgs) { + if (DIE_FLAG) { + try { + writeFileSync(DIE_FLAG, '1') + } catch { + /* best-effort */ + } + } + progress({ k: 'dying', kind: dieKind, msgs }) + process.kill(process.pid, 'SIGKILL') +} + // ── Fixture streaming ────────────────────────────────────────────────── let streaming = false async function streamFixture() { @@ -130,12 +179,19 @@ async function streamFixture() { const drained = emitEvent(item.v) if (drained) await drained events++ + // tool-kill: die exactly as a tool-call event goes over the wire (the + // first tool.* event after the armed msg index — the UI is left with a + // started, never-completed tool). + if (dieAtMsgs !== null && dieKind === 'tool-kill' && msgs >= dieAtMsgs && typeof item.v?.type === 'string' && item.v.type.startsWith('tool.')) { + dieNow(msgs) + } } else if (item.k === 't') { msgs = item.msgs if (msgs >= nextBoundary) { progress({ k: 'boundary', msgs, events }) while (nextBoundary <= msgs) nextBoundary += SAMPLE_EVERY } + if (dieAtMsgs !== null && dieKind === 'kill' && msgs >= dieAtMsgs) dieNow(msgs) } // {"k":"r"} row markers: composer-local rows, nothing on the wire. } diff --git a/bench/harness.mjs b/bench/harness.mjs index 86314eeaf27..156e16f7ad8 100644 --- a/bench/harness.mjs +++ b/bench/harness.mjs @@ -97,6 +97,27 @@ function commOf(pid) { } } +function procStateOf(pid) { + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8') + return stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0] + } catch { + return null + } +} + +// Alive = signalable AND not a zombie (a destroyed-PTY child may linger as Z +// until reaped — that is not an orphan, just an unreaped corpse). +function pidAlive(pid) { + if (!pid) return false + try { + process.kill(pid, 0) + } catch { + return false + } + return procStateOf(pid) !== 'Z' +} + // ── ANSI strip for the determinism digest ────────────────────────────── // Removes CSI/OSC/DCS/SOS/PM/APC sequences, single ESC sequences, and control // chars, then normalizes whitespace. Good enough to compare final rendered @@ -165,7 +186,9 @@ function uiArgv(ui) { * ui: 'ink' | 'opentui' * configName: 'ink' | 'otui-capped' | 'otui-uncapped' * opentuiCap: number|null (HERMES_TUI_MAX_MESSAGES) - * mode: 'mem' | 'cpu-paced' | 'scroll' | 'startup' | 'digest' + * mode: 'mem' | 'cpu-paced' | 'scroll' | 'startup' | 'digest' | 'chaos' + * chaos: { scenario, dieAt?, stopAt?, flaps?, fakeMode? } (chaos mode) + * scenario: 'gw-kill-stream' | 'gw-kill-tool' | 'gw-stop' | 'resize-storm' | 'pty-eof' * fixturePath, fixtureMsgs, fixtureSha * memoryMax: string|null ('2G' → systemd-run --user --scope) * heapMb: number (--max-old-space-size) @@ -202,14 +225,31 @@ export async function runScenario(opts) { const activeSessionFile = join(tmpdir(), `hermes-bench-session-${runId}.json`) writeFileSync(progressFile, '') + const chaosSpec = mode === 'chaos' ? { flaps: 30, ...(opts.chaos ?? {}) } : null const fakeEnv = { HERMES_FAKE_FIXTURE: fixturePath, - HERMES_FAKE_MODE: mode === 'cpu-paced' ? 'paced' : mode === 'scroll' ? 'load-then-idle' : 'burst', + HERMES_FAKE_MODE: + mode === 'cpu-paced' + ? 'paced' + : mode === 'scroll' + ? 'load-then-idle' + : mode === 'chaos' + ? (chaosSpec.fakeMode ?? 'burst') + : 'burst', HERMES_FAKE_RATE: String(pacedRate), HERMES_FAKE_START_DELAY_MS: String(startDelayMs), HERMES_FAKE_SAMPLE_EVERY: String(sampleEvery), HERMES_FAKE_PROGRESS: progressFile } + // Gateway pid discovery (the UI spawns the gateway, env flows through): the + // fake gateway writes its pid here at startup; an auto-heal respawn REWRITES + // it — that rewrite is the respawn-detection signal in chaos cells. + const gwPidFile = mode === 'chaos' ? join(tmpdir(), `hermes-bench-gwpid-${runId}`) : null + if (gwPidFile) fakeEnv.HERMES_FAKE_PIDFILE = gwPidFile + if (chaosSpec?.dieAt) { + fakeEnv.HERMES_FAKE_DIE_AT = chaosSpec.dieAt + fakeEnv.HERMES_FAKE_DIE_FLAG = `${gwPidFile}.dieflag` + } const env = composeEnv({ ui, opentuiCap, heapMb, fakeEnv, activeSessionFile }) const { file, args, cwd } = uiArgv(ui) @@ -315,16 +355,16 @@ export async function runScenario(opts) { // Wait for that flip (scope registration / sh exec take a moment); fall back // to a child walk in case a future wrapper forks instead. let uiPid = term.pid + // Node 26 names its main thread: comm is 'node-MainThread'. + const isNodeComm = pid => commOf(pid).startsWith('node') if (memoryMax || opts.inkNodeSampler) { uiPid = null - // Node 26 names its main thread: comm is 'node-MainThread'. - const isNode = pid => commOf(pid).startsWith('node') for (let i = 0; i < 200 && !exited; i++) { - if (isNode(term.pid)) { + if (isNodeComm(term.pid)) { uiPid = term.pid break } - const nodeKid = childrenOf(term.pid).find(k => isNode(k)) + const nodeKid = childrenOf(term.pid).find(k => isNodeComm(k)) if (nodeKid) { uiPid = nodeKid break @@ -345,6 +385,26 @@ export async function runScenario(opts) { let doneInfo = null let sessionCreateAt = null let gwPid = null + let lastBoundaryMsgs = 0 + let streamStarts = 0 + let dyingItem = null // {k:'dying', kind, msgs, wall} from the fake gateway's last gasp + + // Gateway pid tracking via the pidfile (chaos/pipeline). A respawned + // gateway rewrites the file → a new entry appears here. + const gwPidHistory = [] // {pid, at} (at = epoch ms first observed) + const readGwPidfile = () => { + if (!gwPidFile) return null + try { + const v = Number(readFileSync(gwPidFile, 'utf8').trim()) + return Number.isFinite(v) && v > 0 ? v : null + } catch { + return null + } + } + const pollGwPid = () => { + const p = readGwPidfile() + if (p && gwPidHistory[gwPidHistory.length - 1]?.pid !== p) gwPidHistory.push({ pid: p, at: now() }) + } const takeSample = (kind, msgs, evCount) => { if (!uiPid) return @@ -407,8 +467,18 @@ export async function runScenario(opts) { events.push({ kind: 'rpc', method: item.method, t_ms: now() - t0 }) if (item.method === 'session.create' && sessionCreateAt === null) sessionCreateAt = now() } - if (item.k === 'stream_start') streamStartT = now() - if (item.k === 'boundary') takeSample('boundary', item.msgs, item.events) + if (item.k === 'stream_start') { + if (streamStartT === null) streamStartT = now() + streamStarts++ + } + if (item.k === 'boundary') { + lastBoundaryMsgs = item.msgs + takeSample('boundary', item.msgs, item.events) + } + if (item.k === 'dying' && dyingItem === null) { + dyingItem = item + events.push({ kind: 'gw-dying', die_kind: item.kind, msgs: item.msgs, t_ms: now() - t0 }) + } if (item.k === 'done') { streamDone = true doneInfo = { msgs: item.msgs, events: item.events } @@ -422,6 +492,7 @@ export async function runScenario(opts) { let lastPeriodic = 0 pollTimer = setInterval(() => { for (const item of tailProgress()) handleProgress(item) + if (gwPidFile) pollGwPid() const t = now() if (t - lastPeriodic >= 1000) { lastPeriodic = t @@ -491,6 +562,225 @@ export async function runScenario(opts) { let digest = null let digestText = null + // ── chaos helpers ───────────────────────────────────────────────────── + const sha256 = text => createHash('sha256').update(text).digest('hex') + // Resize-jiggle forces a full repaint (like digest mode) so the captured + // tail is the CURRENT screen, not just incremental damage since the reset. + const forcedScreen = async () => { + if (!exited) { + try { + resetTail() + term.resize(120, 39) + await sleep(350) + resetTail() + term.resize(120, 40) + await sleep(900) + } catch { + /* pty gone mid-jiggle */ + } + } + return stripAnsi(tailBuf.join('')) + } + // Fixture code blocks carry `const xN = M` tokens whose (N,M) pairs are + // unique and appear in fixture order — a position marker. Build the ordered + // key list from the fixture file; the highest ordinal rendered pre-kill is + // the transcript-preservation marker. Preservation = the post-event + // full-repaint screen still shows a marker from a recent pre-kill turn. + // Rendered screens collapse whitespace (`constx4=58`), hence the \s* regex. + const CODE_TOKEN_RE = /const\s*x(\d+)\s*=\s*(\d+)/g + const markerOrder = (() => { + const order = new Map() + try { + const raw = readFileSync(fixturePath, 'utf8') + let i = 0 + for (const m of raw.matchAll(/const x(\d+) = (\d+)/g)) { + const key = `${m[1]}=${m[2]}` + if (!order.has(key)) order.set(key, i++) + } + } catch { + /* no fixture */ + } + return order + })() + const lastCodeIdx = text => { + let max = null + for (const m of text.matchAll(CODE_TOKEN_RE)) { + const ord = markerOrder.get(`${m[1]}=${m[2]}`) + if (ord !== undefined && (max === null || ord > max)) max = ord + } + return max + } + const screenPreserves = (screen, idx) => { + // No pre marker, or a screen window that happens to show no code block at + // all (≈0.9 blocks/turn): fall back to "screen not blank". + const seen = lastCodeIdx(screen) + if (idx === null || seen === null) return screen.length > 500 + // Preserved = the screen shows content at-or-after the pre-event region + // (>= idx-10). No upper bound: the UI legitimately keeps painting events + // it already buffered (pipe + coalesce queue), so the screen may be AHEAD + // of the captured pre-event tail. Only a transcript reset — a re-stream + // from scratch showing early-fixture markers — fails this. + return seen >= idx - 10 + } + + const runChaos = async () => { + const scen = chaosSpec.scenario + const chaos = { scenario: scen } + + if (scen === 'gw-kill-stream' || scen === 'gw-kill-tool') { + // The fake gateway self-SIGKILLs (HERMES_FAKE_DIE_AT) and leaves a + // 'dying' progress line — the precise kill wall-clock. + chaos.kill_seen = await waitFor(() => dyingItem !== null, 90_000) + const preIdx = lastCodeIdx(stripAnsi(tailBuf.join(''))) + chaos.pre_kill_code_idx = preIdx + chaos.died_at_msgs = dyingItem?.msgs ?? null + chaos.first_gw_pid = gwPidHistory[0]?.pid ?? null + // Snapshot the screen post-kill but pre-restream: did the transcript survive? + await sleep(250) + const postKill = await forcedScreen() + chaos.transcript_preserved = screenPreserves(postKill, preIdx) + chaos.post_kill_screen_sha = sha256(postKill) + chaos.post_kill_screen_chars = postKill.length + // Auto-heal detection: the respawned gateway rewrites the pidfile. + await waitFor(() => gwPidHistory.length >= 2, 30_000) + chaos.gateway_respawned = gwPidHistory.length >= 2 + chaos.respawn_gw_pid = gwPidHistory[1]?.pid ?? null + chaos.time_to_respawn_ms = chaos.gateway_respawned && dyingItem ? gwPidHistory[1].at - dyingItem.wall : null + // Resume: the respawned gateway re-streams to completion ('done'). + const resumed = await waitFor(() => streamDone, 120_000) + chaos.stream_resumed = resumed && streamStarts >= 2 + if (resumed) { + await quiesce(quiesceMs) + takeSample('final', doneInfo?.msgs ?? null, doneInfo?.events ?? null) + } + const fin = await forcedScreen() + chaos.final_screen_sha = sha256(fin) + chaos.final_screen_code_idx = lastCodeIdx(fin) + chaos.final_screen_has_fixture = fin.length > 500 + } else if (scen === 'gw-stop') { + // SIGSTOP must be external (a stopped process can't stop itself): the + // harness reads the pidfile and signals at the stopAt boundary. + const stopAt = chaosSpec.stopAt ?? 150 + chaos.stop_at_msgs = stopAt + chaos.stop_reached = await waitFor(() => lastBoundaryMsgs >= stopAt, 180_000) + const gwp = gwPidHistory[gwPidHistory.length - 1]?.pid ?? gwPid + chaos.gw_pid = gwp + if (chaos.stop_reached && gwp) { + const preIdx = lastCodeIdx(stripAnsi(tailBuf.join(''))) + chaos.pre_stop_code_idx = preIdx + try { + process.kill(gwp, 'SIGSTOP') + } catch { + /* gone */ + } + await sleep(100) + chaos.gw_state_after_stop = procStateOf(gwp) // expect 'T' + const tFreeze = now() + let checks = 0 + let aliveOk = 0 + while (now() - tFreeze < 30_000 && !exited) { + await sleep(1000) + checks++ + if (uiPid && readProcSample(uiPid)) aliveOk++ + } + chaos.freeze_observe_s = 30 + chaos.ui_alive_during_freeze = `${aliveOk}/${checks}` + chaos.ui_survived_freeze = !exited + const frozenScreen = await forcedScreen() + chaos.transcript_preserved = screenPreserves(frozenScreen, preIdx) + chaos.frozen_screen_sha = sha256(frozenScreen) + // End the run: CONT first (cleanup needs a running process), then KILL. + try { + process.kill(gwp, 'SIGCONT') + } catch { + /* gone */ + } + await sleep(200) + const tKill = now() + try { + process.kill(gwp, 'SIGKILL') + } catch { + /* gone */ + } + await waitFor(() => gwPidHistory[gwPidHistory.length - 1]?.pid !== gwp, 25_000) + chaos.gateway_respawned = gwPidHistory[gwPidHistory.length - 1]?.pid !== gwp + chaos.time_to_respawn_ms = chaos.gateway_respawned ? gwPidHistory[gwPidHistory.length - 1].at - tKill : null + chaos.stream_resumed = await waitFor(() => streamStarts >= 2, 30_000) + } + } else if (scen === 'resize-storm') { + chaos.loaded = await waitFor(() => streamDone, 180_000) + if (chaos.loaded) await quiesce(quiesceMs) + const pre = await forcedScreen() + const preIdx = lastCodeIdx(pre) + chaos.pre_storm_code_idx = preIdx + const flaps = chaosSpec.flaps ?? 30 + const tStorm = now() + let flapped = 0 + for (let i = 0; i < flaps && !exited; i++) { + try { + term.resize(i % 2 === 0 ? 80 : 120, i % 2 === 0 ? 20 : 40) + flapped++ + } catch { + break + } + await sleep(100) + } + try { + if (!exited) term.resize(120, 40) + } catch { + /* gone */ + } + chaos.flaps = flapped + chaos.storm_ms = now() - tStorm + await sleep(10_000) // settle + chaos.ui_survived_storm = !exited + const post = await forcedScreen() + chaos.transcript_preserved = screenPreserves(post, preIdx) + chaos.post_storm_screen_sha = sha256(post) + } else if (scen === 'pty-eof') { + chaos.loaded = await waitFor(() => streamDone, 180_000) + if (chaos.loaded) await quiesce(quiesceMs) + const gwp = gwPidHistory[gwPidHistory.length - 1]?.pid ?? gwPid + chaos.gw_pid = gwp + // Destroy the PTY master: the UI sees EOF/EIO + SIGHUP — the "terminal + // window closed" case. Does it exit cleanly and reap its gateway? + let how = null + try { + if (typeof term.destroy === 'function') { + term.destroy() + how = 'master destroy()' + } + } catch { + /* ignore */ + } + if (!how) { + try { + term.kill('SIGHUP') + how = 'SIGHUP (no destroy())' + } catch { + how = 'failed' + } + } + chaos.master_close = how + const tEof = now() + let uiGoneAt = null + let gwGoneAt = null + while (now() - tEof < 15_000) { + if (uiGoneAt === null && !pidAlive(uiPid)) uiGoneAt = now() + if (gwGoneAt === null && gwp && !pidAlive(gwp)) gwGoneAt = now() + if (uiGoneAt !== null && (gwGoneAt !== null || !gwp)) break + await sleep(100) + } + chaos.ui_exited_after_eof = uiGoneAt !== null + chaos.ui_exit_after_eof_ms = uiGoneAt !== null ? uiGoneAt - tEof : null + chaos.gateway_reaped = gwGoneAt !== null + chaos.gateway_reaped_ms = gwGoneAt !== null ? gwGoneAt - tEof : null + } + + chaos.ui_survived = uiPid ? pidAlive(uiPid) : !exited + return chaos + } + const sessionStarted = await waitFor(() => sessionCreateAt !== null, 30_000) if (!sessionStarted && !exited) { events.push({ kind: 'error', message: 'no session.create within 30s', t_ms: now() - t0 }) @@ -500,6 +790,8 @@ export async function runScenario(opts) { // settle: boot RPCs done + paint quiet await quiesce(quiesceMs) takeSample('final', 0, 0) + } else if (mode === 'chaos') { + result.chaos = await runChaos() } else { // wait for the stream to finish (or the UI to die — cap-hit IS a result) const ok = await waitFor(() => streamDone, runTimeoutMs, 100) @@ -557,6 +849,36 @@ export async function runScenario(opts) { clearInterval(pollTimer) clearInterval(lagTimer) + // chaos: orphan sweep — any gateway pid ever recorded (or the UI itself) + // still alive after teardown is an orphan: record, then reap (specific pids + // only — never a broad pkill). + if (mode === 'chaos' && result.chaos) { + await sleep(1000) + const orphans = [] + const seen = new Set(gwPidHistory.map(e => e.pid)) + if (gwPid) seen.add(gwPid) + for (const pid of seen) { + if (pidAlive(pid)) { + orphans.push({ pid, comm: commOf(pid), role: 'gateway' }) + try { + process.kill(pid, 'SIGKILL') + } catch { + /* raced */ + } + } + } + if (uiPid && pidAlive(uiPid)) { + orphans.push({ pid: uiPid, comm: commOf(uiPid), role: 'ui' }) + try { + process.kill(uiPid, 'SIGKILL') + } catch { + /* raced */ + } + } + result.chaos.orphans = orphans + result.chaos.ui_exit = exited + } + // cap-hit determination const finalCg = lastCg let capHit = false @@ -638,7 +960,7 @@ export async function runScenario(opts) { opentui_cap: opentuiCap, fixture: { path: fixturePath, msgs: fixtureMsgs, sha256: fixtureSha }, sample_every: sampleEvery, - mode_params: mode === 'cpu-paced' ? { rate: pacedRate } : mode === 'scroll' ? scroll : {}, + mode_params: mode === 'cpu-paced' ? { rate: pacedRate } : mode === 'scroll' ? scroll : mode === 'chaos' ? chaosSpec : {}, ui_pid: uiPid, gw_pid: gwPid, cgroup: cgPath, @@ -679,6 +1001,15 @@ export async function runScenario(opts) { } catch { /* ignore */ } + if (gwPidFile) { + for (const f of [gwPidFile, `${gwPidFile}.dieflag`]) { + try { + unlinkSync(f) + } catch { + /* ignore */ + } + } + } if (outFile) { mkdirSync(dirname(outFile), { recursive: true }) diff --git a/bench/run.mjs b/bench/run.mjs index 5399de9abce..4ee59f0475b 100644 --- a/bench/run.mjs +++ b/bench/run.mjs @@ -6,7 +6,7 @@ // bench/results/-----r.json. // // Usage: -// node run.mjs --cell gate|mem3000|slope10k|nodes|cpu|scroll|startup +// node run.mjs --cell gate|mem3000|slope10k|nodes|cpu|scroll|startup|chaos // node run.mjs --all (the full E1 host sequence, gate first) // Knobs: --reps N, --msgs N, --cap 2G|none, --seed N @@ -300,6 +300,47 @@ async function cellScroll(opts) { } } +// ── chaos/stability cell ───────────────────────────────────────────────── +// 5 scenarios × {ink, otui-capped} = 10 sequential runs. The fake gateway +// self-SIGKILLs deterministically (HERMES_FAKE_DIE_AT) for the kill scenarios; +// SIGSTOP is external (harness reads HERMES_FAKE_PIDFILE). Auto-heal detection +// = pidfile rewrite by the respawned gateway. Results carry summary.chaos. +const CHAOS_SCENARIOS = ['gw-kill-stream', 'gw-kill-tool', 'gw-stop', 'resize-storm', 'pty-eof'] + +async function cellChaos(opts) { + const msgs = opts.msgs ?? 300 + const half = Math.floor(msgs / 2) + const fx = await ensureFixture(msgs) + const configs = opts.configs ?? ['ink', 'otui-capped'] + const scenarios = opts.scenarios ?? CHAOS_SCENARIOS + for (const config of configs) { + for (const scenario of scenarios) { + const chaos = { scenario } + let extra = {} + if (scenario === 'gw-kill-stream') chaos.dieAt = `${half}:kill` + if (scenario === 'gw-kill-tool') chaos.dieAt = `${half}:tool-kill` + if (scenario === 'gw-stop') { + // paced so "mid-stream" exists long enough to land an external SIGSTOP + chaos.stopAt = half + chaos.fakeMode = 'paced' + extra = { pacedRate: 120 } + } + await doRun('chaos', config, scenario, { + mode: 'chaos', + chaos, + fixturePath: fx.path, + fixtureMsgs: fx.msgs, + fixtureSha: fx.sha256, + memoryMax: '2G', + heapMb: 8192, + sampleEvery: 25, + runTimeoutMs: 10 * 60 * 1000, + ...extra + }) + } + } +} + async function cellStartup(opts) { const reps = opts.reps ?? 10 const rand = rng(opts.seed ?? 4242) @@ -331,7 +372,8 @@ const opts = { msgs: opt('msgs') ? Number(opt('msgs')) : undefined, cap: opt('cap'), seed: opt('seed') ? Number(opt('seed')) : undefined, - configs: opt('configs') ? opt('configs').split(',').filter(c => CONFIGS[c]) : undefined + configs: opt('configs') ? opt('configs').split(',').filter(c => CONFIGS[c]) : undefined, + scenarios: opt('scenarios') ? opt('scenarios').split(',').filter(s => CHAOS_SCENARIOS.includes(s)) : undefined } const cell = opt('cell') mkdirSync(RESULTS_DIR, { recursive: true }) @@ -343,7 +385,8 @@ const CELLS = { nodes: cellNodes, cpu: cellCpu, scroll: cellScroll, - startup: cellStartup + startup: cellStartup, + chaos: cellChaos } if (args.includes('--all')) {