opentui(bench): realistic heavy-session fixture (fat tool-turns) + multi-cap matrix

Replaces the synthetic ~5.5-node/msg pushes with a deterministic generator
(scripts/fixture.ts): lorem-ipsum user turns + fat assistant turns (markdown +
reasoning + 1-15 tool parts with multi-line results) driven through the real
apply()/commitSnapshot paths. mem-bench.tsx pumps it + checks the resume path.
Realistic cost is ~20.4 renderables/msg (3.7x synthetic); informed the cap tune.
This commit is contained in:
alt-glitch 2026-06-09 10:25:16 +00:00
parent c40d3172ac
commit f205dc2a3b
2 changed files with 372 additions and 25 deletions

View file

@ -0,0 +1,288 @@
/**
* DEV BENCH FIXTURE NOT a test, NOT production code. A deterministic generator
* for a REALISTIC heavy session, consumed by `scripts/mem-bench.tsx`. Excluded
* from `bun test` (not a *.test.ts) and lint-clean.
*
* The old synthetic bench pushed tiny 3-delta turns (~5.5 mounted nodes each)
* an unrealistic per-message cost. Real transcripts are LUMPY: an assistant turn
* is ONE `message` but a fat node subtree (markdown blocks + a reasoning block +
* several tool headers, each a multi-line result). That makes message-count a
* LOOSE proxy for memory, which is exactly what we're trying to quantify before
* picking a `HERMES_TUI_MAX_MESSAGES` default.
*
* Design: a turn is modeled as a small typed `TurnAction` union (user / system /
* gateway-event). The driver maps user`pushUser`, system`pushSystem`, and every
* gateway event through the SAME `apply()` reducer real usage takes so the
* mounted result is identical to a live session. The same action stream also
* materializes a settled `Message[]` (via `materialize`) for the resume-path check
* (`commitSnapshot`). Everything is seeded by index (no `Math.random`
* unavailable here), so a given `total` reproduces byte-for-byte.
*/
import type { GatewayEvent } from '../src/boundary/schema/GatewayEvent.ts'
import { createSessionStore, type Message } from '../src/logic/store.ts'
/** One scripted action in a turn: a composer push or a decoded gateway event. */
type TurnAction =
| { kind: 'user'; text: string }
| { kind: 'system'; text: string }
| { kind: 'event'; event: GatewayEvent }
/** A pool of lorem-ipsum words — varied content is selected by index from here. */
const WORDS = [
'lorem',
'ipsum',
'dolor',
'sit',
'amet',
'consectetur',
'adipiscing',
'elit',
'sed',
'eiusmod',
'tempor',
'incididunt',
'labore',
'magna',
'aliqua',
'enim',
'minim',
'veniam',
'quis',
'nostrud',
'exercitation',
'ullamco',
'laboris',
'aliquip',
'commodo',
'consequat',
'duis',
'aute',
'irure',
'reprehenderit',
'voluptate',
'velit',
'esse',
'cillum',
'fugiat',
'nulla',
'pariatur',
'excepteur',
'occaecat',
'cupidatat',
'proident',
'sunt',
'culpa',
'officia',
'deserunt',
'mollit',
'anim'
] as const
/** Deterministic pseudo-word stream: pick from WORDS by a seeded index. */
function word(seed: number, k: number): string {
return WORDS[(seed * 31 + k * 7) % WORDS.length] ?? 'lorem'
}
/** A lorem sentence of `n` words, capitalized + terminated. */
function sentence(seed: number, n: number): string {
const parts: string[] = []
for (let k = 0; k < n; k++) parts.push(word(seed + k, k))
const text = parts.join(' ')
return text.charAt(0).toUpperCase() + text.slice(1) + '.'
}
/** A paragraph of `s` sentences (varying length by index). */
function paragraph(seed: number, s: number): string {
const out: string[] = []
for (let i = 0; i < s; i++) out.push(sentence(seed + i * 13, 6 + ((seed + i) % 9)))
return out.join(' ')
}
/** N lorem-ipsum lines (for tool result bodies), each varying in length. */
function lines(seed: number, n: number): string {
const out: string[] = []
for (let i = 0; i < n; i++) out.push(sentence(seed + i * 5, 4 + ((seed + i) % 11)))
return out.join('\n')
}
/** A markdown assistant body: paragraphs + a list + a fenced code block. */
function assistantMarkdown(seed: number): string {
const lead = paragraph(seed, 1 + (seed % 3))
const bullets = [`- ${sentence(seed + 1, 5)}`, `- ${sentence(seed + 2, 7)}`, `- ${sentence(seed + 3, 4)}`].join('\n')
const code = [
'```ts',
`const x${seed % 7} = ${seed % 100}`,
`function f${seed % 5}() {`,
' return x',
'}',
'```'
].join('\n')
const tail = paragraph(seed + 17, 1 + ((seed + 1) % 2))
return `${lead}\n\n${bullets}\n\n${code}\n\n${tail}`
}
/** Tool names cycled by index (mirrors a real tool mix). */
const TOOL_NAMES = ['terminal', 'read_file', 'edit_file', 'grep', 'web_search', 'write_file'] as const
/** A tool.start + tool.complete pair for tool `t` in turn `seed`. */
function toolEvents(seed: number, t: number): GatewayEvent[] {
const id = `tool-${seed}-${t}`
const name = TOOL_NAMES[(seed + t) % TOOL_NAMES.length] ?? 'terminal'
const variant = (seed + t) % 3
// short / capped-16-line / medium result bodies, mixing the render-cost cases.
const bodyLines = variant === 0 ? 2 : variant === 1 ? 18 : 7
const resultText = lines(seed + t * 3, bodyLines)
const context = sentence(seed + t, 4)
// ~half the tools carry a multi-line args block (the expanded-view cost).
const withArgs = (seed + t) % 2 === 0
const start: GatewayEvent = {
type: 'tool.start',
payload: withArgs ? { tool_id: id, name, context, args_text: lines(seed + t, 5) } : { tool_id: id, name, context }
}
const complete: GatewayEvent = {
type: 'tool.complete',
payload: {
tool_id: id,
name,
result_text: resultText,
duration_s: 0.1 + ((seed + t) % 40) / 10,
args: { command: context, index: seed + t }
}
}
return [start, complete]
}
/** One USER message (14 lorem paragraphs; some very short, some RFC-sized). */
function userText(seed: number): string {
const shape = seed % 7
if (shape === 0) return 'yes do that'
if (shape === 1) return 'ok'
if (shape === 6) {
// an RFC-sized pasted block: many paragraphs.
const out: string[] = []
for (let p = 0; p < 8; p++) out.push(paragraph(seed + p * 23, 4 + (p % 3)))
return out.join('\n\n')
}
const n = 1 + (seed % 4)
const out: string[] = []
for (let p = 0; p < n; p++) out.push(paragraph(seed + p * 11, 1 + ((seed + p) % 3)))
return out.join('\n\n')
}
/**
* Build the scripted actions for ONE turn. Most turns are a plain user+assistant
* exchange; a deterministic subset are tool-heavy (115 tool calls) or a system
* slash-output line. Returns the actions for the whole turn in order.
*/
function turnActions(turn: number): TurnAction[] {
const actions: TurnAction[] = []
// Occasional system slash-output line (≈ every 9th turn) instead of a user line.
if (turn % 9 === 4) {
actions.push({ kind: 'system', text: sentence(turn, 8) })
return actions
}
actions.push({ kind: 'user', text: userText(turn) })
actions.push({ kind: 'event', event: { type: 'message.start' } })
// Reasoning on ≈ every 3rd assistant turn.
if (turn % 3 === 0) {
actions.push({
kind: 'event',
event: {
type: 'reasoning.delta',
payload: { text: `**${sentence(turn, 3).replace(/\.$/, '')}**\n\n${paragraph(turn + 5, 2)}` }
}
})
}
// Leading text part.
actions.push({ kind: 'event', event: { type: 'message.delta', payload: { text: assistantMarkdown(turn) } } })
// Tool-heavy turns: ≈ every 4th assistant turn carries several tool calls,
// interleaved with a follow-up text part (the fat-turn stress case).
if (turn % 4 === 0) {
const toolCount = 1 + (turn % 15) // 1..15 tools
for (let t = 0; t < toolCount; t++) {
for (const ev of toolEvents(turn, t)) actions.push({ kind: 'event', event: ev })
}
actions.push({ kind: 'event', event: { type: 'message.delta', payload: { text: paragraph(turn + 31, 2) } } })
}
actions.push({ kind: 'event', event: { type: 'message.complete' } })
return actions
}
/** How many transcript ROWS a turn produces (user/system + at most one assistant). */
export function rowsPerTurn(turn: number): number {
return turn % 9 === 4 ? 1 : 2
}
/** Apply ONE turn's actions to a store via the same paths real usage takes. */
export function applyTurn(store: ReturnType<typeof createSessionStore>, turn: number): void {
for (const action of turnActions(turn)) {
if (action.kind === 'user') store.pushUser(action.text)
else if (action.kind === 'system') store.pushSystem(action.text)
else store.apply(action.event)
}
}
/**
* Drive at least `total` MESSAGES into the live store, calling `onSample(pushes)`
* each time the cumulative produced-row count crosses a `sampleEvery` boundary.
* `pushes` counts MESSAGES (rows produced, pre-cap), so the matrix samples on a
* raw message cadence regardless of the rolling cap.
*/
export function drive(
store: ReturnType<typeof createSessionStore>,
total: number,
sampleEvery: number,
onSample: (pushes: number) => void
): number {
let pushed = 0
let nextSample = sampleEvery
let turn = 0
while (pushed < total) {
applyTurn(store, turn)
pushed += rowsPerTurn(turn)
turn++
while (pushed >= nextSample && nextSample <= total) {
onSample(Math.min(pushed, total))
nextSample += sampleEvery
}
}
return turn
}
/**
* Materialize the FULL settled `Message[]` for the resume path: replay the same
* action stream into a FRESH, EFFECTIVELY-UNCAPPED store and snapshot its rows.
* This guarantees the resume fixture is byte-identical to what the live push
* path produces (minus the rolling cap), so `commitSnapshot` mounts the real shape.
*/
export function materialize(total: number): Message[] {
const prev = process.env.HERMES_TUI_MAX_MESSAGES
process.env.HERMES_TUI_MAX_MESSAGES = String(Number.MAX_SAFE_INTEGER)
const store = createSessionStore()
store.apply({ type: 'gateway.ready' })
let pushed = 0
let turn = 0
while (pushed < total) {
applyTurn(store, turn)
pushed += rowsPerTurn(turn)
turn++
}
// Restore the env so the bench's own cap (read per-store) is unaffected.
if (prev === undefined) delete process.env.HERMES_TUI_MAX_MESSAGES
else process.env.HERMES_TUI_MAX_MESSAGES = prev
// Deep-copy out of the solid store proxy into plain objects (the resume path
// takes a plain Message[]).
return store.state.messages.slice(0, total).map(cloneMessage)
}
/** Plain deep copy of a store Message (drop the solid proxy + streaming flag). */
function cloneMessage(m: Message): Message {
const copy: Message = { role: m.role, text: m.text }
if (m.parts) copy.parts = m.parts.map(p => ({ ...p }))
return copy
}

View file

@ -1,23 +1,37 @@
/**
* DEV BENCH NOT a test, NOT production code. Throwaway memory-measurement
* harness for the Epic 5 comparison doc. Empirically checks whether the rolling
* MESSAGE_CAP bounds the native (Yoga/renderable) allocation footprint as the
* transcript grows. Excluded from `bun test` (not a *.test.ts) and lint-clean.
* harness for tuning the rolling `HERMES_TUI_MAX_MESSAGES` cap. Mounts the
* production `<App store={createSessionStore()}>` under the `@opentui/solid` test
* renderer and samples `process.memoryUsage()` + the mounted-renderable count +
* `getAllocatorStats().activeAllocations`, forcing `Bun.gc(true)` before each
* sample. Excluded from `bun test` (not a *.test.ts) and lint-clean.
*
* Uncapped: HERMES_TUI_MAX_MESSAGES=100000 bun scripts/mem-bench.ts
* Capped: HERMES_TUI_MAX_MESSAGES=400 bun scripts/mem-bench.ts
* It pushes a REALISTIC heavy-session fixture (scripts/fixture.ts) varied user
* turns + fat multi-part assistant turns (markdown + reasoning + several tool
* headers) because per-message size varies hugely, so message-count is only a
* LOOSE memory proxy and we're choosing a cap default.
*
* Run each as a SEPARATE bun invocation so the WASM/native heap starts fresh.
* Uncapped: MEM_BENCH_TOTAL=8000 HERMES_TUI_MAX_MESSAGES=100000 bun scripts/mem-bench.tsx
* Capped: MEM_BENCH_TOTAL=8000 HERMES_TUI_MAX_MESSAGES=1500 bun scripts/mem-bench.tsx
*
* Run each cap as a SEPARATE bun invocation so the WASM/native heap starts fresh.
* The matrix loop:
* for cap in 400 1500 3000 6000 100000; do \
* MEM_BENCH_TOTAL=8000 HERMES_TUI_MAX_MESSAGES=$cap bun scripts/mem-bench.tsx; done
*
* Signal: native `getAllocatorStats().activeAllocations` (the Zig-side allocator
* count every live renderable/Yoga subtree contributes) and the recursive
* renderable descendant count under `renderer.root`. RSS is reported too but is
* noisy and grow-only (WASM linear memory never returns to the OS), so the
* meaningful comparison is the SLOPE of activeAllocations / descendant count:
* capped should plateau after ~CAP messages; uncapped should keep climbing.
* meaningful comparison is the STEADY-STATE plateau: capped should flatten after
* ~CAP messages; uncapped should keep climbing.
*
* GC: forces `Bun.gc(true)` (synchronous) before each sample to measure RETAINED
* memory, not garbage. (`--expose-gc`/`global.gc` is unavailable under Bun.)
*
* RESUME PATH: after the live push matrix, builds the full fixture as a settled
* Message[] and `commitSnapshot`s it (the resume path), reporting mounted nodes +
* RSS verifying the slice-before-set fix bounds resume mounting to cap.
*/
import { resolveRenderLib } from '@opentui/core'
import type { Renderable } from '@opentui/core'
@ -26,11 +40,12 @@ import { testRender } from '@opentui/solid'
import { createSessionStore } from '../src/logic/store.ts'
import { App } from '../src/view/App.tsx'
import { ThemeProvider } from '../src/view/theme.tsx'
import { applyTurn, materialize, rowsPerTurn } from './fixture.ts'
const lib = resolveRenderLib()
const TOTAL = Number.parseInt(process.env.MEM_BENCH_TOTAL ?? '5000', 10)
const SAMPLE_EVERY = Number.parseInt(process.env.MEM_BENCH_SAMPLE ?? '250', 10)
const TOTAL = Number.parseInt(process.env.MEM_BENCH_TOTAL ?? '8000', 10)
const SAMPLE_EVERY = Number.parseInt(process.env.MEM_BENCH_SAMPLE ?? '500', 10)
const cap = process.env.HERMES_TUI_MAX_MESSAGES ?? '(default 400)'
const MB = (bytes: number) => (bytes / 1024 / 1024).toFixed(1)
@ -42,16 +57,6 @@ function descendantCount(node: Renderable): number {
return n
}
/** One streamed assistant turn = a few text parts (a realistic multi-node subtree). */
function pushTurn(store: ReturnType<typeof createSessionStore>, i: number): void {
store.pushUser(`user message ${i}: please summarize the situation in a few lines`)
store.apply({ type: 'message.start' })
store.apply({ type: 'message.delta', payload: { text: `Sure — point one for turn ${i}. ` } })
store.apply({ type: 'message.delta', payload: { text: `Here is point two with a bit more detail. ` } })
store.apply({ type: 'message.delta', payload: { text: `And a closing point three for turn ${i}.` } })
store.apply({ type: 'message.complete' })
}
async function main(): Promise<void> {
const store = createSessionStore()
store.apply({ type: 'gateway.ready' })
@ -67,8 +72,9 @@ async function main(): Promise<void> {
await setup.renderOnce()
await setup.flush()
// header: pad to fixed widths for a readable table
process.stdout.write(`\n=== mem-bench cap=${cap} total=${TOTAL} sampleEvery=${SAMPLE_EVERY} ===\n`)
process.stdout.write(
`\n=== mem-bench (REALISTIC fixture) cap=${cap} total=${TOTAL} sampleEvery=${SAMPLE_EVERY} ===\n`
)
process.stdout.write(
'pushes | msgs | rss(MB) | heapUsed(MB) | external(MB) | arrayBuf(MB) | activeAllocs | renderables\n'
)
@ -97,12 +103,65 @@ async function main(): Promise<void> {
}
await sample(0)
for (let i = 1; i <= TOTAL; i++) {
pushTurn(store, i)
if (i % SAMPLE_EVERY === 0) await sample(i)
// Pump turns inline, sampling each time the cumulative produced-row count crosses
// a SAMPLE_EVERY boundary. Sampling is async (renderOnce/flush/gc), so it lives
// in the loop rather than a sync callback. Mounting is synchronous in Solid, so a
// render pass at the boundary reflects the just-pushed turns.
let pushed = 0
let nextSample = SAMPLE_EVERY
let turn = 0
while (pushed < TOTAL) {
applyTurn(store, turn)
pushed += rowsPerTurn(turn)
turn++
if (pushed >= nextSample) {
await sample(Math.min(pushed, TOTAL))
while (nextSample <= pushed) nextSample += SAMPLE_EVERY
}
}
// Tear down the live push tree BEFORE the resume path so its mounted nodes don't
// pollute the process-wide RSS the resume sample reads. (The renderable COUNT is
// already isolated per-renderer-root, but RSS is process-global.)
store.clearTranscript()
setup.renderer.destroy()
Bun.gc(true)
// ── RESUME PATH: build the full settled fixture and commitSnapshot it (the
// resume hydrate path). Verifies the slice-before-set fix bounds resume mounting
// to ≤ cap — mounting 8000 settled msgs at cap=1500 should mount ~1500-worth of
// rows, NOT 8000-worth. Done on a FRESH store + renderer so the live-push history
// above doesn't skew the count.
const resumeStore = createSessionStore()
resumeStore.apply({ type: 'gateway.ready' })
const resumeSetup = await testRender(
() => (
<ThemeProvider theme={() => resumeStore.state.theme}>
<App store={resumeStore} />
</ThemeProvider>
),
{ width: 100, height: 40, exitOnCtrlC: false }
)
await resumeSetup.renderOnce()
await resumeSetup.flush()
const fullFixture = materialize(TOTAL)
resumeStore.beginBuffer()
resumeStore.commitSnapshot(fullFixture)
await resumeSetup.renderOnce()
await resumeSetup.flush()
Bun.gc(true)
const rm = process.memoryUsage()
const ralloc = lib.getAllocatorStats()
const rrenderables = descendantCount(resumeSetup.renderer.root)
process.stdout.write('\n--- resume path (commitSnapshot of the full fixture) ---\n')
process.stdout.write(`fixture msgs built : ${fullFixture.length}\n`)
process.stdout.write(`mounted msgs (cap) : ${resumeStore.state.messages.length}\n`)
process.stdout.write(`mounted renderables: ${rrenderables}\n`)
process.stdout.write(`activeAllocations : ${ralloc.activeAllocations}\n`)
process.stdout.write(`rss(MB) : ${MB(rm.rss)}\n`)
resumeSetup.renderer.destroy()
}
await main()