diff --git a/ui-tui-opentui-v2/scripts/fixture.ts b/ui-tui-opentui-v2/scripts/fixture.ts new file mode 100644 index 00000000000..6e966f733a4 --- /dev/null +++ b/ui-tui-opentui-v2/scripts/fixture.ts @@ -0,0 +1,288 @@ +/** + * DEV BENCH FIXTURE — NOT a test, NOT production code. A deterministic generator + * for a REALISTIC heavy session, consumed by `scripts/mem-bench.tsx`. Excluded + * from `bun test` (not a *.test.ts) and lint-clean. + * + * The old synthetic bench pushed tiny 3-delta turns (~5.5 mounted nodes each) — + * an unrealistic per-message cost. Real transcripts are LUMPY: an assistant turn + * is ONE `message` but a fat node subtree (markdown blocks + a reasoning block + + * several tool headers, each a multi-line result). That makes message-count a + * LOOSE proxy for memory, which is exactly what we're trying to quantify before + * picking a `HERMES_TUI_MAX_MESSAGES` default. + * + * Design: a turn is modeled as a small typed `TurnAction` union (user / system / + * gateway-event). The driver maps user→`pushUser`, system→`pushSystem`, and every + * gateway event through the SAME `apply()` reducer real usage takes — so the + * mounted result is identical to a live session. The same action stream also + * materializes a settled `Message[]` (via `materialize`) for the resume-path check + * (`commitSnapshot`). Everything is seeded by index (no `Math.random` — + * unavailable here), so a given `total` reproduces byte-for-byte. + */ +import type { GatewayEvent } from '../src/boundary/schema/GatewayEvent.ts' +import { createSessionStore, type Message } from '../src/logic/store.ts' + +/** One scripted action in a turn: a composer push or a decoded gateway event. */ +type TurnAction = + | { kind: 'user'; text: string } + | { kind: 'system'; text: string } + | { kind: 'event'; event: GatewayEvent } + +/** A pool of lorem-ipsum words — varied content is selected by index from here. */ +const WORDS = [ + 'lorem', + 'ipsum', + 'dolor', + 'sit', + 'amet', + 'consectetur', + 'adipiscing', + 'elit', + 'sed', + 'eiusmod', + 'tempor', + 'incididunt', + 'labore', + 'magna', + 'aliqua', + 'enim', + 'minim', + 'veniam', + 'quis', + 'nostrud', + 'exercitation', + 'ullamco', + 'laboris', + 'aliquip', + 'commodo', + 'consequat', + 'duis', + 'aute', + 'irure', + 'reprehenderit', + 'voluptate', + 'velit', + 'esse', + 'cillum', + 'fugiat', + 'nulla', + 'pariatur', + 'excepteur', + 'occaecat', + 'cupidatat', + 'proident', + 'sunt', + 'culpa', + 'officia', + 'deserunt', + 'mollit', + 'anim' +] as const + +/** Deterministic pseudo-word stream: pick from WORDS by a seeded index. */ +function word(seed: number, k: number): string { + return WORDS[(seed * 31 + k * 7) % WORDS.length] ?? 'lorem' +} + +/** A lorem sentence of `n` words, capitalized + terminated. */ +function sentence(seed: number, n: number): string { + const parts: string[] = [] + for (let k = 0; k < n; k++) parts.push(word(seed + k, k)) + const text = parts.join(' ') + return text.charAt(0).toUpperCase() + text.slice(1) + '.' +} + +/** A paragraph of `s` sentences (varying length by index). */ +function paragraph(seed: number, s: number): string { + const out: string[] = [] + for (let i = 0; i < s; i++) out.push(sentence(seed + i * 13, 6 + ((seed + i) % 9))) + return out.join(' ') +} + +/** N lorem-ipsum lines (for tool result bodies), each varying in length. */ +function lines(seed: number, n: number): string { + const out: string[] = [] + for (let i = 0; i < n; i++) out.push(sentence(seed + i * 5, 4 + ((seed + i) % 11))) + return out.join('\n') +} + +/** A markdown assistant body: paragraphs + a list + a fenced code block. */ +function assistantMarkdown(seed: number): string { + const lead = paragraph(seed, 1 + (seed % 3)) + const bullets = [`- ${sentence(seed + 1, 5)}`, `- ${sentence(seed + 2, 7)}`, `- ${sentence(seed + 3, 4)}`].join('\n') + const code = [ + '```ts', + `const x${seed % 7} = ${seed % 100}`, + `function f${seed % 5}() {`, + ' return x', + '}', + '```' + ].join('\n') + const tail = paragraph(seed + 17, 1 + ((seed + 1) % 2)) + return `${lead}\n\n${bullets}\n\n${code}\n\n${tail}` +} + +/** Tool names cycled by index (mirrors a real tool mix). */ +const TOOL_NAMES = ['terminal', 'read_file', 'edit_file', 'grep', 'web_search', 'write_file'] as const + +/** A tool.start + tool.complete pair for tool `t` in turn `seed`. */ +function toolEvents(seed: number, t: number): GatewayEvent[] { + const id = `tool-${seed}-${t}` + const name = TOOL_NAMES[(seed + t) % TOOL_NAMES.length] ?? 'terminal' + const variant = (seed + t) % 3 + // short / capped-16-line / medium result bodies, mixing the render-cost cases. + const bodyLines = variant === 0 ? 2 : variant === 1 ? 18 : 7 + const resultText = lines(seed + t * 3, bodyLines) + const context = sentence(seed + t, 4) + // ~half the tools carry a multi-line args block (the expanded-view cost). + const withArgs = (seed + t) % 2 === 0 + const start: GatewayEvent = { + type: 'tool.start', + payload: withArgs ? { tool_id: id, name, context, args_text: lines(seed + t, 5) } : { tool_id: id, name, context } + } + const complete: GatewayEvent = { + type: 'tool.complete', + payload: { + tool_id: id, + name, + result_text: resultText, + duration_s: 0.1 + ((seed + t) % 40) / 10, + args: { command: context, index: seed + t } + } + } + return [start, complete] +} + +/** One USER message (1–4 lorem paragraphs; some very short, some RFC-sized). */ +function userText(seed: number): string { + const shape = seed % 7 + if (shape === 0) return 'yes do that' + if (shape === 1) return 'ok' + if (shape === 6) { + // an RFC-sized pasted block: many paragraphs. + const out: string[] = [] + for (let p = 0; p < 8; p++) out.push(paragraph(seed + p * 23, 4 + (p % 3))) + return out.join('\n\n') + } + const n = 1 + (seed % 4) + const out: string[] = [] + for (let p = 0; p < n; p++) out.push(paragraph(seed + p * 11, 1 + ((seed + p) % 3))) + return out.join('\n\n') +} + +/** + * Build the scripted actions for ONE turn. Most turns are a plain user+assistant + * exchange; a deterministic subset are tool-heavy (1–15 tool calls) or a system + * slash-output line. Returns the actions for the whole turn in order. + */ +function turnActions(turn: number): TurnAction[] { + const actions: TurnAction[] = [] + // Occasional system slash-output line (≈ every 9th turn) instead of a user line. + if (turn % 9 === 4) { + actions.push({ kind: 'system', text: sentence(turn, 8) }) + return actions + } + + actions.push({ kind: 'user', text: userText(turn) }) + actions.push({ kind: 'event', event: { type: 'message.start' } }) + + // Reasoning on ≈ every 3rd assistant turn. + if (turn % 3 === 0) { + actions.push({ + kind: 'event', + event: { + type: 'reasoning.delta', + payload: { text: `**${sentence(turn, 3).replace(/\.$/, '')}**\n\n${paragraph(turn + 5, 2)}` } + } + }) + } + + // Leading text part. + actions.push({ kind: 'event', event: { type: 'message.delta', payload: { text: assistantMarkdown(turn) } } }) + + // Tool-heavy turns: ≈ every 4th assistant turn carries several tool calls, + // interleaved with a follow-up text part (the fat-turn stress case). + if (turn % 4 === 0) { + const toolCount = 1 + (turn % 15) // 1..15 tools + for (let t = 0; t < toolCount; t++) { + for (const ev of toolEvents(turn, t)) actions.push({ kind: 'event', event: ev }) + } + actions.push({ kind: 'event', event: { type: 'message.delta', payload: { text: paragraph(turn + 31, 2) } } }) + } + + actions.push({ kind: 'event', event: { type: 'message.complete' } }) + return actions +} + +/** How many transcript ROWS a turn produces (user/system + at most one assistant). */ +export function rowsPerTurn(turn: number): number { + return turn % 9 === 4 ? 1 : 2 +} + +/** Apply ONE turn's actions to a store via the same paths real usage takes. */ +export function applyTurn(store: ReturnType, turn: number): void { + for (const action of turnActions(turn)) { + if (action.kind === 'user') store.pushUser(action.text) + else if (action.kind === 'system') store.pushSystem(action.text) + else store.apply(action.event) + } +} + +/** + * Drive at least `total` MESSAGES into the live store, calling `onSample(pushes)` + * each time the cumulative produced-row count crosses a `sampleEvery` boundary. + * `pushes` counts MESSAGES (rows produced, pre-cap), so the matrix samples on a + * raw message cadence regardless of the rolling cap. + */ +export function drive( + store: ReturnType, + total: number, + sampleEvery: number, + onSample: (pushes: number) => void +): number { + let pushed = 0 + let nextSample = sampleEvery + let turn = 0 + while (pushed < total) { + applyTurn(store, turn) + pushed += rowsPerTurn(turn) + turn++ + while (pushed >= nextSample && nextSample <= total) { + onSample(Math.min(pushed, total)) + nextSample += sampleEvery + } + } + return turn +} + +/** + * Materialize the FULL settled `Message[]` for the resume path: replay the same + * action stream into a FRESH, EFFECTIVELY-UNCAPPED store and snapshot its rows. + * This guarantees the resume fixture is byte-identical to what the live push + * path produces (minus the rolling cap), so `commitSnapshot` mounts the real shape. + */ +export function materialize(total: number): Message[] { + const prev = process.env.HERMES_TUI_MAX_MESSAGES + process.env.HERMES_TUI_MAX_MESSAGES = String(Number.MAX_SAFE_INTEGER) + const store = createSessionStore() + store.apply({ type: 'gateway.ready' }) + let pushed = 0 + let turn = 0 + while (pushed < total) { + applyTurn(store, turn) + pushed += rowsPerTurn(turn) + turn++ + } + // Restore the env so the bench's own cap (read per-store) is unaffected. + if (prev === undefined) delete process.env.HERMES_TUI_MAX_MESSAGES + else process.env.HERMES_TUI_MAX_MESSAGES = prev + // Deep-copy out of the solid store proxy into plain objects (the resume path + // takes a plain Message[]). + return store.state.messages.slice(0, total).map(cloneMessage) +} + +/** Plain deep copy of a store Message (drop the solid proxy + streaming flag). */ +function cloneMessage(m: Message): Message { + const copy: Message = { role: m.role, text: m.text } + if (m.parts) copy.parts = m.parts.map(p => ({ ...p })) + return copy +} diff --git a/ui-tui-opentui-v2/scripts/mem-bench.tsx b/ui-tui-opentui-v2/scripts/mem-bench.tsx index efd24dc6122..581a9df8b2d 100644 --- a/ui-tui-opentui-v2/scripts/mem-bench.tsx +++ b/ui-tui-opentui-v2/scripts/mem-bench.tsx @@ -1,23 +1,37 @@ /** * DEV BENCH — NOT a test, NOT production code. Throwaway memory-measurement - * harness for the Epic 5 comparison doc. Empirically checks whether the rolling - * MESSAGE_CAP bounds the native (Yoga/renderable) allocation footprint as the - * transcript grows. Excluded from `bun test` (not a *.test.ts) and lint-clean. + * harness for tuning the rolling `HERMES_TUI_MAX_MESSAGES` cap. Mounts the + * production `` under the `@opentui/solid` test + * renderer and samples `process.memoryUsage()` + the mounted-renderable count + + * `getAllocatorStats().activeAllocations`, forcing `Bun.gc(true)` before each + * sample. Excluded from `bun test` (not a *.test.ts) and lint-clean. * - * Uncapped: HERMES_TUI_MAX_MESSAGES=100000 bun scripts/mem-bench.ts - * Capped: HERMES_TUI_MAX_MESSAGES=400 bun scripts/mem-bench.ts + * It pushes a REALISTIC heavy-session fixture (scripts/fixture.ts) — varied user + * turns + fat multi-part assistant turns (markdown + reasoning + several tool + * headers) — because per-message size varies hugely, so message-count is only a + * LOOSE memory proxy and we're choosing a cap default. * - * Run each as a SEPARATE bun invocation so the WASM/native heap starts fresh. + * Uncapped: MEM_BENCH_TOTAL=8000 HERMES_TUI_MAX_MESSAGES=100000 bun scripts/mem-bench.tsx + * Capped: MEM_BENCH_TOTAL=8000 HERMES_TUI_MAX_MESSAGES=1500 bun scripts/mem-bench.tsx + * + * Run each cap as a SEPARATE bun invocation so the WASM/native heap starts fresh. + * The matrix loop: + * for cap in 400 1500 3000 6000 100000; do \ + * MEM_BENCH_TOTAL=8000 HERMES_TUI_MAX_MESSAGES=$cap bun scripts/mem-bench.tsx; done * * Signal: native `getAllocatorStats().activeAllocations` (the Zig-side allocator * count — every live renderable/Yoga subtree contributes) and the recursive * renderable descendant count under `renderer.root`. RSS is reported too but is * noisy and grow-only (WASM linear memory never returns to the OS), so the - * meaningful comparison is the SLOPE of activeAllocations / descendant count: - * capped should plateau after ~CAP messages; uncapped should keep climbing. + * meaningful comparison is the STEADY-STATE plateau: capped should flatten after + * ~CAP messages; uncapped should keep climbing. * * GC: forces `Bun.gc(true)` (synchronous) before each sample to measure RETAINED * memory, not garbage. (`--expose-gc`/`global.gc` is unavailable under Bun.) + * + * RESUME PATH: after the live push matrix, builds the full fixture as a settled + * Message[] and `commitSnapshot`s it (the resume path), reporting mounted nodes + + * RSS — verifying the slice-before-set fix bounds resume mounting to ≤ cap. */ import { resolveRenderLib } from '@opentui/core' import type { Renderable } from '@opentui/core' @@ -26,11 +40,12 @@ import { testRender } from '@opentui/solid' import { createSessionStore } from '../src/logic/store.ts' import { App } from '../src/view/App.tsx' import { ThemeProvider } from '../src/view/theme.tsx' +import { applyTurn, materialize, rowsPerTurn } from './fixture.ts' const lib = resolveRenderLib() -const TOTAL = Number.parseInt(process.env.MEM_BENCH_TOTAL ?? '5000', 10) -const SAMPLE_EVERY = Number.parseInt(process.env.MEM_BENCH_SAMPLE ?? '250', 10) +const TOTAL = Number.parseInt(process.env.MEM_BENCH_TOTAL ?? '8000', 10) +const SAMPLE_EVERY = Number.parseInt(process.env.MEM_BENCH_SAMPLE ?? '500', 10) const cap = process.env.HERMES_TUI_MAX_MESSAGES ?? '(default 400)' const MB = (bytes: number) => (bytes / 1024 / 1024).toFixed(1) @@ -42,16 +57,6 @@ function descendantCount(node: Renderable): number { return n } -/** One streamed assistant turn = a few text parts (a realistic multi-node subtree). */ -function pushTurn(store: ReturnType, i: number): void { - store.pushUser(`user message ${i}: please summarize the situation in a few lines`) - store.apply({ type: 'message.start' }) - store.apply({ type: 'message.delta', payload: { text: `Sure — point one for turn ${i}. ` } }) - store.apply({ type: 'message.delta', payload: { text: `Here is point two with a bit more detail. ` } }) - store.apply({ type: 'message.delta', payload: { text: `And a closing point three for turn ${i}.` } }) - store.apply({ type: 'message.complete' }) -} - async function main(): Promise { const store = createSessionStore() store.apply({ type: 'gateway.ready' }) @@ -67,8 +72,9 @@ async function main(): Promise { await setup.renderOnce() await setup.flush() - // header: pad to fixed widths for a readable table - process.stdout.write(`\n=== mem-bench cap=${cap} total=${TOTAL} sampleEvery=${SAMPLE_EVERY} ===\n`) + process.stdout.write( + `\n=== mem-bench (REALISTIC fixture) cap=${cap} total=${TOTAL} sampleEvery=${SAMPLE_EVERY} ===\n` + ) process.stdout.write( 'pushes | msgs | rss(MB) | heapUsed(MB) | external(MB) | arrayBuf(MB) | activeAllocs | renderables\n' ) @@ -97,12 +103,65 @@ async function main(): Promise { } await sample(0) - for (let i = 1; i <= TOTAL; i++) { - pushTurn(store, i) - if (i % SAMPLE_EVERY === 0) await sample(i) + // Pump turns inline, sampling each time the cumulative produced-row count crosses + // a SAMPLE_EVERY boundary. Sampling is async (renderOnce/flush/gc), so it lives + // in the loop rather than a sync callback. Mounting is synchronous in Solid, so a + // render pass at the boundary reflects the just-pushed turns. + let pushed = 0 + let nextSample = SAMPLE_EVERY + let turn = 0 + while (pushed < TOTAL) { + applyTurn(store, turn) + pushed += rowsPerTurn(turn) + turn++ + if (pushed >= nextSample) { + await sample(Math.min(pushed, TOTAL)) + while (nextSample <= pushed) nextSample += SAMPLE_EVERY + } } + // Tear down the live push tree BEFORE the resume path so its mounted nodes don't + // pollute the process-wide RSS the resume sample reads. (The renderable COUNT is + // already isolated per-renderer-root, but RSS is process-global.) + store.clearTranscript() setup.renderer.destroy() + Bun.gc(true) + + // ── RESUME PATH: build the full settled fixture and commitSnapshot it (the + // resume hydrate path). Verifies the slice-before-set fix bounds resume mounting + // to ≤ cap — mounting 8000 settled msgs at cap=1500 should mount ~1500-worth of + // rows, NOT 8000-worth. Done on a FRESH store + renderer so the live-push history + // above doesn't skew the count. + const resumeStore = createSessionStore() + resumeStore.apply({ type: 'gateway.ready' }) + const resumeSetup = await testRender( + () => ( + resumeStore.state.theme}> + + + ), + { width: 100, height: 40, exitOnCtrlC: false } + ) + await resumeSetup.renderOnce() + await resumeSetup.flush() + + const fullFixture = materialize(TOTAL) + resumeStore.beginBuffer() + resumeStore.commitSnapshot(fullFixture) + await resumeSetup.renderOnce() + await resumeSetup.flush() + Bun.gc(true) + const rm = process.memoryUsage() + const ralloc = lib.getAllocatorStats() + const rrenderables = descendantCount(resumeSetup.renderer.root) + process.stdout.write('\n--- resume path (commitSnapshot of the full fixture) ---\n') + process.stdout.write(`fixture msgs built : ${fullFixture.length}\n`) + process.stdout.write(`mounted msgs (cap) : ${resumeStore.state.messages.length}\n`) + process.stdout.write(`mounted renderables: ${rrenderables}\n`) + process.stdout.write(`activeAllocations : ${ralloc.activeAllocations}\n`) + process.stdout.write(`rss(MB) : ${MB(rm.rss)}\n`) + + resumeSetup.renderer.destroy() } await main()