From a4890569a3705f3ee44462cf6a0b60ace9656279 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 18 Jul 2026 21:14:05 -0400 Subject: [PATCH 1/3] perf(tui): render streamed markdown incrementally per block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StreamingMd previously split in-flight text into one memoized stable-prefix plus a re-parsed tail. Every time the stable boundary advanced, the prefix string changed, its memo key missed, and the entire prefix re-tokenized — O(blocks^2) parse work across a long reply — and finding the boundary rescanned fence state from position 0 on every delta. Replace the monolithic prefix with an append-only array of settled top-level blocks, each rendered as its own memoized on text that never changes once committed (every block parses exactly once for the life of the stream), plus a persistent scanner that keeps fence/math open-state and scan position across deltas so each delta only scans the newly arrived complete lines. Boundaries stay at "\n\n" outside code/math fences; partial trailing lines stay in the tail until their newline arrives so a growing "```" can't be misjudged. Replaying a newline-terminated block-heavy stream at width 80 through a real Ink render (one process per strategy so the parse LRU and GC pressure can't cross-contaminate): | Blocks | Appends | naive full Md | monolithic prefix | per-block | |--------|---------|---------------|-------------------|-----------| | 32 | 135 | 279.7 ms | 221.1 ms | 104.7 ms | | 128 | 543 | 3.03 s | 2.39 s | 317.8 ms | | 512 | 2175 | 56.28 s | 35.14 s | 3.05 s | Remaining per-block cost is Ink layout of the growing tree, not parsing. Bench: ui-tui/scripts/bench-streaming-md.tsx. --- ui-tui/scripts/bench-streaming-md.tsx | 276 +++++++++++++++++ .../src/__tests__/streamingMarkdown.test.ts | 236 +++++++++++++-- ui-tui/src/components/streamingMarkdown.tsx | 282 ++++++++++-------- 3 files changed, 657 insertions(+), 137 deletions(-) create mode 100644 ui-tui/scripts/bench-streaming-md.tsx diff --git a/ui-tui/scripts/bench-streaming-md.tsx b/ui-tui/scripts/bench-streaming-md.tsx new file mode 100644 index 000000000000..30a9127ac161 --- /dev/null +++ b/ui-tui/scripts/bench-streaming-md.tsx @@ -0,0 +1,276 @@ +// Benchmark: streamed-markdown render strategies for the TUI. +// +// Replays a newline-terminated, block-heavy synthetic stream at width 80 +// through a real Ink render (renderSync + rerender per update) and compares: +// +// naive — per update (re-tokenizes everything) +// monolithic — the previous StreamingMd: one memoized stable-prefix +// plus a tail . O(total) fence scan per update and a +// full prefix re-parse every time the boundary advances. +// per-block — current StreamingMd: append-only settled block array + +// incremental scanner. Each block parses exactly once. +// +// Each strategy/size pair runs in its OWN child process (orchestrator mode) +// so the parse-tree LRU in markdown.tsx and GC pressure from one strategy +// can't distort another's numbers. Each run also gets a unique text salt and +// a fresh theme object (its own WeakMap cache bucket). +// +// Run: npx tsx scripts/bench-streaming-md.tsx +// Single case: npx tsx scripts/bench-streaming-md.tsx + +import { execFileSync } from 'child_process' +import { PassThrough } from 'stream' + +import { Box, renderSync } from '@hermes/ink' +import React, { memo, useRef } from 'react' + +import { Md } from '../src/components/markdown.js' +import { StreamingMd } from '../src/components/streamingMarkdown.js' +import { DEFAULT_THEME } from '../src/theme.js' + +// ---- previous implementation (monolithic stable prefix), for comparison ---- + +const fenceOpenAt = (s: string, end: number) => { + let codeOpen = false + let mathOpen = false + let mathOpener: '$$' | '\\[' | null = null + let i = 0 + + while (i < end) { + const nl = s.indexOf('\n', i) + const lineEnd = nl < 0 || nl > end ? end : nl + const line = s.slice(i, lineEnd).trim() + + if (/^(?:`{3,}|~{3,})/.test(line)) { + codeOpen = !codeOpen + } else if (!codeOpen) { + if (!mathOpen && /^\$\$/.test(line)) { + if (!(line.length >= 4 && /\$\$$/.test(line))) { + mathOpen = true + mathOpener = '$$' + } + } else if (!mathOpen && /^\\\[/.test(line)) { + if (!/\\\]$/.test(line)) { + mathOpen = true + mathOpener = '\\[' + } + } else if (mathOpen && mathOpener === '$$' && /\$\$$/.test(line)) { + mathOpen = false + mathOpener = null + } else if (mathOpen && mathOpener === '\\[' && /\\\]$/.test(line)) { + mathOpen = false + mathOpener = null + } + } + + if (nl < 0 || nl >= end) { + break + } + + i = nl + 1 + } + + return codeOpen || mathOpen +} + +const findStableBoundaryOld = (text: string) => { + let idx = text.length + + while (idx > 0) { + const boundary = text.lastIndexOf('\n\n', idx - 1) + + if (boundary < 0) { + return -1 + } + + const splitAt = boundary + 2 + + if (!fenceOpenAt(text, splitAt)) { + return splitAt + } + + idx = boundary + } + + return -1 +} + +const MonolithicStreamingMd = memo(function MonolithicStreamingMd({ + t, + text +}: { + t: typeof DEFAULT_THEME + text: string +}) { + const stablePrefixRef = useRef('') + + if (!text.startsWith(stablePrefixRef.current)) { + stablePrefixRef.current = '' + } + + const boundary = findStableBoundaryOld(text) + + if (boundary > stablePrefixRef.current.length) { + stablePrefixRef.current = text.slice(0, boundary) + } + + const stablePrefix = stablePrefixRef.current + const unstableSuffix = text.slice(stablePrefix.length) + + if (!stablePrefix) { + return + } + + if (!unstableSuffix) { + return + } + + return ( + + + + + ) +}) + +// ---- synthetic stream ---- + +const makeBlocks = (count: number, salt: string) => { + const blocks: string[] = [] + + for (let i = 0; i < count; i++) { + switch (i % 4) { + case 0: + blocks.push(`Paragraph ${salt}-${i} explaining step ${i} with **bold** and \`code\` inline.\n`) + + break + + case 1: + blocks.push(`- item one ${salt}-${i}\n- item two with _emphasis_\n- item three\n`) + + break + + case 2: + blocks.push(`\`\`\`ts\nconst v${i} = compute${salt}(${i})\nif (v${i} > 0) {\n emit(v${i})\n}\n\`\`\`\n`) + + break + + default: + blocks.push(`### Heading ${salt}-${i}\n\nSome follow-up prose for section ${i}.\n`) + } + } + + return blocks +} + +// Newline-terminated updates: the stream grows one line per rerender. +const makeUpdates = (blocks: string[]) => { + const full = blocks.join('\n') + const updates: string[] = [] + + let pos = 0 + + while (pos < full.length) { + const nl = full.indexOf('\n', pos) + + pos = nl < 0 ? full.length : nl + 1 + updates.push(full.slice(0, pos)) + } + + return updates +} + +const nullStream = () => { + const s = new PassThrough() + + Object.assign(s, { columns: 80, isTTY: false, rows: 24 }) + s.on('data', () => {}) + + return s +} + +const bench = (label: string, updates: string[], node: (t: typeof DEFAULT_THEME, text: string) => React.ReactNode) => { + // Fresh theme per run → fresh (collectable) mdCache bucket. + const runTheme = { ...DEFAULT_THEME } + + const instance = renderSync(node(runTheme, ''), { + patchConsole: false, + stderr: nullStream() as unknown as NodeJS.WriteStream, + stdin: nullStream() as unknown as NodeJS.ReadStream, + stdout: nullStream() as unknown as NodeJS.WriteStream + }) + + const start = performance.now() + + let n = 0 + + for (const text of updates) { + instance.rerender(node(runTheme, text)) + + // Something in the render path emits performance measures; unbounded, + // the entry buffer itself becomes a memory leak over thousands of + // rerenders and skews long runs. + if (++n % 64 === 0) { + performance.clearMeasures() + performance.clearMarks() + } + } + + const elapsed = performance.now() - start + + instance.unmount() + instance.cleanup() + + return { elapsed, label } +} + +const STRATEGIES = { + monolithic: (t: typeof DEFAULT_THEME, text: string) => , + naive: (t: typeof DEFAULT_THEME, text: string) => , + 'per-block': (t: typeof DEFAULT_THEME, text: string) => +} as const + +const [strategyArg, sizeArg] = process.argv.slice(2) + +if (strategyArg) { + // Child mode: run one strategy/size and print elapsed ms as JSON. + const size = Number(sizeArg) + const updates = makeUpdates(makeBlocks(size, `${strategyArg}${size}`)) + const { elapsed } = bench(strategyArg, updates, STRATEGIES[strategyArg as keyof typeof STRATEGIES]) + + console.log(JSON.stringify({ appends: updates.length, elapsed })) +} else { + // Orchestrator mode: one child process per strategy/size. + const sizes = [32, 128, 512] + + const run = (strategy: string, size: number): { appends: number; elapsed: number } => { + const out = execFileSync('npx', ['tsx', 'scripts/bench-streaming-md.tsx', strategy, String(size)], { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=8192' }, + timeout: 3_600_000 + }) + + return JSON.parse(out.trim().split('\n').at(-1)!) + } + + const fmt = (ms: number) => (ms >= 1000 ? `${(ms / 1000).toFixed(2)} s` : `${ms.toFixed(1)} ms`) + + console.log( + '| Blocks | Append calls | naive (full Md) | monolithic prefix | per-block (new) | new vs naive | new vs monolithic |' + ) + console.log( + '|--------|--------------|-----------------|-------------------|-----------------|--------------|-------------------|' + ) + + for (const size of sizes) { + const naive = run('naive', size) + const mono = run('monolithic', size) + const perBlock = run('per-block', size) + + console.log( + `| ${size} | ${perBlock.appends} | ${fmt(naive.elapsed)} | ${fmt(mono.elapsed)} | ${fmt(perBlock.elapsed)} | ${( + naive.elapsed / perBlock.elapsed + ).toFixed(1)}x | ${(mono.elapsed / perBlock.elapsed).toFixed(1)}x |` + ) + } +} diff --git a/ui-tui/src/__tests__/streamingMarkdown.test.ts b/ui-tui/src/__tests__/streamingMarkdown.test.ts index 1a825a62f127..0c134a0c0b9e 100644 --- a/ui-tui/src/__tests__/streamingMarkdown.test.ts +++ b/ui-tui/src/__tests__/streamingMarkdown.test.ts @@ -1,19 +1,48 @@ +import { PassThrough } from 'stream' + +import { Box, renderSync } from '@hermes/ink' +import React from 'react' import { describe, expect, it } from 'vitest' -import { findStableBoundary } from '../components/streamingMarkdown.js' -// We test the pure boundary logic by rendering the component's ref -// behaviour through repeated calls. Since React isn't being rendered here, -// we reach into the module to test findStableBoundary via its exported -// behaviour — but the pure helper isn't exported. So test the component's -// observable output: pass sequential text values and verify the stable -// prefix never retreats. -// -// Strategy: mount StreamingMd in isolation and observe which -// instances it renders (by text prop). Without a DOM renderer that's -// heavy, so we validate the helper behaviour by directly invoking the -// fence/boundary logic via a re-exported surface. +import { Md } from '../components/markdown.js' +import { advanceScan, createScanState, findStableBoundary } from '../components/streamingMarkdown.js' +import { stripAnsi } from '../lib/text.js' import { DEFAULT_THEME } from '../theme.js' +const BEL = String.fromCharCode(7) +const ESC = String.fromCharCode(27) +const CSI_RE = new RegExp(`${ESC}\\[[0-?]*[ -/]*[@-~]`, 'g') +const OSC_RE = new RegExp(`${ESC}\\][\\s\\S]*?(?:${BEL}|${ESC}\\\\)`, 'g') + +const renderPlain = (node: React.ReactNode) => { + const stdout = new PassThrough() + const stdin = new PassThrough() + const stderr = new PassThrough() + let output = '' + + Object.assign(stdout, { columns: 80, isTTY: false, rows: 24 }) + Object.assign(stdin, { isTTY: false }) + Object.assign(stderr, { isTTY: false }) + stdout.on('data', chunk => { + output += chunk.toString() + }) + + const instance = renderSync(node, { + patchConsole: false, + stderr: stderr as NodeJS.WriteStream, + stdin: stdin as NodeJS.ReadStream, + stdout: stdout as NodeJS.WriteStream + }) + + instance.unmount() + instance.cleanup() + + return output + .replace(OSC_RE, '') + .split('\n') + .map(line => stripAnsi(line).replace(CSI_RE, '').trimEnd()) +} + describe('findStableBoundary', () => { it('returns -1 when no blank line exists yet', () => { expect(findStableBoundary('partial line with no newline yet')).toBe(-1) @@ -111,11 +140,182 @@ describe('findStableBoundary', () => { }) }) -describe('streaming theme assumption', () => { - it('theme is exportable (component import sanity check)', () => { - // Sanity that the theme we pass doesn't change shape. Component import - // already happens above — this is a smoke test that the module graph - // for streamingMarkdown wires up without cycles. - expect(DEFAULT_THEME.color.accent).toBeTruthy() +// A corpus exercising every construct the boundary scanner must respect: +// paragraphs, fenced code (with blank lines and $$ bait inside), display +// math ($$ and \[), setext headings, tables, lists, quotes, headings. +const CORPUS = [ + 'Intro paragraph explaining the plan in some detail.\n', + '\nSection Title\n=============\n', + '\nA paragraph before code.\n', + '\n```ts\nconst a = 1\n\nconst b = 2\n// $$ not math $$\n```\n', + '\nBetween-blocks narration.\n', + '\n$$\nE = mc^2\n\n\\sum_i x_i\n$$\n', + '\n- item one\n- item two\n\n1. first\n2. second\n', + '\n| a | b |\n|---|---|\n| 1 | 2 |\n', + '\n> quoted wisdom\n> second line\n', + '\n\\[\nx^2 + y^2 = z^2\n\\]\n', + '\n## Closing heading\n', + '\nFinal paragraph without a trailing newline' +].join('') + +const mulberry32 = (seed: number) => () => { + seed |= 0 + seed = (seed + 0x6d2b79f5) | 0 + + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed) + + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 +} + +describe('advanceScan (incremental scanner)', () => { + it('reconstructs the input exactly: blocks + tail === text', () => { + const state = createScanState() + + advanceScan(CORPUS, state) + + expect(state.blocks.join('')).toHaveLength(state.settledLen) + expect(state.blocks.join('') + CORPUS.slice(state.settledLen)).toBe(CORPUS) + expect(state.blocks.length).toBeGreaterThan(5) + }) + + it('produces identical blocks fed incrementally at arbitrary cut points', () => { + const oneShot = createScanState() + + advanceScan(CORPUS, oneShot) + + for (let seed = 1; seed <= 8; seed++) { + const rand = mulberry32(seed) + const state = createScanState() + + let pos = 0 + + while (pos < CORPUS.length) { + pos = Math.min(CORPUS.length, pos + 1 + Math.floor(rand() * 7)) + + const prevBlocks = state.blocks.length + + advanceScan(CORPUS.slice(0, pos), state) + + // Append-only: previously committed blocks never change. + expect(state.blocks.length).toBeGreaterThanOrEqual(prevBlocks) + } + + expect(state.blocks).toEqual(oneShot.blocks) + expect(state.settledLen).toBe(oneShot.settledLen) + } + }) + + it('is idempotent when called again with the same text', () => { + const state = createScanState() + + advanceScan(CORPUS, state) + + const blocks = [...state.blocks] + + advanceScan(CORPUS, state) + + expect(state.blocks).toEqual(blocks) + }) + + it('holds a partial trailing line in the tail even if it looks fence-like', () => { + // "``" could grow into "```ts" — the scanner must not judge the line + // until its newline arrives. + const state = createScanState() + + advanceScan('para\n\n``', state) + + expect(state.blocks).toEqual(['para\n\n']) + expect(state.codeOpen).toBe(false) + + advanceScan('para\n\n```ts\ncode\n\nstill code', state) + + // The blank line inside the now-open fence must not commit a block. + expect(state.blocks).toEqual(['para\n\n']) + expect(state.codeOpen).toBe(true) + + advanceScan('para\n\n```ts\ncode\n\nstill code\n```\n\nafter\n\n', state) + + expect(state.blocks).toEqual(['para\n\n', '```ts\ncode\n\nstill code\n```\n\n', 'after\n\n']) + expect(state.codeOpen).toBe(false) + }) + + it('does not commit whitespace-only blocks on 3+ newline runs', () => { + const state = createScanState() + + advanceScan('alpha\n\n\n\nbeta\n\n', state) + + expect(state.blocks).toEqual(['alpha\n\n', '\n\nbeta\n\n']) + expect(state.blocks.join('')).toBe('alpha\n\n\n\nbeta\n\n') + }) + + it('keeps a setext heading contiguous with its paragraph', () => { + // A setext underline attaches to the line above; the only committed + // boundary is the blank line, so the pair can never be torn apart or + // retroactively merged (the desktop splitter needed a fix for this, + // #67176 — blank-line boundaries are immune by construction). + const state = createScanState() + + advanceScan('Title\n', state) + advanceScan('Title\n====\n', state) + advanceScan('Title\n====\n\nbody\n\n', state) + + expect(state.blocks).toEqual(['Title\n====\n\n', 'body\n\n']) + }) +}) + +describe('StreamingMd rendering equivalence', () => { + it('settled blocks + tail render identically to one combined Md', () => { + const state = createScanState() + + advanceScan(CORPUS, state) + + const tail = CORPUS.slice(state.settledLen) + const t = DEFAULT_THEME + + const split = renderPlain( + React.createElement( + Box, + { flexDirection: 'column' }, + ...state.blocks.map((block, i) => React.createElement(Md, { key: i, t, text: block })), + tail ? React.createElement(Md, { key: 'tail', t, text: tail }) : null + ) + ) + + const combined = renderPlain(React.createElement(Md, { t, text: CORPUS })) + + expect(split).toEqual(combined) + }) + + it('renders split/combined identically at every streamed step', () => { + const rand = mulberry32(42) + const state = createScanState() + const t = DEFAULT_THEME + + let pos = 0 + + while (pos < CORPUS.length) { + pos = Math.min(CORPUS.length, pos + 24 + Math.floor(rand() * 200)) + + const text = CORPUS.slice(0, pos) + + advanceScan(text, state) + + const tail = text.slice(state.settledLen) + + const split = renderPlain( + React.createElement( + Box, + { flexDirection: 'column' }, + ...state.blocks.map((block, i) => React.createElement(Md, { key: i, t, text: block })), + tail ? React.createElement(Md, { key: 'tail', t, text: tail }) : null + ) + ) + + const combined = renderPlain(React.createElement(Md, { t, text })) + + expect(split).toEqual(combined) + } }) }) diff --git a/ui-tui/src/components/streamingMarkdown.tsx b/ui-tui/src/components/streamingMarkdown.tsx index 786a38124611..1acda990b781 100644 --- a/ui-tui/src/components/streamingMarkdown.tsx +++ b/ui-tui/src/components/streamingMarkdown.tsx @@ -4,29 +4,57 @@ // on every stream delta. At 20-char batches over a 3 KB response that's 150 // full re-parses. // -// This splits `text` at the last stable top-level block boundary (blank -// line outside a fenced code span) into: -// stablePrefix — passed to an inner , memoized on its exact text -// value. During the turn, the prefix only grows monotonically, -// so its memo key matches the previous render and React -// reuses the cached subtree — zero re-tokenization. -// unstableSuffix — the in-flight block(s). A separate re-parses just -// this tail on every delta (O(unstable length) vs. -// O(total length)). +// The previous incarnation split `text` at the last stable top-level block +// boundary into ONE memoized stable-prefix plus a re-parsed tail . +// That removed the per-delta cost but kept a per-block cliff: every time the +// boundary advanced, the prefix string changed, its memo key missed, and the +// ENTIRE prefix re-tokenized from scratch — O(blocks²) work across a long +// reply, plus an O(total) fence scan on every delta to find the boundary. // -// The boundary is stored in a ref so it only advances — idempotent under -// StrictMode double-render. Component unmounts between turns (isStreaming -// flips off → message moves to history and renders via directly), so -// the ref resets naturally. +// This version is fully incremental: +// settled blocks — an append-only array of top-level block strings. Each +// block renders as its own , memoized on its exact +// text, which never changes once committed. Every block +// is tokenized exactly once for the life of the stream. +// unstable tail — the in-flight block(s) after the last committed +// boundary. A single re-parses just this tail on +// every delta (O(tail) vs. O(total)). +// scanner state — fence/math open-state and scan position persist across +// deltas in a ref, so each delta only scans the newly +// arrived complete lines instead of the whole text. // -// Layout: the two subtrees MUST render stacked (column). The parent +// A block boundary is a literal "\n\n" outside any fenced code or display +// math block. Only complete lines (terminated by "\n") are scanned — a +// partial trailing line could still grow into a fence opener, so it stays +// in the tail until its newline arrives. Blank-line boundaries also make +// the split immune to retroactive block merges (e.g. a setext underline +// only attaches to the contiguous paragraph above it — it cannot reach +// across a committed "\n\n"). +// +// The scanner treats an unmatched `$$` / `\[` opener as open forever. This +// is INTENTIONALLY more conservative than `markdown.tsx`'s parser, which +// falls back to paragraph rendering when an opener has no closer. The +// renderer can do that safely because it always sees the full text on +// every call. The streaming chunker cannot — once a block is committed it +// is frozen, so prematurely deciding "this `$$` is just prose" would +// permanently commit a paragraph rendering that becomes wrong the instant +// the closer streams in. Parking the boundary keeps everything from the +// opener onward in the mutable tail until the closer arrives (or the +// stream ends and the non-streaming takes over, at which point the +// renderer's fallback kicks in correctly). +// +// State is stored in a ref and only ever advances — idempotent under +// StrictMode double-render. The component unmounts between turns +// (isStreaming flips off → message moves to history and renders via a +// single ), so the ref resets naturally. If `text` stops extending the +// scanned prefix anyway (turn reuse, or the bounded live-render window +// trimming the front of a very long reply), the scanner resets and the +// per-text LRU inside still absorbs most of the re-parse. +// +// Layout: the subtrees MUST render stacked (column). The parent // container in messageLine.tsx is a default `flexDirection: 'row'` Box -// (Ink's default), so returning a bare Fragment of two siblings -// laid them out side-by-side — producing the "two jumbled columns while -// streaming" rendering bug. Wrapping in a flexDirection="column" Box -// here localizes the fix to the streaming path; the non-streaming -// already returns its own column Box, so its single-child case was never -// affected. +// (Ink's default), so returning a bare Fragment of sibling s would lay +// them out side-by-side — the "two jumbled columns while streaming" bug. import { Box } from '@hermes/ink' import { memo, useRef } from 'react' @@ -35,133 +63,149 @@ import type { Theme } from '../theme.js' import { Md } from './markdown.js' -// Count ``` / ~~~ AND `$$` / `\[…\]` fence toggles in `s` up to `end`. Odd -// = currently inside a fenced block; splitting the prefix there would -// orphan the fence and let the unstable suffix re-render as broken -// markdown. Math fences only toggle when the code fence is closed so -// snippets like ` ```\n$$x$$\n``` ` (math example inside a code block) -// don't double-count. A `$$x$$` line that opens AND closes on its own -// produces zero net toggles; that's `len >= 4` plus `endsDollar`. -// -// NB: this is INTENTIONALLY more conservative than `markdown.tsx`'s -// parser, which falls back to paragraph rendering when an `$$` opener -// has no matching closer. The renderer can do that safely because it -// always sees the full text on every call. The streaming chunker -// cannot — once a chunk is committed to the monotonic stable prefix it -// is frozen, so prematurely deciding "this `$$` is just prose" would -// permanently commit a paragraph rendering that becomes wrong the -// instant the closer streams in. Treating any unmatched `$$` opener -// as still-open keeps the boundary parked behind it until the closer -// arrives (or the stream ends and the non-streaming `` takes over, -// at which point the renderer's fallback kicks in correctly). -const fenceOpenAt = (s: string, end: number) => { - let codeOpen = false - let mathOpen = false - let mathOpener: '$$' | '\\[' | null = null - let i = 0 +export interface StreamScanState { + /** Settled top-level block strings, in order. Append-only. */ + blocks: string[] + /** True while inside an unclosed ``` / ~~~ fence at the scan position. */ + codeOpen: boolean + /** Non-null while inside an unclosed display-math block at the scan position. */ + mathOpener: '$$' | '\\[' | null + /** + * The prefix of the text whose complete lines have been scanned — + * text.slice(0, scanPos), with scanPos === scanned.length. Kept as a + * string (not just an index) so the reset guard can verify the scanned + * region still matches, fence state included. + */ + scanned: string + /** Length of the committed prefix — blocks.join('').length. */ + settledLen: number +} - while (i < end) { - const nl = s.indexOf('\n', i) - const lineEnd = nl < 0 || nl > end ? end : nl - const line = s.slice(i, lineEnd).trim() +export const createScanState = (): StreamScanState => ({ + blocks: [], + codeOpen: false, + mathOpener: null, + scanned: '', + settledLen: 0 +}) - if (/^(?:`{3,}|~{3,})/.test(line)) { - codeOpen = !codeOpen - } else if (!codeOpen) { - if (!mathOpen && /^\$\$/.test(line)) { - const isSingleLine = line.length >= 4 && /\$\$$/.test(line) +// Apply one complete (newline-terminated) line to the fence/math state. +// Mirrors the toggle rules the old fenceOpenAt() used: ``` / ~~~ toggle the +// code fence; `$$` / `\[` open display math only when the line doesn't also +// close it; closers only count when the matching opener is pending. Math +// markers inside an open code fence are ignored (a `$$` example in a code +// block must not open math). +const applyLine = (state: StreamScanState, line: string) => { + if (/^(?:`{3,}|~{3,})/.test(line)) { + state.codeOpen = !state.codeOpen - if (!isSingleLine) { - mathOpen = true - mathOpener = '$$' - } - } else if (!mathOpen && /^\\\[/.test(line)) { - const isSingleLine = /\\\]$/.test(line) + return + } - if (!isSingleLine) { - mathOpen = true - mathOpener = '\\[' - } - } else if (mathOpen && mathOpener === '$$' && /\$\$$/.test(line)) { - mathOpen = false - mathOpener = null - } else if (mathOpen && mathOpener === '\\[' && /\\\]$/.test(line)) { - mathOpen = false - mathOpener = null + if (state.codeOpen) { + return + } + + if (!state.mathOpener) { + if (/^\$\$/.test(line)) { + const singleLine = line.length >= 4 && /\$\$$/.test(line) + + if (!singleLine) { + state.mathOpener = '$$' + } + } else if (/^\\\[/.test(line)) { + const singleLine = /\\\]$/.test(line) + + if (!singleLine) { + state.mathOpener = '\\[' } } + } else if (state.mathOpener === '$$' && /\$\$$/.test(line)) { + state.mathOpener = null + } else if (state.mathOpener === '\\[' && /\\\]$/.test(line)) { + state.mathOpener = null + } +} - if (nl < 0 || nl >= end) { +// Consume newly arrived COMPLETE lines from the scan position, committing a +// block at every "\n\n" boundary reached outside fences. Whitespace-only +// candidates (runs of 3+ newlines) are left for the next block rather than +// committed as empty s. Mutates `state`; calling again with the same +// text is a no-op (idempotent). +export const advanceScan = (text: string, state: StreamScanState) => { + const start = state.scanned.length + + let i = start + + while (i < text.length) { + const nl = text.indexOf('\n', i) + + if (nl < 0) { + // Partial trailing line — could still become a fence opener. Leave it + // (and its whole block) in the unstable tail. break } + if (nl === i) { + // Empty line. If it's the second half of a "\n\n" pair (i.e. not the + // very first character) and no fence is open, the text before it is a + // settled top-level block. + if (i > 0 && !state.codeOpen && !state.mathOpener) { + const block = text.slice(state.settledLen, nl + 1) + + if (/\S/.test(block)) { + state.blocks.push(block) + state.settledLen = nl + 1 + } + } + } else { + applyLine(state, text.slice(i, nl).trim()) + } + i = nl + 1 } - return codeOpen || mathOpen + if (i > start) { + state.scanned += text.slice(start, i) + } } -// Find the last "\n\n" boundary before `end` that is OUTSIDE a fenced code -// block. Returns the index AFTER the second newline (start of the next -// block), or -1 if no safe boundary exists yet. +// Compatibility shim over the incremental scanner: index just past the last +// committed block boundary in `text`, or -1 when nothing has settled yet. +// Kept for boundary-semantics tests; StreamingMd itself keeps scanner state +// across deltas instead of recomputing from scratch. export const findStableBoundary = (text: string) => { - let idx = text.length + const state = createScanState() - while (idx > 0) { - const boundary = text.lastIndexOf('\n\n', idx - 1) + advanceScan(text, state) - if (boundary < 0) { - return -1 - } - - // Boundary candidate: end of stable prefix is boundary + 2 (start of - // next block). Check fence balance up to that point. - const splitAt = boundary + 2 - - if (!fenceOpenAt(text, splitAt)) { - return splitAt - } - - idx = boundary - } - - return -1 + return state.settledLen > 0 ? state.settledLen : -1 } export const StreamingMd = memo(function StreamingMd({ cols, compact, t, text }: StreamingMdProps) { - const stablePrefixRef = useRef('') + const scanRef = useRef(createScanState()) - // Reset if the text no longer starts with our recorded prefix (defensive; - // normally the component unmounts between turns so this shouldn't trigger). - if (!text.startsWith(stablePrefixRef.current)) { - stablePrefixRef.current = '' + let state = scanRef.current + + // Reset when the text no longer extends the scanned prefix: a new turn + // reusing the mounted component, or boundedLiveRenderText trimming the + // front of a very long reply. Comparing the full scanned region (not just + // the settled blocks) guarantees the persisted fence state is still valid. + if (!text.startsWith(state.scanned)) { + state = scanRef.current = createScanState() } - const boundary = findStableBoundary(text) + advanceScan(text, state) - // Only advance the prefix — never retreat. The boundary math looks at the - // FULL text each call; if it returns a larger index than before, we grow - // the cached prefix. Monotonic growth makes the memo key stable across - // deltas (identical string → same subtree → no re-render). - if (boundary > stablePrefixRef.current.length) { - stablePrefixRef.current = text.slice(0, boundary) - } - - const stablePrefix = stablePrefixRef.current - const unstableSuffix = text.slice(stablePrefix.length) - - if (!stablePrefix) { - return - } - - if (!unstableSuffix) { - return - } + const tail = text.slice(state.settledLen) return ( - - + {state.blocks.map((block, i) => ( + + ))} + + {tail ? : null} ) }) From fbabdfbe15b1a425745b3e633d6f4c4114e3cbb4 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 18 Jul 2026 22:16:23 -0400 Subject: [PATCH 2/3] bench(tui): add per-append timing series mode to streaming-md bench --- ui-tui/scripts/bench-streaming-md.tsx | 32 ++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/ui-tui/scripts/bench-streaming-md.tsx b/ui-tui/scripts/bench-streaming-md.tsx index 30a9127ac161..3655969db92b 100644 --- a/ui-tui/scripts/bench-streaming-md.tsx +++ b/ui-tui/scripts/bench-streaming-md.tsx @@ -189,7 +189,12 @@ const nullStream = () => { return s } -const bench = (label: string, updates: string[], node: (t: typeof DEFAULT_THEME, text: string) => React.ReactNode) => { +const bench = ( + label: string, + updates: string[], + node: (t: typeof DEFAULT_THEME, text: string) => React.ReactNode, + captureSeries = false +) => { // Fresh theme per run → fresh (collectable) mdCache bucket. const runTheme = { ...DEFAULT_THEME } @@ -200,13 +205,20 @@ const bench = (label: string, updates: string[], node: (t: typeof DEFAULT_THEME, stdout: nullStream() as unknown as NodeJS.WriteStream }) + const times: number[] = [] const start = performance.now() let n = 0 for (const text of updates) { + const t0 = captureSeries ? performance.now() : 0 + instance.rerender(node(runTheme, text)) + if (captureSeries) { + times.push(performance.now() - t0) + } + // Something in the render path emits performance measures; unbounded, // the entry buffer itself becomes a memory leak over thousands of // rerenders and skews long runs. @@ -221,7 +233,7 @@ const bench = (label: string, updates: string[], node: (t: typeof DEFAULT_THEME, instance.unmount() instance.cleanup() - return { elapsed, label } + return { elapsed, label, times } } const STRATEGIES = { @@ -232,7 +244,21 @@ const STRATEGIES = { const [strategyArg, sizeArg] = process.argv.slice(2) -if (strategyArg) { +if (strategyArg === 'series') { + // Series mode: per-append render times for one strategy/size, as JSON. + // Usage: npx tsx scripts/bench-streaming-md.tsx series + const [, seriesStrategy, seriesSize] = process.argv.slice(2) + const size = Number(seriesSize) + const updates = makeUpdates(makeBlocks(size, `${seriesStrategy}${size}`)) + const { elapsed, times } = bench( + seriesStrategy!, + updates, + STRATEGIES[seriesStrategy as keyof typeof STRATEGIES], + true + ) + + console.log(JSON.stringify({ appends: updates.length, elapsed, times })) +} else if (strategyArg) { // Child mode: run one strategy/size and print elapsed ms as JSON. const size = Number(sizeArg) const updates = makeUpdates(makeBlocks(size, `${strategyArg}${size}`)) From 24b44d56a3f989dbdfdceaf346ec07e3d8e9e281 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 18 Jul 2026 23:42:10 -0400 Subject: [PATCH 3/3] refactor(tui): tighten StreamingMd comments and fence folding Condense the header rationale to the load-bearing invariants and inline the single-line math-fence guards. No behavior change; scanner logic and public surface (createScanState / advanceScan / findStableBoundary) unchanged. --- ui-tui/src/components/streamingMarkdown.tsx | 144 +++++++------------- 1 file changed, 46 insertions(+), 98 deletions(-) diff --git a/ui-tui/src/components/streamingMarkdown.tsx b/ui-tui/src/components/streamingMarkdown.tsx index 1acda990b781..992bf9a67b65 100644 --- a/ui-tui/src/components/streamingMarkdown.tsx +++ b/ui-tui/src/components/streamingMarkdown.tsx @@ -1,60 +1,32 @@ // StreamingMd — incremental markdown renderer for in-flight assistant text. // -// Naive approach (render ) re-tokenizes the entire message -// on every stream delta. At 20-char batches over a 3 KB response that's 150 -// full re-parses. +// Rendering per delta re-tokenizes the whole message every +// time (O(total) × deltas). The prior stable-prefix split fixed the per-delta +// cost but not the per-block cliff: each advanced boundary re-tokenized the +// entire prefix from scratch — O(blocks²) — plus an O(total) fence rescan. // -// The previous incarnation split `text` at the last stable top-level block -// boundary into ONE memoized stable-prefix plus a re-parsed tail . -// That removed the per-delta cost but kept a per-block cliff: every time the -// boundary advanced, the prefix string changed, its memo key missed, and the -// ENTIRE prefix re-tokenized from scratch — O(blocks²) work across a long -// reply, plus an O(total) fence scan on every delta to find the boundary. +// This is fully incremental. A forward scanner keeps fence/math state + scan +// position in a ref across deltas, so each delta touches only newly-arrived +// complete lines. Settled top-level blocks (split on "\n\n" outside a fence) +// are frozen into an append-only array, each its own memoized on text +// that never changes — every block tokenizes exactly once. Only the in-flight +// tail re-parses per delta (O(tail)). // -// This version is fully incremental: -// settled blocks — an append-only array of top-level block strings. Each -// block renders as its own , memoized on its exact -// text, which never changes once committed. Every block -// is tokenized exactly once for the life of the stream. -// unstable tail — the in-flight block(s) after the last committed -// boundary. A single re-parses just this tail on -// every delta (O(tail) vs. O(total)). -// scanner state — fence/math open-state and scan position persist across -// deltas in a ref, so each delta only scans the newly -// arrived complete lines instead of the whole text. +// Invariants that keep it correct: +// · Only newline-terminated lines are scanned; a partial trailing line may +// yet become a fence opener, so it stays in the tail. +// · Blank-line boundaries can't be retroactively merged (a setext underline +// only binds the contiguous line above it, never across a committed "\n\n"). +// · An unmatched `$$` / `\[` opener is treated as open forever — more +// conservative than markdown.tsx's full-text fallback, because a committed +// block is frozen and can't be un-decided once the closer streams in. +// · State only advances (idempotent under StrictMode). The component +// unmounts between turns; if `text` stops extending `scanned` (turn reuse, +// or boundedLiveRenderText front-trimming a huge reply) the scanner resets +// and 's LRU absorbs the re-parse. // -// A block boundary is a literal "\n\n" outside any fenced code or display -// math block. Only complete lines (terminated by "\n") are scanned — a -// partial trailing line could still grow into a fence opener, so it stays -// in the tail until its newline arrives. Blank-line boundaries also make -// the split immune to retroactive block merges (e.g. a setext underline -// only attaches to the contiguous paragraph above it — it cannot reach -// across a committed "\n\n"). -// -// The scanner treats an unmatched `$$` / `\[` opener as open forever. This -// is INTENTIONALLY more conservative than `markdown.tsx`'s parser, which -// falls back to paragraph rendering when an opener has no closer. The -// renderer can do that safely because it always sees the full text on -// every call. The streaming chunker cannot — once a block is committed it -// is frozen, so prematurely deciding "this `$$` is just prose" would -// permanently commit a paragraph rendering that becomes wrong the instant -// the closer streams in. Parking the boundary keeps everything from the -// opener onward in the mutable tail until the closer arrives (or the -// stream ends and the non-streaming takes over, at which point the -// renderer's fallback kicks in correctly). -// -// State is stored in a ref and only ever advances — idempotent under -// StrictMode double-render. The component unmounts between turns -// (isStreaming flips off → message moves to history and renders via a -// single ), so the ref resets naturally. If `text` stops extending the -// scanned prefix anyway (turn reuse, or the bounded live-render window -// trimming the front of a very long reply), the scanner resets and the -// per-text LRU inside still absorbs most of the re-parse. -// -// Layout: the subtrees MUST render stacked (column). The parent -// container in messageLine.tsx is a default `flexDirection: 'row'` Box -// (Ink's default), so returning a bare Fragment of sibling s would lay -// them out side-by-side — the "two jumbled columns while streaming" bug. +// Layout: the subtrees MUST stack in a column — the messageLine.tsx +// parent is a default row Box, so bare siblings render side-by-side. import { Box } from '@hermes/ink' import { memo, useRef } from 'react' @@ -66,18 +38,14 @@ import { Md } from './markdown.js' export interface StreamScanState { /** Settled top-level block strings, in order. Append-only. */ blocks: string[] - /** True while inside an unclosed ``` / ~~~ fence at the scan position. */ + /** Inside an unclosed ``` / ~~~ fence at the scan position. */ codeOpen: boolean - /** Non-null while inside an unclosed display-math block at the scan position. */ + /** Non-null inside an unclosed display-math block at the scan position. */ mathOpener: '$$' | '\\[' | null - /** - * The prefix of the text whose complete lines have been scanned — - * text.slice(0, scanPos), with scanPos === scanned.length. Kept as a - * string (not just an index) so the reset guard can verify the scanned - * region still matches, fence state included. - */ + /** Prefix whose complete lines have been scanned (kept as text so the reset + * guard can confirm the scanned region — and thus fence state — still holds). */ scanned: string - /** Length of the committed prefix — blocks.join('').length. */ + /** Length of the committed prefix (blocks.join('').length). */ settledLen: number } @@ -89,12 +57,9 @@ export const createScanState = (): StreamScanState => ({ settledLen: 0 }) -// Apply one complete (newline-terminated) line to the fence/math state. -// Mirrors the toggle rules the old fenceOpenAt() used: ``` / ~~~ toggle the -// code fence; `$$` / `\[` open display math only when the line doesn't also -// close it; closers only count when the matching opener is pending. Math -// markers inside an open code fence are ignored (a `$$` example in a code -// block must not open math). +// Fold one complete line into the fence/math state: ``` / ~~~ toggle the code +// fence; `$$` / `\[` open display math unless the line also closes it; closers +// count only against a pending opener; math inside an open code fence is inert. const applyLine = (state: StreamScanState, line: string) => { if (/^(?:`{3,}|~{3,})/.test(line)) { state.codeOpen = !state.codeOpen @@ -107,18 +72,10 @@ const applyLine = (state: StreamScanState, line: string) => { } if (!state.mathOpener) { - if (/^\$\$/.test(line)) { - const singleLine = line.length >= 4 && /\$\$$/.test(line) - - if (!singleLine) { - state.mathOpener = '$$' - } - } else if (/^\\\[/.test(line)) { - const singleLine = /\\\]$/.test(line) - - if (!singleLine) { - state.mathOpener = '\\[' - } + if (/^\$\$/.test(line) && !(line.length >= 4 && /\$\$$/.test(line))) { + state.mathOpener = '$$' + } else if (/^\\\[/.test(line) && !/\\\]$/.test(line)) { + state.mathOpener = '\\[' } } else if (state.mathOpener === '$$' && /\$\$$/.test(line)) { state.mathOpener = null @@ -127,11 +84,10 @@ const applyLine = (state: StreamScanState, line: string) => { } } -// Consume newly arrived COMPLETE lines from the scan position, committing a -// block at every "\n\n" boundary reached outside fences. Whitespace-only -// candidates (runs of 3+ newlines) are left for the next block rather than -// committed as empty s. Mutates `state`; calling again with the same -// text is a no-op (idempotent). +// Consume newly-arrived complete lines, committing a settled block at every +// "\n\n" outside a fence. Whitespace-only runs stay with the next block, never +// committed as empty s. Mutates `state`; re-calling with the same text is +// a no-op (idempotent). export const advanceScan = (text: string, state: StreamScanState) => { const start = state.scanned.length @@ -141,15 +97,11 @@ export const advanceScan = (text: string, state: StreamScanState) => { const nl = text.indexOf('\n', i) if (nl < 0) { - // Partial trailing line — could still become a fence opener. Leave it - // (and its whole block) in the unstable tail. - break + break // partial trailing line — could still open a fence; keep in tail } if (nl === i) { - // Empty line. If it's the second half of a "\n\n" pair (i.e. not the - // very first character) and no fence is open, the text before it is a - // settled top-level block. + // Second half of a "\n\n" outside any fence → prior text is a block. if (i > 0 && !state.codeOpen && !state.mathOpener) { const block = text.slice(state.settledLen, nl + 1) @@ -170,10 +122,8 @@ export const advanceScan = (text: string, state: StreamScanState) => { } } -// Compatibility shim over the incremental scanner: index just past the last -// committed block boundary in `text`, or -1 when nothing has settled yet. -// Kept for boundary-semantics tests; StreamingMd itself keeps scanner state -// across deltas instead of recomputing from scratch. +// Index just past the last committed boundary, or -1 if nothing has settled. +// Thin wrapper over the scanner for boundary-semantics tests. export const findStableBoundary = (text: string) => { const state = createScanState() @@ -187,10 +137,8 @@ export const StreamingMd = memo(function StreamingMd({ cols, compact, t, text }: let state = scanRef.current - // Reset when the text no longer extends the scanned prefix: a new turn - // reusing the mounted component, or boundedLiveRenderText trimming the - // front of a very long reply. Comparing the full scanned region (not just - // the settled blocks) guarantees the persisted fence state is still valid. + // Reset if `text` no longer extends the scanned prefix (turn reuse, or a + // front-trim), which would invalidate the persisted fence state. if (!text.startsWith(state.scanned)) { state = scanRef.current = createScanState() }