mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-20 15:33:54 +00:00
refactor: replace custom lib/remend-tail.ts with built-in tailBoundedRemend
delete lib/remend-tail.ts (108 lines) and lib/remend-tail.test.ts (105 lines). the tailBoundedRemend export from @assistant-ui/react-streamdown 0.3.4 is algorithmically identical — same findRemendWindowStart boundary scan, same fence/math tracking, same slice-and-repair strategy. the only differences are improvements: the built-in handles \r (CR) in line endings for Windows compatibility, and accepts an optional RemendOptions parameter passed through to remend. the import in markdown-text.tsx moves from @/lib/remend-tail to @assistant-ui/react-streamdown. the call site (preprocessWithTailRepair) is unchanged. verified: tsc 0 errors, eslint clean, vitest 0 new failures (15 pre-existing, 786 passing — 6 fewer than before because the deleted remend-tail.test.ts had 6 cases), manual verification of incomplete markdown repair during streaming.
This commit is contained in:
parent
6c95740d9e
commit
4e10a38c9e
3 changed files with 4 additions and 217 deletions
|
|
@ -5,7 +5,8 @@ import {
|
|||
parseMarkdownIntoBlocks,
|
||||
type StreamdownTextComponents,
|
||||
StreamdownTextPrimitive,
|
||||
type SyntaxHighlighterProps
|
||||
type SyntaxHighlighterProps,
|
||||
tailBoundedRemend
|
||||
} from '@assistant-ui/react-streamdown'
|
||||
import { code } from '@streamdown/code'
|
||||
import { type ComponentProps, memo, useEffect, useMemo, useState } from 'react'
|
||||
|
|
@ -29,7 +30,6 @@ import {
|
|||
mediaStreamUrl
|
||||
} from '@/lib/media'
|
||||
import { previewTargetFromMarkdownHref } from '@/lib/preview-targets'
|
||||
import { tailBoundedRemend } from '@/lib/remend-tail'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
import { detectEmbed, extractAlert, MarkdownAlert, RichCodeBlock, UrlEmbed } from './embeds'
|
||||
|
|
@ -49,8 +49,8 @@ import { detectEmbed, extractAlert, MarkdownAlert, RichCodeBlock, UrlEmbed } fro
|
|||
const mathPlugin = createMemoizedMathPlugin({ singleDollarTextMath: true })
|
||||
|
||||
// Replaces Streamdown's `parseIncompleteMarkdown` (full-text remend per
|
||||
// flush) with a tail-bounded repair — see lib/remend-tail.ts. Must stay
|
||||
// module-scope so the prop identity is stable across renders.
|
||||
// flush) with a tail-bounded repair. Must stay module-scope so the prop
|
||||
// identity is stable across renders.
|
||||
function preprocessWithTailRepair(text: string): string {
|
||||
try {
|
||||
return tailBoundedRemend(preprocessMarkdown(text))
|
||||
|
|
|
|||
|
|
@ -1,105 +0,0 @@
|
|||
import { parseMarkdownIntoBlocks } from '@assistant-ui/react-streamdown'
|
||||
import remend from 'remend'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { findRemendWindowStart, tailBoundedRemend } from './remend-tail'
|
||||
|
||||
const CORPUS = `# Heading one
|
||||
|
||||
Intro paragraph with **bold**, *italic*, \`inline code\`, and a [link](https://example.com).
|
||||
|
||||
## Code
|
||||
|
||||
\`\`\`python
|
||||
def main():
|
||||
cost = "$5"
|
||||
print(f"total: $\{cost}")
|
||||
\`\`\`
|
||||
|
||||
Some text after the fence with $x^2 + y^2$ inline math.
|
||||
|
||||
$$
|
||||
\\int_0^1 f(x) dx
|
||||
$$
|
||||
|
||||
- list item one with **bold**
|
||||
- list item two
|
||||
|
||||
| col a | col b |
|
||||
| ----- | ----- |
|
||||
| 1 | 2 |
|
||||
|
||||
~~~js
|
||||
const s = \`template \${value}\`
|
||||
~~~
|
||||
|
||||
Final paragraph with ~~strike~~ and unfinished [link text](https://exa
|
||||
`
|
||||
|
||||
/**
|
||||
* Render-equivalence oracle: full-text remend and tail-bounded remend may
|
||||
* differ in raw string output ONLY in ways that cannot affect rendering —
|
||||
* i.e. after block splitting, every block must be identical. (Streamdown
|
||||
* renders blocks independently, so block-level equality IS render equality.)
|
||||
*/
|
||||
function blocksOf(text: string): string[] {
|
||||
return parseMarkdownIntoBlocks(text)
|
||||
}
|
||||
|
||||
describe('tailBoundedRemend', () => {
|
||||
it('matches full remend block output at every streaming prefix', () => {
|
||||
for (let end = 1; end <= CORPUS.length; end++) {
|
||||
const prefix = CORPUS.slice(0, end)
|
||||
const full = blocksOf(remend(prefix))
|
||||
const tail = blocksOf(tailBoundedRemend(prefix))
|
||||
|
||||
expect(tail, `prefix length ${end}: ${JSON.stringify(prefix.slice(-60))}`).toEqual(full)
|
||||
}
|
||||
})
|
||||
|
||||
it('repairs an unclosed fence opened early in a long message', () => {
|
||||
const text = `intro\n\n\`\`\`python\n${'x = 1\n'.repeat(500)}print("$dollar")`
|
||||
const repaired = tailBoundedRemend(text)
|
||||
|
||||
expect(blocksOf(repaired)).toEqual(blocksOf(remend(text)))
|
||||
// the window must reach back to the fence opener
|
||||
expect(findRemendWindowStart(text)).toBe(text.indexOf('```python'))
|
||||
})
|
||||
|
||||
it('bounds the window to the tail paragraph when no fence is open', () => {
|
||||
const text = `para one\n\npara two\n\npara three with **bold`
|
||||
const start = findRemendWindowStart(text)
|
||||
|
||||
expect(start).toBe(text.indexOf('para three'))
|
||||
expect(tailBoundedRemend(text)).toBe(remend(text))
|
||||
})
|
||||
|
||||
it('widens the window across an open $$ math block', () => {
|
||||
const text = `before\n\n$$\n\\frac{a}{b}`
|
||||
const start = findRemendWindowStart(text)
|
||||
|
||||
expect(start).toBeLessThanOrEqual(text.indexOf('$$'))
|
||||
expect(blocksOf(tailBoundedRemend(text))).toEqual(blocksOf(remend(text)))
|
||||
})
|
||||
|
||||
it('handles closed constructs without modification', () => {
|
||||
const text = `done **bold** and \`code\`\n\n\`\`\`js\nconst a = 1\n\`\`\`\n\nlast line.`
|
||||
|
||||
expect(tailBoundedRemend(text)).toBe(text)
|
||||
})
|
||||
|
||||
it('intentionally diverges from full remend on cross-block dangling openers', () => {
|
||||
// Full remend scans the whole document and appends `**` for an opener
|
||||
// left dangling in an EARLIER block, dumping stray asterisks into the
|
||||
// unrelated tail block ("|**"). Because Streamdown splits into blocks
|
||||
// after the repair, that opener never renders as bold either way — the
|
||||
// tail-bounded result is the cleaner of the two. This test documents
|
||||
// the divergence so a future remend upgrade that changes the behavior
|
||||
// gets noticed.
|
||||
const text = `- item with **dangling\n- item two\n\n|`
|
||||
|
||||
expect(remend(text).endsWith('|**')).toBe(true)
|
||||
expect(tailBoundedRemend(text).endsWith('|')).toBe(true)
|
||||
expect(tailBoundedRemend(text).endsWith('|**')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
import remend from 'remend'
|
||||
|
||||
// Tail-bounded incomplete-markdown repair.
|
||||
//
|
||||
// Streamdown's built-in `parseIncompleteMarkdown` runs `remend` over the whole
|
||||
// accumulated message on every streaming flush (~18% of script time on 50KB+
|
||||
// messages). But repairs only ever matter in the trailing block: inline
|
||||
// constructs can't cross a blank line, and Streamdown splits into blocks AFTER
|
||||
// the repair, so a dangling opener in an earlier block can't reach the tail.
|
||||
// We run `remend` on just that block instead.
|
||||
|
||||
const BACKTICK = 96 // `
|
||||
const TILDE = 126 // ~
|
||||
const SPACE = 32
|
||||
const TAB = 9
|
||||
const BACKSLASH = 92
|
||||
|
||||
const isSpace = (c: number) => c === SPACE || c === TAB
|
||||
|
||||
/**
|
||||
* Index of the last top-level block start — the char after the most recent
|
||||
* blank line that sits outside any open code fence or `$$` math block. An
|
||||
* unclosed fence/math always begins after that blank, so it stays wholly
|
||||
* inside the window without separate tracking. One cheap char pass, no regex.
|
||||
*/
|
||||
export function findRemendWindowStart(text: string): number {
|
||||
const n = text.length
|
||||
let inFence = false
|
||||
let fenceChar = 0
|
||||
let fenceRun = 0
|
||||
let inMath = false
|
||||
let boundary = 0
|
||||
let pending = -1 // a blank line, committed to `boundary` once content follows
|
||||
|
||||
for (let lineStart = 0; lineStart <= n; ) {
|
||||
let lineEnd = text.indexOf('\n', lineStart)
|
||||
|
||||
if (lineEnd === -1) {
|
||||
lineEnd = n
|
||||
}
|
||||
|
||||
let i = lineStart
|
||||
|
||||
while (i < lineEnd && isSpace(text.charCodeAt(i))) {
|
||||
i += 1
|
||||
}
|
||||
|
||||
const first = i < lineEnd ? text.charCodeAt(i) : -1
|
||||
let marker = false
|
||||
|
||||
// Fence open/close (``` or ~~~, ≤3 spaces indent).
|
||||
if ((first === BACKTICK || first === TILDE) && i - lineStart <= 3) {
|
||||
let run = i
|
||||
|
||||
while (run < lineEnd && text.charCodeAt(run) === first) {
|
||||
run += 1
|
||||
}
|
||||
|
||||
if (run - i >= 3) {
|
||||
marker = true
|
||||
|
||||
if (!inFence) {
|
||||
inFence = true
|
||||
fenceChar = first
|
||||
fenceRun = run - i
|
||||
} else if (first === fenceChar && run - i >= fenceRun && onlyWhitespace(text, run, lineEnd)) {
|
||||
inFence = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Toggle `$$` math state on plain lines ($$ inside a fence is literal).
|
||||
if (!inFence && !marker) {
|
||||
for (let s = text.indexOf('$$', lineStart); s !== -1 && s < lineEnd - 1; s = text.indexOf('$$', s + 2)) {
|
||||
if (s === 0 || text.charCodeAt(s - 1) !== BACKSLASH) {
|
||||
inMath = !inMath
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (first === -1 && !inFence && !inMath) {
|
||||
pending = lineEnd + 1
|
||||
} else if (pending !== -1) {
|
||||
boundary = pending
|
||||
pending = -1
|
||||
}
|
||||
|
||||
lineStart = lineEnd + 1
|
||||
}
|
||||
|
||||
return boundary
|
||||
}
|
||||
|
||||
function onlyWhitespace(text: string, from: number, to: number): boolean {
|
||||
for (let i = from; i < to; i += 1) {
|
||||
if (!isSpace(text.charCodeAt(i))) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
export function tailBoundedRemend(text: string): string {
|
||||
const start = findRemendWindowStart(text)
|
||||
|
||||
return start <= 0 ? remend(text) : text.slice(0, start) + remend(text.slice(start))
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue