mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-11 13:41:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

opentui(v6): tier-A latex — unicode math with fence-aware preprocessing

Browse source
This commit is contained in:
alt-glitch 2026-06-11 01:44:48 +05:30
parent e9af6a5110
commit 50e34713b6
5 changed files with 1601 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

279
ui-opentui/src/logic/mathPreprocess.ts Normal file
View file

@ -0,0 +1,279 @@
/**
* Fence-aware LaTeX→Unicode span converter. Runs on the raw markdown string
* BEFORE it reaches the native `<markdown>` renderable (the one seam in
* view/markdown.tsx), so the native parser only ever sees already-converted
* unicode text. Tier-A: text-only — no styled spans, no accent color on math
* (that needs renderNode hooks into MarkdownRenderable; deferred).
*
* Span detection ports the Ink tokenizer's EXACT rules (ui-tui/src/components/
* markdown.tsx — keep in sync):
*
* • inline `$…$` — INLINE_RE group 17:
* (?<!\$)\$([^\s$](?:[^$\n]*?[^\s$])?)\$(?!\$)
* content starts AND ends with a non-space-non-`$`, contains no `$` or
* newline. This is the currency guard: in `I paid $5 and $10` the closing
* `$` is preceded by a space, so nothing matches and the prose survives.
* • inline `\(…\)` — INLINE_RE group 18: `\\\(([^\n]+?)\\\)` (single line).
* • display `$$…$$` / `\[…\]` — MATH_BLOCK_OPEN_RE: opener only at the start
* of a (whitespace-trimmed) line; closes on the same line (`$$x$$`) or on a
* later line ENDING with the closer. No closer anywhere → the line passes
* through verbatim (Ink renders it as a plain paragraph). That rule is what
* makes streaming safe for free: an unclosed `$$`/`$` mid-stream stays
* verbatim and converts exactly once, when the closing delimiter arrives
* (the whole text re-feeds per delta).
*
* Because the markdown parser hasn't run yet, fence / inline-code state is
* tracked here:
* • fenced blocks: ``` or ~~~ runs (3+, any info string) open; a line that is
* only a run of the SAME character, at least as long, closes (CommonMark).
* Everything inside, including the fence lines, passes through untouched.
* • inline code: per-line backtick scan — a run of N backticks opens a span
* closed by the next run of EXACTLY N backticks on the same line
* (CommonMark rule); unmatched runs are literal text. Multi-line inline
* code spans are NOT supported (the Ink tokenizer was per-line too).
*
* Known, documented deviations from full markdown awareness (both rare, both
* shared with or narrower than the Ink renderer's behavior):
* • a paired `$…$` inside a link destination (`[x](http://a$b$c)`) converts;
* Ink's tokenizer matched the link first.
* • 4-space-indented code blocks are not tracked (fences only).
*
* `\boxed{…}` sentinels (U+0001/U+0002 from texToUnicode) are STRIPPED to the
* inner text — injecting a styled span into the native renderable needs
* renderer hooks; deferred with the rest of tier-B.
*
* Perf: this runs over the FULL text on every streaming delta. Early-exit
* fast path returns the same string reference when no `$` / `\(` / `\[`
* appears at all, and when a scan converts nothing the original reference is
* returned too (so the renderable's content prop stays identity-stable).
*/
import { BOX_RE, texToUnicode } from './mathUnicode.ts'
const FENCE_OPEN_RE = /^\s*(`{3,}|~{3,})/
const FENCE_CLOSE_RE = /^\s*(`{3,}|~{3,})\s*$/
// Display math openers/closers — ported verbatim from Ink's markdown.tsx.
const MATH_BLOCK_OPEN_RE = /^\s*(\$\$|\\\[)(.*)$/
const MATH_BLOCK_CLOSE_DOLLAR_RE = /^(.*?)\$\$\s*$/
const MATH_BLOCK_CLOSE_BRACKET_RE = /^(.*?)\\\]\s*$/
// Ink INLINE_RE group 17 / 18, anchored (sticky). The `(?<!\$)` lookbehind is
// checked by the caller (prev char), everything else is byte-for-byte Ink's.
const INLINE_DOLLAR_RE = /\$([^\s$](?:[^$\n]*?[^\s$])?)\$(?!\$)/y
const INLINE_PAREN_RE = /\\\(([^\n]+?)\\\)/y
/** texToUnicode + strip the \boxed highlight sentinels down to plain text. */
const toUnicode = (tex: string): string => texToUnicode(tex).replace(BOX_RE, '$1')
/** Index of the next run of EXACTLY `len` backticks at/after `from`, or -1. */
const findBacktickClose = (line: string, from: number, len: number): number => {
let i = from
while (i < line.length) {
if (line[i] !== '`') {
i++
continue
}
let j = i + 1
while (j < line.length && line[j] === '`') j++
if (j - i === len) {
return i
}
i = j
}
return -1
}
// Convert inline `$…$` / `\(…\)` spans in one prose line, skipping inline
// code spans. Returns the SAME string reference when nothing converted.
const convertInline = (line: string): string => {
let out = ''
let i = 0
let changed = false
while (i < line.length) {
const ch = line[i]
if (ch === '`') {
let j = i + 1
while (j < line.length && line[j] === '`') j++
const close = findBacktickClose(line, j, j - i)
if (close >= 0) {
out += line.slice(i, close + (j - i))
i = close + (j - i)
} else {
out += line.slice(i, j)
i = j
}
continue
}
if (ch === '$' && line[i - 1] !== '$') {
INLINE_DOLLAR_RE.lastIndex = i
const m = INLINE_DOLLAR_RE.exec(line)
if (m) {
out += toUnicode(m[1] ?? '')
i = INLINE_DOLLAR_RE.lastIndex
changed = true
continue
}
}
if (ch === '\\' && line[i + 1] === '(') {
INLINE_PAREN_RE.lastIndex = i
const m = INLINE_PAREN_RE.exec(line)
if (m) {
out += toUnicode(m[1] ?? '')
i = INLINE_PAREN_RE.lastIndex
changed = true
continue
}
}
out += ch
i++
}
return changed ? out : line
}
export function preprocessMath(markdown: string, _opts?: { streaming?: boolean | undefined }): string {
// Fast path — REQUIRED, this runs on every streaming delta. No math trigger
// characters anywhere → hand back the exact same string (identity).
if (!markdown.includes('$') && !markdown.includes('\\(') && !markdown.includes('\\[')) {
return markdown
}
const lines = markdown.split('\n')
const out: string[] = []
let changed = false
let fence: { char: string; len: number } | null = null
let i = 0
// Emit a converted display block as its own paragraph: blank-line separated
// from surrounding prose (only where a separator is actually missing).
const pushDisplay = (block: string[], nextIdx: number) => {
if (out.length > 0 && out[out.length - 1]?.trim()) {
out.push('')
}
out.push(...block)
if (nextIdx < lines.length && lines[nextIdx]?.trim()) {
out.push('')
}
changed = true
}
while (i < lines.length) {
const line = lines[i] ?? ''
if (fence) {
out.push(line)
const close = line.match(FENCE_CLOSE_RE)?.[1]
if (close && close.charAt(0) === fence.char && close.length >= fence.len) {
fence = null
}
i++
continue
}
const open = line.match(FENCE_OPEN_RE)?.[1]
if (open) {
fence = { char: open.charAt(0), len: open.length }
out.push(line)
i++
continue
}
const mathOpen = line.match(MATH_BLOCK_OPEN_RE)
if (mathOpen) {
const closeRe = mathOpen[1] === '$$' ? MATH_BLOCK_CLOSE_DOLLAR_RE : MATH_BLOCK_CLOSE_BRACKET_RE
const headRest = mathOpen[2] ?? ''
// Single-line block: `$$x + y = z$$` or `\[x\]`.
const sameLineClose = headRest.match(closeRe)
if (sameLineClose) {
const inner = (sameLineClose[1] ?? '').trim()
pushDisplay(inner ? [toUnicode(inner)] : [], i + 1)
i++
continue
}
// Multi-line block: scan ahead for a real closer before committing. If
// none exists in the rest of the (possibly still-streaming) document,
// the line stays verbatim — Ink's paragraph fallback.
let closeIdx = -1
let closeTail = ''
for (let j = i + 1; j < lines.length; j++) {
const m = (lines[j] ?? '').match(closeRe)
if (m) {
closeIdx = j
closeTail = m[1] ?? ''
break
}
}
if (closeIdx >= 0) {
const block: string[] = []
if (headRest.trim()) {
block.push(headRest)
}
for (let j = i + 1; j < closeIdx; j++) {
block.push(lines[j] ?? '')
}
const tail = closeTail.trimEnd()
if (tail.trim()) {
block.push(tail)
}
pushDisplay(
block.map(l => toUnicode(l)),
closeIdx + 1
)
i = closeIdx + 1
continue
}
}
const converted = convertInline(line)
if (converted !== line) {
changed = true
}
out.push(converted)
i++
}
return changed ? out.join('\n') : markdown
}

773
ui-opentui/src/logic/mathUnicode.ts Normal file
View file

@ -0,0 +1,773 @@
// Best-effort LaTeX → Unicode for inline / display math (ported verbatim from
// ui-tui/src/lib/mathUnicode.ts — keep the two in sync). The terminal can't
// typeset LaTeX, but Unicode covers
// most of what models actually emit: Greek letters, blackboard / fraktur /
// calligraphic capitals, set theory + logic operators, common arrows,
// sub/superscripts, and `\frac{a}{b}` collapsed to `a/b`.
//
// Design rules:
// • Pure regex pipeline. Anything we don't recognise is preserved
// verbatim (so a `\foo{bar}` we've never heard of still survives).
// A real LaTeX parser would be more correct but throws on partial
// input — terminal users would rather see the raw command than a
// parse-error placeholder.
// • Longest-match-first ordering on commands so `\le` doesn't shadow
// `\leq`, `\sub` doesn't shadow `\subseteq`, etc.
// • Word-boundary lookahead `(?![A-Za-z])` after each command so
// `\pix` (made-up command) doesn't get partially substituted as `π`.
// • `\mathbb{X}`, `\mathcal{X}`, `\mathfrak{X}` only handle a single
// letter argument — multi-letter `\mathbb{NN}` is rare and would
// need a real parser to do correctly.
// • Sub/super scripts only convert if EVERY character has a Unicode
// equivalent. Mixed content like `^{n+1}` falls back to the raw
// LaTeX so we don't emit `ⁿ+¹` (which has no `+` superscript glyph
// in some fonts and reads worse than the source).
const SYMBOLS: Record<string, string> = {
// Greek lowercase
'\\alpha': 'α',
'\\beta': 'β',
'\\gamma': 'γ',
'\\delta': 'δ',
'\\epsilon': 'ε',
'\\varepsilon': 'ε',
'\\zeta': 'ζ',
'\\eta': 'η',
'\\theta': 'θ',
'\\vartheta': 'ϑ',
'\\iota': 'ι',
'\\kappa': 'κ',
'\\lambda': 'λ',
'\\mu': 'μ',
'\\nu': 'ν',
'\\xi': 'ξ',
'\\pi': 'π',
'\\varpi': 'ϖ',
'\\rho': 'ρ',
'\\varrho': 'ϱ',
'\\sigma': 'σ',
'\\varsigma': 'ς',
'\\tau': 'τ',
'\\upsilon': 'υ',
'\\phi': 'φ',
'\\varphi': 'φ',
'\\chi': 'χ',
'\\psi': 'ψ',
'\\omega': 'ω',
// Greek uppercase
'\\Gamma': 'Γ',
'\\Delta': 'Δ',
'\\Theta': 'Θ',
'\\Lambda': 'Λ',
'\\Xi': 'Ξ',
'\\Pi': 'Π',
'\\Sigma': 'Σ',
'\\Upsilon': 'Υ',
'\\Phi': 'Φ',
'\\Psi': 'Ψ',
'\\Omega': 'Ω',
// Big operators
'\\sum': '∑',
'\\prod': '∏',
'\\coprod': '∐',
'\\int': '∫',
'\\iint': '∬',
'\\iiint': '∭',
'\\oint': '∮',
'\\bigcup': '⋃',
'\\bigcap': '⋂',
'\\bigvee': '⋁',
'\\bigwedge': '⋀',
'\\bigoplus': '⨁',
'\\bigotimes': '⨂',
// Calculus
'\\partial': '∂',
'\\nabla': '∇',
'\\sqrt': '√',
// Sets
'\\emptyset': '∅',
'\\varnothing': '∅',
'\\infty': '∞',
'\\in': '∈',
'\\notin': '∉',
'\\ni': '∋',
'\\subset': '⊂',
'\\supset': '⊃',
'\\subseteq': '⊆',
'\\supseteq': '⊇',
'\\subsetneq': '⊊',
'\\supsetneq': '⊋',
'\\cup': '∪',
'\\cap': '∩',
'\\setminus': '∖',
'\\complement': '∁',
// Logic
'\\forall': '∀',
'\\exists': '∃',
'\\nexists': '∄',
'\\land': '∧',
'\\lor': '∨',
'\\lnot': '¬',
'\\neg': '¬',
'\\therefore': '∴',
'\\because': '∵',
// Relations
'\\le': '≤',
'\\leq': '≤',
'\\ge': '≥',
'\\geq': '≥',
'\\ne': '≠',
'\\neq': '≠',
'\\ll': '≪',
'\\gg': '≫',
'\\approx': '≈',
'\\equiv': '≡',
'\\cong': '≅',
'\\sim': '∼',
'\\simeq': '≃',
'\\propto': '∝',
'\\perp': '⊥',
'\\parallel': '∥',
'\\models': '⊨',
'\\vdash': '⊢',
'\\mid': '∣',
'\\nmid': '∤',
'\\divides': '∣',
// Common standalone glyphs
'\\blacksquare': '■',
'\\square': '□',
'\\Box': '□',
'\\qed': '∎',
'\\bigstar': '★',
// Modular arithmetic — the `\pmod{p}` form (with arg) is handled below;
// the bare `\bmod` / `\mod` commands are simple text substitutions.
'\\bmod': 'mod',
'\\mod': 'mod',
// Brackets / fences (named delimiter commands; the `\left\X` / `\right\X`
// unwrapping below leaves these behind for the symbol pass to resolve).
'\\langle': '⟨',
'\\rangle': '⟩',
'\\lceil': '⌈',
'\\rceil': '⌉',
'\\lfloor': '⌊',
'\\rfloor': '⌋',
'\\|': '‖',
// Arrows
'\\to': '→',
'\\rightarrow': '→',
'\\leftarrow': '←',
'\\leftrightarrow': '↔',
'\\Rightarrow': '⇒',
'\\Leftarrow': '⇐',
'\\Leftrightarrow': '⇔',
'\\implies': '⟹',
'\\impliedby': '⟸',
'\\iff': '⟺',
'\\mapsto': '↦',
'\\hookrightarrow': '↪',
'\\hookleftarrow': '↩',
'\\uparrow': '↑',
'\\downarrow': '↓',
'\\updownarrow': '↕',
// Binary operators
'\\cdot': '⋅',
'\\cdots': '⋯',
'\\ldots': '…',
'\\dots': '…',
'\\dotsb': '…',
'\\dotsc': '…',
'\\vdots': '⋮',
'\\ddots': '⋱',
'\\times': '×',
'\\div': '÷',
'\\pm': '±',
'\\mp': '∓',
'\\circ': '∘',
'\\bullet': '•',
'\\star': '⋆',
'\\ast': '∗',
'\\oplus': '⊕',
'\\ominus': '⊖',
'\\otimes': '⊗',
'\\odot': '⊙',
'\\diamond': '⋄',
'\\angle': '∠',
'\\triangle': '△',
// Spacing — collapse to varying widths of regular space
'\\,': ' ',
'\\;': ' ',
'\\:': ' ',
'\\!': '',
'\\ ': ' ',
'\\quad': ' ',
'\\qquad': ' ',
// Functions (LaTeX renders these in roman; we just keep the name)
'\\sin': 'sin',
'\\cos': 'cos',
'\\tan': 'tan',
'\\cot': 'cot',
'\\sec': 'sec',
'\\csc': 'csc',
'\\arcsin': 'arcsin',
'\\arccos': 'arccos',
'\\arctan': 'arctan',
'\\sinh': 'sinh',
'\\cosh': 'cosh',
'\\tanh': 'tanh',
'\\log': 'log',
'\\ln': 'ln',
'\\exp': 'exp',
'\\det': 'det',
'\\dim': 'dim',
'\\ker': 'ker',
'\\lim': 'lim',
'\\liminf': 'liminf',
'\\limsup': 'limsup',
'\\sup': 'sup',
'\\inf': 'inf',
'\\max': 'max',
'\\min': 'min',
'\\arg': 'arg',
'\\gcd': 'gcd',
// Escaped literals — model occasionally emits these for display
'\\&': '&',
'\\%': '%',
'\\$': '$',
'\\#': '#',
'\\_': '_',
'\\{': '{',
'\\}': '}'
}
const BB: Record<string, string> = {
A: '𝔸',
B: '𝔹',
C: 'ℂ',
D: '𝔻',
E: '𝔼',
F: '𝔽',
G: '𝔾',
H: 'ℍ',
I: '𝕀',
J: '𝕁',
K: '𝕂',
L: '𝕃',
M: '𝕄',
N: 'ℕ',
O: '𝕆',
P: 'ℙ',
Q: 'ℚ',
R: 'ℝ',
S: '𝕊',
T: '𝕋',
U: '𝕌',
V: '𝕍',
W: '𝕎',
X: '𝕏',
Y: '𝕐',
Z: 'ℤ'
}
const CAL: Record<string, string> = {
A: '𝒜',
B: 'ℬ',
C: '𝒞',
D: '𝒟',
E: 'ℰ',
F: 'ℱ',
G: '𝒢',
H: 'ℋ',
I: 'ℐ',
J: '𝒥',
K: '𝒦',
L: 'ℒ',
M: 'ℳ',
N: '𝒩',
O: '𝒪',
P: '𝒫',
Q: '𝒬',
R: 'ℛ',
S: '𝒮',
T: '𝒯',
U: '𝒰',
V: '𝒱',
W: '𝒲',
X: '𝒳',
Y: '𝒴',
Z: '𝒵'
}
const FRAK: Record<string, string> = {
A: '𝔄',
B: '𝔅',
C: 'ℭ',
D: '𝔇',
E: '𝔈',
F: '𝔉',
G: '𝔊',
H: 'ℌ',
I: 'ℑ',
J: '𝔍',
K: '𝔎',
L: '𝔏',
M: '𝔐',
N: '𝔑',
O: '𝔒',
P: '𝔓',
Q: '𝔔',
R: 'ℜ',
S: '𝔖',
T: '𝔗',
U: '𝔘',
V: '𝔙',
W: '𝔚',
X: '𝔛',
Y: '𝔜',
Z: 'ℨ'
}
const SUPERSCRIPT: Record<string, string> = {
'0': '⁰',
'1': '¹',
'2': '²',
'3': '³',
'4': '⁴',
'5': '⁵',
'6': '⁶',
'7': '⁷',
'8': '⁸',
'9': '⁹',
'+': '⁺',
'-': '⁻',
'=': '⁼',
'(': '⁽',
')': '⁾',
a: 'ᵃ',
b: 'ᵇ',
c: 'ᶜ',
d: 'ᵈ',
e: 'ᵉ',
f: 'ᶠ',
g: 'ᵍ',
h: 'ʰ',
i: 'ⁱ',
j: 'ʲ',
k: 'ᵏ',
l: 'ˡ',
m: 'ᵐ',
n: 'ⁿ',
o: 'ᵒ',
p: 'ᵖ',
r: 'ʳ',
s: 'ˢ',
t: 'ᵗ',
u: 'ᵘ',
v: 'ᵛ',
w: 'ʷ',
x: 'ˣ',
y: 'ʸ',
z: 'ᶻ'
}
const SUBSCRIPT: Record<string, string> = {
'0': '₀',
'1': '₁',
'2': '₂',
'3': '₃',
'4': '₄',
'5': '₅',
'6': '₆',
'7': '₇',
'8': '₈',
'9': '₉',
'+': '₊',
'-': '₋',
'=': '₌',
'(': '₍',
')': '₎',
a: 'ₐ',
e: 'ₑ',
h: 'ₕ',
i: 'ᵢ',
j: 'ⱼ',
k: 'ₖ',
l: 'ₗ',
m: 'ₘ',
n: 'ₙ',
o: 'ₒ',
p: 'ₚ',
r: 'ᵣ',
s: 'ₛ',
t: 'ₜ',
u: 'ᵤ',
v: 'ᵥ',
x: 'ₓ'
}
// Sentinel control characters used to mark `\boxed` / `\fbox` regions in
// the converted output. The renderer splits on these to apply a highlight
// style; consumers that don't want highlighting can strip them with the
// exported `BOX_RE` below.
export const BOX_OPEN = '\u0001'
export const BOX_CLOSE = '\u0002'
// eslint-disable-next-line no-control-regex
export const BOX_RE = /\u0001([^\u0001\u0002]*)\u0002/g
const escapeRe = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
// Pre-compile two symbol regexes: one for letter-ending commands (`\pi`,
// `\sum`) which need a `(?![A-Za-z])` lookahead so they don't partially
// match `\pix` or `\summa`, and one for punctuation-ending commands
// (`\{`, `\,`, `\|`) which must NOT have the lookahead — otherwise
// `\{p` would refuse to substitute because `p` is a letter.
//
// Longest commands first inside each group so `\leq` beats `\le`.
const splitByEnding = (keys: string[]) => {
const letter: string[] = []
const punct: string[] = []
for (const k of keys) {
if (/[A-Za-z]$/.test(k)) {
letter.push(k)
} else {
punct.push(k)
}
}
return { letter, punct }
}
const buildAlt = (cmds: string[]) =>
cmds
.sort((a, b) => b.length - a.length)
.map(escapeRe)
.join('|')
const { letter: LETTER_CMDS, punct: PUNCT_CMDS } = splitByEnding(Object.keys(SYMBOLS))
const SYMBOL_LETTER_RE = new RegExp('(?:' + buildAlt(LETTER_CMDS) + ')(?![A-Za-z])', 'g')
const SYMBOL_PUNCT_RE = new RegExp('(?:' + buildAlt(PUNCT_CMDS) + ')', 'g')
const convertScript = (input: string, table: Record<string, string>, sigil: '^' | '_'): string => {
let out = ''
let allMapped = true
for (const ch of input) {
const mapped = table[ch]
if (!mapped) {
allMapped = false
break
}
out += mapped
}
if (allMapped) {
return out
}
// Fallback: if the body is a single visible character (e.g. `∞` after
// earlier symbol substitution), render it without braces — `^∞` reads
// far better than `^{∞}` in a terminal. Multi-char bodies that don't
// fully convert use parens (`e^(iπ)`) instead of braces (`e^{iπ}`)
// because parens are normal punctuation while braces look like
// unrendered LaTeX.
const trimmed = input.trim()
if ([...trimmed].length === 1) {
return `${sigil}${trimmed}`
}
return `${sigil}(${trimmed})`
}
// Walk the string and parse `{...}` honouring nested braces. Unlike a
// `\{[^{}]*\}` regex this survives `\frac{|t|^{p-1}|P(t)|^p}{...}` where
// the numerator contains its own braces from a superscript. Returns the
// inner content (without the outer braces) and the offset just past the
// closing `}`. Returns null if there is no balanced brace at `start`.
const readBraced = (s: string, start: number): { content: string; end: number } | null => {
if (s[start] !== '{') {
return null
}
let depth = 1
let i = start + 1
while (i < s.length && depth > 0) {
const c = s[i]
// Skip escapes — `\{` and `\}` inside a body are literal braces and
// should not change the brace counter.
if (c === '\\' && i + 1 < s.length) {
i += 2
continue
}
if (c === '{') {
depth++
} else if (c === '}') {
depth--
}
if (depth > 0) {
i++
}
}
if (depth !== 0) {
return null
}
return { content: s.slice(start + 1, i), end: i + 1 }
}
// Replace every occurrence of `\command{arg}` using balanced-brace parsing
// (so `\boxed{x^{n+1}}` works where a `[^{}]*` regex would fail). The
// `render` callback receives the inner content already recursed-into, so
// `\boxed{\boxed{x}}` resolves outside-in cleanly. Unmatched `\command`
// (no following `{...}`) is preserved verbatim.
const replaceBracedCommand = (input: string, command: string, render: (content: string) => string): string => {
const cmdLen = command.length
let out = ''
let i = 0
while (i < input.length) {
const idx = input.indexOf(command, i)
if (idx < 0) {
out += input.slice(i)
return out
}
const after = input[idx + cmdLen]
if (after && /[A-Za-z]/.test(after)) {
out += input.slice(i, idx + cmdLen)
i = idx + cmdLen
continue
}
out += input.slice(i, idx)
let p = idx + cmdLen
while (input[p] === ' ' || input[p] === '\t') p++
const arg = readBraced(input, p)
if (!arg) {
out += input.slice(idx, p + 1)
i = p + 1
continue
}
out += render(replaceBracedCommand(arg.content, command, render))
i = arg.end
}
return out
}
// Replace every `\frac{num}{den}` with `num/den` (parens around either
// side when its precedence demands it). The recursion handles nested
// fractions naturally: `\frac{1}{\frac{1}{x}}` collapses to `1/(1/x)`
// because we recurse into `den` before deciding whether to parenthesise.
const replaceFracs = (input: string): string => {
let out = ''
let i = 0
while (i < input.length) {
const idx = input.indexOf('\\frac', i)
if (idx < 0) {
out += input.slice(i)
return out
}
const after = input[idx + 5]
// `(?![A-Za-z])` — protect hypothetical commands like `\fraction`.
if (after && /[A-Za-z]/.test(after)) {
out += input.slice(i, idx + 5)
i = idx + 5
continue
}
out += input.slice(i, idx)
let p = idx + 5
while (input[p] === ' ' || input[p] === '\t') p++
const num = readBraced(input, p)
if (!num) {
out += input.slice(idx, p + 1)
i = p + 1
continue
}
p = num.end
while (input[p] === ' ' || input[p] === '\t') p++
const den = readBraced(input, p)
if (!den) {
out += input.slice(idx, p + 1)
i = p + 1
continue
}
out += `${wrapForFrac(replaceFracs(num.content))}/${wrapForFrac(replaceFracs(den.content))}`
i = den.end
}
return out
}
// Wrap multi-token expressions in parens so `\frac{a+b}{c}` becomes
// `(a+b)/c` rather than `a+b/c`. We wrap whenever inline `/` would
// change the meaning — that's any binary operator (`+`, `-`, `*`, `/`)
// or whitespace separating tokens. `*` and `/` matter because nested
// fractions and products like `\frac{a*b}{c}` and `\frac{1/x}{y}` would
// otherwise read as `a*b/c` (right-associative ambiguity) and `1/x/y`.
// Atomic factors like `n!`, `x^2`, `\sin x` don't trigger any of these
// and stay un-parenthesised — wrapping them just clutters the output.
const wrapForFrac = (expr: string) => {
const trimmed = expr.trim()
if (!trimmed) {
return trimmed
}
if (/^\(.*\)$/.test(trimmed)) {
return trimmed
}
if (/[+\-/*]|\s/.test(trimmed)) {
return `(${trimmed})`
}
return trimmed
}
export function texToUnicode(input: string): string {
let s = input
s = s.replace(/\\mathbb\s*\{([A-Za-z])\}/g, (raw, c: string) => BB[c] ?? raw)
s = s.replace(/\\mathcal\s*\{([A-Za-z])\}/g, (raw, c: string) => CAL[c] ?? raw)
s = s.replace(/\\mathfrak\s*\{([A-Za-z])\}/g, (raw, c: string) => FRAK[c] ?? raw)
s = s.replace(/\\mathbf\s*\{([^{}]+)\}/g, (_, c: string) => c)
s = s.replace(/\\mathit\s*\{([^{}]+)\}/g, (_, c: string) => c)
s = s.replace(/\\mathrm\s*\{([^{}]+)\}/g, (_, c: string) => c)
s = s.replace(/\\text\s*\{([^{}]+)\}/g, (_, c: string) => c)
s = s.replace(/\\operatorname\s*\{([^{}]+)\}/g, (_, c: string) => c)
s = s.replace(/\\overline\s*\{([^{}]+)\}/g, (_, c: string) => `${c}\u0305`)
s = s.replace(/\\hat\s*\{([^{}]+)\}/g, (_, c: string) => `${c}\u0302`)
s = s.replace(/\\bar\s*\{([^{}]+)\}/g, (_, c: string) => `${c}\u0304`)
s = s.replace(/\\tilde\s*\{([^{}]+)\}/g, (_, c: string) => `${c}\u0303`)
s = s.replace(/\\vec\s*\{([^{}]+)\}/g, (_, c: string) => `${c}\u20D7`)
s = s.replace(/\\dot\s*\{([^{}]+)\}/g, (_, c: string) => `${c}\u0307`)
s = s.replace(/\\ddot\s*\{([^{}]+)\}/g, (_, c: string) => `${c}\u0308`)
s = replaceFracs(s)
// `\boxed{X}` / `\fbox{X}` highlight a final answer. Terminals can't
// draw a real box, so we wrap the content in U+0001 / U+0002 control
// characters — non-printable, never present in real text — so a renderer
// with styled-span hooks can apply a highlight style (inverse video) to
// the bracketed region. The OpenTUI engine's preprocessor currently
// strips them via BOX_RE (styled-span injection into the native
// `<markdown>` renderable is deferred); `texToUnicode` stays pure-string.
// Argument is parsed with balanced braces so nested `{...}` from
// superscripts / fractions inside the box survive.
s = replaceBracedCommand(s, '\\boxed', body => `${BOX_OPEN}${body.trim()}${BOX_CLOSE}`)
s = replaceBracedCommand(s, '\\fbox', body => `${BOX_OPEN}${body.trim()}${BOX_CLOSE}`)
// `\xrightarrow{label}` / `\xleftarrow{label}` collapse to an arrow with
// the label inline. LaTeX renders the label above the arrow; in monospace
// we put it adjacent — `─label→` is the closest readable approximation.
// Run before the symbol pass so the label can still pick up Greek and
// operator substitutions afterwards.
s = s.replace(/\\xrightarrow\s*\{([^{}]*)\}/g, (_, label: string) => `─${label.trim()}→`)
s = s.replace(/\\xleftarrow\s*\{([^{}]*)\}/g, (_, label: string) => `←${label.trim()}─`)
s = s.replace(/\\Longrightarrow/g, '⟹')
s = s.replace(/\\Longleftarrow/g, '⟸')
s = s.replace(/\\Longleftrightarrow/g, '⟺')
// `\pmod{p}` → ` (mod p)` (LaTeX adds parens automatically); `\pod{p}`
// is a paren-less variant; `\tag{n}` is the equation-number annotation
// shown to the right of an equation. Collapse to a single-space-prefixed
// bracketed form. The leading `\s*` in the pattern absorbs any whitespace
// already in the source so we don't end up with `b (mod p)` (double
// space) when the user wrote `b \pmod{p}`.
s = s.replace(/\s*\\pmod\s*\{([^{}]*)\}/g, (_, p: string) => ` (mod ${p.trim()})`)
s = s.replace(/\s*\\pod\s*\{([^{}]*)\}/g, (_, p: string) => ` (${p.trim()})`)
s = s.replace(/\s*\\tag\s*\{([^{}]*)\}/g, (_, n: string) => ` (${n.trim()})`)
// `\big`, `\Big`, `\bigg`, `\Bigg` (with optional `l`/`r`/`m` suffix)
// are sizing wrappers analogous to `\left`/`\right` but without the
// automatic-pairing semantics. Strip them and leave whatever delimiter
// follows. The trailing `(?![A-Za-z])` protects `\bigtriangleup` and
// any other letter-continuation command from being shaved.
s = s.replace(/\\(?:Bigg|bigg|Big|big)[lrm]?(?![A-Za-z])/g, '')
// Style / size hints that don't typeset any glyph and only affect how
// things would be sized in a real LaTeX engine. In a terminal every
// glyph is one monospace cell, so there's nothing to do — drop them
// (with any trailing whitespace) so they don't leak through as raw
// `\displaystyle` in the output.
s = s.replace(/\\(?:scriptscriptstyle|displaystyle|scriptstyle|textstyle|nolimits|limits)(?![A-Za-z])\s*/g, '')
// `\left` and `\right` are sizing wrappers around any delimiter — bare
// (`\left(`), escaped (`\left\{`), or named (`\left\langle`). Strip the
// wrapper unconditionally and let the rest of the pipeline (or the
// upcoming symbol pass) handle whatever delimiter follows. The optional
// `.?` consumes `\left.` / `\right.` which mean "no delimiter".
// Lookahead `(?![A-Za-z])` keeps `\leftarrow` / `\leftrightarrow` safe.
s = s.replace(/\\left(?![A-Za-z])\.?/g, '')
s = s.replace(/\\right(?![A-Za-z])\.?/g, '')
// Run symbol substitution BEFORE scripts so a body like `^{\infty}`
// becomes `^{∞}` first; convertScript can then either map ∞ to a
// superscript (it can't — Unicode lacks one) or fall back to `^∞`
// by stripping braces around the now-single-character body.
//
// Punctuation pass first — these can be followed by letters (`\{p`
// is "open-brace then p"), so the letter pass's `(?![A-Za-z])` rule
// would wrongly block them.
s = s.replace(SYMBOL_PUNCT_RE, m => SYMBOLS[m] ?? m)
s = s.replace(SYMBOL_LETTER_RE, m => SYMBOLS[m] ?? m)
// Bare `^c` / `_c` handles ONLY alphanumerics and `+`/`-`/`=`. Parens
// are intentionally excluded because the braced-fallback above can
// emit `(...)` and we don't want a second pass to greedily convert
// its opening paren into `⁽` and orphan the closing one.
s = s.replace(/\^\s*\{([^{}]+)\}/g, (_, body: string) => convertScript(body, SUPERSCRIPT, '^'))
s = s.replace(/\^([A-Za-z0-9+\-=])/g, (raw, ch: string) => SUPERSCRIPT[ch] ?? raw)
s = s.replace(/_\s*\{([^{}]+)\}/g, (_, body: string) => convertScript(body, SUBSCRIPT, '_'))
s = s.replace(/_([A-Za-z0-9+\-=])/g, (raw, ch: string) => SUBSCRIPT[ch] ?? raw)
return s
}

245
ui-opentui/src/test/mathPreprocess.test.ts Normal file
View file

@ -0,0 +1,245 @@
/**
* preprocessMath — the fence-aware LaTeX→Unicode span converter that runs on the
* raw markdown BEFORE it reaches the native `<markdown>` renderable.
*
* Span detection ports the Ink tokenizer's exact rules (ui-tui markdown.tsx):
* • inline `$…$` — INLINE_RE group 17: `(?<!\$)\$([^\s$](?:[^$\n]*?[^\s$])?)\$(?!\$)`
* (content starts/ends non-space-non-$, no `$`/newline inside → currency-safe)
* • inline `\(…\)` — INLINE_RE group 18 (single line, lazy)
* • display `$$…$$` / `\[…\]` — opener only at line start (MATH_BLOCK_OPEN_RE),
* same-line close or scan-ahead; NO closer → line passes through verbatim
* (which is exactly what makes mid-stream unclosed math safe).
*
* Fence/inline-code state is tracked by hand because the markdown parser hasn't
* run yet: ``` / ~~~ fences (any info string, closer same char + >= length) and
* per-line backtick code spans (run of N backticks closed by exactly N).
*/
import { describe, expect, it } from 'vitest'
import { preprocessMath } from '../logic/mathPreprocess.ts'
import { BOX_CLOSE, BOX_OPEN } from '../logic/mathUnicode.ts'
const hasSentinels = (s: string) => s.includes(BOX_OPEN) || s.includes(BOX_CLOSE)
describe('preprocessMath — early exit', () => {
it('returns the SAME string reference when no math trigger chars exist', () => {
const s = 'plain prose with **bold** and a [link](https://x.dev), no math at all'
expect(preprocessMath(s)).toBe(s)
})
it('returns the same reference when $ exists but nothing converts (pure currency)', () => {
const s = 'I paid $5 and $10'
expect(preprocessMath(s)).toBe(s)
})
it('handles the empty string', () => {
expect(preprocessMath('')).toBe('')
})
})
describe('preprocessMath — inline $…$', () => {
it('converts a simple inline span', () => {
expect(preprocessMath('Einstein: $E=mc^2$ wow')).toBe('Einstein: E=mc² wow')
})
it('converts Greek and blackboard inside inline math', () => {
expect(preprocessMath('so $\\pi \\in \\mathbb{R}$ holds')).toBe('so π ∈ ℝ holds')
})
it('converts multiple spans on one line', () => {
expect(preprocessMath('$\\alpha$ then $\\beta$')).toBe('α then β')
})
it('keeps unknown commands verbatim inside the span (delimiters still removed)', () => {
expect(preprocessMath('see $\\circledast$ here')).toBe('see \\circledast here')
})
})
describe('preprocessMath — currency anti-jank (Ink rule parity)', () => {
it('does not mathify "$5 and $10" (closing $ preceded by space)', () => {
expect(preprocessMath('it costs $5 and $10 today')).toBe('it costs $5 and $10 today')
})
it('does not mathify a lone unclosed dollar amount', () => {
expect(preprocessMath('paid $5.')).toBe('paid $5.')
})
it('does not mathify when content starts with a space', () => {
expect(preprocessMath('weird $ x$ thing')).toBe('weird $ x$ thing')
})
it('does not treat $$ as two inline delimiters', () => {
expect(preprocessMath('a $$ b')).toBe('a $$ b')
})
})
describe('preprocessMath — inline \\(…\\)', () => {
it('converts paren-delimited inline math', () => {
expect(preprocessMath('foo \\(x + y\\) bar')).toBe('foo x + y bar')
expect(preprocessMath('\\(\\pi\\)')).toBe('π')
})
it('leaves an unclosed \\( verbatim', () => {
expect(preprocessMath('foo \\(x + y bar')).toBe('foo \\(x + y bar')
})
})
describe('preprocessMath — display math', () => {
it('converts a single-line $$…$$ to its own plain paragraph text', () => {
expect(preprocessMath('$$E = mc^2$$')).toBe('E = mc²')
})
it('converts a multi-line $$ block, blank-line separated from prose', () => {
const input = 'Quadratic:\n$$\nx = \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a}\n$$\nDone.'
expect(preprocessMath(input)).toBe('Quadratic:\n\nx = (-b ± √{b² - 4ac})/2a\n\nDone.')
})
it('does not double blank lines when the block is already separated', () => {
const input = 'Prose.\n\n$$\n\\sum_{n=0}^{\\infty} \\frac{1}{n!}\n$$\n\nMore.'
expect(preprocessMath(input)).toBe('Prose.\n\n∑ₙ₌₀^∞ 1/n!\n\nMore.')
})
it('converts \\[ … \\] single-line and multi-line', () => {
expect(preprocessMath('\\[ \\alpha + \\beta \\]')).toBe('α + β')
expect(preprocessMath('\\[\n\\frac{a}{b}\n\\]')).toBe('a/b')
})
it('keeps content on the opener/closer lines (`$$x +` … `y$$`)', () => {
expect(preprocessMath('$$x^2 +\ny^2$$')).toBe('x² +\ny²')
})
it('leaves `$$x$$ trailing prose` verbatim (opener must close on its own line)', () => {
const s = '$$x+y$$ followed by more'
expect(preprocessMath(s)).toBe(s)
})
it('leaves an unclosed $$ block verbatim (no partial conversion)', () => {
const s = 'Before\n$$\nE = mc^2'
expect(preprocessMath(s)).toBe(s)
})
})
describe('preprocessMath — \\boxed sentinels are stripped to plain text', () => {
it('strips the U+0001/U+0002 sentinels, keeping the inner text', () => {
const out = preprocessMath('$$\\boxed{x = 0}$$')
expect(out).toBe('x = 0')
expect(hasSentinels(out)).toBe(false)
})
it('strips sentinels in inline spans too', () => {
expect(preprocessMath('answer $\\boxed{42}$ found')).toBe('answer 42 found')
})
})
describe('preprocessMath — fence passthrough', () => {
it('leaves $ spans inside ``` fences untouched (any info string)', () => {
const s =
'Look:\n\n```python title=x.py\nprice = "$5 and $10"\nmath = "$E=mc^2$"\n$$\nnot math\n$$\n```\n\nBut $\\pi$ converts.'
expect(preprocessMath(s)).toBe(
'Look:\n\n```python title=x.py\nprice = "$5 and $10"\nmath = "$E=mc^2$"\n$$\nnot math\n$$\n```\n\nBut π converts.'
)
})
it('respects ~~~ fences and longer-run closers', () => {
const s = '~~~\n$E=mc^2$\n~~~~\nafter $\\pi$'
expect(preprocessMath(s)).toBe('~~~\n$E=mc^2$\n~~~~\nafter π')
})
it('a shorter or different-char run does NOT close the fence', () => {
const s = '````\n```\n$E=mc^2$\n~~~\n````\n$\\pi$'
expect(preprocessMath(s)).toBe('````\n```\n$E=mc^2$\n~~~\n````\nπ')
})
it('an unclosed fence protects everything after it (streaming code block)', () => {
const s = '```sh\necho "$HOME and $PATH$"\nstill inside $x$'
expect(preprocessMath(s)).toBe(s)
})
})
describe('preprocessMath — inline code passthrough', () => {
it('leaves `$x$` in single-backtick code untouched, converts math outside', () => {
expect(preprocessMath('use `$x$` to write $\\pi$')).toBe('use `$x$` to write π')
})
it('handles double-backtick spans containing a backtick (exact-N closer)', () => {
expect(preprocessMath('``code $a$ with ` tick`` then $\\pi$')).toBe('``code $a$ with ` tick`` then π')
})
it('an unmatched backtick run is literal; math after it still converts', () => {
expect(preprocessMath('`unclosed and $\\pi$')).toBe('`unclosed and π')
})
it('display opener inside inline code is not display math', () => {
const s = 'the `$$` delimiter is TeX'
expect(preprocessMath(s)).toBe(s)
})
})
describe('preprocessMath — streaming safety (growing prefixes)', () => {
const doc = 'Euler: $e^{i\\pi}$ neat.\n\n$$\n\\sum_{n=0}^{\\infty} \\frac{1}{n!}\n$$\n\nDone $5 cheap.'
const final = 'Euler: e^(iπ) neat.\n\n∑ₙ₌₀^∞ 1/n!\n\nDone $5 cheap.'
it('every prefix produces sane output: no sentinels, no partial conversion of an open span', () => {
for (let n = 1; n <= doc.length; n++) {
const out = preprocessMath(doc.slice(0, n), { streaming: true })
expect(hasSentinels(out)).toBe(false)
// an inline span still open (single `$` so far) must stay verbatim
if (n >= 8 && n < 'Euler: $e^{i\\pi}$'.length) {
expect(out).toBe(doc.slice(0, n))
}
}
})
it('the inline span converts exactly once its closer arrives', () => {
const upto = 'Euler: $e^{i\\pi}$'.length
expect(preprocessMath(doc.slice(0, upto), { streaming: true })).toBe('Euler: e^(iπ)')
})
it('an open $$ block stays verbatim until the closing line arrives', () => {
const open = 'Euler: $e^{i\\pi}$ neat.\n\n$$\n\\sum_{n=0}^{\\infty} \\frac{1}{n!}'
expect(preprocessMath(open, { streaming: true })).toBe(
'Euler: e^(iπ) neat.\n\n$$\n\\sum_{n=0}^{\\infty} \\frac{1}{n!}'
)
const closed = open + '\n$$'
expect(preprocessMath(closed, { streaming: true })).toBe('Euler: e^(iπ) neat.\n\n∑ₙ₌₀^∞ 1/n!')
})
it('the full document converts and is stable across the streaming flag', () => {
expect(preprocessMath(doc, { streaming: true })).toBe(final)
expect(preprocessMath(doc)).toBe(final)
})
})
describe('preprocessMath — mixed document', () => {
it('prose + fence + inline code + currency + inline and display math', () => {
const input = [
'# Math $\\Sigma$ report',
'',
'Costs $5 and $10, but $\\alpha + \\beta$ converts and \\(x^2\\) too.',
'',
'```tex',
'keep $\\alpha$ raw $$',
'```',
'',
'$$',
'\\boxed{\\frac{n+1}{2}}',
'$$',
'',
'End `$ literal` here.'
].join('\n')
const expected = [
'# Math Σ report',
'',
'Costs $5 and $10, but α + β converts and x² too.',
'',
'```tex',
'keep $\\alpha$ raw $$',
'```',
'',
'(n+1)/2',
'',
'End `$ literal` here.'
].join('\n')
expect(preprocessMath(input)).toBe(expected)
})
})

295
ui-opentui/src/test/mathUnicode.test.ts Normal file
View file

@ -0,0 +1,295 @@
import { describe, expect, it } from 'vitest'
import { BOX_CLOSE, BOX_OPEN, BOX_RE, texToUnicode } from '../logic/mathUnicode.ts'
const stripBox = (s: string) => s.replace(BOX_RE, '$1')
describe('texToUnicode — symbols', () => {
it('substitutes lowercase Greek', () => {
expect(texToUnicode('\\alpha + \\beta + \\pi')).toBe('α + β + π')
expect(texToUnicode('\\omega')).toBe('ω')
})
it('substitutes uppercase Greek', () => {
expect(texToUnicode('\\Sigma \\Omega \\Pi')).toBe('Σ Ω Π')
})
it('substitutes set theory and logic operators', () => {
expect(texToUnicode('A \\cup B \\cap C')).toBe('A ∪ B ∩ C')
expect(texToUnicode('\\forall x \\in \\emptyset')).toBe('∀ x ∈ ∅')
expect(texToUnicode('p \\implies q \\iff r')).toBe('p ⟹ q ⟺ r')
})
it('substitutes relations and arrows', () => {
expect(texToUnicode('a \\le b \\ge c \\ne d')).toBe('a ≤ b ≥ c ≠ d')
expect(texToUnicode('f: A \\to B')).toBe('f: A → B')
})
it('uses longest-match-first so \\leq beats \\le', () => {
expect(texToUnicode('\\leq')).toBe('≤')
})
it('preserves unknown commands that share a prefix with known ones', () => {
// `\leqq` is a real LaTeX command (≦) we don't have in our table.
// The word-boundary lookahead prevents `\le` from matching, so the
// whole thing is preserved verbatim — much better than `≤qq`.
expect(texToUnicode('\\leqq')).toBe('\\leqq')
})
it('refuses to substitute a partial command (word boundary)', () => {
expect(texToUnicode('\\alphabet')).toBe('\\alphabet')
expect(texToUnicode('\\pin')).toBe('\\pin')
})
})
describe('texToUnicode — blackboard / calligraphic / fraktur', () => {
it('renders \\mathbb capitals', () => {
expect(texToUnicode('\\mathbb{R}')).toBe('ℝ')
expect(texToUnicode('\\mathbb{N} \\subset \\mathbb{Z} \\subset \\mathbb{Q} \\subset \\mathbb{R}')).toBe(
'ℕ ⊂ ℤ ⊂ ℚ ⊂ ℝ'
)
})
it('renders \\mathcal and \\mathfrak', () => {
expect(texToUnicode('\\mathcal{F} \\subset \\mathfrak{A}')).toBe('ℱ ⊂ 𝔄')
})
it('preserves \\mathbb{...} when argument is multi-letter or non-letter', () => {
expect(texToUnicode('\\mathbb{NN}')).toBe('\\mathbb{NN}')
expect(texToUnicode('\\mathbb{1}')).toBe('\\mathbb{1}')
})
it('strips \\mathbf / \\mathit / \\mathrm / \\text wrappers (no Unicode bold/italic in monospace)', () => {
expect(texToUnicode('\\mathbf{x}')).toBe('x')
expect(texToUnicode('\\text{if } x > 0')).toBe('if x > 0')
expect(texToUnicode('\\operatorname{rank}(A)')).toBe('rank(A)')
})
})
describe('texToUnicode — sub / superscripts', () => {
it('converts simple superscripts', () => {
expect(texToUnicode('x^2 + y^2')).toBe('x² + y²')
expect(texToUnicode('e^{n}')).toBe('eⁿ')
})
it('converts simple subscripts', () => {
expect(texToUnicode('a_1 + a_2 + a_n')).toBe('a₁ + a₂ + aₙ')
expect(texToUnicode('x_{0}')).toBe('x₀')
})
it('converts mixed-content scripts when every glyph has a Unicode form', () => {
// `+`, digits, and lowercase letters all have superscript glyphs,
// so `n+1` → `ⁿ⁺¹`. Comma has no subscript form, so `i,j` falls
// back to `_(i,j)` (parens) rather than partially substituting —
// parens read as ordinary grouping while braces look like leftover
// unrendered LaTeX.
expect(texToUnicode('x^{n+1}')).toBe('xⁿ⁺¹')
expect(texToUnicode('a_{i,j}')).toBe('a_(i,j)')
})
it('uses parens (not braces) when the body has Greek with no superscript form', () => {
// π has no Unicode superscript, so `e^{i\pi}` after symbol pass is
// `e^{iπ}` and the script fallback emits `e^(iπ)` — much more
// readable than the LaTeX-looking `e^{iπ}`.
expect(texToUnicode('e^{i\\pi}')).toBe('e^(iπ)')
})
it('strips braces on script fallback when body collapses to a single char', () => {
// `^{\infty}` → symbol pass produces `^{∞}` → convertScript can't
// find ∞ in SUPERSCRIPT, but the body is one char so we drop the
// braces and emit `^∞` (much more readable than `^{∞}`).
expect(texToUnicode('e^{\\infty}')).toBe('e^∞')
})
it('handles a real-world sum', () => {
expect(texToUnicode('\\sum_{n=0}^{\\infty} \\frac{1}{n!}')).toBe('∑ₙ₌₀^∞ 1/n!')
})
})
describe('texToUnicode — fractions', () => {
it('collapses \\frac to a/b', () => {
expect(texToUnicode('\\frac{1}{2}')).toBe('1/2')
expect(texToUnicode('\\frac{a}{b}')).toBe('a/b')
})
it('parenthesises multi-token numerator / denominator', () => {
expect(texToUnicode('\\frac{n+1}{2}')).toBe('(n+1)/2')
expect(texToUnicode('\\frac{a + b}{c - d}')).toBe('(a + b)/(c - d)')
})
it('handles nested fractions', () => {
expect(texToUnicode('\\frac{1}{\\frac{1}{x}}')).toBe('1/(1/x)')
})
it("handles braces inside numerator / denominator (regression: regex \\frac couldn't)", () => {
// The regex-only `\frac` matcher used `[^{}]*` for each arg, which
// failed the moment a numerator contained its own braces (here the
// `{p-1}` from a superscript). The balanced-brace parser handles it.
expect(texToUnicode('\\frac{|t|^{p-1}|P(t)|^p}{(p-1)!}')).toBe('(|t|ᵖ⁻¹|P(t)|ᵖ)/((p-1)!)')
})
it('preserves \\frac when arguments are malformed', () => {
expect(texToUnicode('\\frac{a}')).toBe('\\frac{a}')
expect(texToUnicode('\\fraction{a}{b}')).toBe('\\fraction{a}{b}')
})
})
describe('texToUnicode — typography no-ops', () => {
it('strips \\displaystyle / \\textstyle / \\scriptstyle / \\scriptscriptstyle', () => {
expect(texToUnicode('\\displaystyle\\sum_{i=1}^n x_i')).toBe('∑ᵢ₌₁ⁿ xᵢ')
expect(texToUnicode('f(x) = \\displaystyle \\frac{1}{2}')).toBe('f(x) = 1/2')
expect(texToUnicode('\\textstyle x + y')).toBe('x + y')
})
it('strips \\limits / \\nolimits which only affect bound positioning', () => {
expect(texToUnicode('\\sum\\limits_{k=1}^n a_k')).toBe('∑ₖ₌₁ⁿ aₖ')
expect(texToUnicode('\\int\\nolimits_0^1 f(x) dx')).toBe('∫₀¹ f(x) dx')
})
it('does not eat letter-continuation commands like \\limit_inf', () => {
// The `(?![A-Za-z])` lookahead protects hypothetical commands that
// start with `\limit` / `\display` / etc. The bare names are stripped
// but anything longer is preserved verbatim.
expect(texToUnicode('\\limitinf x')).toBe('\\limitinf x')
})
})
describe('texToUnicode — sizing wrappers', () => {
it('strips \\big / \\Big / \\bigg / \\Bigg before delimiters', () => {
expect(texToUnicode('\\bigl[ x \\bigr]')).toBe('[ x ]')
expect(texToUnicode('\\Big( y \\Big)')).toBe('( y )')
expect(texToUnicode('\\bigg| z \\bigg|')).toBe('| z |')
expect(texToUnicode('\\Biggl\\{ a \\Biggr\\}')).toBe('{ a }')
})
it('does not eat \\bigtriangleup or other letter-continuations', () => {
expect(texToUnicode('A \\bigtriangleup B')).toBe('A \\bigtriangleup B')
})
})
describe('texToUnicode — modular arithmetic and tags', () => {
it('renders \\pmod{p} as " (mod p)"', () => {
expect(texToUnicode('a \\equiv b \\pmod{p}')).toBe('a ≡ b (mod p)')
})
it('renders \\bmod / \\mod inline', () => {
expect(texToUnicode('a \\bmod n')).toBe('a mod n')
})
it('collapses \\tag{n} to " (n)"', () => {
expect(texToUnicode('x = y \\tag{24}')).toBe('x = y (24)')
})
})
describe('texToUnicode — newly added symbols', () => {
it('renders \\nmid, \\blacksquare, \\qed', () => {
expect(texToUnicode('p \\nmid q')).toBe('p ∤ q')
expect(texToUnicode('Therefore \\blacksquare')).toBe('Therefore ■')
expect(texToUnicode('done \\qed')).toBe('done ∎')
})
})
describe('texToUnicode — \\boxed / \\fbox', () => {
// `\boxed` produces non-printable U+0001 / U+0002 sentinels around its
// content so the markdown renderer can apply highlight styling. These
// tests assert both the sentinel form and the human-readable
// strip-fallback (BOX_RE).
it('wraps simple boxed content in BOX_OPEN/BOX_CLOSE sentinels', () => {
expect(texToUnicode('\\boxed{x = 0}')).toBe(`${BOX_OPEN}x = 0${BOX_CLOSE}`)
expect(stripBox(texToUnicode('\\boxed{x = 0}'))).toBe('x = 0')
expect(stripBox(texToUnicode('\\fbox{answer}'))).toBe('answer')
})
it("handles boxed expressions with nested braces (regression: regex couldn't)", () => {
// A `[^{}]*` regex would stop at the first `{` inside the body. The
// balanced-brace parser walks past it.
expect(stripBox(texToUnicode('\\boxed{x^{n+1}}'))).toBe('xⁿ⁺¹')
expect(stripBox(texToUnicode('\\boxed{\\frac{a}{b}}'))).toBe('a/b')
})
it('handles real-world boxed final answer', () => {
expect(stripBox(texToUnicode('\\boxed{J = -\\sum_{k=0}^n a_k F(k)}'))).toBe('J = -∑ₖ₌₀ⁿ aₖ F(k)')
})
it('preserves \\boxed without a brace argument', () => {
expect(texToUnicode('\\boxed something')).toBe('\\boxed something')
})
})
describe('texToUnicode — combining marks', () => {
it('applies \\overline / \\bar / \\hat / \\vec / \\tilde', () => {
expect(texToUnicode('\\overline{x}')).toBe('x\u0305')
expect(texToUnicode('\\hat{y}')).toBe('y\u0302')
expect(texToUnicode('\\vec{v}')).toBe('v\u20D7')
})
})
describe('texToUnicode — left/right delimiters', () => {
it('strips \\left and \\right keeping the delimiter character', () => {
expect(texToUnicode('\\left( x + y \\right)')).toBe('( x + y )')
expect(texToUnicode('\\left| x \\right|')).toBe('| x |')
})
it('handles escaped delimiters \\left\\{ ... \\right\\}', () => {
expect(texToUnicode('\\left\\{p/q \\mid q \\neq 0\\right\\}')).toBe('{p/q ∣ q ≠ 0}')
})
it('handles named delimiters via \\left\\langle / \\right\\rangle', () => {
expect(texToUnicode('\\left\\langle u, v \\right\\rangle')).toBe('⟨ u, v ⟩')
})
it('drops \\left. and \\right. (which are explicit "no delimiter")', () => {
expect(texToUnicode('\\left. f \\right|')).toBe(' f |')
})
it('preserves \\leftarrow / \\rightarrow (word boundary blocks the strip)', () => {
expect(texToUnicode('A \\leftarrow B \\rightarrow C')).toBe('A ← B → C')
})
})
describe('texToUnicode — labelled arrows', () => {
it('renders \\xrightarrow{label} as ─label→', () => {
expect(texToUnicode('a \\xrightarrow{x=1} b')).toBe('a ─x=1→ b')
})
it('renders \\xleftarrow{label} as ←label─', () => {
expect(texToUnicode('a \\xleftarrow{n} b')).toBe('a ←n─ b')
})
it('still applies symbol substitution inside the label', () => {
expect(texToUnicode('a \\xrightarrow{n \\to \\infty} L')).toBe('a ─n → ∞→ L')
})
})
describe('texToUnicode — punctuation commands without lookahead', () => {
it('substitutes \\{ even when immediately followed by a letter', () => {
// Regression: with a global `(?![A-Za-z])` lookahead, `\{p` refused
// to substitute (because `p` is a letter) and rendered as `\{p`.
expect(texToUnicode('\\{p, q\\}')).toBe('{p, q}')
})
it('substitutes thin-space \\, before a letter', () => {
expect(texToUnicode('a\\,b')).toBe('a b')
})
})
describe('texToUnicode — round-trip realism', () => {
it('renders a typical model-emitted formula', () => {
expect(texToUnicode('\\alpha \\in \\mathbb{R}, \\alpha \\notin \\mathbb{Q}')).toBe('α ∈ ℝ, α ∉ ℚ')
})
it('preserves unknown commands verbatim', () => {
expect(texToUnicode('\\bigtriangleup \\circledast')).toBe('\\bigtriangleup \\circledast')
})
it('handles commands without delimiters between', () => {
// Word-boundary lookahead means `\alpha\beta` doesn't accidentally
// match `\alphabeta` as one ungrouped token.
expect(texToUnicode('\\alpha\\beta')).toBe('αβ')
})
it('leaves plain text alone', () => {
expect(texToUnicode('hello world')).toBe('hello world')
expect(texToUnicode('')).toBe('')
})
})

10
ui-opentui/src/view/markdown.tsx
View file

@ -19,7 +19,9 @@
* it's rebuilt only when the skin changes (a new `Theme` object). * it's rebuilt only when the skin changes (a new `Theme` object).
*/ */
import { RGBA, SyntaxStyle } from '@opentui/core' import { RGBA, SyntaxStyle } from '@opentui/core'
import { createMemo } from 'solid-js'
import { preprocessMath } from '../logic/mathPreprocess.ts'
import type { Theme } from '../logic/theme.ts' import type { Theme } from '../logic/theme.ts'
import { useTheme } from './theme.tsx' import { useTheme } from './theme.tsx'
@ -92,6 +94,12 @@ export function syntaxStyleFor(theme: Theme): SyntaxStyle {
export function Markdown(props: { text: string; streaming?: boolean; fg?: string }) { export function Markdown(props: { text: string; streaming?: boolean; fg?: string }) {
const theme = useTheme() const theme = useTheme()
// Tier-A LaTeX: convert `$…$` / `$$…$$` / `\(...\)` / `\[...\]` spans to
// unicode BEFORE the native parser sees the text (fences and inline code pass
// through untouched; unclosed delimiters stay verbatim mid-stream — see
// logic/mathPreprocess.ts). Memoized so it recomputes only when the text
// delta arrives, and the no-math fast path returns the same string identity.
const content = createMemo(() => preprocessMath(props.text, { streaming: props.streaming }))
// `internalBlockMode="top-level"` is the anti-flicker mode (stable head blocks // `internalBlockMode="top-level"` is the anti-flicker mode (stable head blocks
// aren't re-rendered per delta); `tableOptions` gives native GFM tables with // aren't re-rendered per delta); `tableOptions` gives native GFM tables with
// inline formatting; `fg` overrides the base text color (muted for reasoning). // inline formatting; `fg` overrides the base text color (muted for reasoning).
@ -99,7 +107,7 @@ export function Markdown(props: { text: string; streaming?: boolean; fg?: string
// copies the RENDERED text (markers gone) via native selection, by design. // copies the RENDERED text (markers gone) via native selection, by design.
return ( return (
<markdown <markdown
content={props.text} content={content()}
syntaxStyle={syntaxStyleFor(theme())} syntaxStyle={syntaxStyleFor(theme())}
streaming={props.streaming ?? false} streaming={props.streaming ?? false}
internalBlockMode="top-level" internalBlockMode="top-level"