mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-14 14:12:44 +00:00
338b5275be
@opentui/core@0.4.0 bundles only 5 grammars (ts/js/markdown/markdown_inline/ zig) and Hermes registered none of its own — Python/Rust/Go/bash/JSON/C/HTML/ CSS/YAML/TOML tool bodies and fences rendered plain text (never a regression: no addDefaultParsers existed anywhere in branch history). Now: parsers/manifest.json curates the 10 grammars (cpp deliberately dropped — 3.28MB alone); scripts/update-parsers.mjs vendors wasm+highlights.scm with magic/content validation (plain Node fetch — core's update-assets generator is Bun-flavored and its import-module won't bundle under esbuild, so registration skips it and points at the vendored files by runtime-resolved path instead); boundary/parsers.ts registers via the public addDefaultParsers() at entry module load, before the first <code>/<markdown> mount initializes the global tree-sitter client. ~4MB vendored, committed (build inputs, offline-safe). Markdown fence injections need no infoStringMap: fence labels resolve as filetype ids and core's ext maps already normalize py→python, zsh→bash, h→c. Live-smoked in a real renderer: python tool body draws 6 distinct token colors; ```python and ```yaml fences inside markdown highlight too. 6 new tests pin the wiring (vendored assets valid, registration set, filetype routing); visuals stay live-smoke territory per codeBlock.tsx.
71 lines
2.9 KiB
JavaScript
71 lines
2.9 KiB
JavaScript
/**
|
|
* Vendor the extra Tree-sitter grammars listed in parsers/manifest.json into
|
|
* parsers/<filetype>/ (wasm + highlights.scm). Plain Node fetch — no Bun
|
|
* (core's update-assets.js is bun-shebanged; its download logic is just
|
|
* fetch+fs, so we do the same two writes ourselves and keep the generated
|
|
* import-module out of the esbuild bundle entirely — registration reads the
|
|
* vendored files by PATH at runtime, see src/boundary/parsers.ts).
|
|
*
|
|
* Idempotent: existing valid files are kept unless --force. Validates wasm
|
|
* magic and a non-empty query so a bad download can never be committed.
|
|
*
|
|
* node scripts/update-parsers.mjs [--force]
|
|
*
|
|
* The vendored files are COMMITTED (build inputs, like @opentui/core's own
|
|
* assets/) — builds and offline machines never re-download.
|
|
*/
|
|
import { Buffer } from 'node:buffer'
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
import { dirname, join, resolve } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
|
const parsersDir = join(root, 'parsers')
|
|
const force = process.argv.includes('--force')
|
|
|
|
const manifest = JSON.parse(await readFile(join(parsersDir, 'manifest.json'), 'utf8'))
|
|
|
|
const wasmUrl = p =>
|
|
`https://github.com/${p.org}/tree-sitter-${p.filetype}/releases/download/${p.tag}/tree-sitter-${p.filetype}.wasm`
|
|
const scmUrl = p =>
|
|
`https://raw.githubusercontent.com/${p.org}/tree-sitter-${p.filetype}/${p.tag}/queries/highlights.scm`
|
|
|
|
async function fetchBytes(url) {
|
|
const response = await globalThis.fetch(url)
|
|
if (!response.ok) throw new Error(`${response.status} ${response.statusText} for ${url}`)
|
|
return Buffer.from(await response.arrayBuffer())
|
|
}
|
|
|
|
async function haveValid(path, validate) {
|
|
try {
|
|
return validate(await readFile(path))
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
const isWasm = bytes => bytes.length > 8 && bytes.subarray(0, 4).toString('latin1') === '\0asm'
|
|
const isQuery = bytes => bytes.length > 0 && bytes.toString('utf8').trim().length > 0
|
|
|
|
let downloaded = 0
|
|
for (const parser of manifest.parsers) {
|
|
const dir = join(parsersDir, parser.filetype)
|
|
await mkdir(dir, { recursive: true })
|
|
const targets = [
|
|
{ name: `tree-sitter-${parser.filetype}.wasm`, url: wasmUrl(parser), validate: isWasm },
|
|
{ name: 'highlights.scm', url: scmUrl(parser), validate: isQuery }
|
|
]
|
|
for (const target of targets) {
|
|
const path = join(dir, target.name)
|
|
if (!force && (await haveValid(path, target.validate))) {
|
|
console.log(`✓ kept ${parser.filetype}/${target.name}`)
|
|
continue
|
|
}
|
|
const bytes = await fetchBytes(target.url)
|
|
if (!target.validate(bytes)) throw new Error(`validation failed for ${target.url}`)
|
|
await writeFile(path, bytes)
|
|
downloaded += 1
|
|
console.log(`↓ fetched ${parser.filetype}/${target.name} (${(bytes.length / 1024).toFixed(0)} KB)`)
|
|
}
|
|
}
|
|
console.log(`done — ${downloaded} file(s) fetched, ${manifest.parsers.length} grammars vendored`)
|