mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-30 01:41:43 +00:00
feat: just more cleaning
This commit is contained in:
parent
46cef4b7fa
commit
4b4b4d47bc
24 changed files with 2852 additions and 829 deletions
|
|
@ -29,8 +29,16 @@ const dim = (s: string) => DIM + s + DIM_OFF
|
|||
|
||||
let _seg: Intl.Segmenter | null = null
|
||||
const seg = () => (_seg ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' }))
|
||||
const STOP_CACHE_MAX = 32
|
||||
const stopCache = new Map<string, number[]>()
|
||||
|
||||
function graphemeStops(s: string) {
|
||||
const hit = stopCache.get(s)
|
||||
|
||||
if (hit) {
|
||||
return hit
|
||||
}
|
||||
|
||||
const stops = [0]
|
||||
|
||||
for (const { index } of seg().segment(s)) {
|
||||
|
|
@ -43,6 +51,16 @@ function graphemeStops(s: string) {
|
|||
stops.push(s.length)
|
||||
}
|
||||
|
||||
stopCache.set(s, stops)
|
||||
|
||||
if (stopCache.size > STOP_CACHE_MAX) {
|
||||
const oldest = stopCache.keys().next().value
|
||||
|
||||
if (oldest !== undefined) {
|
||||
stopCache.delete(oldest)
|
||||
}
|
||||
}
|
||||
|
||||
return stops
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue