diff --git a/bench/memwatch-report.mjs b/bench/memwatch-report.mjs new file mode 100644 index 000000000000..f6b54f4462eb --- /dev/null +++ b/bench/memwatch-report.mjs @@ -0,0 +1,71 @@ +#!/usr/bin/env node +// memwatch-report — aggregate the per-session NDJSON written by the TUI's +// in-process sampler (ui-opentui/src/boundary/memlog.ts) into one fleet table. +// +// Usage: node memwatch-report.mjs [dir] (default ~/.hermes/logs/memwatch) +// Output: one row per session file — start, duration, baseline/peak/last RSS, +// peak mounted rows, and a crude steady-state slope (MB/h over the last half) — +// plus anomaly flags: SLOPE (last-half slope > 20MB/h), PEAK (> 450MB), +// MOUNTED (peak mounted rows > 200 — windowing should bound ~30-120). +import { readdirSync, readFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +const dir = process.argv[2] ?? join(homedir(), '.hermes', 'logs', 'memwatch') + +let files = [] +try { + files = readdirSync(dir).filter(f => f.endsWith('.jsonl')).sort() +} catch { + console.error(`no memwatch dir at ${dir} — enable with HERMES_TUI_DIAGNOSTICS=1 (or HERMES_TUI_MEMLOG=1)`) + process.exit(1) +} +if (!files.length) { + console.error(`no sessions logged yet in ${dir}`) + process.exit(1) +} + +const rows = [] +for (const f of files) { + const samples = [] + for (const line of readFileSync(join(dir, f), 'utf8').split('\n')) { + if (!line.trim()) continue + try { samples.push(JSON.parse(line)) } catch { /* torn write */ } + } + if (samples.length < 2) continue + const rss = samples.map(s => s.rss_kb / 1024) + const peak = Math.max(...rss) + const durMin = (samples.at(-1).t - samples[0].t) / 60 + // steady-state slope: least-squares over the last half of the samples + const half = samples.slice(Math.floor(samples.length / 2)) + const t0 = half[0].t + const xs = half.map(s => (s.t - t0) / 3600) + const ys = half.map(s => s.rss_kb / 1024) + const n = xs.length + const mx = xs.reduce((a, b) => a + b, 0) / n + const my = ys.reduce((a, b) => a + b, 0) / n + const denom = xs.reduce((a, x) => a + (x - mx) ** 2, 0) + const slope = denom > 0 ? xs.reduce((a, x, i) => a + (x - mx) * (ys[i] - my), 0) / denom : 0 + const peakMounted = Math.max(...samples.map(s => s.peak_mounted ?? 0)) + const flags = [] + if (slope > 20 && durMin > 10) flags.push('SLOPE') + if (peak > 450) flags.push('PEAK') + if (peakMounted > 200) flags.push('MOUNTED') + rows.push({ + session: f.replace('.jsonl', ''), + start: new Date(samples[0].t * 1000).toISOString().slice(0, 16), + min: Math.round(durMin), + base: Math.round(rss[0]), + peak: Math.round(peak), + last: Math.round(rss.at(-1)), + mounted: peakMounted, + 'MB/h': Math.round(slope * 10) / 10, + flags: flags.join(',') || '—' + }) +} + +console.table(rows) +const flagged = rows.filter(r => r.flags !== '—') +console.log(flagged.length + ? `\n${flagged.length} session(s) flagged — investigate with bench/live-attach.sh --heap on a live one.` + : `\nall ${rows.length} sessions healthy (no slope/peak/mounted anomalies).`) diff --git a/docs/opentui-env-flags.md b/docs/opentui-env-flags.md index 5f947a53ce41..23b7ba86ec51 100644 --- a/docs/opentui-env-flags.md +++ b/docs/opentui-env-flags.md @@ -28,6 +28,7 @@ classified by who should ever touch it. The design rule shipped with this doc: | `HERMES_TUI_WINDOWING` | **on** | `0` = bit-exact pre-windowing renderer (every row mounts; cap clamps back to 1000). The A/B + regression escape hatch. | | `HERMES_TUI_WINDOW_IDLE_MS` | ~1000 | Idle-measure pulse cadence (the spacer-exactness march). Test knob. | | `HERMES_TUI_WINDOW_STATS` | = `HERMES_TUI_DIAGNOSTICS` | Exposes live/peak mounted-row counters (`globalThis.__hermesTuiWindowStats`) for bench/live-attach reads. | +| `HERMES_TUI_MEMLOG` | = `HERMES_TUI_DIAGNOSTICS` | In-process 1Hz memory self-sampling (`boundary/memlog.ts`) → `~/.hermes/logs/memwatch/-.jsonl` (rss/heap/external + mounted rows; 14-day retention). Fleet view: `node bench/memwatch-report.mjs`. The "monitor all my sessions" answer: one `export HERMES_TUI_DIAGNOSTICS=1` in your shell rc covers every session. | | `HERMES_TUI_LOG_LEVEL` / `HERMES_TUI_LOG_FILE` | engine defaults | Logging verbosity/destination (`/logs` reads the ring buffer regardless). Deliberately independent of the master switch — support often wants logs without the full diag surface. | ## 4. Internal plumbing (set by the launcher/bench/tests — humans never set these) diff --git a/ui-opentui/src/boundary/memlog.ts b/ui-opentui/src/boundary/memlog.ts new file mode 100644 index 000000000000..0f832f5e82f7 --- /dev/null +++ b/ui-opentui/src/boundary/memlog.ts @@ -0,0 +1,91 @@ +/** + * memlog — in-process 1Hz memory self-sampling to NDJSON. + * + * The fleet-monitoring answer to "attach live-attach.sh to all 5–10 of my + * sessions": instead of an external watcher chasing pids, every TUI session + * logs its OWN samples when enabled, keyed by pid + boot time, into + * `~/.hermes/logs/memwatch/`. Aggregate across sessions with + * `bench/memwatch-report.mjs`. + * + * Gating (docs/opentui-env-flags.md): `HERMES_TUI_MEMLOG` — defaults to the + * `HERMES_TUI_DIAGNOSTICS` master switch, individually overridable either way. + * One `export HERMES_TUI_DIAGNOSTICS=1` in a dev's shell rc therefore covers + * every session they ever start; regular users write nothing. + * + * Cost when on: one `process.memoryUsage()` + one short append per second + * (~60 bytes/s, ~5MB/day across ten busy sessions). The interval is unref'd — + * it never keeps the process alive. Every failure path disables the logger + * silently (diagnostics must never break the TUI). Retention: files older + * than 14 days are pruned at start, best-effort. + * + * Sample shape (one JSON object per line): + * { t, rss_kb, heap_used_kb, external_kb, mounted, peak_mounted } + * `mounted`/`peak_mounted` come from the windowing DEV counters + * (logic/window.ts) — they update whenever windowing is active, independent + * of the WINDOW_STATS exposure flag. + */ +import { appendFileSync, mkdirSync, readdirSync, statSync, unlinkSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +import { diagnosticsEnabled, envFlag } from '../logic/env.ts' +import { windowRowStats } from '../logic/window.ts' + +const RETENTION_DAYS = 14 +const SAMPLE_MS = 1000 + +function memwatchDir(): string { + const home = process.env.HERMES_HOME?.trim() + const base = home && home.length > 0 ? home : join(homedir(), '.hermes') + return join(base, 'logs', 'memwatch') +} + +function pruneOld(dir: string): void { + const cutoff = Date.now() - RETENTION_DAYS * 24 * 3600 * 1000 + try { + for (const name of readdirSync(dir)) { + if (!name.endsWith('.jsonl')) continue + const p = join(dir, name) + try { + if (statSync(p).mtimeMs < cutoff) unlinkSync(p) + } catch { + /* best-effort */ + } + } + } catch { + /* best-effort */ + } +} + +/** Start the self-sampler (no-op unless enabled). Returns a stop function. */ +export function startMemlog(): () => void { + if (!envFlag(process.env.HERMES_TUI_MEMLOG, diagnosticsEnabled())) return () => {} + try { + const dir = memwatchDir() + mkdirSync(dir, { recursive: true }) + pruneOld(dir) + const boot = new Date().toISOString().replace(/[:.]/g, '').slice(0, 15) + const file = join(dir, `${boot}-${process.pid}.jsonl`) + const timer = setInterval(() => { + try { + const m = process.memoryUsage() + const w = windowRowStats() + const line = JSON.stringify({ + t: Math.floor(Date.now() / 1000), + rss_kb: Math.floor(m.rss / 1024), + heap_used_kb: Math.floor(m.heapUsed / 1024), + external_kb: Math.floor(m.external / 1024), + mounted: w.mounted, + peak_mounted: w.peakMounted + }) + appendFileSync(file, line + '\n') + } catch { + clearInterval(timer) // a failing diagnostic must not retry forever + } + }, SAMPLE_MS) + timer.unref?.() + return () => clearInterval(timer) + } catch { + return () => {} + } +} diff --git a/ui-opentui/src/entry/main.tsx b/ui-opentui/src/entry/main.tsx index a57071520f07..6578da969e17 100644 --- a/ui-opentui/src/entry/main.tsx +++ b/ui-opentui/src/entry/main.tsx @@ -29,6 +29,7 @@ import { readClipboardImage, writeClipboard } from '../boundary/clipboard.ts' import { GatewayService, type GatewayServiceShape } from '../boundary/gateway/GatewayService.ts' import { liveGatewayLayer } from '../boundary/gateway/liveGateway.ts' import { getLog } from '../boundary/log.ts' +import { startMemlog } from '../boundary/memlog.ts' import { acquireRenderer } from '../boundary/renderer.ts' import { makeAppLayer } from '../boundary/runtime.ts' import { nthAssistantResponse } from '../logic/copy.ts' @@ -381,6 +382,10 @@ export const run = Effect.fn('Tui.run')(function* (input: TuiInput) { onCtrlC, onCopySelection }) + // Fleet memory self-sampling (HERMES_TUI_MEMLOG / diagnostics master + // switch — boundary/memlog.ts). Scoped acquire→release like the renderer. + const stopMemlog = startMemlog() + yield* Effect.addFinalizer(() => Effect.sync(stopMemlog)) doQuit = () => { if (!renderer.isDestroyed) renderer.destroy() }