opentui(v6): fleet memory self-sampling — HERMES_TUI_MEMLOG + memwatch-report aggregator

Instead of an external watcher chasing 5-10 concurrent session pids,
every TUI samples ITSELF at 1Hz (rss/heap/external + windowing
mounted/peak-mounted counters) into ~/.hermes/logs/memwatch/, gated by
HERMES_TUI_MEMLOG (defaults to the HERMES_TUI_DIAGNOSTICS master
switch) — one shell-rc export covers every session a dev ever starts.
Unref'd timer, every failure path silently disables, 14-day retention.
bench/memwatch-report.mjs aggregates the fleet: per-session
baseline/peak/last, steady-state MB/h slope, peak mounted rows, and
SLOPE/PEAK/MOUNTED anomaly flags. Verified live: two fake-gateway
smoke sessions logged and aggregated (102MB base, mounted ≤60).

bench/memwatch-report.mjs

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+// memwatch-report — aggregate the per-session NDJSON written by the TUI's
+// in-process sampler (ui-opentui/src/boundary/memlog.ts) into one fleet table.
+//
+// Usage: node memwatch-report.mjs [dir]    (default ~/.hermes/logs/memwatch)
+// Output: one row per session file — start, duration, baseline/peak/last RSS,
+// peak mounted rows, and a crude steady-state slope (MB/h over the last half) —
+// plus anomaly flags: SLOPE (last-half slope > 20MB/h), PEAK (> 450MB),
+// MOUNTED (peak mounted rows > 200 — windowing should bound ~30-120).
+import { readdirSync, readFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+const dir = process.argv[2] ?? join(homedir(), '.hermes', 'logs', 'memwatch')
+
+let files = []
+try {
+  files = readdirSync(dir).filter(f => f.endsWith('.jsonl')).sort()
+} catch {
+  console.error(`no memwatch dir at ${dir} — enable with HERMES_TUI_DIAGNOSTICS=1 (or HERMES_TUI_MEMLOG=1)`)
+  process.exit(1)
+}
+if (!files.length) {
+  console.error(`no sessions logged yet in ${dir}`)
+  process.exit(1)
+}
+
+const rows = []
+for (const f of files) {
+  const samples = []
+  for (const line of readFileSync(join(dir, f), 'utf8').split('\n')) {
+    if (!line.trim()) continue
+    try { samples.push(JSON.parse(line)) } catch { /* torn write */ }
+  }
+  if (samples.length < 2) continue
+  const rss = samples.map(s => s.rss_kb / 1024)
+  const peak = Math.max(...rss)
+  const durMin = (samples.at(-1).t - samples[0].t) / 60
+  // steady-state slope: least-squares over the last half of the samples
+  const half = samples.slice(Math.floor(samples.length / 2))
+  const t0 = half[0].t
+  const xs = half.map(s => (s.t - t0) / 3600)
+  const ys = half.map(s => s.rss_kb / 1024)
+  const n = xs.length
+  const mx = xs.reduce((a, b) => a + b, 0) / n
+  const my = ys.reduce((a, b) => a + b, 0) / n
+  const denom = xs.reduce((a, x) => a + (x - mx) ** 2, 0)
+  const slope = denom > 0 ? xs.reduce((a, x, i) => a + (x - mx) * (ys[i] - my), 0) / denom : 0
+  const peakMounted = Math.max(...samples.map(s => s.peak_mounted ?? 0))
+  const flags = []
+  if (slope > 20 && durMin > 10) flags.push('SLOPE')
+  if (peak > 450) flags.push('PEAK')
+  if (peakMounted > 200) flags.push('MOUNTED')
+  rows.push({
+    session: f.replace('.jsonl', ''),
+    start: new Date(samples[0].t * 1000).toISOString().slice(0, 16),
+    min: Math.round(durMin),
+    base: Math.round(rss[0]),
+    peak: Math.round(peak),
+    last: Math.round(rss.at(-1)),
+    mounted: peakMounted,
+    'MB/h': Math.round(slope * 10) / 10,
+    flags: flags.join(',') || '—'
+  })
+}
+
+console.table(rows)
+const flagged = rows.filter(r => r.flags !== '—')
+console.log(flagged.length
+  ? `\n${flagged.length} session(s) flagged — investigate with bench/live-attach.sh <pid> --heap on a live one.`
+  : `\nall ${rows.length} sessions healthy (no slope/peak/mounted anomalies).`)

docs/opentui-env-flags.md

@@ -28,6 +28,7 @@ classified by who should ever touch it. The design rule shipped with this doc:
 | `HERMES_TUI_WINDOWING` | **on** | `0` = bit-exact pre-windowing renderer (every row mounts; cap clamps back to 1000). The A/B + regression escape hatch. |
 | `HERMES_TUI_WINDOW_IDLE_MS` | ~1000 | Idle-measure pulse cadence (the spacer-exactness march). Test knob. |
 | `HERMES_TUI_WINDOW_STATS` | = `HERMES_TUI_DIAGNOSTICS` | Exposes live/peak mounted-row counters (`globalThis.__hermesTuiWindowStats`) for bench/live-attach reads. |
+| `HERMES_TUI_MEMLOG` | = `HERMES_TUI_DIAGNOSTICS` | In-process 1Hz memory self-sampling (`boundary/memlog.ts`) → `~/.hermes/logs/memwatch/<boot>-<pid>.jsonl` (rss/heap/external + mounted rows; 14-day retention). Fleet view: `node bench/memwatch-report.mjs`. The "monitor all my sessions" answer: one `export HERMES_TUI_DIAGNOSTICS=1` in your shell rc covers every session. |
 | `HERMES_TUI_LOG_LEVEL` / `HERMES_TUI_LOG_FILE` | engine defaults | Logging verbosity/destination (`/logs` reads the ring buffer regardless). Deliberately independent of the master switch — support often wants logs without the full diag surface. |
 
 ## 4. Internal plumbing (set by the launcher/bench/tests — humans never set these)

ui-opentui/src/boundary/memlog.ts

@@ -0,0 +1,91 @@
+/**
+ * memlog — in-process 1Hz memory self-sampling to NDJSON.
+ *
+ * The fleet-monitoring answer to "attach live-attach.sh to all 5–10 of my
+ * sessions": instead of an external watcher chasing pids, every TUI session
+ * logs its OWN samples when enabled, keyed by pid + boot time, into
+ * `~/.hermes/logs/memwatch/`. Aggregate across sessions with
+ * `bench/memwatch-report.mjs`.
+ *
+ * Gating (docs/opentui-env-flags.md): `HERMES_TUI_MEMLOG` — defaults to the
+ * `HERMES_TUI_DIAGNOSTICS` master switch, individually overridable either way.
+ * One `export HERMES_TUI_DIAGNOSTICS=1` in a dev's shell rc therefore covers
+ * every session they ever start; regular users write nothing.
+ *
+ * Cost when on: one `process.memoryUsage()` + one short append per second
+ * (~60 bytes/s, ~5MB/day across ten busy sessions). The interval is unref'd —
+ * it never keeps the process alive. Every failure path disables the logger
+ * silently (diagnostics must never break the TUI). Retention: files older
+ * than 14 days are pruned at start, best-effort.
+ *
+ * Sample shape (one JSON object per line):
+ *   { t, rss_kb, heap_used_kb, external_kb, mounted, peak_mounted }
+ * `mounted`/`peak_mounted` come from the windowing DEV counters
+ * (logic/window.ts) — they update whenever windowing is active, independent
+ * of the WINDOW_STATS exposure flag.
+ */
+import { appendFileSync, mkdirSync, readdirSync, statSync, unlinkSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+import { diagnosticsEnabled, envFlag } from '../logic/env.ts'
+import { windowRowStats } from '../logic/window.ts'
+
+const RETENTION_DAYS = 14
+const SAMPLE_MS = 1000
+
+function memwatchDir(): string {
+  const home = process.env.HERMES_HOME?.trim()
+  const base = home && home.length > 0 ? home : join(homedir(), '.hermes')
+  return join(base, 'logs', 'memwatch')
+}
+
+function pruneOld(dir: string): void {
+  const cutoff = Date.now() - RETENTION_DAYS * 24 * 3600 * 1000
+  try {
+    for (const name of readdirSync(dir)) {
+      if (!name.endsWith('.jsonl')) continue
+      const p = join(dir, name)
+      try {
+        if (statSync(p).mtimeMs < cutoff) unlinkSync(p)
+      } catch {
+        /* best-effort */
+      }
+    }
+  } catch {
+    /* best-effort */
+  }
+}
+
+/** Start the self-sampler (no-op unless enabled). Returns a stop function. */
+export function startMemlog(): () => void {
+  if (!envFlag(process.env.HERMES_TUI_MEMLOG, diagnosticsEnabled())) return () => {}
+  try {
+    const dir = memwatchDir()
+    mkdirSync(dir, { recursive: true })
+    pruneOld(dir)
+    const boot = new Date().toISOString().replace(/[:.]/g, '').slice(0, 15)
+    const file = join(dir, `${boot}-${process.pid}.jsonl`)
+    const timer = setInterval(() => {
+      try {
+        const m = process.memoryUsage()
+        const w = windowRowStats()
+        const line = JSON.stringify({
+          t: Math.floor(Date.now() / 1000),
+          rss_kb: Math.floor(m.rss / 1024),
+          heap_used_kb: Math.floor(m.heapUsed / 1024),
+          external_kb: Math.floor(m.external / 1024),
+          mounted: w.mounted,
+          peak_mounted: w.peakMounted
+        })
+        appendFileSync(file, line + '\n')
+      } catch {
+        clearInterval(timer) // a failing diagnostic must not retry forever
+      }
+    }, SAMPLE_MS)
+    timer.unref?.()
+    return () => clearInterval(timer)
+  } catch {
+    return () => {}
+  }
+}

ui-opentui/src/entry/main.tsx

@@ -29,6 +29,7 @@ import { readClipboardImage, writeClipboard } from '../boundary/clipboard.ts'
 import { GatewayService, type GatewayServiceShape } from '../boundary/gateway/GatewayService.ts'
 import { liveGatewayLayer } from '../boundary/gateway/liveGateway.ts'
 import { getLog } from '../boundary/log.ts'
+import { startMemlog } from '../boundary/memlog.ts'
 import { acquireRenderer } from '../boundary/renderer.ts'
 import { makeAppLayer } from '../boundary/runtime.ts'
 import { nthAssistantResponse } from '../logic/copy.ts'
@@ -381,6 +382,10 @@ export const run = Effect.fn('Tui.run')(function* (input: TuiInput) {
         onCtrlC,
         onCopySelection
       })
+      // Fleet memory self-sampling (HERMES_TUI_MEMLOG / diagnostics master
+      // switch — boundary/memlog.ts). Scoped acquire→release like the renderer.
+      const stopMemlog = startMemlog()
+      yield* Effect.addFinalizer(() => Effect.sync(stopMemlog))
       doQuit = () => {
         if (!renderer.isDestroyed) renderer.destroy()
       }