From 41a5bbf3e849623e16fa2e55315e7041191e7c48 Mon Sep 17 00:00:00 2001 From: alt-glitch Date: Tue, 9 Jun 2026 07:39:36 +0000 Subject: [PATCH] opentui(harden): gateway recovery policy (count-cap + exp backoff) --- .../src/logic/gatewayRecovery.ts | 52 +++++++++++++ .../src/test/gatewayRecovery.test.ts | 76 +++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 ui-tui-opentui-v2/src/logic/gatewayRecovery.ts create mode 100644 ui-tui-opentui-v2/src/test/gatewayRecovery.test.ts diff --git a/ui-tui-opentui-v2/src/logic/gatewayRecovery.ts b/ui-tui-opentui-v2/src/logic/gatewayRecovery.ts new file mode 100644 index 00000000000..41da30d8e56 --- /dev/null +++ b/ui-tui-opentui-v2/src/logic/gatewayRecovery.ts @@ -0,0 +1,52 @@ +/** + * Pure recovery-budget policy for the gateway exit handler (LOGIC side — no + * Effect, no refs, no UI). Ported from Ink's `ui-tui/src/app/gatewayRecovery.ts` + * and EXTENDED with opencode-style exponential backoff. + * + * A gateway that crash-loops on startup must not let the TUI spawn-storm, so + * respawn+resume attempts are capped to GATEWAY_RECOVERY_LIMIT within a sliding + * GATEWAY_RECOVERY_WINDOW_MS; past the budget the app falls back to the inert + * "gateway exited" state. Kept pure (no refs/UI) so the bound — including the + * crash-loop case — is unit-testable. + */ +export const GATEWAY_RECOVERY_LIMIT = 3 +export const GATEWAY_RECOVERY_WINDOW_MS = 60_000 + +export interface RecoveryPlan { + /** Attempt timestamps to persist (the pruned window, plus `now` iff recovering). */ + attempts: number[] + recover: boolean + /** + * Session to resume — the live sid, or the not-yet-consumed recovery target + * when the live sid was already cleared by a prior exit. + */ + sid: null | string +} + +/** + * Decide whether to respawn+resume after a gateway death. `liveSid` is the + * current session (nulled on the first exit); `recoverSid` is a pending + * recovery target carried across a respawn that died before gateway.ready — + * so a startup crash-loop keeps retrying the same session up to the budget + * instead of stranding it after one attempt. + */ +export function planGatewayRecovery( + liveSid: null | string, + recoverSid: null | string, + attempts: number[], + now: number +): RecoveryPlan { + const sid = liveSid ?? recoverSid + const recent = attempts.filter(t => now - t < GATEWAY_RECOVERY_WINDOW_MS) + const recover = Boolean(sid) && recent.length < GATEWAY_RECOVERY_LIMIT + + return { attempts: recover ? [...recent, now] : recent, recover, sid } +} + +/** + * Exponential backoff between respawn attempts (opencode-style): 1s, 2s, 4s, … + * capped at 30s. `attempt` is 1-based (the first respawn waits 1s). + */ +export function backoffMs(attempt: number): number { + return Math.min(1000 * 2 ** Math.max(0, attempt - 1), 30_000) +} diff --git a/ui-tui-opentui-v2/src/test/gatewayRecovery.test.ts b/ui-tui-opentui-v2/src/test/gatewayRecovery.test.ts new file mode 100644 index 00000000000..9f23389376a --- /dev/null +++ b/ui-tui-opentui-v2/src/test/gatewayRecovery.test.ts @@ -0,0 +1,76 @@ +/** + * Recovery-budget policy test (LOGIC side, pure). The crash-loop bound: attempts + * are capped within a sliding window, stale attempts are pruned, and recovery is + * refused with no session. Plus opencode-style exponential backoff (1s→30s cap). + */ +import { describe, expect, test } from 'bun:test' + +import { + backoffMs, + GATEWAY_RECOVERY_LIMIT, + GATEWAY_RECOVERY_WINDOW_MS, + planGatewayRecovery +} from '../logic/gatewayRecovery.ts' + +describe('planGatewayRecovery — crash-loop budget', () => { + test('allows GATEWAY_RECOVERY_LIMIT attempts within the window, refuses the next', () => { + const sid = 'sess-1' + let attempts: number[] = [] + const now = 1_000_000 + + // The first LIMIT exits all recover, each recording its timestamp. + for (let i = 0; i < GATEWAY_RECOVERY_LIMIT; i++) { + const plan = planGatewayRecovery(sid, null, attempts, now + i) + expect(plan.recover).toBe(true) + expect(plan.sid).toBe(sid) + attempts = plan.attempts + } + expect(attempts).toHaveLength(GATEWAY_RECOVERY_LIMIT) + + // The (LIMIT+1)th within the window is refused; attempts are NOT extended. + const refused = planGatewayRecovery(sid, null, attempts, now + GATEWAY_RECOVERY_LIMIT) + expect(refused.recover).toBe(false) + expect(refused.attempts).toHaveLength(GATEWAY_RECOVERY_LIMIT) + }) + + test('prunes attempts older than GATEWAY_RECOVERY_WINDOW_MS, freeing the budget', () => { + const sid = 'sess-1' + const now = 1_000_000 + // Three stale attempts (all outside the window) + one fresh. + const stale = [now - GATEWAY_RECOVERY_WINDOW_MS - 5, now - GATEWAY_RECOVERY_WINDOW_MS - 4, now - 30_000] + const plan = planGatewayRecovery(sid, null, stale, now) + // The two truly-stale ones are pruned; the in-window one survives + `now` added. + expect(plan.recover).toBe(true) + expect(plan.attempts).toEqual([now - 30_000, now]) + }) + + test('refuses recovery when there is no session id (live nor recover)', () => { + const plan = planGatewayRecovery(null, null, [], 1_000_000) + expect(plan.recover).toBe(false) + expect(plan.sid).toBeNull() + expect(plan.attempts).toEqual([]) + }) + + test('falls back to the recoverSid when the live sid was already cleared', () => { + const plan = planGatewayRecovery(null, 'pending-sess', [], 1_000_000) + expect(plan.recover).toBe(true) + expect(plan.sid).toBe('pending-sess') + }) +}) + +describe('backoffMs — exponential delay (1s→30s cap)', () => { + test('doubles per attempt (1-based) and caps at 30000ms', () => { + expect(backoffMs(1)).toBe(1000) + expect(backoffMs(2)).toBe(2000) + expect(backoffMs(3)).toBe(4000) + expect(backoffMs(4)).toBe(8000) + expect(backoffMs(5)).toBe(16000) + expect(backoffMs(6)).toBe(30000) // 32000 clamped to the cap + expect(backoffMs(10)).toBe(30000) // stays at the cap + }) + + test('clamps a non-positive attempt to the first delay', () => { + expect(backoffMs(0)).toBe(1000) + expect(backoffMs(-3)).toBe(1000) + }) +})