opentui(harden): gateway recovery policy (count-cap + exp backoff)

This commit is contained in:
alt-glitch 2026-06-09 07:39:36 +00:00
parent 84b77f68e5
commit 41a5bbf3e8
2 changed files with 128 additions and 0 deletions

View file

@ -0,0 +1,52 @@
/**
* Pure recovery-budget policy for the gateway exit handler (LOGIC side no
* Effect, no refs, no UI). Ported from Ink's `ui-tui/src/app/gatewayRecovery.ts`
* and EXTENDED with opencode-style exponential backoff.
*
* A gateway that crash-loops on startup must not let the TUI spawn-storm, so
* respawn+resume attempts are capped to GATEWAY_RECOVERY_LIMIT within a sliding
* GATEWAY_RECOVERY_WINDOW_MS; past the budget the app falls back to the inert
* "gateway exited" state. Kept pure (no refs/UI) so the bound including the
* crash-loop case is unit-testable.
*/
export const GATEWAY_RECOVERY_LIMIT = 3
export const GATEWAY_RECOVERY_WINDOW_MS = 60_000
export interface RecoveryPlan {
/** Attempt timestamps to persist (the pruned window, plus `now` iff recovering). */
attempts: number[]
recover: boolean
/**
* Session to resume the live sid, or the not-yet-consumed recovery target
* when the live sid was already cleared by a prior exit.
*/
sid: null | string
}
/**
* Decide whether to respawn+resume after a gateway death. `liveSid` is the
* current session (nulled on the first exit); `recoverSid` is a pending
* recovery target carried across a respawn that died before gateway.ready
* so a startup crash-loop keeps retrying the same session up to the budget
* instead of stranding it after one attempt.
*/
export function planGatewayRecovery(
liveSid: null | string,
recoverSid: null | string,
attempts: number[],
now: number
): RecoveryPlan {
const sid = liveSid ?? recoverSid
const recent = attempts.filter(t => now - t < GATEWAY_RECOVERY_WINDOW_MS)
const recover = Boolean(sid) && recent.length < GATEWAY_RECOVERY_LIMIT
return { attempts: recover ? [...recent, now] : recent, recover, sid }
}
/**
* Exponential backoff between respawn attempts (opencode-style): 1s, 2s, 4s,
* capped at 30s. `attempt` is 1-based (the first respawn waits 1s).
*/
export function backoffMs(attempt: number): number {
return Math.min(1000 * 2 ** Math.max(0, attempt - 1), 30_000)
}

View file

@ -0,0 +1,76 @@
/**
* Recovery-budget policy test (LOGIC side, pure). The crash-loop bound: attempts
* are capped within a sliding window, stale attempts are pruned, and recovery is
* refused with no session. Plus opencode-style exponential backoff (1s30s cap).
*/
import { describe, expect, test } from 'bun:test'
import {
backoffMs,
GATEWAY_RECOVERY_LIMIT,
GATEWAY_RECOVERY_WINDOW_MS,
planGatewayRecovery
} from '../logic/gatewayRecovery.ts'
describe('planGatewayRecovery — crash-loop budget', () => {
test('allows GATEWAY_RECOVERY_LIMIT attempts within the window, refuses the next', () => {
const sid = 'sess-1'
let attempts: number[] = []
const now = 1_000_000
// The first LIMIT exits all recover, each recording its timestamp.
for (let i = 0; i < GATEWAY_RECOVERY_LIMIT; i++) {
const plan = planGatewayRecovery(sid, null, attempts, now + i)
expect(plan.recover).toBe(true)
expect(plan.sid).toBe(sid)
attempts = plan.attempts
}
expect(attempts).toHaveLength(GATEWAY_RECOVERY_LIMIT)
// The (LIMIT+1)th within the window is refused; attempts are NOT extended.
const refused = planGatewayRecovery(sid, null, attempts, now + GATEWAY_RECOVERY_LIMIT)
expect(refused.recover).toBe(false)
expect(refused.attempts).toHaveLength(GATEWAY_RECOVERY_LIMIT)
})
test('prunes attempts older than GATEWAY_RECOVERY_WINDOW_MS, freeing the budget', () => {
const sid = 'sess-1'
const now = 1_000_000
// Three stale attempts (all outside the window) + one fresh.
const stale = [now - GATEWAY_RECOVERY_WINDOW_MS - 5, now - GATEWAY_RECOVERY_WINDOW_MS - 4, now - 30_000]
const plan = planGatewayRecovery(sid, null, stale, now)
// The two truly-stale ones are pruned; the in-window one survives + `now` added.
expect(plan.recover).toBe(true)
expect(plan.attempts).toEqual([now - 30_000, now])
})
test('refuses recovery when there is no session id (live nor recover)', () => {
const plan = planGatewayRecovery(null, null, [], 1_000_000)
expect(plan.recover).toBe(false)
expect(plan.sid).toBeNull()
expect(plan.attempts).toEqual([])
})
test('falls back to the recoverSid when the live sid was already cleared', () => {
const plan = planGatewayRecovery(null, 'pending-sess', [], 1_000_000)
expect(plan.recover).toBe(true)
expect(plan.sid).toBe('pending-sess')
})
})
describe('backoffMs — exponential delay (1s→30s cap)', () => {
test('doubles per attempt (1-based) and caps at 30000ms', () => {
expect(backoffMs(1)).toBe(1000)
expect(backoffMs(2)).toBe(2000)
expect(backoffMs(3)).toBe(4000)
expect(backoffMs(4)).toBe(8000)
expect(backoffMs(5)).toBe(16000)
expect(backoffMs(6)).toBe(30000) // 32000 clamped to the cap
expect(backoffMs(10)).toBe(30000) // stays at the cap
})
test('clamps a non-positive attempt to the first delay', () => {
expect(backoffMs(0)).toBe(1000)
expect(backoffMs(-3)).toBe(1000)
})
})