mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-09 13:21:42 +00:00
opentui(harden): gateway recovery policy (count-cap + exp backoff)
This commit is contained in:
parent
84b77f68e5
commit
41a5bbf3e8
2 changed files with 128 additions and 0 deletions
52
ui-tui-opentui-v2/src/logic/gatewayRecovery.ts
Normal file
52
ui-tui-opentui-v2/src/logic/gatewayRecovery.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
/**
|
||||
* Pure recovery-budget policy for the gateway exit handler (LOGIC side — no
|
||||
* Effect, no refs, no UI). Ported from Ink's `ui-tui/src/app/gatewayRecovery.ts`
|
||||
* and EXTENDED with opencode-style exponential backoff.
|
||||
*
|
||||
* A gateway that crash-loops on startup must not let the TUI spawn-storm, so
|
||||
* respawn+resume attempts are capped to GATEWAY_RECOVERY_LIMIT within a sliding
|
||||
* GATEWAY_RECOVERY_WINDOW_MS; past the budget the app falls back to the inert
|
||||
* "gateway exited" state. Kept pure (no refs/UI) so the bound — including the
|
||||
* crash-loop case — is unit-testable.
|
||||
*/
|
||||
export const GATEWAY_RECOVERY_LIMIT = 3
|
||||
export const GATEWAY_RECOVERY_WINDOW_MS = 60_000
|
||||
|
||||
export interface RecoveryPlan {
|
||||
/** Attempt timestamps to persist (the pruned window, plus `now` iff recovering). */
|
||||
attempts: number[]
|
||||
recover: boolean
|
||||
/**
|
||||
* Session to resume — the live sid, or the not-yet-consumed recovery target
|
||||
* when the live sid was already cleared by a prior exit.
|
||||
*/
|
||||
sid: null | string
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether to respawn+resume after a gateway death. `liveSid` is the
|
||||
* current session (nulled on the first exit); `recoverSid` is a pending
|
||||
* recovery target carried across a respawn that died before gateway.ready —
|
||||
* so a startup crash-loop keeps retrying the same session up to the budget
|
||||
* instead of stranding it after one attempt.
|
||||
*/
|
||||
export function planGatewayRecovery(
|
||||
liveSid: null | string,
|
||||
recoverSid: null | string,
|
||||
attempts: number[],
|
||||
now: number
|
||||
): RecoveryPlan {
|
||||
const sid = liveSid ?? recoverSid
|
||||
const recent = attempts.filter(t => now - t < GATEWAY_RECOVERY_WINDOW_MS)
|
||||
const recover = Boolean(sid) && recent.length < GATEWAY_RECOVERY_LIMIT
|
||||
|
||||
return { attempts: recover ? [...recent, now] : recent, recover, sid }
|
||||
}
|
||||
|
||||
/**
|
||||
* Exponential backoff between respawn attempts (opencode-style): 1s, 2s, 4s, …
|
||||
* capped at 30s. `attempt` is 1-based (the first respawn waits 1s).
|
||||
*/
|
||||
export function backoffMs(attempt: number): number {
|
||||
return Math.min(1000 * 2 ** Math.max(0, attempt - 1), 30_000)
|
||||
}
|
||||
76
ui-tui-opentui-v2/src/test/gatewayRecovery.test.ts
Normal file
76
ui-tui-opentui-v2/src/test/gatewayRecovery.test.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
/**
|
||||
* Recovery-budget policy test (LOGIC side, pure). The crash-loop bound: attempts
|
||||
* are capped within a sliding window, stale attempts are pruned, and recovery is
|
||||
* refused with no session. Plus opencode-style exponential backoff (1s→30s cap).
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
|
||||
import {
|
||||
backoffMs,
|
||||
GATEWAY_RECOVERY_LIMIT,
|
||||
GATEWAY_RECOVERY_WINDOW_MS,
|
||||
planGatewayRecovery
|
||||
} from '../logic/gatewayRecovery.ts'
|
||||
|
||||
describe('planGatewayRecovery — crash-loop budget', () => {
|
||||
test('allows GATEWAY_RECOVERY_LIMIT attempts within the window, refuses the next', () => {
|
||||
const sid = 'sess-1'
|
||||
let attempts: number[] = []
|
||||
const now = 1_000_000
|
||||
|
||||
// The first LIMIT exits all recover, each recording its timestamp.
|
||||
for (let i = 0; i < GATEWAY_RECOVERY_LIMIT; i++) {
|
||||
const plan = planGatewayRecovery(sid, null, attempts, now + i)
|
||||
expect(plan.recover).toBe(true)
|
||||
expect(plan.sid).toBe(sid)
|
||||
attempts = plan.attempts
|
||||
}
|
||||
expect(attempts).toHaveLength(GATEWAY_RECOVERY_LIMIT)
|
||||
|
||||
// The (LIMIT+1)th within the window is refused; attempts are NOT extended.
|
||||
const refused = planGatewayRecovery(sid, null, attempts, now + GATEWAY_RECOVERY_LIMIT)
|
||||
expect(refused.recover).toBe(false)
|
||||
expect(refused.attempts).toHaveLength(GATEWAY_RECOVERY_LIMIT)
|
||||
})
|
||||
|
||||
test('prunes attempts older than GATEWAY_RECOVERY_WINDOW_MS, freeing the budget', () => {
|
||||
const sid = 'sess-1'
|
||||
const now = 1_000_000
|
||||
// Three stale attempts (all outside the window) + one fresh.
|
||||
const stale = [now - GATEWAY_RECOVERY_WINDOW_MS - 5, now - GATEWAY_RECOVERY_WINDOW_MS - 4, now - 30_000]
|
||||
const plan = planGatewayRecovery(sid, null, stale, now)
|
||||
// The two truly-stale ones are pruned; the in-window one survives + `now` added.
|
||||
expect(plan.recover).toBe(true)
|
||||
expect(plan.attempts).toEqual([now - 30_000, now])
|
||||
})
|
||||
|
||||
test('refuses recovery when there is no session id (live nor recover)', () => {
|
||||
const plan = planGatewayRecovery(null, null, [], 1_000_000)
|
||||
expect(plan.recover).toBe(false)
|
||||
expect(plan.sid).toBeNull()
|
||||
expect(plan.attempts).toEqual([])
|
||||
})
|
||||
|
||||
test('falls back to the recoverSid when the live sid was already cleared', () => {
|
||||
const plan = planGatewayRecovery(null, 'pending-sess', [], 1_000_000)
|
||||
expect(plan.recover).toBe(true)
|
||||
expect(plan.sid).toBe('pending-sess')
|
||||
})
|
||||
})
|
||||
|
||||
describe('backoffMs — exponential delay (1s→30s cap)', () => {
|
||||
test('doubles per attempt (1-based) and caps at 30000ms', () => {
|
||||
expect(backoffMs(1)).toBe(1000)
|
||||
expect(backoffMs(2)).toBe(2000)
|
||||
expect(backoffMs(3)).toBe(4000)
|
||||
expect(backoffMs(4)).toBe(8000)
|
||||
expect(backoffMs(5)).toBe(16000)
|
||||
expect(backoffMs(6)).toBe(30000) // 32000 clamped to the cap
|
||||
expect(backoffMs(10)).toBe(30000) // stays at the cap
|
||||
})
|
||||
|
||||
test('clamps a non-positive attempt to the first delay', () => {
|
||||
expect(backoffMs(0)).toBe(1000)
|
||||
expect(backoffMs(-3)).toBe(1000)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue