diff --git a/ui-tui-opentui-v2/src/boundary/gateway/client.ts b/ui-tui-opentui-v2/src/boundary/gateway/client.ts index eeff84f38aa..1d9e3f2a4ad 100644 --- a/ui-tui-opentui-v2/src/boundary/gateway/client.ts +++ b/ui-tui-opentui-v2/src/boundary/gateway/client.ts @@ -74,6 +74,9 @@ export class RawGatewayClient { stdout: 'pipe', stderr: 'pipe', onExit: (_p, code, signal) => { + // Identity guard: a stale child's late exit must not act after a restart + // has already installed a new `this.proc` (else it'd null the live child). + if (this.proc !== proc) return const reason = `gateway exited (code=${code ?? 'null'} signal=${signal ?? 'null'})` this.log.warn('gateway', reason) this.rejectAll(reason) @@ -171,7 +174,9 @@ export class RawGatewayClient { /** Send a JSON-RPC request; resolves with `result` (long handlers reply async). */ request(method: string, params: unknown): Promise { - if (!this.proc) this.start() + // Do NOT auto-start here: during the recovery backoff window `this.proc` is + // null, and a respawn here would BYPASS the backoff (the first spawn always + // comes from subscribe() → client.start()). A null proc rejects below. const proc = this.proc const stdin = proc?.stdin if (!stdin || typeof stdin === 'number') return Promise.reject(new Error('gateway not running')) diff --git a/ui-tui-opentui-v2/src/boundary/gateway/liveGateway.ts b/ui-tui-opentui-v2/src/boundary/gateway/liveGateway.ts index 7e3f0cd7b4f..92bc78d9ae4 100644 --- a/ui-tui-opentui-v2/src/boundary/gateway/liveGateway.ts +++ b/ui-tui-opentui-v2/src/boundary/gateway/liveGateway.ts @@ -15,6 +15,7 @@ import { Effect, Layer, Option, Schema } from 'effect' import { batch } from 'solid-js' +import { backoffMs, planGatewayRecovery } from '../../logic/gatewayRecovery.ts' import { GatewayError } from '../errors.ts' import { getLog } from '../log.ts' import { GatewayEventSchema, type GatewayEvent } from '../schema/GatewayEvent.ts' @@ -30,6 +31,14 @@ function makeLiveGateway(): { service: GatewayServiceShape; stop: () => void } { const handlers = new Set<(event: GatewayEvent) => void>() let sessionId: string | undefined + // Auto-heal recovery state (driver below). `recoverSid` is the resume target + // carried across a respawn that died before gateway.ready; `recoveryAttempts` + // is the sliding crash-loop budget window; `restartTimer` is the pending + // backoff respawn (cleared on teardown so it can't fire post-stop). + let recoverSid: string | undefined + let recoveryAttempts: number[] = [] + let restartTimer: ReturnType | undefined + // 16ms event coalescing → one batched repaint (opencode sdk.tsx model). let queue: GatewayEvent[] = [] let timer: ReturnType | undefined @@ -69,10 +78,37 @@ function makeLiveGateway(): { service: GatewayServiceShape; stop: () => void } { enqueue(decoded.value) } + // Recovery driver: on a child exit, clear the frozen spinner (via the store's + // gateway.exited case), then — under the crash-loop budget — respawn the child + // on exponential backoff. The post-respawn gateway.ready triggers the re-resume + // (driven from entry's subscribe callback). Hoisted so it can be passed to + // `new RawGatewayClient` below while itself referencing the `client` const — + // `client` is assigned by the time onExit ever fires at runtime. + function onExit(reason: string): void { + log.warn('gateway', 'transport exited', { reason }) + // Clears the frozen spinner + shows status (store handles gateway.exited). + enqueue({ type: 'gateway.exited', payload: { reason } }) + const plan = planGatewayRecovery(sessionId ?? null, recoverSid ?? null, recoveryAttempts, Date.now()) + recoveryAttempts = plan.attempts + if (!plan.recover || plan.sid === null) { + enqueue({ type: 'error', payload: { message: 'gateway exited repeatedly — type /resume to retry' } }) + return + } + recoverSid = plan.sid + const attempt = recoveryAttempts.length + const delay = backoffMs(attempt) + enqueue({ type: 'gateway.recovering', payload: { attempt, delay_ms: delay } }) + if (restartTimer) clearTimeout(restartTimer) + restartTimer = setTimeout(() => { + restartTimer = undefined + client.start() + }, delay) + } + const client = new RawGatewayClient({ log, onEvent: onRawEvent, - onExit: reason => log.warn('gateway', 'transport exited', { reason }) + onExit }) const service: GatewayServiceShape = { @@ -118,6 +154,9 @@ function makeLiveGateway(): { service: GatewayServiceShape; stop: () => void } { const stop = () => { if (timer) clearTimeout(timer) timer = undefined + // Also kill any pending backoff respawn so it can't fire after teardown. + if (restartTimer) clearTimeout(restartTimer) + restartTimer = undefined client.stop() } return { service, stop } diff --git a/ui-tui-opentui-v2/src/entry/main.tsx b/ui-tui-opentui-v2/src/entry/main.tsx index c2420ea0176..be1dfb9b275 100644 --- a/ui-tui-opentui-v2/src/entry/main.tsx +++ b/ui-tui-opentui-v2/src/entry/main.tsx @@ -194,8 +194,28 @@ export const run = Effect.fn('Tui.run')(function* (input: TuiInput) { const pasteStore = createPasteStore() // Contact point #2: boundary pushes decoded events into the Solid store. + // The callback ALSO drives auto-heal re-resume: a post-crash gateway.ready + // (i.e. one that follows a gateway.exited, so `recoverSid` is set) re-resumes + // the session so the transcript continues. The INITIAL gateway.ready has + // `recoverSid === undefined`, so the normal bootstrap path is untouched. const gateway = yield* GatewayService - yield* gateway.subscribe(event => store.apply(event)) + let recoverSid: string | undefined + yield* gateway.subscribe(event => { + store.apply(event) + if (event.type === 'gateway.exited') { + recoverSid = gateway.sessionId() ?? recoverSid + } else if (event.type === 'gateway.ready' && recoverSid !== undefined) { + const sid = recoverSid + recoverSid = undefined + Effect.runFork( + resumeInto(gateway, store, sid, input.cols).pipe( + Effect.catchCause(cause => + Effect.sync(() => getLog().warn('recover', 'resume failed', { cause: String(cause) })) + ) + ) + ) + } + }) // ── Ctrl+C state machine (item 11) ────────────────────────────────── // While a turn runs, the first Ctrl+C STOPS the agent (session.interrupt); diff --git a/ui-tui-opentui-v2/src/logic/store.ts b/ui-tui-opentui-v2/src/logic/store.ts index b2801e46e13..ec6a8913d4b 100644 --- a/ui-tui-opentui-v2/src/logic/store.ts +++ b/ui-tui-opentui-v2/src/logic/store.ts @@ -685,7 +685,10 @@ export function createSessionStore() { // the user their in-flight reply was lost, and show a recovering status. case 'gateway.exited': { setState('info', prev => ({ ...prev, running: false })) - setState('status', 'gateway exited — recovering…') + // Neutral status: we don't ALWAYS recover (budget exhaustion). The + // "recovering…" wording now comes from the gateway.recovering case, + // which fires only when a respawn is actually scheduled. + setState('status', 'gateway exited') const reason = event.payload?.reason const base = 'gateway exited — recovering your session (any in-flight reply was lost)' pushSystem(reason ? `${base}: ${reason}` : base) diff --git a/ui-tui-opentui-v2/src/test/store.test.ts b/ui-tui-opentui-v2/src/test/store.test.ts index 3efba650619..46f6851daae 100644 --- a/ui-tui-opentui-v2/src/test/store.test.ts +++ b/ui-tui-opentui-v2/src/test/store.test.ts @@ -339,7 +339,8 @@ describe('session store — gateway lifecycle / transport errors (auto-heal foun store.apply({ type: 'gateway.exited' }) // THE key bug fix: the spinner is cleared even though no message.complete arrived. expect(store.state.info.running).toBe(false) - expect(store.state.status).toBe('gateway exited — recovering…') + // Neutral status — "recovering…" now comes from gateway.recovering only. + expect(store.state.status).toBe('gateway exited') const sys = store.state.messages.filter(m => m.role === 'system') expect(sys).toHaveLength(1) expect(sys[0]!.text).toContain('in-flight reply was lost')