From 8f33e39682ae27963c93c87ea13bd0a3c2a55ed8 Mon Sep 17 00:00:00 2001 From: UnathiCodex Date: Mon, 20 Jul 2026 16:51:27 +0200 Subject: [PATCH] fix(desktop): prevent false runtime-not-ready under gateway load (#66174) * fix(desktop): stabilize runtime readiness polling * fix(tui_gateway): pool live-session status polling * fix(desktop): clear readiness when gateway disconnects --- .../shell/hooks/use-status-snapshot.test.ts | 170 ++++++++++++++++++ .../app/shell/hooks/use-status-snapshot.ts | 59 ++++-- .../test_inline_rpc_gil_starvation.py | 1 + tui_gateway/server.py | 6 + 4 files changed, 221 insertions(+), 15 deletions(-) create mode 100644 apps/desktop/src/app/shell/hooks/use-status-snapshot.test.ts diff --git a/apps/desktop/src/app/shell/hooks/use-status-snapshot.test.ts b/apps/desktop/src/app/shell/hooks/use-status-snapshot.test.ts new file mode 100644 index 000000000000..9ffbfe4b0a51 --- /dev/null +++ b/apps/desktop/src/app/shell/hooks/use-status-snapshot.test.ts @@ -0,0 +1,170 @@ +import { act, cleanup, renderHook } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { getStatus } from '@/hermes' + +import { useStatusSnapshot } from './use-status-snapshot' + +vi.mock('@/hermes', () => ({ + getStatus: vi.fn() +})) + +type GatewayRequester = (method: string, params?: Record) => Promise + +function deferred() { + let resolve: (value: T) => void = () => undefined + let reject: (reason?: unknown) => void = () => undefined + + const promise = new Promise((nextResolve, nextReject) => { + resolve = nextResolve + reject = nextReject + }) + + return { promise, reject, resolve } +} + +async function flushAsync() { + await act(async () => { + await vi.advanceTimersByTimeAsync(0) + }) +} + +beforeEach(() => { + vi.useFakeTimers() + vi.mocked(getStatus) + .mockReset() + .mockResolvedValue({} as never) +}) + +afterEach(() => { + cleanup() + vi.useRealTimers() +}) + +describe('useStatusSnapshot', () => { + it('keeps the last authoritative readiness through a transient RPC failure', async () => { + let refresh = 0 + + const requestGatewayMock = vi.fn(async (method: string) => { + const cycle = Math.floor(refresh / 2) + refresh += 1 + + if (cycle > 0) { + throw new Error(`${method} timed out`) + } + + return (method === 'setup.runtime_check' ? { ok: true } : { provider_configured: true }) as never + }) + + const requestGateway = requestGatewayMock as unknown as GatewayRequester + + const { result } = renderHook(() => useStatusSnapshot('open', requestGateway)) + + await flushAsync() + expect(result.current.inferenceStatus).toMatchObject({ ready: true, source: 'runtime_check' }) + + await act(async () => { + await vi.advanceTimersByTimeAsync(15_000) + }) + + expect(result.current.inferenceStatus).toMatchObject({ ready: true, source: 'runtime_check' }) + }) + + it('does not present an initial transport failure as inference not ready', async () => { + const requestGatewayMock = vi.fn(async (method: string) => { + throw new Error(`${method} connection closed`) + }) + + const requestGateway = requestGatewayMock as unknown as GatewayRequester + + const { result } = renderHook(() => useStatusSnapshot('open', requestGateway)) + + await flushAsync() + + expect(result.current.inferenceStatus).toBeNull() + }) + + it('still publishes an authoritative runtime failure', async () => { + const requestGatewayMock = vi.fn( + async (method: string) => + (method === 'setup.runtime_check' + ? { error: 'No usable credentials found for nous.', ok: false } + : { provider_configured: true }) as never + ) + + const requestGateway = requestGatewayMock as unknown as GatewayRequester + + const { result } = renderHook(() => useStatusSnapshot('open', requestGateway)) + + await flushAsync() + + expect(result.current.inferenceStatus).toMatchObject({ + ready: false, + reason: expect.stringContaining('No usable credentials found for nous.'), + source: 'runtime_check' + }) + }) + + it('clears readiness immediately when the gateway disconnects', async () => { + const pendingStatus = deferred() + + vi.mocked(getStatus) + .mockResolvedValueOnce({} as never) + .mockReturnValueOnce(pendingStatus.promise) + + const requestGateway = vi.fn( + async (method: string) => + (method === 'setup.runtime_check' ? { ok: true } : { provider_configured: true }) as never + ) as unknown as GatewayRequester + + const { rerender, result } = renderHook(({ gatewayState }) => useStatusSnapshot(gatewayState, requestGateway), { + initialProps: { gatewayState: 'open' } + }) + + await flushAsync() + expect(result.current.inferenceStatus).toMatchObject({ ready: true, source: 'runtime_check' }) + + rerender({ gatewayState: 'connecting' }) + + expect(getStatus).toHaveBeenCalledTimes(2) + expect(result.current.inferenceStatus).toBeNull() + }) + + it('waits for a slow refresh to settle before scheduling another one', async () => { + const setup = deferred() + const runtime = deferred() + + const requestGatewayMock = vi.fn( + (method: string) => (method === 'setup.runtime_check' ? runtime.promise : setup.promise) as never + ) + + const requestGateway = requestGatewayMock as unknown as GatewayRequester + + renderHook(() => useStatusSnapshot('open', requestGateway)) + await flushAsync() + + expect(requestGatewayMock).toHaveBeenCalledTimes(2) + + await act(async () => { + await vi.advanceTimersByTimeAsync(60_000) + }) + + expect(requestGatewayMock).toHaveBeenCalledTimes(2) + + await act(async () => { + setup.resolve({ provider_configured: true }) + runtime.resolve({ ok: true }) + await vi.advanceTimersByTimeAsync(0) + }) + + await act(async () => { + await vi.advanceTimersByTimeAsync(14_999) + }) + expect(requestGatewayMock).toHaveBeenCalledTimes(2) + + await act(async () => { + await vi.advanceTimersByTimeAsync(1) + }) + expect(requestGatewayMock).toHaveBeenCalledTimes(4) + }) +}) diff --git a/apps/desktop/src/app/shell/hooks/use-status-snapshot.ts b/apps/desktop/src/app/shell/hooks/use-status-snapshot.ts index 0cdbc5b98e2c..7c531d9cf811 100644 --- a/apps/desktop/src/app/shell/hooks/use-status-snapshot.ts +++ b/apps/desktop/src/app/shell/hooks/use-status-snapshot.ts @@ -14,38 +14,67 @@ export function useStatusSnapshot(gatewayState: string | undefined, requestGatew useEffect(() => { let cancelled = false + let timer: number | undefined + + // A closed/connecting gateway cannot have an authoritative live-runtime + // result. Clear readiness before starting the REST status leg so a hung + // getStatus() cannot leave a stale "ready" state visible after disconnect. + if (gatewayState !== 'open') { + setInferenceStatus(null) + } + + const scheduleRefresh = () => { + if (!cancelled) { + timer = window.setTimeout(() => void refresh(), REFRESH_MS) + } + } const refresh = async () => { try { - const [next, inference] = await Promise.all([ + // Wait for both legs before scheduling the next refresh. setInterval + // allowed a slow runtime check to overlap with later polls, which + // multiplied load on an already-busy gateway and let stale failures + // race newer healthy results. + const [statusResult, inferenceResult] = await Promise.allSettled([ getStatus(), - gatewayState === 'open' - ? evaluateRuntimeReadiness(requestGateway).catch(error => ({ - checksDisagree: false, - ready: false, - reason: error instanceof Error ? error.message : String(error), - source: 'fallback' as const - })) - : Promise.resolve(null) + gatewayState === 'open' ? evaluateRuntimeReadiness(requestGateway) : Promise.resolve(null) ]) if (cancelled) { return } - setStatusSnapshot(next) - setInferenceStatus(inference) - } catch { - // Keep last snapshot through transient gateway flaps. + if (statusResult.status === 'fulfilled') { + setStatusSnapshot(statusResult.value) + } + + if (inferenceResult.status === 'fulfilled') { + const inference = inferenceResult.value + + if (inference === null) { + setInferenceStatus(null) + } else if (inference.source !== 'fallback') { + // runtime_check/setup_status returned an authoritative boolean. + // A fallback means both RPCs failed or returned no boolean, so it + // is a transient/unknown transport state, not proof that inference + // became unconfigured. Keep the last authoritative result instead + // of flashing "Inference not ready" during a gateway flap. + setInferenceStatus(inference) + } + } + } finally { + scheduleRefresh() } } void refresh() - const timer = window.setInterval(() => void refresh(), REFRESH_MS) return () => { cancelled = true - window.clearInterval(timer) + + if (timer !== undefined) { + window.clearTimeout(timer) + } } }, [gatewayState, requestGateway]) diff --git a/tests/tui_gateway/test_inline_rpc_gil_starvation.py b/tests/tui_gateway/test_inline_rpc_gil_starvation.py index 99c63c746280..32aa60b0f00d 100644 --- a/tests/tui_gateway/test_inline_rpc_gil_starvation.py +++ b/tests/tui_gateway/test_inline_rpc_gil_starvation.py @@ -64,6 +64,7 @@ def capture(server): # seconds when the GIL is contended by concurrent agent turns. FRONTEND_POLLED_RPCS = [ + "session.active_list", # live-session rehydrate — in-memory registry "session.list", # loads session list — SQLite query "pet.info", # petdex poll — file/network read "process.list", # background process status — process registry scan diff --git a/tui_gateway/server.py b/tui_gateway/server.py index e6f32e078baa..798abaedb7df 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -242,6 +242,12 @@ _LONG_HANDLERS = frozenset( # the WS read loop and causing false "needs setup" (#50005 family). "setup.runtime_check", "setup.status", + # Desktop also polls the in-memory live-session registry every 15s. + # The handler is normally cheap, but under heavy agent GIL pressure it + # can still stall for tens of seconds. Keep it off the WS reader thread + # so a delayed status rehydrate cannot block runtime readiness, prompt + # submission, or interrupts queued behind it on the same socket. + "session.active_list", "session.branch", "session.compress", "session.list",