mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-29 18:46:59 +00:00
fix(desktop): prevent false runtime-not-ready under gateway load (#66174)
* fix(desktop): stabilize runtime readiness polling * fix(tui_gateway): pool live-session status polling * fix(desktop): clear readiness when gateway disconnects
This commit is contained in:
parent
3e6cead363
commit
8f33e39682
4 changed files with 221 additions and 15 deletions
170
apps/desktop/src/app/shell/hooks/use-status-snapshot.test.ts
Normal file
170
apps/desktop/src/app/shell/hooks/use-status-snapshot.test.ts
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
import { act, cleanup, renderHook } from '@testing-library/react'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { getStatus } from '@/hermes'
|
||||
|
||||
import { useStatusSnapshot } from './use-status-snapshot'
|
||||
|
||||
vi.mock('@/hermes', () => ({
|
||||
getStatus: vi.fn()
|
||||
}))
|
||||
|
||||
type GatewayRequester = <T = unknown>(method: string, params?: Record<string, unknown>) => Promise<T>
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve: (value: T) => void = () => undefined
|
||||
let reject: (reason?: unknown) => void = () => undefined
|
||||
|
||||
const promise = new Promise<T>((nextResolve, nextReject) => {
|
||||
resolve = nextResolve
|
||||
reject = nextReject
|
||||
})
|
||||
|
||||
return { promise, reject, resolve }
|
||||
}
|
||||
|
||||
async function flushAsync() {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.mocked(getStatus)
|
||||
.mockReset()
|
||||
.mockResolvedValue({} as never)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
cleanup()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
describe('useStatusSnapshot', () => {
|
||||
it('keeps the last authoritative readiness through a transient RPC failure', async () => {
|
||||
let refresh = 0
|
||||
|
||||
const requestGatewayMock = vi.fn(async (method: string) => {
|
||||
const cycle = Math.floor(refresh / 2)
|
||||
refresh += 1
|
||||
|
||||
if (cycle > 0) {
|
||||
throw new Error(`${method} timed out`)
|
||||
}
|
||||
|
||||
return (method === 'setup.runtime_check' ? { ok: true } : { provider_configured: true }) as never
|
||||
})
|
||||
|
||||
const requestGateway = requestGatewayMock as unknown as GatewayRequester
|
||||
|
||||
const { result } = renderHook(() => useStatusSnapshot('open', requestGateway))
|
||||
|
||||
await flushAsync()
|
||||
expect(result.current.inferenceStatus).toMatchObject({ ready: true, source: 'runtime_check' })
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(15_000)
|
||||
})
|
||||
|
||||
expect(result.current.inferenceStatus).toMatchObject({ ready: true, source: 'runtime_check' })
|
||||
})
|
||||
|
||||
it('does not present an initial transport failure as inference not ready', async () => {
|
||||
const requestGatewayMock = vi.fn(async (method: string) => {
|
||||
throw new Error(`${method} connection closed`)
|
||||
})
|
||||
|
||||
const requestGateway = requestGatewayMock as unknown as GatewayRequester
|
||||
|
||||
const { result } = renderHook(() => useStatusSnapshot('open', requestGateway))
|
||||
|
||||
await flushAsync()
|
||||
|
||||
expect(result.current.inferenceStatus).toBeNull()
|
||||
})
|
||||
|
||||
it('still publishes an authoritative runtime failure', async () => {
|
||||
const requestGatewayMock = vi.fn(
|
||||
async (method: string) =>
|
||||
(method === 'setup.runtime_check'
|
||||
? { error: 'No usable credentials found for nous.', ok: false }
|
||||
: { provider_configured: true }) as never
|
||||
)
|
||||
|
||||
const requestGateway = requestGatewayMock as unknown as GatewayRequester
|
||||
|
||||
const { result } = renderHook(() => useStatusSnapshot('open', requestGateway))
|
||||
|
||||
await flushAsync()
|
||||
|
||||
expect(result.current.inferenceStatus).toMatchObject({
|
||||
ready: false,
|
||||
reason: expect.stringContaining('No usable credentials found for nous.'),
|
||||
source: 'runtime_check'
|
||||
})
|
||||
})
|
||||
|
||||
it('clears readiness immediately when the gateway disconnects', async () => {
|
||||
const pendingStatus = deferred<never>()
|
||||
|
||||
vi.mocked(getStatus)
|
||||
.mockResolvedValueOnce({} as never)
|
||||
.mockReturnValueOnce(pendingStatus.promise)
|
||||
|
||||
const requestGateway = vi.fn(
|
||||
async (method: string) =>
|
||||
(method === 'setup.runtime_check' ? { ok: true } : { provider_configured: true }) as never
|
||||
) as unknown as GatewayRequester
|
||||
|
||||
const { rerender, result } = renderHook(({ gatewayState }) => useStatusSnapshot(gatewayState, requestGateway), {
|
||||
initialProps: { gatewayState: 'open' }
|
||||
})
|
||||
|
||||
await flushAsync()
|
||||
expect(result.current.inferenceStatus).toMatchObject({ ready: true, source: 'runtime_check' })
|
||||
|
||||
rerender({ gatewayState: 'connecting' })
|
||||
|
||||
expect(getStatus).toHaveBeenCalledTimes(2)
|
||||
expect(result.current.inferenceStatus).toBeNull()
|
||||
})
|
||||
|
||||
it('waits for a slow refresh to settle before scheduling another one', async () => {
|
||||
const setup = deferred<unknown>()
|
||||
const runtime = deferred<unknown>()
|
||||
|
||||
const requestGatewayMock = vi.fn(
|
||||
(method: string) => (method === 'setup.runtime_check' ? runtime.promise : setup.promise) as never
|
||||
)
|
||||
|
||||
const requestGateway = requestGatewayMock as unknown as GatewayRequester
|
||||
|
||||
renderHook(() => useStatusSnapshot('open', requestGateway))
|
||||
await flushAsync()
|
||||
|
||||
expect(requestGatewayMock).toHaveBeenCalledTimes(2)
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
})
|
||||
|
||||
expect(requestGatewayMock).toHaveBeenCalledTimes(2)
|
||||
|
||||
await act(async () => {
|
||||
setup.resolve({ provider_configured: true })
|
||||
runtime.resolve({ ok: true })
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(14_999)
|
||||
})
|
||||
expect(requestGatewayMock).toHaveBeenCalledTimes(2)
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
})
|
||||
expect(requestGatewayMock).toHaveBeenCalledTimes(4)
|
||||
})
|
||||
})
|
||||
|
|
@ -14,38 +14,67 @@ export function useStatusSnapshot(gatewayState: string | undefined, requestGatew
|
|||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
let timer: number | undefined
|
||||
|
||||
// A closed/connecting gateway cannot have an authoritative live-runtime
|
||||
// result. Clear readiness before starting the REST status leg so a hung
|
||||
// getStatus() cannot leave a stale "ready" state visible after disconnect.
|
||||
if (gatewayState !== 'open') {
|
||||
setInferenceStatus(null)
|
||||
}
|
||||
|
||||
const scheduleRefresh = () => {
|
||||
if (!cancelled) {
|
||||
timer = window.setTimeout(() => void refresh(), REFRESH_MS)
|
||||
}
|
||||
}
|
||||
|
||||
const refresh = async () => {
|
||||
try {
|
||||
const [next, inference] = await Promise.all([
|
||||
// Wait for both legs before scheduling the next refresh. setInterval
|
||||
// allowed a slow runtime check to overlap with later polls, which
|
||||
// multiplied load on an already-busy gateway and let stale failures
|
||||
// race newer healthy results.
|
||||
const [statusResult, inferenceResult] = await Promise.allSettled([
|
||||
getStatus(),
|
||||
gatewayState === 'open'
|
||||
? evaluateRuntimeReadiness(requestGateway).catch(error => ({
|
||||
checksDisagree: false,
|
||||
ready: false,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
source: 'fallback' as const
|
||||
}))
|
||||
: Promise.resolve(null)
|
||||
gatewayState === 'open' ? evaluateRuntimeReadiness(requestGateway) : Promise.resolve(null)
|
||||
])
|
||||
|
||||
if (cancelled) {
|
||||
return
|
||||
}
|
||||
|
||||
setStatusSnapshot(next)
|
||||
setInferenceStatus(inference)
|
||||
} catch {
|
||||
// Keep last snapshot through transient gateway flaps.
|
||||
if (statusResult.status === 'fulfilled') {
|
||||
setStatusSnapshot(statusResult.value)
|
||||
}
|
||||
|
||||
if (inferenceResult.status === 'fulfilled') {
|
||||
const inference = inferenceResult.value
|
||||
|
||||
if (inference === null) {
|
||||
setInferenceStatus(null)
|
||||
} else if (inference.source !== 'fallback') {
|
||||
// runtime_check/setup_status returned an authoritative boolean.
|
||||
// A fallback means both RPCs failed or returned no boolean, so it
|
||||
// is a transient/unknown transport state, not proof that inference
|
||||
// became unconfigured. Keep the last authoritative result instead
|
||||
// of flashing "Inference not ready" during a gateway flap.
|
||||
setInferenceStatus(inference)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
scheduleRefresh()
|
||||
}
|
||||
}
|
||||
|
||||
void refresh()
|
||||
const timer = window.setInterval(() => void refresh(), REFRESH_MS)
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
window.clearInterval(timer)
|
||||
|
||||
if (timer !== undefined) {
|
||||
window.clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
}, [gatewayState, requestGateway])
|
||||
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ def capture(server):
|
|||
# seconds when the GIL is contended by concurrent agent turns.
|
||||
|
||||
FRONTEND_POLLED_RPCS = [
|
||||
"session.active_list", # live-session rehydrate — in-memory registry
|
||||
"session.list", # loads session list — SQLite query
|
||||
"pet.info", # petdex poll — file/network read
|
||||
"process.list", # background process status — process registry scan
|
||||
|
|
|
|||
|
|
@ -242,6 +242,12 @@ _LONG_HANDLERS = frozenset(
|
|||
# the WS read loop and causing false "needs setup" (#50005 family).
|
||||
"setup.runtime_check",
|
||||
"setup.status",
|
||||
# Desktop also polls the in-memory live-session registry every 15s.
|
||||
# The handler is normally cheap, but under heavy agent GIL pressure it
|
||||
# can still stall for tens of seconds. Keep it off the WS reader thread
|
||||
# so a delayed status rehydrate cannot block runtime readiness, prompt
|
||||
# submission, or interrupts queued behind it on the same socket.
|
||||
"session.active_list",
|
||||
"session.branch",
|
||||
"session.compress",
|
||||
"session.list",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue