export const REMOTE_LIVENESS_TIMEOUT_MS = 10_000 export const REMOTE_LIVENESS_FAILURE_LIMIT = 3 // Even at the capped retry path, consecutive liveness observations are at most // about 48s apart (ticket mint + socket open + backoff + the next status probe). // One minute keeps a continuous outage together without carrying old failures. export const REMOTE_LIVENESS_FAILURE_WINDOW_MS = 60_000 export interface RemoteLivenessFailure { failures: number shouldReset: boolean } interface RemoteConnectionDescriptor { baseUrl?: null | string mode?: null | string } export interface RevalidateRemoteConnectionOptions { connectionPromise: Promise currentConnectionPromise: () => null | Promise log: (message: string) => void probe: (url: string, options: { timeoutMs: number }) => Promise resetConnection: () => void tracker: RemoteLivenessTracker } export interface RemoteRevalidationResult { ok: true rebuilt: boolean } /** * Coalesces revalidation work for one cached connection promise. * * Every Desktop BrowserWindow owns a renderer gateway loop. When several * windows observe the same disconnect they can all ask the Electron main * process to revalidate the shared primary connection at once. Those calls * must count as one probe, not several consecutive failures. */ export class RemoteRevalidationCoordinator { readonly #inflightByConnection = new WeakMap>() run(connection: object, task: () => Promise): Promise { const existing = this.#inflightByConnection.get(connection) as Promise | undefined if (existing) { return existing } const pending = Promise.resolve().then(task) const clear = () => { if (this.#inflightByConnection.get(connection) === pending) { this.#inflightByConnection.delete(connection) } } this.#inflightByConnection.set(connection, pending) // Clean up on both outcomes without creating an unhandled rejected branch. void pending.then(clear, clear) return pending } } /** * Tracks consecutive remote liveness failures independently per gateway. * A successful probe clears the streak, and reaching the limit consumes it so * a rebuilt connection starts from a clean state. */ export class RemoteLivenessTracker { readonly #failureLimit: number readonly #failureWindowMs: number readonly #failuresByBaseUrl = new Map() readonly #now: () => number constructor( failureLimit = REMOTE_LIVENESS_FAILURE_LIMIT, failureWindowMs = REMOTE_LIVENESS_FAILURE_WINDOW_MS, now: () => number = Date.now ) { if (!Number.isInteger(failureLimit) || failureLimit < 1) { throw new Error('Remote liveness failure limit must be a positive integer.') } if (!Number.isFinite(failureWindowMs) || failureWindowMs < 1) { throw new Error('Remote liveness failure window must be positive.') } this.#failureLimit = failureLimit this.#failureWindowMs = failureWindowMs this.#now = now } recordSuccess(baseUrl: string): void { this.#failuresByBaseUrl.delete(baseUrl) } recordFailure(baseUrl: string): RemoteLivenessFailure { const now = this.#now() const previous = this.#failuresByBaseUrl.get(baseUrl) const withinFailureWindow = previous && now - previous.lastFailureAt <= this.#failureWindowMs const failures = (withinFailureWindow ? previous.failures : 0) + 1 const shouldReset = failures >= this.#failureLimit if (shouldReset) { this.#failuresByBaseUrl.delete(baseUrl) } else { this.#failuresByBaseUrl.set(baseUrl, { failures, lastFailureAt: now }) } return { failures, shouldReset } } clear(): void { this.#failuresByBaseUrl.clear() } } /** * Probe the cached primary remote connection and apply the failure policy. * The caller owns single-flight coordination; identity checks here ensure an * old async result cannot mutate or reset a replacement connection. */ export async function revalidateRemoteConnection({ connectionPromise, currentConnectionPromise, log, probe, resetConnection, tracker }: RevalidateRemoteConnectionOptions): Promise { let connection: TConnection try { connection = await connectionPromise } catch { // The cached boot already rejected; its own recovery path will clear it. return { ok: true, rebuilt: false } } if (currentConnectionPromise() !== connectionPromise) { return { ok: true, rebuilt: false } } if (connection.mode !== 'remote' || !connection.baseUrl) { return { ok: true, rebuilt: false } } const baseUrl = connection.baseUrl.replace(/\/+$/, '') try { await probe(`${baseUrl}/api/status`, { timeoutMs: REMOTE_LIVENESS_TIMEOUT_MS }) if (currentConnectionPromise() !== connectionPromise) { return { ok: true, rebuilt: false } } tracker.recordSuccess(baseUrl) return { ok: true, rebuilt: false } } catch { if (currentConnectionPromise() !== connectionPromise) { return { ok: true, rebuilt: false } } const failure = tracker.recordFailure(baseUrl) if (!failure.shouldReset) { log( `Cached remote Hermes backend failed liveness probe (${failure.failures}/${REMOTE_LIVENESS_FAILURE_LIMIT}); keeping connection for retry.` ) return { ok: true, rebuilt: false } } log('Cached remote Hermes backend failed liveness probe; dropping stale connection.') resetConnection() return { ok: true, rebuilt: true } } }