From ce77488d2f715cec07b199ecbd7b3a28a948b77a Mon Sep 17 00:00:00 2001 From: Dark <107168337+JDis03@users.noreply.github.com> Date: Mon, 8 Jun 2026 12:20:10 -0400 Subject: [PATCH] fix(ui): add retry logic to SSE pong to improve connection resilience (#519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem When the client receives a ping from the server, it responds with a pong via HTTP POST. On unstable networks (mobile, WiFi with poor signal, network switches), this POST can fail in multiple ways: - **Hung requests**: `fetch()` never rejects, blocking retries indefinitely - **Network disconnection**: `Failed to fetch` - **Brief disconnections**: transient network errors Previously, a single missed pong would cause the server to close the SSE connection after 45s, leaving message responses stuck in queue until the next message triggered a reconnection. ## Solution Three improvements to make the pong POST resilient: ### 1. Request timeout (10s) Each pong POST is now bounded with a 10s `AbortSignal` timeout. Hung requests fail fast instead of blocking indefinitely, allowing retries to start before the server's stale connection sweep. ### 2. Selective retry with `isRetryableError()` Only retries transient failures where recovery is possible: - `AbortError` / `TimeoutError` (hung or timed-out requests) - `Failed to fetch` (network disconnected) - `NetworkError` (browser network errors) Non-retryable errors like `404 Client connection not found` (permanently closed connection) fail immediately instead of wasting retry attempts. ### 3. Exponential backoff (3 attempts, 100ms → 2000ms) Handles burst failures gracefully without hammering the server. ## Changes - **`packages/ui/src/lib/retry-utils.ts`** (new): Reusable retry utility with `timeoutMs` and `shouldRetry` predicate support - **`packages/ui/src/lib/server-events.ts`**: Updated pong handler to use bounded timeout + selective retry ## Verification - Build passes: `npm run build:ui` ✅ - Manually verified in production logs: retries now visible as `Pong failed after retries` instead of single immediate failure - SSE monitor log at `~/.codenomad/logs/sse-monitor.log` shows `PONG_OK` / `PONG_FAIL` / `STALE` events for ongoing monitoring --- packages/ui/src/lib/api-client.ts | 10 +++-- packages/ui/src/lib/retry-utils.ts | 64 ++++++++++++++++++++++++++++ packages/ui/src/lib/server-events.ts | 24 +++++++---- 3 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 packages/ui/src/lib/retry-utils.ts diff --git a/packages/ui/src/lib/api-client.ts b/packages/ui/src/lib/api-client.ts index dfc512ea..353360ea 100644 --- a/packages/ui/src/lib/api-client.ts +++ b/packages/ui/src/lib/api-client.ts @@ -520,11 +520,15 @@ export const serverApi = { body: JSON.stringify({ ...identity, enabled }), }) }, - sendClientConnectionPong(payload: { clientId: string; connectionId: string; pingTs?: number }): Promise { - return request("/api/client-connections/pong", { + sendClientConnectionPong(payload: { clientId: string; connectionId: string; pingTs?: number }, signal?: AbortSignal): Promise { + const init: RequestInit = { method: "POST", body: JSON.stringify(payload), - }) + } + if (signal) { + init.signal = signal + } + return request("/api/client-connections/pong", init) }, fetchBackgroundProcessOutput( instanceId: string, diff --git a/packages/ui/src/lib/retry-utils.ts b/packages/ui/src/lib/retry-utils.ts new file mode 100644 index 00000000..d644cbe9 --- /dev/null +++ b/packages/ui/src/lib/retry-utils.ts @@ -0,0 +1,64 @@ +interface RetryOptions { + maxAttempts?: number + initialDelayMs?: number + maxDelayMs?: number + backoffMultiplier?: number + timeoutMs?: number + shouldRetry?: (error: Error, attempt: number) => boolean +} + +export async function retryWithBackoff( + fn: (signal?: AbortSignal) => Promise, + options: RetryOptions = {}, +): Promise { + const { + maxAttempts = 3, + initialDelayMs = 100, + maxDelayMs = 5000, + backoffMultiplier = 2, + timeoutMs, + shouldRetry = () => true, + } = options + + let lastError: Error | null = null + let delayMs = initialDelayMs + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + if (timeoutMs) { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), timeoutMs) + try { + const result = await fn(controller.signal) + clearTimeout(timer) + return result + } catch (error) { + clearTimeout(timer) + throw error + } + } + + return await fn() + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)) + lastError = err + + if (attempt < maxAttempts && shouldRetry(err, attempt)) { + await new Promise((resolve) => setTimeout(resolve, delayMs)) + delayMs = Math.min(delayMs * backoffMultiplier, maxDelayMs) + } else { + throw err + } + } + } + + throw lastError || new Error("Failed after retries") +} + +export function isRetryableError(error: Error): boolean { + if (error.name === "AbortError" || error.name === "TimeoutError") return true + if (error.message.includes("Failed to fetch")) return true + if (error.message.includes("NetworkError")) return true + if (error.message.includes("timeout")) return true + return false +} diff --git a/packages/ui/src/lib/server-events.ts b/packages/ui/src/lib/server-events.ts index 833e6c2a..0791d675 100644 --- a/packages/ui/src/lib/server-events.ts +++ b/packages/ui/src/lib/server-events.ts @@ -2,6 +2,7 @@ import type { WorkspaceEventPayload, WorkspaceEventType } from "../../../server/ import { serverApi } from "./api-client" import { getClientIdentity } from "./client-identity" import { getLogger } from "./logger" +import { retryWithBackoff, isRetryableError } from "./retry-utils" const RETRY_BASE_DELAY = 1000 const RETRY_MAX_DELAY = 10000 @@ -39,14 +40,21 @@ class ServerEvents { (event) => this.dispatch(event), () => this.scheduleReconnect(), (payload) => { - void serverApi - .sendClientConnectionPong({ - ...getClientIdentity(), - pingTs: payload.ts, - }) - .catch((error) => { - log.error("Failed to send client connection pong", error) - }) + const identity = getClientIdentity() + const pongPayload = { ...identity, pingTs: payload.ts } + + void retryWithBackoff( + (signal) => serverApi.sendClientConnectionPong(pongPayload, signal), + { + maxAttempts: 3, + initialDelayMs: 100, + maxDelayMs: 2000, + timeoutMs: 10000, + shouldRetry: (error) => isRetryableError(error), + }, + ).catch((error) => { + log.warn("Failed to send client connection pong after retries", error) + }) }, ) this.source.onopen = () => {