fix(ui): add retry logic to SSE pong to improve connection resilience (#519)

## Problem

When the client receives a ping from the server, it responds with a pong
via HTTP POST. On unstable networks (mobile, WiFi with poor signal,
network switches), this POST can fail in multiple ways:

- **Hung requests**: `fetch()` never rejects, blocking retries
indefinitely
- **Network disconnection**: `Failed to fetch`
- **Brief disconnections**: transient network errors

Previously, a single missed pong would cause the server to close the SSE
connection after 45s, leaving message responses stuck in queue until the
next message triggered a reconnection.

## Solution

Three improvements to make the pong POST resilient:

### 1. Request timeout (10s)
Each pong POST is now bounded with a 10s `AbortSignal` timeout. Hung
requests fail fast instead of blocking indefinitely, allowing retries to
start before the server's stale connection sweep.

### 2. Selective retry with `isRetryableError()`
Only retries transient failures where recovery is possible:
- `AbortError` / `TimeoutError` (hung or timed-out requests)
- `Failed to fetch` (network disconnected)
- `NetworkError` (browser network errors)

Non-retryable errors like `404 Client connection not found` (permanently
closed connection) fail immediately instead of wasting retry attempts.

### 3. Exponential backoff (3 attempts, 100ms → 2000ms)
Handles burst failures gracefully without hammering the server.

## Changes

- **`packages/ui/src/lib/retry-utils.ts`** (new): Reusable retry utility
with `timeoutMs` and `shouldRetry` predicate support
- **`packages/ui/src/lib/server-events.ts`**: Updated pong handler to
use bounded timeout + selective retry

## Verification

- Build passes: `npm run build:ui` 
- Manually verified in production logs: retries now visible as `Pong
failed after retries` instead of single immediate failure
- SSE monitor log at `~/.codenomad/logs/sse-monitor.log` shows `PONG_OK`
/ `PONG_FAIL` / `STALE` events for ongoing monitoring
This commit is contained in:
Dark 2026-06-08 12:20:10 -04:00 committed by GitHub
parent ff10c1f395
commit ce77488d2f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 87 additions and 11 deletions

View file

@ -520,11 +520,15 @@ export const serverApi = {
body: JSON.stringify({ ...identity, enabled }),
})
},
sendClientConnectionPong(payload: { clientId: string; connectionId: string; pingTs?: number }): Promise<void> {
return request<void>("/api/client-connections/pong", {
sendClientConnectionPong(payload: { clientId: string; connectionId: string; pingTs?: number }, signal?: AbortSignal): Promise<void> {
const init: RequestInit = {
method: "POST",
body: JSON.stringify(payload),
})
}
if (signal) {
init.signal = signal
}
return request<void>("/api/client-connections/pong", init)
},
fetchBackgroundProcessOutput(
instanceId: string,

View file

@ -0,0 +1,64 @@
interface RetryOptions {
maxAttempts?: number
initialDelayMs?: number
maxDelayMs?: number
backoffMultiplier?: number
timeoutMs?: number
shouldRetry?: (error: Error, attempt: number) => boolean
}
export async function retryWithBackoff<T>(
fn: (signal?: AbortSignal) => Promise<T>,
options: RetryOptions = {},
): Promise<T> {
const {
maxAttempts = 3,
initialDelayMs = 100,
maxDelayMs = 5000,
backoffMultiplier = 2,
timeoutMs,
shouldRetry = () => true,
} = options
let lastError: Error | null = null
let delayMs = initialDelayMs
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
if (timeoutMs) {
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), timeoutMs)
try {
const result = await fn(controller.signal)
clearTimeout(timer)
return result
} catch (error) {
clearTimeout(timer)
throw error
}
}
return await fn()
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error))
lastError = err
if (attempt < maxAttempts && shouldRetry(err, attempt)) {
await new Promise((resolve) => setTimeout(resolve, delayMs))
delayMs = Math.min(delayMs * backoffMultiplier, maxDelayMs)
} else {
throw err
}
}
}
throw lastError || new Error("Failed after retries")
}
export function isRetryableError(error: Error): boolean {
if (error.name === "AbortError" || error.name === "TimeoutError") return true
if (error.message.includes("Failed to fetch")) return true
if (error.message.includes("NetworkError")) return true
if (error.message.includes("timeout")) return true
return false
}

View file

@ -2,6 +2,7 @@ import type { WorkspaceEventPayload, WorkspaceEventType } from "../../../server/
import { serverApi } from "./api-client"
import { getClientIdentity } from "./client-identity"
import { getLogger } from "./logger"
import { retryWithBackoff, isRetryableError } from "./retry-utils"
const RETRY_BASE_DELAY = 1000
const RETRY_MAX_DELAY = 10000
@ -39,14 +40,21 @@ class ServerEvents {
(event) => this.dispatch(event),
() => this.scheduleReconnect(),
(payload) => {
void serverApi
.sendClientConnectionPong({
...getClientIdentity(),
pingTs: payload.ts,
})
.catch((error) => {
log.error("Failed to send client connection pong", error)
})
const identity = getClientIdentity()
const pongPayload = { ...identity, pingTs: payload.ts }
void retryWithBackoff(
(signal) => serverApi.sendClientConnectionPong(pongPayload, signal),
{
maxAttempts: 3,
initialDelayMs: 100,
maxDelayMs: 2000,
timeoutMs: 10000,
shouldRetry: (error) => isRetryableError(error),
},
).catch((error) => {
log.warn("Failed to send client connection pong after retries", error)
})
},
)
this.source.onopen = () => {