mirror of
https://github.com/NeuralNomadsAI/CodeNomad.git
synced 2026-08-31 01:44:52 +00:00
fix(ui): add retry logic to SSE pong to improve connection resilience (#519)
## Problem
When the client receives a ping from the server, it responds with a pong
via HTTP POST. On unstable networks (mobile, WiFi with poor signal,
network switches), this POST can fail in multiple ways:
- **Hung requests**: `fetch()` never rejects, blocking retries
indefinitely
- **Network disconnection**: `Failed to fetch`
- **Brief disconnections**: transient network errors
Previously, a single missed pong would cause the server to close the SSE
connection after 45s, leaving message responses stuck in queue until the
next message triggered a reconnection.
## Solution
Three improvements to make the pong POST resilient:
### 1. Request timeout (10s)
Each pong POST is now bounded with a 10s `AbortSignal` timeout. Hung
requests fail fast instead of blocking indefinitely, allowing retries to
start before the server's stale connection sweep.
### 2. Selective retry with `isRetryableError()`
Only retries transient failures where recovery is possible:
- `AbortError` / `TimeoutError` (hung or timed-out requests)
- `Failed to fetch` (network disconnected)
- `NetworkError` (browser network errors)
Non-retryable errors like `404 Client connection not found` (permanently
closed connection) fail immediately instead of wasting retry attempts.
### 3. Exponential backoff (3 attempts, 100ms → 2000ms)
Handles burst failures gracefully without hammering the server.
## Changes
- **`packages/ui/src/lib/retry-utils.ts`** (new): Reusable retry utility
with `timeoutMs` and `shouldRetry` predicate support
- **`packages/ui/src/lib/server-events.ts`**: Updated pong handler to
use bounded timeout + selective retry
## Verification
- Build passes: `npm run build:ui` ✅
- Manually verified in production logs: retries now visible as `Pong
failed after retries` instead of single immediate failure
- SSE monitor log at `~/.codenomad/logs/sse-monitor.log` shows `PONG_OK`
/ `PONG_FAIL` / `STALE` events for ongoing monitoring
This commit is contained in:
parent
ff10c1f395
commit
ce77488d2f
3 changed files with 87 additions and 11 deletions
|
|
@ -520,11 +520,15 @@ export const serverApi = {
|
|||
body: JSON.stringify({ ...identity, enabled }),
|
||||
})
|
||||
},
|
||||
sendClientConnectionPong(payload: { clientId: string; connectionId: string; pingTs?: number }): Promise<void> {
|
||||
return request<void>("/api/client-connections/pong", {
|
||||
sendClientConnectionPong(payload: { clientId: string; connectionId: string; pingTs?: number }, signal?: AbortSignal): Promise<void> {
|
||||
const init: RequestInit = {
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
if (signal) {
|
||||
init.signal = signal
|
||||
}
|
||||
return request<void>("/api/client-connections/pong", init)
|
||||
},
|
||||
fetchBackgroundProcessOutput(
|
||||
instanceId: string,
|
||||
|
|
|
|||
64
packages/ui/src/lib/retry-utils.ts
Normal file
64
packages/ui/src/lib/retry-utils.ts
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
interface RetryOptions {
|
||||
maxAttempts?: number
|
||||
initialDelayMs?: number
|
||||
maxDelayMs?: number
|
||||
backoffMultiplier?: number
|
||||
timeoutMs?: number
|
||||
shouldRetry?: (error: Error, attempt: number) => boolean
|
||||
}
|
||||
|
||||
export async function retryWithBackoff<T>(
|
||||
fn: (signal?: AbortSignal) => Promise<T>,
|
||||
options: RetryOptions = {},
|
||||
): Promise<T> {
|
||||
const {
|
||||
maxAttempts = 3,
|
||||
initialDelayMs = 100,
|
||||
maxDelayMs = 5000,
|
||||
backoffMultiplier = 2,
|
||||
timeoutMs,
|
||||
shouldRetry = () => true,
|
||||
} = options
|
||||
|
||||
let lastError: Error | null = null
|
||||
let delayMs = initialDelayMs
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
if (timeoutMs) {
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs)
|
||||
try {
|
||||
const result = await fn(controller.signal)
|
||||
clearTimeout(timer)
|
||||
return result
|
||||
} catch (error) {
|
||||
clearTimeout(timer)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
return await fn()
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error))
|
||||
lastError = err
|
||||
|
||||
if (attempt < maxAttempts && shouldRetry(err, attempt)) {
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs))
|
||||
delayMs = Math.min(delayMs * backoffMultiplier, maxDelayMs)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error("Failed after retries")
|
||||
}
|
||||
|
||||
export function isRetryableError(error: Error): boolean {
|
||||
if (error.name === "AbortError" || error.name === "TimeoutError") return true
|
||||
if (error.message.includes("Failed to fetch")) return true
|
||||
if (error.message.includes("NetworkError")) return true
|
||||
if (error.message.includes("timeout")) return true
|
||||
return false
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ import type { WorkspaceEventPayload, WorkspaceEventType } from "../../../server/
|
|||
import { serverApi } from "./api-client"
|
||||
import { getClientIdentity } from "./client-identity"
|
||||
import { getLogger } from "./logger"
|
||||
import { retryWithBackoff, isRetryableError } from "./retry-utils"
|
||||
|
||||
const RETRY_BASE_DELAY = 1000
|
||||
const RETRY_MAX_DELAY = 10000
|
||||
|
|
@ -39,14 +40,21 @@ class ServerEvents {
|
|||
(event) => this.dispatch(event),
|
||||
() => this.scheduleReconnect(),
|
||||
(payload) => {
|
||||
void serverApi
|
||||
.sendClientConnectionPong({
|
||||
...getClientIdentity(),
|
||||
pingTs: payload.ts,
|
||||
})
|
||||
.catch((error) => {
|
||||
log.error("Failed to send client connection pong", error)
|
||||
})
|
||||
const identity = getClientIdentity()
|
||||
const pongPayload = { ...identity, pingTs: payload.ts }
|
||||
|
||||
void retryWithBackoff(
|
||||
(signal) => serverApi.sendClientConnectionPong(pongPayload, signal),
|
||||
{
|
||||
maxAttempts: 3,
|
||||
initialDelayMs: 100,
|
||||
maxDelayMs: 2000,
|
||||
timeoutMs: 10000,
|
||||
shouldRetry: (error) => isRetryableError(error),
|
||||
},
|
||||
).catch((error) => {
|
||||
log.warn("Failed to send client connection pong after retries", error)
|
||||
})
|
||||
},
|
||||
)
|
||||
this.source.onopen = () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue