From 371adcf37130629ffb9bbeed9f5548ce08ffa93b Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 24 Jun 2026 18:12:25 +0200 Subject: [PATCH] fix(coding-agent): retry explicit provider retry errors closes #6019 --- packages/ai/CHANGELOG.md | 4 + packages/ai/src/index.ts | 1 + packages/ai/src/utils/retry.ts | 96 +++++++++++++++++++ packages/ai/test/retry.test.ts | 38 ++++++++ packages/coding-agent/CHANGELOG.md | 4 + .../coding-agent/src/core/agent-session.ts | 22 +---- ...19-explicit-provider-retry-message.test.ts | 31 ++++++ 7 files changed, 178 insertions(+), 18 deletions(-) create mode 100644 packages/ai/src/utils/retry.ts create mode 100644 packages/ai/test/retry.test.ts create mode 100644 packages/coding-agent/test/suite/regressions/6019-explicit-provider-retry-message.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index b03b0353b..03818dc31 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed retry classification for provider errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)). + ## [0.80.2] - 2026-06-23 ### Changed diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 6c3f9675c..57c448ccc 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -43,5 +43,6 @@ export type { OAuthSelectPrompt, } from "./utils/oauth/types.ts"; export * from "./utils/overflow.ts"; +export * from "./utils/retry.ts"; export * from "./utils/typebox-helpers.ts"; export * from "./utils/validation.ts"; diff --git a/packages/ai/src/utils/retry.ts b/packages/ai/src/utils/retry.ts new file mode 100644 index 000000000..d3a95a0a4 --- /dev/null +++ b/packages/ai/src/utils/retry.ts @@ -0,0 +1,96 @@ +import type { AssistantMessage } from "../types.ts"; + +function buildProviderErrorPattern(patterns: readonly string[]): RegExp { + return new RegExp(patterns.join("|"), "i"); +} + +const NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN = buildProviderErrorPattern([ + // OpenCode Go/free-tier limits returned as 429 JSON error types by OpenCode's + // Zen API. These are subscription/account limits, not transient throttles. + "GoUsageLimitError", + "FreeUsageLimitError", + + // OpenCode Go subscription-limit text asks users to enable available-balance + // usage after rolling/weekly/monthly limits are reached. + "Monthly usage limit reached", + "available balance", + + // Generic quota/budget/billing exhaustion. `insufficient_quota` is OpenAI's + // quota/billing error code; the other strings cover common gateway wording. + "insufficient_quota", + "out of budget", + "quota exceeded", + "billing", +]); + +const RETRYABLE_PROVIDER_ERROR_PATTERN = buildProviderErrorPattern([ + // Generic provider load, HTTP status, and server-side transient failures. + "overloaded", + "rate.?limit", + "too many requests", + "429", + "500", + "502", + "503", + "504", + "service.?unavailable", + "server.?error", + "internal.?error", + + // Wrapper/provider text for transient upstream failures, including OpenRouter + // "Provider returned error" responses (#2264). + "provider.?returned.?error", + + // Network, proxy, and fetch transport failures. This includes OpenAI Codex + // raw-fetch failures such as "upstream connect", "connection refused", and + // "reset before headers" (#733), plus OpenRouter connection drops (#3317). + "network.?error", + "connection.?error", + "connection.?refused", + "connection.?lost", + "other side closed", + "fetch failed", + "upstream.?connect", + "reset before headers", + "socket hang up", + "timed? out", + "timeout", + "terminated", + + // WebSocket transports can report close/error text instead of HTTP/fetch text. + "websocket.?closed", + "websocket.?error", + + // Premature stream endings from SDKs and transports. Anthropic can throw + // "stream ended without ..." and "Anthropic stream ended before message_stop" + // (#4433); Bedrock/Smithy can throw an HTTP/2 no-response error (#3594). + "ended without", + "stream ended before message_stop", + "http2 request did not get a response", + + // Provider-requested retry delay cap failures should flow through the outer + // retry policy so callers can surface/abort the backoff (#1123). + "retry delay", + + // Explicit retry guidance emitted mid-stream by OpenAI Responses and Bedrock + // stream exceptions (#6019). + "you can retry your request", + "try your request again", + "please retry your request", +]); + +/** + * Classifies whether a failed assistant message looks like a transient provider + * or transport error, so callers can decide if the last assistant turn should be + * restarted. + * + * This does not implement retry policy. Callers should first handle context + * overflow separately, then apply their own retry budget, backoff, and reporting + * before restarting the assistant turn. + */ +export function isRetryableAssistantError(message: AssistantMessage): boolean { + if (message.stopReason !== "error" || !message.errorMessage) return false; + const errorMessage = message.errorMessage; + if (NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN.test(errorMessage)) return false; + return RETRYABLE_PROVIDER_ERROR_PATTERN.test(errorMessage); +} diff --git a/packages/ai/test/retry.test.ts b/packages/ai/test/retry.test.ts new file mode 100644 index 000000000..17273c70b --- /dev/null +++ b/packages/ai/test/retry.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { fauxAssistantMessage } from "../src/providers/faux.ts"; +import { isRetryableAssistantError } from "../src/utils/retry.ts"; + +const openAIExplicitRetryMessage = + "An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message."; +const bedrockExplicitRetryMessage = + '{"message":"The system encountered an unexpected error during processing. Try your request again."}'; + +describe("provider retry classification", () => { + it("matches explicit provider retry guidance", () => { + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: openAIExplicitRetryMessage }), + ), + ).toBe(true); + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: bedrockExplicitRetryMessage }), + ), + ).toBe(true); + }); + + it("keeps provider limit errors non-retryable", () => { + expect( + isRetryableAssistantError( + fauxAssistantMessage("", { stopReason: "error", errorMessage: "429 quota exceeded" }), + ), + ).toBe(false); + }); + + it("classifies assistant error messages", () => { + expect( + isRetryableAssistantError(fauxAssistantMessage("", { stopReason: "error", errorMessage: "overloaded_error" })), + ).toBe(true); + expect(isRetryableAssistantError(fauxAssistantMessage("not an error"))).toBe(false); + }); +}); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index a432a068a..3dd5d2e19 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed auto-retry for provider stream errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)). + ## [0.80.2] - 2026-06-23 ### Changed diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 65a6ebd88..fafa1031e 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -29,6 +29,7 @@ import { cleanupSessionResources, getSupportedThinkingLevels, isContextOverflow, + isRetryableAssistantError, modelsAreEqual, resetApiProviders, streamSimple, @@ -2482,29 +2483,14 @@ export class AgentSession { // Auto-Retry // ========================================================================= - private _isNonRetryableProviderLimitError(errorMessage: string): boolean { - return /GoUsageLimitError|FreeUsageLimitError|Monthly usage limit reached|available balance|insufficient_quota|out of budget|quota exceeded|billing/i.test( - errorMessage, - ); - } - /** * Check if an error is retryable (overloaded, rate limit, server errors). * Context overflow errors are NOT retryable (handled by compaction instead). */ private _isRetryableError(message: AssistantMessage): boolean { - if (message.stopReason !== "error" || !message.errorMessage) return false; - - // Context overflow is handled by compaction, not retry - const contextWindow = this.model?.contextWindow ?? 0; - if (isContextOverflow(message, contextWindow)) return false; - - const err = message.errorMessage; - if (this._isNonRetryableProviderLimitError(err)) return false; - // Match: overloaded_error, provider returned error, rate limit, 429, 500, 502, 503, 504, service unavailable, network/connection errors (including connection lost), WebSocket transport closes/errors, fetch failed, premature stream endings, HTTP/2 closed before response, terminated, retry delay exceeded - return /overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|stream ended before message_stop|http2 request did not get a response|timed? out|timeout|terminated|retry delay/i.test( - err, - ); + // Context overflow is handled by compaction, not retry. + if (isContextOverflow(message, this.model?.contextWindow ?? 0)) return false; + return isRetryableAssistantError(message); } /** diff --git a/packages/coding-agent/test/suite/regressions/6019-explicit-provider-retry-message.test.ts b/packages/coding-agent/test/suite/regressions/6019-explicit-provider-retry-message.test.ts new file mode 100644 index 000000000..aa4889c3e --- /dev/null +++ b/packages/coding-agent/test/suite/regressions/6019-explicit-provider-retry-message.test.ts @@ -0,0 +1,31 @@ +import { fauxAssistantMessage } from "@earendil-works/pi-ai"; +import { describe, expect, it } from "vitest"; +import { createHarness } from "../harness.ts"; + +const openAIExplicitRetryMessage = + "An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message."; +const bedrockExplicitRetryMessage = + '{"message":"The system encountered an unexpected error during processing. Try your request again."}'; + +describe("regression: issue 6019 explicit provider retry messages", () => { + it.each([ + ["openai", openAIExplicitRetryMessage], + ["bedrock", bedrockExplicitRetryMessage], + ])("retries %s explicit retry guidance", async (_provider, errorMessage) => { + const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } }); + try { + harness.setResponses([ + fauxAssistantMessage("", { stopReason: "error", errorMessage }), + fauxAssistantMessage("recovered"), + ]); + + await harness.session.prompt("test"); + + expect(harness.faux.state.callCount).toBe(2); + expect(harness.eventsOfType("auto_retry_start").map((event) => event.errorMessage)).toEqual([errorMessage]); + expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([true]); + } finally { + harness.cleanup(); + } + }); +});