fix(coding-agent): retry explicit provider retry errors
Some checks are pending
CI / build-check-test (push) Waiting to run

closes #6019
This commit is contained in:
Armin Ronacher 2026-06-24 18:12:25 +02:00
parent c29bbc0958
commit 371adcf371
7 changed files with 178 additions and 18 deletions

View file

@ -2,6 +2,10 @@
## [Unreleased]
### Fixed
- Fixed retry classification for provider errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)).
## [0.80.2] - 2026-06-23
### Changed

View file

@ -43,5 +43,6 @@ export type {
OAuthSelectPrompt,
} from "./utils/oauth/types.ts";
export * from "./utils/overflow.ts";
export * from "./utils/retry.ts";
export * from "./utils/typebox-helpers.ts";
export * from "./utils/validation.ts";

View file

@ -0,0 +1,96 @@
import type { AssistantMessage } from "../types.ts";
function buildProviderErrorPattern(patterns: readonly string[]): RegExp {
return new RegExp(patterns.join("|"), "i");
}
const NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN = buildProviderErrorPattern([
// OpenCode Go/free-tier limits returned as 429 JSON error types by OpenCode's
// Zen API. These are subscription/account limits, not transient throttles.
"GoUsageLimitError",
"FreeUsageLimitError",
// OpenCode Go subscription-limit text asks users to enable available-balance
// usage after rolling/weekly/monthly limits are reached.
"Monthly usage limit reached",
"available balance",
// Generic quota/budget/billing exhaustion. `insufficient_quota` is OpenAI's
// quota/billing error code; the other strings cover common gateway wording.
"insufficient_quota",
"out of budget",
"quota exceeded",
"billing",
]);
const RETRYABLE_PROVIDER_ERROR_PATTERN = buildProviderErrorPattern([
// Generic provider load, HTTP status, and server-side transient failures.
"overloaded",
"rate.?limit",
"too many requests",
"429",
"500",
"502",
"503",
"504",
"service.?unavailable",
"server.?error",
"internal.?error",
// Wrapper/provider text for transient upstream failures, including OpenRouter
// "Provider returned error" responses (#2264).
"provider.?returned.?error",
// Network, proxy, and fetch transport failures. This includes OpenAI Codex
// raw-fetch failures such as "upstream connect", "connection refused", and
// "reset before headers" (#733), plus OpenRouter connection drops (#3317).
"network.?error",
"connection.?error",
"connection.?refused",
"connection.?lost",
"other side closed",
"fetch failed",
"upstream.?connect",
"reset before headers",
"socket hang up",
"timed? out",
"timeout",
"terminated",
// WebSocket transports can report close/error text instead of HTTP/fetch text.
"websocket.?closed",
"websocket.?error",
// Premature stream endings from SDKs and transports. Anthropic can throw
// "stream ended without ..." and "Anthropic stream ended before message_stop"
// (#4433); Bedrock/Smithy can throw an HTTP/2 no-response error (#3594).
"ended without",
"stream ended before message_stop",
"http2 request did not get a response",
// Provider-requested retry delay cap failures should flow through the outer
// retry policy so callers can surface/abort the backoff (#1123).
"retry delay",
// Explicit retry guidance emitted mid-stream by OpenAI Responses and Bedrock
// stream exceptions (#6019).
"you can retry your request",
"try your request again",
"please retry your request",
]);
/**
* Classifies whether a failed assistant message looks like a transient provider
* or transport error, so callers can decide if the last assistant turn should be
* restarted.
*
* This does not implement retry policy. Callers should first handle context
* overflow separately, then apply their own retry budget, backoff, and reporting
* before restarting the assistant turn.
*/
export function isRetryableAssistantError(message: AssistantMessage): boolean {
if (message.stopReason !== "error" || !message.errorMessage) return false;
const errorMessage = message.errorMessage;
if (NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN.test(errorMessage)) return false;
return RETRYABLE_PROVIDER_ERROR_PATTERN.test(errorMessage);
}

View file

@ -0,0 +1,38 @@
import { describe, expect, it } from "vitest";
import { fauxAssistantMessage } from "../src/providers/faux.ts";
import { isRetryableAssistantError } from "../src/utils/retry.ts";
const openAIExplicitRetryMessage =
"An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message.";
const bedrockExplicitRetryMessage =
'{"message":"The system encountered an unexpected error during processing. Try your request again."}';
describe("provider retry classification", () => {
it("matches explicit provider retry guidance", () => {
expect(
isRetryableAssistantError(
fauxAssistantMessage("", { stopReason: "error", errorMessage: openAIExplicitRetryMessage }),
),
).toBe(true);
expect(
isRetryableAssistantError(
fauxAssistantMessage("", { stopReason: "error", errorMessage: bedrockExplicitRetryMessage }),
),
).toBe(true);
});
it("keeps provider limit errors non-retryable", () => {
expect(
isRetryableAssistantError(
fauxAssistantMessage("", { stopReason: "error", errorMessage: "429 quota exceeded" }),
),
).toBe(false);
});
it("classifies assistant error messages", () => {
expect(
isRetryableAssistantError(fauxAssistantMessage("", { stopReason: "error", errorMessage: "overloaded_error" })),
).toBe(true);
expect(isRetryableAssistantError(fauxAssistantMessage("not an error"))).toBe(false);
});
});

View file

@ -2,6 +2,10 @@
## [Unreleased]
### Fixed
- Fixed auto-retry for provider stream errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)).
## [0.80.2] - 2026-06-23
### Changed

View file

@ -29,6 +29,7 @@ import {
cleanupSessionResources,
getSupportedThinkingLevels,
isContextOverflow,
isRetryableAssistantError,
modelsAreEqual,
resetApiProviders,
streamSimple,
@ -2482,29 +2483,14 @@ export class AgentSession {
// Auto-Retry
// =========================================================================
private _isNonRetryableProviderLimitError(errorMessage: string): boolean {
return /GoUsageLimitError|FreeUsageLimitError|Monthly usage limit reached|available balance|insufficient_quota|out of budget|quota exceeded|billing/i.test(
errorMessage,
);
}
/**
* Check if an error is retryable (overloaded, rate limit, server errors).
* Context overflow errors are NOT retryable (handled by compaction instead).
*/
private _isRetryableError(message: AssistantMessage): boolean {
if (message.stopReason !== "error" || !message.errorMessage) return false;
// Context overflow is handled by compaction, not retry
const contextWindow = this.model?.contextWindow ?? 0;
if (isContextOverflow(message, contextWindow)) return false;
const err = message.errorMessage;
if (this._isNonRetryableProviderLimitError(err)) return false;
// Match: overloaded_error, provider returned error, rate limit, 429, 500, 502, 503, 504, service unavailable, network/connection errors (including connection lost), WebSocket transport closes/errors, fetch failed, premature stream endings, HTTP/2 closed before response, terminated, retry delay exceeded
return /overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|stream ended before message_stop|http2 request did not get a response|timed? out|timeout|terminated|retry delay/i.test(
err,
);
// Context overflow is handled by compaction, not retry.
if (isContextOverflow(message, this.model?.contextWindow ?? 0)) return false;
return isRetryableAssistantError(message);
}
/**

View file

@ -0,0 +1,31 @@
import { fauxAssistantMessage } from "@earendil-works/pi-ai";
import { describe, expect, it } from "vitest";
import { createHarness } from "../harness.ts";
const openAIExplicitRetryMessage =
"An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message.";
const bedrockExplicitRetryMessage =
'{"message":"The system encountered an unexpected error during processing. Try your request again."}';
describe("regression: issue 6019 explicit provider retry messages", () => {
it.each([
["openai", openAIExplicitRetryMessage],
["bedrock", bedrockExplicitRetryMessage],
])("retries %s explicit retry guidance", async (_provider, errorMessage) => {
const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });
try {
harness.setResponses([
fauxAssistantMessage("", { stopReason: "error", errorMessage }),
fauxAssistantMessage("recovered"),
]);
await harness.session.prompt("test");
expect(harness.faux.state.callCount).toBe(2);
expect(harness.eventsOfType("auto_retry_start").map((event) => event.errorMessage)).toEqual([errorMessage]);
expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([true]);
} finally {
harness.cleanup();
}
});
});