mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-07-09 17:28:45 +00:00
fix(coding-agent): retry explicit provider retry errors
Some checks are pending
CI / build-check-test (push) Waiting to run
Some checks are pending
CI / build-check-test (push) Waiting to run
closes #6019
This commit is contained in:
parent
c29bbc0958
commit
371adcf371
7 changed files with 178 additions and 18 deletions
|
|
@ -2,6 +2,10 @@
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed retry classification for provider errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)).
|
||||
|
||||
## [0.80.2] - 2026-06-23
|
||||
|
||||
### Changed
|
||||
|
|
|
|||
|
|
@ -43,5 +43,6 @@ export type {
|
|||
OAuthSelectPrompt,
|
||||
} from "./utils/oauth/types.ts";
|
||||
export * from "./utils/overflow.ts";
|
||||
export * from "./utils/retry.ts";
|
||||
export * from "./utils/typebox-helpers.ts";
|
||||
export * from "./utils/validation.ts";
|
||||
|
|
|
|||
96
packages/ai/src/utils/retry.ts
Normal file
96
packages/ai/src/utils/retry.ts
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import type { AssistantMessage } from "../types.ts";
|
||||
|
||||
function buildProviderErrorPattern(patterns: readonly string[]): RegExp {
|
||||
return new RegExp(patterns.join("|"), "i");
|
||||
}
|
||||
|
||||
const NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN = buildProviderErrorPattern([
|
||||
// OpenCode Go/free-tier limits returned as 429 JSON error types by OpenCode's
|
||||
// Zen API. These are subscription/account limits, not transient throttles.
|
||||
"GoUsageLimitError",
|
||||
"FreeUsageLimitError",
|
||||
|
||||
// OpenCode Go subscription-limit text asks users to enable available-balance
|
||||
// usage after rolling/weekly/monthly limits are reached.
|
||||
"Monthly usage limit reached",
|
||||
"available balance",
|
||||
|
||||
// Generic quota/budget/billing exhaustion. `insufficient_quota` is OpenAI's
|
||||
// quota/billing error code; the other strings cover common gateway wording.
|
||||
"insufficient_quota",
|
||||
"out of budget",
|
||||
"quota exceeded",
|
||||
"billing",
|
||||
]);
|
||||
|
||||
const RETRYABLE_PROVIDER_ERROR_PATTERN = buildProviderErrorPattern([
|
||||
// Generic provider load, HTTP status, and server-side transient failures.
|
||||
"overloaded",
|
||||
"rate.?limit",
|
||||
"too many requests",
|
||||
"429",
|
||||
"500",
|
||||
"502",
|
||||
"503",
|
||||
"504",
|
||||
"service.?unavailable",
|
||||
"server.?error",
|
||||
"internal.?error",
|
||||
|
||||
// Wrapper/provider text for transient upstream failures, including OpenRouter
|
||||
// "Provider returned error" responses (#2264).
|
||||
"provider.?returned.?error",
|
||||
|
||||
// Network, proxy, and fetch transport failures. This includes OpenAI Codex
|
||||
// raw-fetch failures such as "upstream connect", "connection refused", and
|
||||
// "reset before headers" (#733), plus OpenRouter connection drops (#3317).
|
||||
"network.?error",
|
||||
"connection.?error",
|
||||
"connection.?refused",
|
||||
"connection.?lost",
|
||||
"other side closed",
|
||||
"fetch failed",
|
||||
"upstream.?connect",
|
||||
"reset before headers",
|
||||
"socket hang up",
|
||||
"timed? out",
|
||||
"timeout",
|
||||
"terminated",
|
||||
|
||||
// WebSocket transports can report close/error text instead of HTTP/fetch text.
|
||||
"websocket.?closed",
|
||||
"websocket.?error",
|
||||
|
||||
// Premature stream endings from SDKs and transports. Anthropic can throw
|
||||
// "stream ended without ..." and "Anthropic stream ended before message_stop"
|
||||
// (#4433); Bedrock/Smithy can throw an HTTP/2 no-response error (#3594).
|
||||
"ended without",
|
||||
"stream ended before message_stop",
|
||||
"http2 request did not get a response",
|
||||
|
||||
// Provider-requested retry delay cap failures should flow through the outer
|
||||
// retry policy so callers can surface/abort the backoff (#1123).
|
||||
"retry delay",
|
||||
|
||||
// Explicit retry guidance emitted mid-stream by OpenAI Responses and Bedrock
|
||||
// stream exceptions (#6019).
|
||||
"you can retry your request",
|
||||
"try your request again",
|
||||
"please retry your request",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Classifies whether a failed assistant message looks like a transient provider
|
||||
* or transport error, so callers can decide if the last assistant turn should be
|
||||
* restarted.
|
||||
*
|
||||
* This does not implement retry policy. Callers should first handle context
|
||||
* overflow separately, then apply their own retry budget, backoff, and reporting
|
||||
* before restarting the assistant turn.
|
||||
*/
|
||||
export function isRetryableAssistantError(message: AssistantMessage): boolean {
|
||||
if (message.stopReason !== "error" || !message.errorMessage) return false;
|
||||
const errorMessage = message.errorMessage;
|
||||
if (NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN.test(errorMessage)) return false;
|
||||
return RETRYABLE_PROVIDER_ERROR_PATTERN.test(errorMessage);
|
||||
}
|
||||
38
packages/ai/test/retry.test.ts
Normal file
38
packages/ai/test/retry.test.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { fauxAssistantMessage } from "../src/providers/faux.ts";
|
||||
import { isRetryableAssistantError } from "../src/utils/retry.ts";
|
||||
|
||||
const openAIExplicitRetryMessage =
|
||||
"An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message.";
|
||||
const bedrockExplicitRetryMessage =
|
||||
'{"message":"The system encountered an unexpected error during processing. Try your request again."}';
|
||||
|
||||
describe("provider retry classification", () => {
|
||||
it("matches explicit provider retry guidance", () => {
|
||||
expect(
|
||||
isRetryableAssistantError(
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: openAIExplicitRetryMessage }),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isRetryableAssistantError(
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: bedrockExplicitRetryMessage }),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps provider limit errors non-retryable", () => {
|
||||
expect(
|
||||
isRetryableAssistantError(
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: "429 quota exceeded" }),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("classifies assistant error messages", () => {
|
||||
expect(
|
||||
isRetryableAssistantError(fauxAssistantMessage("", { stopReason: "error", errorMessage: "overloaded_error" })),
|
||||
).toBe(true);
|
||||
expect(isRetryableAssistantError(fauxAssistantMessage("not an error"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -2,6 +2,10 @@
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed auto-retry for provider stream errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)).
|
||||
|
||||
## [0.80.2] - 2026-06-23
|
||||
|
||||
### Changed
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import {
|
|||
cleanupSessionResources,
|
||||
getSupportedThinkingLevels,
|
||||
isContextOverflow,
|
||||
isRetryableAssistantError,
|
||||
modelsAreEqual,
|
||||
resetApiProviders,
|
||||
streamSimple,
|
||||
|
|
@ -2482,29 +2483,14 @@ export class AgentSession {
|
|||
// Auto-Retry
|
||||
// =========================================================================
|
||||
|
||||
private _isNonRetryableProviderLimitError(errorMessage: string): boolean {
|
||||
return /GoUsageLimitError|FreeUsageLimitError|Monthly usage limit reached|available balance|insufficient_quota|out of budget|quota exceeded|billing/i.test(
|
||||
errorMessage,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an error is retryable (overloaded, rate limit, server errors).
|
||||
* Context overflow errors are NOT retryable (handled by compaction instead).
|
||||
*/
|
||||
private _isRetryableError(message: AssistantMessage): boolean {
|
||||
if (message.stopReason !== "error" || !message.errorMessage) return false;
|
||||
|
||||
// Context overflow is handled by compaction, not retry
|
||||
const contextWindow = this.model?.contextWindow ?? 0;
|
||||
if (isContextOverflow(message, contextWindow)) return false;
|
||||
|
||||
const err = message.errorMessage;
|
||||
if (this._isNonRetryableProviderLimitError(err)) return false;
|
||||
// Match: overloaded_error, provider returned error, rate limit, 429, 500, 502, 503, 504, service unavailable, network/connection errors (including connection lost), WebSocket transport closes/errors, fetch failed, premature stream endings, HTTP/2 closed before response, terminated, retry delay exceeded
|
||||
return /overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|stream ended before message_stop|http2 request did not get a response|timed? out|timeout|terminated|retry delay/i.test(
|
||||
err,
|
||||
);
|
||||
// Context overflow is handled by compaction, not retry.
|
||||
if (isContextOverflow(message, this.model?.contextWindow ?? 0)) return false;
|
||||
return isRetryableAssistantError(message);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
import { fauxAssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createHarness } from "../harness.ts";
|
||||
|
||||
const openAIExplicitRetryMessage =
|
||||
"An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_******** in your message.";
|
||||
const bedrockExplicitRetryMessage =
|
||||
'{"message":"The system encountered an unexpected error during processing. Try your request again."}';
|
||||
|
||||
describe("regression: issue 6019 explicit provider retry messages", () => {
|
||||
it.each([
|
||||
["openai", openAIExplicitRetryMessage],
|
||||
["bedrock", bedrockExplicitRetryMessage],
|
||||
])("retries %s explicit retry guidance", async (_provider, errorMessage) => {
|
||||
const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } });
|
||||
try {
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage }),
|
||||
fauxAssistantMessage("recovered"),
|
||||
]);
|
||||
|
||||
await harness.session.prompt("test");
|
||||
|
||||
expect(harness.faux.state.callCount).toBe(2);
|
||||
expect(harness.eventsOfType("auto_retry_start").map((event) => event.errorMessage)).toEqual([errorMessage]);
|
||||
expect(harness.eventsOfType("auto_retry_end").map((event) => event.success)).toEqual([true]);
|
||||
} finally {
|
||||
harness.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue