From 58fa6cf85b994e214722ba1b116308455eff5995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E8=89=AF?= <1204183885@qq.com> Date: Tue, 28 Jul 2026 09:23:38 +0800 Subject: [PATCH] fix(core): fast-fail permanent quota-exhaustion 429s instead of silent retry (#7842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(core): fast-fail permanent quota-exhaustion 429s instead of silent retry A 429 whose body signals a permanently exhausted quota — one that carries a reset time, e.g. 'quota has been exhausted ... will reset at ...' — is a non-retryable condition. qwen-code classified it as a transient rate-limit, retried silently through the full retry budget, and surfaced no error, so the session appeared to hang with no output. Detect such errors and fast-fail on the first attempt, surfacing a friendly message that preserves the provider's reset time and points at switching API key / auth. Transient 429s without a reset time still retry as before. - quotaErrorDetection: isQuotaExhaustedError + formatQuotaExhaustedMessage - retry: fast-fail block (mirrors the existing Qwen-OAuth quota path) - errorParsing: surface the friendly message verbatim, no [API Error:] wrap * test(core): cover ApiError quota exhaustion * fix(core): preserve quota error status * fix(core): remove status from quota fast-fail, unwrap JSON error bodies (#7842) - Remove .status=429 from the thrown quota-exhaustion error: it made isRateLimitError() return true, re-triggering the stream-side retry loop in geminiChat.ts (the exact bug this PR fixes). Use cause to preserve the original error for diagnostics instead. - Unwrap JSON error bodies in getQuotaMessage so openai-compatible providers that emit 429s as JSON surface the human-readable message instead of raw JSON. - Add idempotency guard to formatQuotaExhaustedMessage. - Pin all three detection clauses with negative fixtures, add isRateLimitError(thrown)===false regression test, move quota tests out of the Qwen OAuth describe block. - Update stale isAlreadyFormatted JSDoc; simplify single-element it.each to plain it. * fix(core): fast-fail mid-stream quota exhaustion before rate-limit retry (#7842) --------- Co-authored-by: Qwen Code Autofix --- packages/core/src/core/geminiChat.test.ts | 41 ++++++ packages/core/src/core/geminiChat.ts | 24 ++++ packages/core/src/utils/errorParsing.test.ts | 15 ++ packages/core/src/utils/errorParsing.ts | 21 ++- .../src/utils/quotaErrorDetection.test.ts | 132 ++++++++++++++++++ .../core/src/utils/quotaErrorDetection.ts | 80 +++++++++++ packages/core/src/utils/retry.test.ts | 82 +++++++++++ packages/core/src/utils/retry.ts | 29 +++- 8 files changed, 420 insertions(+), 4 deletions(-) diff --git a/packages/core/src/core/geminiChat.test.ts b/packages/core/src/core/geminiChat.test.ts index e4340f2626..da083a74d4 100644 --- a/packages/core/src/core/geminiChat.test.ts +++ b/packages/core/src/core/geminiChat.test.ts @@ -7952,6 +7952,47 @@ describe('GeminiChat', async () => { } }); + it('fast-fails a mid-stream quota-exhaustion error instead of scheduling a rate-limit retry', async () => { + // A permanent quota-exhaustion 429 can arrive mid-stream as a + // StreamContentError while reading, bypassing the retryWithBackoff + // fast-fail that only wraps stream establishment. The stream-side + // catch must fast-fail it before the rate-limit branch; otherwise + // isRateLimitError (code 429) schedules a 1-5 minute delay on an + // error that cannot succeed until the reset time. + vi.useFakeTimers(); + + try { + const quotaError = new StreamContentError( + '{"error":{"code":"429","message":"Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC."}}', + ); + vi.mocked( + mockContentGenerator.generateContentStream, + ).mockResolvedValueOnce( + (async function* () { + throw quotaError; + + yield {} as GenerateContentResponse; + })(), + ); + + const stream = await chat.sendMessageStream( + 'test-model', + { message: 'test' }, + 'prompt-quota-fastfail', + ); + const iterator = stream[Symbol.asyncIterator](); + + // Fast-fail: the first pull rejects with the friendly message. No + // RETRY event is yielded and no rate-limit delay is scheduled. + await expect(iterator.next()).rejects.toThrow(/Quota exhausted/); + expect( + mockContentGenerator.generateContentStream, + ).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + it('should use Retry-After delay for streamed rate-limit errors', async () => { vi.useFakeTimers(); diff --git a/packages/core/src/core/geminiChat.ts b/packages/core/src/core/geminiChat.ts index 23ebea5be0..c922f81fee 100644 --- a/packages/core/src/core/geminiChat.ts +++ b/packages/core/src/core/geminiChat.ts @@ -24,6 +24,10 @@ import { isUnattendedMode, type HeartbeatInfo, } from '../utils/retry.js'; +import { + isQuotaExhaustedError, + formatQuotaExhaustedMessage, +} from '../utils/quotaErrorDetection.js'; import { getErrorStatus, isAbortError } from '../utils/errors.js'; import { createDebugLogger } from '../utils/debugLogger.js'; import { parseAndFormatApiError } from '../utils/errorParsing.js'; @@ -2504,6 +2508,26 @@ export class GeminiChat { extraRetryErrorCodes, }); + // Permanent quota exhaustion (e.g. Bailian token-plan "1-week + // quota has been exhausted, will reset at ...") can arrive + // mid-stream as a StreamContentError, bypassing retryWithBackoff + // (which only wraps stream establishment). Fast-fail before the + // rate-limit branch: its 429 code would otherwise schedule a 1-5 + // minute delay on an error that cannot succeed until the reset + // time. Throws a plain Error (no .status) and skips model + // fallback, matching the retryWithBackoff fast-fail. + if (isQuotaExhaustedError(error)) { + debugLogger.warn('Quota exhausted mid-stream, fast-failing', { + retryPath: 'stream', + retryDecision: 'fail-fast', + errorKind: classification.kind, + classificationReason: classification.reason, + }); + throw new Error(formatQuotaExhaustedMessage(error), { + cause: error, + }); + } + const isRateLimit = isRateLimitError(error, extraRetryErrorCodes); if (isRateLimit) { const details = getRateLimitErrorDetails(error); diff --git a/packages/core/src/utils/errorParsing.test.ts b/packages/core/src/utils/errorParsing.test.ts index e13aa72623..eb91b9ef14 100644 --- a/packages/core/src/utils/errorParsing.test.ts +++ b/packages/core/src/utils/errorParsing.test.ts @@ -153,6 +153,21 @@ describe('parseAndFormatApiError', () => { expect(parseAndFormatApiError(error)).toBe(message); }); + it('should surface a friendly quota-exhaustion message verbatim (string)', () => { + const message = + 'Quota exhausted: Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC.\n\nPlease retry after the reset time, or switch to another API key / auth method.'; + // Must NOT be re-wrapped in "[API Error: …]" and must NOT pick up the + // misleading "please wait and try again later" transient-throttle suffix. + expect(parseAndFormatApiError(message)).toBe(message); + }); + + it('should surface a friendly quota-exhaustion StructuredError.message verbatim', () => { + const message = + 'Quota exhausted: Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC.\n\nPlease retry after the reset time, or switch to another API key / auth method.'; + const error: StructuredError = { message, status: 429 }; + expect(parseAndFormatApiError(error, AuthType.USE_OPENAI)).toBe(message); + }); + // Idempotency — added after a customer report where a 4xx in non-interactive // mode produced "[API Error: [API Error: ...]]". The non-interactive runner // formats once, prints, then throws an Error whose .message is the formatted diff --git a/packages/core/src/utils/errorParsing.ts b/packages/core/src/utils/errorParsing.ts index 82064686c3..e6561fa85e 100644 --- a/packages/core/src/utils/errorParsing.ts +++ b/packages/core/src/utils/errorParsing.ts @@ -4,7 +4,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { isApiError, isStructuredError } from './quotaErrorDetection.js'; +import { + isApiError, + isStructuredError, + QUOTA_EXHAUSTED_PREFIX, +} from './quotaErrorDetection.js'; import { AuthType } from '../core/contentGenerator.js'; import { getErrorMessage } from './errors.js'; @@ -34,13 +38,17 @@ function getRateLimitMessage(authType?: AuthType): string { const API_ERROR_PREFIX = '[API Error: '; /** - * Returns true when `value` already looks like the output of - * parseAndFormatApiError. + * Returns true when `value` is already in final user-facing form and must not + * be re-wrapped by parseAndFormatApiError. * * Accepts: * 1) base format: "[API Error: ...]" * 2) 429 format: "[API Error: ...]" followed by one of the known quota * guidance suffixes. + * 3) friendly quota-exhaustion messages ("Quota exhausted: ...") built by + * formatQuotaExhaustedMessage — these live here rather than in the + * Qwen-OAuth prefix list because they also arrive via the plain-string + * path (a StreamContentError whose .message is the formatted text). * * Used as an idempotency guard: when an upstream caller has already passed an * Error through parseAndFormatApiError, stuffed the formatted string into @@ -49,6 +57,13 @@ const API_ERROR_PREFIX = '[API Error: '; */ function isAlreadyFormatted(value: string): boolean { const trimmed = value.trimEnd(); + + // Friendly quota-exhaustion messages built by formatQuotaExhaustedMessage + // are already in final form — surface them verbatim, do not wrap. + if (trimmed.startsWith(QUOTA_EXHAUSTED_PREFIX)) { + return true; + } + if (!trimmed.startsWith(API_ERROR_PREFIX)) { return false; } diff --git a/packages/core/src/utils/quotaErrorDetection.test.ts b/packages/core/src/utils/quotaErrorDetection.test.ts index 0da986623c..d7c2372426 100644 --- a/packages/core/src/utils/quotaErrorDetection.test.ts +++ b/packages/core/src/utils/quotaErrorDetection.test.ts @@ -11,6 +11,8 @@ import { isGenericQuotaExceededError, isApiError, isStructuredError, + isQuotaExhaustedError, + formatQuotaExhaustedMessage, type ApiError, } from './quotaErrorDetection.js'; @@ -102,6 +104,136 @@ describe('quotaErrorDetection', () => { }); }); + describe('isQuotaExhaustedError', () => { + it('detects the Bailian token-plan quota-exhaustion error', () => { + const error = Object.assign( + new Error( + '429 Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC.', + ), + { status: 429 }, + ); + expect(isQuotaExhaustedError(error)).toBe(true); + }); + + it('detects a plain-string quota-exhaustion message', () => { + expect( + isQuotaExhaustedError( + 'Your token-plan quota has been exceeded. It will reset at 2026-07-27.', + ), + ).toBe(true); + }); + + it('detects a reset-time message without the word "will"', () => { + expect( + isQuotaExhaustedError( + new Error('Your quota is exhausted. Reset at 2026-08-01 00:00 UTC.'), + ), + ).toBe(true); + }); + + it('detects an ApiError-shaped quota-exhaustion message', () => { + const error: ApiError = { + error: { + code: 429, + message: + 'Your quota has been exhausted. It will reset at 2026-07-28.', + status: 'RESOURCE_EXHAUSTED', + details: [], + }, + }; + expect(isQuotaExhaustedError(error)).toBe(true); + expect(formatQuotaExhaustedMessage(error)).toContain( + 'Your quota has been exhausted. It will reset at 2026-07-28.', + ); + }); + + it('does not match transient throttling without a reset time', () => { + expect( + isQuotaExhaustedError( + new Error('Rate limit exceeded. Please retry later.'), + ), + ).toBe(false); + }); + + it('does not match a 429 that only says quota without a reset time', () => { + expect(isQuotaExhaustedError(new Error('Your quota is exhausted.'))).toBe( + false, + ); + }); + + it('does not match quota + reset time without exhausted/exceeded', () => { + // Pins the (exhausted || exceeded) clause: an OpenAI-style TPM body + // "Rate limit reached … Your quota will reset at …" must stay false. + expect( + isQuotaExhaustedError( + new Error( + 'Rate limit reached for gpt-4. Your quota will reset at 12:00:05Z.', + ), + ), + ).toBe(false); + }); + + it('does not match exhausted + reset time without quota', () => { + // Pins the quota clause. + expect( + isQuotaExhaustedError( + new Error('Resources exhausted. Will reset at 2026-08-01.'), + ), + ).toBe(false); + }); + + it('does not match unrelated errors', () => { + expect(isQuotaExhaustedError(new Error('Network timeout'))).toBe(false); + expect(isQuotaExhaustedError(null)).toBe(false); + expect(isQuotaExhaustedError(undefined)).toBe(false); + }); + }); + + describe('formatQuotaExhaustedMessage', () => { + it('strips the leading HTTP-status prefix and keeps the reset time', () => { + const error = Object.assign( + new Error( + '429 Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC.', + ), + { status: 429 }, + ); + const message = formatQuotaExhaustedMessage(error); + expect(message.startsWith('Quota exhausted: ')).toBe(true); + expect(message).not.toContain('429 Your'); + expect(message).toContain('will reset at 07-27 09:25:00 UTC'); + expect(message).toContain('switch to another API key'); + }); + + it('falls back when no message can be extracted', () => { + const message = formatQuotaExhaustedMessage(42); + expect(message.startsWith('Quota exhausted: ')).toBe(true); + expect(message).toContain('quota has been exhausted'); + }); + + it('unwraps a JSON error body to surface the nested message', () => { + // openai-compatible providers emit 429 bodies as JSON; the friendly + // message must show the human-readable text, not raw JSON. + const error = new Error( + '{"error":{"code":"429","message":"Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC.","status":"RESOURCE_EXHAUSTED","details":[]}}', + ); + const message = formatQuotaExhaustedMessage(error); + expect(message).toContain( + 'Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC.', + ); + expect(message).not.toContain('{"error"'); + }); + + it('is idempotent — does not double-wrap its own output', () => { + const first = formatQuotaExhaustedMessage( + new Error( + 'Your quota has been exhausted. It will reset at 2026-07-28.', + ), + ); + const second = formatQuotaExhaustedMessage(new Error(first)); + expect(second).toBe(first); + }); + }); + describe('type guards', () => { describe('isApiError', () => { it('should detect valid API error', () => { diff --git a/packages/core/src/utils/quotaErrorDetection.ts b/packages/core/src/utils/quotaErrorDetection.ts index 12c15d22b2..77555ac53c 100644 --- a/packages/core/src/utils/quotaErrorDetection.ts +++ b/packages/core/src/utils/quotaErrorDetection.ts @@ -118,3 +118,83 @@ export function isQwenQuotaExceededError(error: unknown): boolean { message.toLowerCase().includes('free allocated quota exceeded') ); } + +/** + * Prefix marking a friendly quota-exhausted message produced by + * {@link formatQuotaExhaustedMessage}. `parseAndFormatApiError` recognizes it + * so the message is surfaced verbatim instead of being re-wrapped in + * "[API Error: …]". + */ +export const QUOTA_EXHAUSTED_PREFIX = 'Quota exhausted: '; + +/** Best-effort extraction of a human-readable message from common error shapes. */ +function getQuotaMessage(error: unknown): string | null { + let message: string | null = null; + if (typeof error === 'string') message = error; + else if (isStructuredError(error)) message = error.message; + else if (isApiError(error)) message = error.error.message; + if (message === null) return null; + // Unwrap a JSON error body ('{"error":{"code":"429","message":"..."}}') + // so the nested human-readable message is surfaced instead of raw JSON — + // the same extraction parseAndFormatApiError performs. + const start = message.indexOf('{'); + if (start !== -1) { + try { + const parsed = JSON.parse(message.substring(start)) as unknown; + if (isApiError(parsed)) return parsed.error.message; + } catch { + /* not a JSON error body */ + } + } + return message; +} + +/** + * Detects permanent quota-exhaustion errors that carry a reset time. + * + * Unlike transient rate-limiting (TPM/RPM throttling, which lifts within + * seconds–minutes), these signal an allocated quota fully spent and only + * resetting at a specific future time. Retrying cannot succeed until then — + * the caller should fast-fail and surface the reset time instead of hanging + * the session through the full retry budget with no output. + * + * Matches e.g. the Bailian token-plan error surfaced via the OpenAI SDK: + * "429 Your token-plan 1-week quota has been exhausted. The quota will + * reset at 07-27 09:25:00 UTC." + * + * @param error The error to check (Error, ApiError, or message string). + * @returns True when the error names a quota that is exhausted/exceeded AND + * carries a reset time — the combination that signals a permanent condition. + */ +export function isQuotaExhaustedError(error: unknown): boolean { + const message = getQuotaMessage(error); + if (!message) return false; + const lower = message.toLowerCase(); + return ( + lower.includes('quota') && + (lower.includes('exhausted') || lower.includes('exceeded')) && + (lower.includes('will reset') || lower.includes('reset at')) + ); +} + +/** + * Builds a friendly, self-contained message for a permanent quota-exhaustion + * error. Preserves the provider's verbatim wording (which names the quota and + * the reset time) and appends an actionable hint. + * + * The result is prefixed with {@link QUOTA_EXHAUSTED_PREFIX} so + * `parseAndFormatApiError` surfaces it verbatim rather than wrapping it. + * + * @param error The error whose message should be surfaced. + * @returns A user-facing message starting with `QUOTA_EXHAUSTED_PREFIX`. + */ +export function formatQuotaExhaustedMessage(error: unknown): string { + const raw = getQuotaMessage(error) ?? ''; + if (raw.startsWith(QUOTA_EXHAUSTED_PREFIX)) return raw; + // Strip a leading "NNN " HTTP-status prefix the OpenAI SDK prepends + // (e.g. "429 Your token-plan …") so the surfaced text does not start with + // a bare status code. + const stripped = + raw.replace(/^\d{3}\s+/, '').trim() || 'quota has been exhausted'; + return `${QUOTA_EXHAUSTED_PREFIX}${stripped}\n\nPlease retry after the reset time, or switch to another API key / auth method.`; +} diff --git a/packages/core/src/utils/retry.test.ts b/packages/core/src/utils/retry.test.ts index c96632dbf1..5660a697e8 100644 --- a/packages/core/src/utils/retry.test.ts +++ b/packages/core/src/utils/retry.test.ts @@ -22,6 +22,7 @@ import { } from './retry.js'; import { retryContext } from './retryContext.js'; import { getErrorStatus } from './errors.js'; +import { isRateLimitError } from './rateLimit.js'; import { setSimulate429 } from './testUtils.js'; import { AuthType } from '../core/contentGenerator.js'; @@ -625,6 +626,87 @@ describe('retryWithBackoff', () => { expect(fn).toHaveBeenCalledTimes(3); }); }); + + describe('permanent quota-exhaustion fast-fail (any auth)', () => { + it('should throw immediately for a permanent quota-exhaustion error', async () => { + // Bailian token-plan "1-week quota has been exhausted" surfaces as a 429 + // from the OpenAI SDK but is permanent — it must fast-fail, not retry. + const quotaError = Object.assign( + new Error( + '429 Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC.', + ), + { status: 429 }, + ); + const fn = vi.fn().mockRejectedValue(quotaError); + + const promise = retryWithBackoff(fn, { + maxAttempts: 5, + initialDelayMs: 1000, + maxDelayMs: 5000, + authType: AuthType.USE_OPENAI, + }); + + await expect(promise).rejects.toMatchObject({ + message: expect.stringContaining('Quota exhausted'), + }); + + // Should be called only once (no retries) + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('thrown error must not be a rate-limit error (pins no-status intent)', async () => { + // The thrown error deliberately carries no .status so that + // isRateLimitError() returns false — preventing the stream-side + // rate-limit retry loop in geminiChat.ts from re-driving it. + const quotaError = Object.assign( + new Error( + '429 Your token-plan 1-week quota has been exhausted. The quota will reset at 07-27 09:25:00 UTC.', + ), + { status: 429 }, + ); + const fn = vi.fn().mockRejectedValue(quotaError); + + const promise = retryWithBackoff(fn, { + maxAttempts: 5, + initialDelayMs: 1000, + maxDelayMs: 5000, + authType: AuthType.USE_OPENAI, + }); + + let thrown: unknown; + const assertionPromise = promise.catch((e: unknown) => { + thrown = e; + }); + await vi.runAllTimersAsync(); + await assertionPromise; + + expect(isRateLimitError(thrown)).toBe(false); + }); + + it('should retry a transient 429 that does not carry a reset time', async () => { + // A plain TPM/RPM 429 (no "will reset at") stays retryable — the + // quota-exhaustion fast-fail must not swallow transient throttling. + const transient429 = Object.assign( + new Error('Rate limit exceeded. Please retry later.'), + { status: 429 }, + ); + const fn = vi + .fn() + .mockRejectedValueOnce(transient429) + .mockResolvedValue('success'); + + const promise = retryWithBackoff(fn, { + maxAttempts: 5, + initialDelayMs: 100, + maxDelayMs: 1000, + authType: AuthType.USE_OPENAI, + }); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe('success'); + expect(fn).toHaveBeenCalledTimes(2); + }); + }); }); describe('isTransientCapacityError', () => { diff --git a/packages/core/src/utils/retry.ts b/packages/core/src/utils/retry.ts index e9174395f1..5258546fde 100644 --- a/packages/core/src/utils/retry.ts +++ b/packages/core/src/utils/retry.ts @@ -6,7 +6,11 @@ import type { GenerateContentResponse } from '@google/genai'; import { AuthType } from '../core/contentGenerator.js'; -import { isQwenQuotaExceededError } from './quotaErrorDetection.js'; +import { + isQwenQuotaExceededError, + isQuotaExhaustedError, + formatQuotaExhaustedMessage, +} from './quotaErrorDetection.js'; import { createDebugLogger } from './debugLogger.js'; import { getErrorStatus } from './errors.js'; import { isRateLimitError } from './rateLimit.js'; @@ -355,6 +359,29 @@ export async function retryWithBackoff( ); } + // Permanent quota exhaustion (e.g. Bailian token-plan "1-week quota has + // been exhausted, will reset at …"). Unlike transient 429 throttling, + // retrying cannot succeed until the reset time — fast-fail and surface a + // friendly message so the session does not hang through the full retry + // budget with no output. Applies to any auth type since a reset time is + // the universal signal of a permanently exhausted quota. + if (isQuotaExhaustedError(error)) { + debugLogger.error( + 'Quota exhausted, fast-failing', + retryDiagnostics, + error, + ); + // Intentionally throws a plain Error with no `.status`: a 429 status + // would make isRateLimitError() return true and re-trigger the + // stream-side rate-limit retry loop in geminiChat.ts (up to 10 retries + // at 1-5 min delays), reintroducing the silent hang this fast-fail + // eliminates. This also skips model fallback — quota exhaustion is + // provider-scoped and temporary, so the user should retry after the + // reset time rather than burning fallback provider quota. `cause` + // preserves the original error for diagnostics. + throw new Error(formatQuotaExhaustedMessage(error), { cause: error }); + } + // Determine if this error qualifies for persistent retry. // Persistent mode still respects shouldRetryOnError — callers can force // fast-fail even for transient errors if they explicitly return false.