From d6cbdc158079ef547005cfbb4af485d66cab7dc8 Mon Sep 17 00:00:00 2001 From: clousky2020 <33016567+clousky2020@users.noreply.github.com> Date: Mon, 20 Apr 2026 08:13:45 +0800 Subject: [PATCH] feat: add ModelScope provider with circuit breaker and daily quota lock (#1430) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.7.0. Thanks @clousky2020 for this massive and important contribution! ๐ŸŽ‰ We've translated the deprecation comments to English for consistency, and it is now officially merged into the release branch. Great work on the ModelScope integration and Circuit Breaker! --- next.config.mjs | 1 + open-sse/config/providerRegistry.ts | 28 +- open-sse/executors/antigravity.ts | 29 +- open-sse/executors/codex.ts | 2 + open-sse/handlers/chatCore.ts | 18 +- open-sse/index.ts | 3 + open-sse/services/accountFallback.ts | 249 +++++++++++++++++- open-sse/services/cloudCodeThinking.ts | 6 + open-sse/services/combo.ts | 60 +++++ open-sse/services/errorClassifier.ts | 5 + .../combos/IntelligentComboPanel.tsx | 3 + .../dashboard/providers/[id]/page.tsx | 28 ++ .../providers/components/ModelStatusBadge.tsx | 162 ++++++++++++ .../components/ModelStatusContext.tsx | 186 +++++++++++++ src/app/(dashboard)/layout.tsx | 7 +- src/i18n/messages/en.json | 11 +- src/i18n/messages/zh-CN.json | 13 +- src/sse/handlers/chat.ts | 6 +- src/sse/handlers/chatHelpers.ts | 7 +- src/sse/services/auth.ts | 26 +- tests/integration/chat-pipeline.test.ts | 12 + tests/unit/account-fallback-service.test.ts | 218 +++++++++++++++ tests/unit/chat-route-coverage.test.ts | 5 + tests/unit/combo-routing-engine.test.ts | 46 ++++ tests/unit/error-classifier.test.ts | 18 ++ 25 files changed, 1129 insertions(+), 20 deletions(-) create mode 100644 src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx create mode 100644 src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx diff --git a/next.config.mjs b/next.config.mjs index db1017ae8..375b966e9 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -28,6 +28,7 @@ const nextConfig = { "./playwright-report/**/*", "./app.__qa_backup/**/*", "./tests/**/*", + "./logs/**/*", ], }, serverExternalPackages: [ diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts index b48b3dc9d..ede6e0cdf 100644 --- a/open-sse/config/providerRegistry.ts +++ b/open-sse/config/providerRegistry.ts @@ -1400,6 +1400,25 @@ export const REGISTRY: Record = { passthroughModels: true, }, + modelscope: { + id: "modelscope", + alias: "ms", + format: "openai", + executor: "default", + baseUrl: "https://api-inference.modelscope.cn/v1/chat/completions", + authType: "apikey", + authHeader: "bearer", + // ModelScope uses per-model quotas. Setting passthroughModels: true ensures 429/404 + // only locks the specific model, not the entire connection. This allows fallback + // to other models on the same API key. + passthroughModels: true, + models: [ + { id: "moonshotai/Kimi-K2.5", name: "Kimi K2.5" }, + { id: "ZhipuAI/GLM-5", name: "GLM-5" }, + { id: "stepfun-ai/Step-3.5-Flash", name: "Step-3.5-Flash" }, + ], + }, + // โ”€โ”€ New Free Providers (2026) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ longcat: { @@ -2128,8 +2147,14 @@ export function getUnsupportedParams(provider: string, modelId: string): readonl const cached = _unsupportedParamsMap.get(modelId); if (cached) return cached; - // 3. Handle prefixed model IDs (e.g., "openai/o3" โ†’ "o3") + // 3. Handle prefixed model IDs (e.g., "openai/o3" โ†’ "o3", "moonshotai/Kimi-K2.5" โ†’ "moonshotai/Kimi-K2.5") + // ModelScope models have slash in ID, check both full ID and bare ID if (modelId.includes("/")) { + // First check full model ID with provider prefix (e.g., "moonshotai/Kimi-K2.5") + const cachedWithPrefix = _unsupportedParamsMap.get(modelId); + if (cachedWithPrefix) return cachedWithPrefix; + + // Fall back to bare ID (e.g., "Kimi-K2.5") const bareId = modelId.split("/").pop() || ""; const bare = _unsupportedParamsMap.get(bareId); if (bare) return bare; @@ -2154,6 +2179,7 @@ export function getProviderCategory(provider: string): "oauth" | "apikey" { * Derive the latest opus/sonnet/haiku model IDs from the `claude` registry entry. * Picks the first model whose ID matches each family pattern โ€” registry order * determines precedence, so newer models should be listed first. + * @deprecated This function will be removed in v4.0, please use REGISTRY.claude?.models directly */ export function getClaudeCodeDefaultModels(): { opus: string; diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index b8dc191b7..b98723273 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -735,18 +735,45 @@ export class AntigravityExecutor extends BaseExecutor { // consuming the stream. The client receives the unmodified SSE data. if (response.body) { let sseBuffer = ""; + const decoder = new TextDecoder(); // Singleton for correct streaming decode + const MAX_BUFFER_SIZE = 16 * 1024; // Limit to prevent OOM on large streams + const passThrough = new TransformStream({ transform(chunk, controller) { controller.enqueue(chunk); // Accumulate text to scan for remainingCredits try { - const text = new TextDecoder().decode(chunk, { stream: true }); + const text = decoder.decode(chunk, { stream: true }); sseBuffer += text; + // Limit buffer size to prevent unbounded growth + // Truncate only after a complete newline to avoid splitting SSE lines mid-payload + if (sseBuffer.length > MAX_BUFFER_SIZE) { + const lastNewline = sseBuffer.lastIndexOf( + "\n", + sseBuffer.length - MAX_BUFFER_SIZE + ); + if (lastNewline !== -1) { + sseBuffer = sseBuffer.slice(lastNewline + 1); + } else { + // No newline found in discard region โ€” buffer contains an incomplete SSE line. + // Discard it entirely to avoid returning malformed data; the remainingCredits + // parser won't find valid data in a truncated line anyway. + sseBuffer = ""; + } + } } catch { /* decoding best-effort */ } }, flush() { + // Final decode for any remaining bytes + try { + const text = decoder.decode(); // Flush pending bytes + sseBuffer += text; + } catch { + /* decoding best-effort */ + } + // Parse the accumulated SSE data for remainingCredits try { const lines = sseBuffer.split("\n"); diff --git a/open-sse/executors/codex.ts b/open-sse/executors/codex.ts index 899c4eb00..fcbcbab1c 100644 --- a/open-sse/executors/codex.ts +++ b/open-sse/executors/codex.ts @@ -271,6 +271,8 @@ function convertSystemToDeveloperRole(body: Record): void { * server-generated prefix (rs_, fc_, resp_, msg_) โ€” so the content is * preserved but the backend won't try to look it up * 4. Always deletes previous_response_id (endpoint doesn't persist responses) + * + * @deprecated This function will be removed in v4.0, Codex executor has updated processing logic */ function stripStoredItemReferences(body: Record): void { // Always strip previous_response_id โ€” the /codex/responses endpoint does not diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 49aefdbc8..820649228 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -14,7 +14,12 @@ import { createRequestLogger } from "../utils/requestLogger.ts"; import { getModelTargetFormat, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.ts"; import { resolveModelAlias } from "../services/modelDeprecation.ts"; import { getUnsupportedParams } from "../config/providerRegistry.ts"; -import { hasPerModelQuota, lockModelIfPerModelQuota } from "../services/accountFallback.ts"; +import { + hasPerModelQuota, + lockModelIfPerModelQuota, + isDailyQuotaExhausted, + getMsUntilTomorrow, +} from "../services/accountFallback.ts"; import { COOLDOWN_MS } from "../config/constants.ts"; import { buildErrorBody, @@ -2109,17 +2114,22 @@ export async function handleChatCore({ } } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { // Providers with per-model quotas โ€” lock the model only, not the connection + // Daily quota exhausted: lock until tomorrow; otherwise use default cooldown + const isDailyQuota = isDailyQuotaExhausted(message); + const quotaCooldownMs = isDailyQuota + ? getMsUntilTomorrow() + : retryAfterMs || COOLDOWN_MS.rateLimit; if ( lockModelIfPerModelQuota( provider, connectionId, model, - "quota_exhausted", - retryAfterMs || COOLDOWN_MS.rateLimit + isDailyQuota ? "daily_quota_exhausted" : "quota_exhausted", + quotaCooldownMs ) ) { console.warn( - `[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil((retryAfterMs || COOLDOWN_MS.rateLimit) / 1000)}s (connection stays active)` + `[provider] Node ${connectionId} ${isDailyQuota ? "daily " : ""}quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` ); } else { await updateProviderConnection(connectionId, { diff --git a/open-sse/index.ts b/open-sse/index.ts index fffb5ea17..2c4941068 100644 --- a/open-sse/index.ts +++ b/open-sse/index.ts @@ -50,6 +50,9 @@ export { isAccountUnavailable, getUnavailableUntil, filterAvailableAccounts, + isProviderInCooldown, + getProviderCooldownRemainingMs, + getProvidersInCooldown, } from "./services/accountFallback.ts"; export { diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 95bdf510d..efadc2c25 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -24,6 +24,30 @@ type ModelFailureState = { resetAfterMs: number; }; +// Provider-level failure tracking for circuit breaker behavior +type ProviderFailureEntry = { + failureCount: number; + lastFailureAt: number; + resetAfterMs: number; + cooldownUntil: number | null; +}; + +// Error codes that count toward provider-level failure threshold +const PROVIDER_FAILURE_ERROR_CODES = new Set([429, 408, 500, 502, 503, 504]); + +// Configuration for provider-level failure tracking +const PROVIDER_FAILURE_THRESHOLD = 5; +const PROVIDER_FAILURE_WINDOW_MS = 20 * 60 * 1000; // 20 minutes +const PROVIDER_COOLDOWN_MS = 10 * 60 * 1000; // 10 minutes cooling + +// Provider-level failure state map: providerId -> failure entry +const providerFailureState = new Map(); +// Guard against synchronous re-entrant calls within the same event-loop tick. +// NOT a true mutex โ€” Node.js is single-threaded, so different SSE streams +// can interleave across ticks. This Set prevents a single call from recursively +// re-entering recordProviderFailure within the same synchronous call stack. +const providerFailureLocks = new Set(); + // T06 (sub2api PR #1037): Signals that indicate permanent account deactivation. // When a 401 body contains these strings, the account is permanently dead // and should NOT be retried after token refresh. @@ -82,8 +106,12 @@ const CONTEXT_OVERFLOW_PATTERNS = [ const MALFORMED_REQUEST_PATTERNS = [ /\bimproperly formed request\b/i, /\binvalid.*message.*format/i, - /\bmessages must alternate/i, - /\bempty (message|content)/i, + /\bmessages must alternate\b/i, + /\bempty (message|content)\b/i, + // Tool call function name errors + /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i, + /function.*name.*(?:blank|empty|missing)/i, + /tool_call.*name.*(?:blank|empty|missing)/i, ]; /** @@ -355,11 +383,21 @@ export function clearModelLock(provider, connectionId, model) { * Compatible and passthrough providers multiplex multiple upstream models behind one * connection, so transient 404/429 responses should stay model-scoped instead of * poisoning the whole connection. + * + * @param provider - Provider ID + * @param _model - Model ID (reserved for future use) + * @param connectionPassthroughModels - Optional per-connection override from providerSpecificData. + * When provided, takes precedence over registry/provider-level logic. */ export function hasPerModelQuota( provider: string | null | undefined, - _model: string | null | undefined = null + _model: string | null | undefined = null, + connectionPassthroughModels?: boolean ): boolean { + // Connection-level override takes precedence (e.g., user-configured ModelScope) + if (typeof connectionPassthroughModels === "boolean") { + return connectionPassthroughModels; + } if (!provider) return false; if (provider === "gemini") return true; if (getPassthroughProviders().has(provider)) return true; @@ -376,18 +414,23 @@ export function lockModelIfPerModelQuota( connectionId: string, model: string | null, reason: string, - cooldownMs: number + cooldownMs: number, + connectionPassthroughModels?: boolean ): boolean { - if (!hasPerModelQuota(provider, model) || !model) return false; + if (!hasPerModelQuota(provider, model, connectionPassthroughModels) || !model) return false; + // Skip model-level lock if the entire provider is in circuit-breaker cooldown. + // The provider cooldown already prevents all requests, so a model lock is redundant. + if (isProviderInCooldown(provider)) return false; lockModel(provider, connectionId, model, reason, cooldownMs); return true; } export function shouldMarkAccountExhaustedFrom429( provider: string | null | undefined, - model: string | null | undefined = null + model: string | null | undefined = null, + connectionPassthroughModels?: boolean ): boolean { - return !hasPerModelQuota(provider, model); + return !hasPerModelQuota(provider, model, connectionPassthroughModels); } /** @@ -442,6 +485,145 @@ export function getAllModelLockouts() { return active; } +// โ”€โ”€โ”€ Provider-Level Failure Tracking โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Track failures at provider level: when a provider has too many transient failures +// across all its connections, cooldown the entire provider temporarily. + +/** + * Check if a provider is currently in cooldown due to too many failures + */ +export function isProviderInCooldown(provider: string | null | undefined): boolean { + if (!provider) return false; + const entry = providerFailureState.get(provider); + if (!entry) return false; + + // If in cooldown, check if it has expired + if (entry.cooldownUntil !== null && Date.now() >= entry.cooldownUntil) { + providerFailureState.delete(provider); + return false; + } + + return entry.cooldownUntil !== null; +} + +/** + * Get remaining cooldown time for a provider + */ +export function getProviderCooldownRemainingMs(provider: string | null | undefined): number | null { + if (!provider) return null; + const entry = providerFailureState.get(provider); + if (!entry || entry.cooldownUntil === null) return null; + + const remaining = entry.cooldownUntil - Date.now(); + return remaining > 0 ? remaining : null; +} + +/** + * Record a failure for a provider. When threshold is reached within the window, + * the provider enters cooldown. + */ +export function recordProviderFailure( + provider: string | null | undefined, + log?: { warn?: (...args: unknown[]) => void } +): void { + if (!provider) return; + + // Guard against concurrent re-entrant calls within the same tick + if (providerFailureLocks.has(provider)) return; + providerFailureLocks.add(provider); + + try { + const now = Date.now(); + const entry = providerFailureState.get(provider); + + // Check if we're in cooldown period + if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) { + return; // Already in cooldown, don't record + } + + // Check if failure window has expired + if (entry && now - entry.lastFailureAt > entry.resetAfterMs) { + // Window expired, reset count + providerFailureState.set(provider, { + failureCount: 1, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil: null, + }); + return; + } + + // Increment failure count + const newCount = entry ? entry.failureCount + 1 : 1; + + if (newCount >= PROVIDER_FAILURE_THRESHOLD) { + // Threshold reached, enter cooldown + const cooldownUntil = now + PROVIDER_COOLDOWN_MS; + providerFailureState.set(provider, { + failureCount: newCount, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil, + }); + log?.warn?.( + `[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s โ€” entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown` + ); + } else { + // Just increment counter + providerFailureState.set(provider, { + failureCount: newCount, + lastFailureAt: now, + resetAfterMs: PROVIDER_FAILURE_WINDOW_MS, + cooldownUntil: null, + }); + } + } finally { + providerFailureLocks.delete(provider); + } +} + +/** + * Clear provider failure state (e.g., after successful request) + */ +export function clearProviderFailure(provider: string | null | undefined): void { + if (!provider) return; + providerFailureState.delete(provider); +} + +/** + * Get all providers currently in cooldown (for debugging/dashboard) + */ +export function getProvidersInCooldown(): Array<{ + provider: string; + failureCount: number; + cooldownRemainingMs: number | null; + lastFailureAt: number; +}> { + const result = []; + for (const [provider, entry] of providerFailureState) { + if (entry.cooldownUntil === null) continue; + const remaining = entry.cooldownUntil - Date.now(); + if (remaining <= 0) { + providerFailureState.delete(provider); + continue; + } + result.push({ + provider, + failureCount: entry.failureCount, + cooldownRemainingMs: remaining, + lastFailureAt: entry.lastFailureAt, + }); + } + return result; +} + +/** + * Check if a status code should be counted toward provider failure threshold + */ +export function isProviderFailureCode(status: number): boolean { + return PROVIDER_FAILURE_ERROR_CODES.has(status); +} + // โ”€โ”€โ”€ Retry-After Parsing โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /** @@ -625,6 +807,40 @@ export function classifyError(status, errorText) { return RateLimitReason.UNKNOWN; } +// โ”€โ”€โ”€ Daily Quota Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +/** + * Calculate milliseconds from now until tomorrow at midnight (00:00:00). + * Used to lock a model until the next day when daily quota is exhausted. + * @returns {number} Milliseconds until tomorrow + */ +export function getMsUntilTomorrow(): number { + const now = new Date(); + const tomorrow = new Date(now); + tomorrow.setDate(tomorrow.getDate() + 1); + tomorrow.setHours(0, 0, 0, 0); + const ms = tomorrow.getTime() - now.getTime(); + // Guard against DST edge cases: if ms is negative (shouldn't happen) or + // unreasonably large (>25h due to spring-forward), cap at 24 hours. + return ms > 0 && ms <= 25 * 60 * 60 * 1000 ? ms : 24 * 60 * 60 * 1000; +} + +/** + * Check if error text indicates daily quota exhaustion (as opposed to rate limiting). + * Daily quota errors typically mention "today's quota" or "try again tomorrow". + * @param {string} errorText - Error message text + * @returns {boolean} True if daily quota is exhausted + */ +export function isDailyQuotaExhausted(errorText: string): boolean { + if (!errorText) return false; + const lower = errorText.toLowerCase(); + return ( + lower.includes("today's quota") || + lower.includes("daily quota") || + lower.includes("try again tomorrow") + ); +} + // โ”€โ”€โ”€ Configurable Backoff โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ /** @@ -681,6 +897,12 @@ export function checkFallbackError( const errorStr = (errorText || "").toString(); const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null); + // Track provider-level failures for circuit breaker behavior + // Only count transient errors that are likely to recover + if (isProviderFailureCode(status)) { + recordProviderFailure(provider); + } + function parseResetFromHeaders(headers, errorStr = "") { if (!headers) return null; @@ -737,6 +959,19 @@ export function checkFallbackError( }; } + // Daily quota exhausted โ€” lock model until tomorrow + if (isDailyQuotaExhausted(errorStr)) { + const msUntilTomorrow = getMsUntilTomorrow(); + // Cap at 24 hours to handle timezone edge cases + const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000); + return { + shouldFallback: true, + cooldownMs, + reason: RateLimitReason.QUOTA_EXHAUSTED, + dailyQuotaExhausted: true, + }; + } + if (lowerError.includes("no credentials")) { return { shouldFallback: true, diff --git a/open-sse/services/cloudCodeThinking.ts b/open-sse/services/cloudCodeThinking.ts index 769c414d7..9d3b3b944 100644 --- a/open-sse/services/cloudCodeThinking.ts +++ b/open-sse/services/cloudCodeThinking.ts @@ -23,6 +23,9 @@ function stripGeminiThinkingConfig(value: unknown): unknown { return next; } +/** + * @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest + */ export function shouldStripCloudCodeThinking(provider: string, model: string): boolean { if (!provider || !model) return false; const normalizedModel = normalizeCloudCodeModel(model); @@ -33,6 +36,9 @@ export function shouldStripCloudCodeThinking(provider: string, model: string): b return CLOUD_CODE_REASONING_UNSUPPORTED_PATTERNS.some((pattern) => pattern.test(normalizedModel)); } +/** + * @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest + */ export function stripCloudCodeThinkingConfig( body: Record ): Record { diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index dcbd63a45..796a8a58d 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -59,6 +59,47 @@ const COMBO_BAD_REQUEST_FALLBACK_PATTERNS = [ /unsupported content part type/i, /tool(?:_call|_use)? .* not (?:available|found)/i, /third-party apps/i, + // Context overflow โ€” model-specific, may succeed on a model with larger context window + /context overflow/i, + /context length exceeded/i, + /prompt too large/i, + /token limit/i, + /too many tokens/i, + /exceeds? context/i, + /maximum context/i, + /input too long/i, + /messages? exceed/i, + // Model not supported/found โ€” permanent model-level error, try next combo target + /no provider supported/i, + /model not found/i, + /model not available/i, + /unsupported model/i, + /model.*has no provider/i, + // Function calling format error โ€” model doesn't support this capability + /function\.?arguments.*(must be|should be|ๅฟ…้กป).*(json|JSON)/i, + /tool.*arguments.*invalid/i, + /function.*parameter.*(invalid|format)/i, + // Input length range error โ€” model-specific context limit + /range of input length/i, + /input length should be/i, + // Transient 400 errors from upstream โ€” should fallback to next combo target + /ๆœๅŠก้‡ๅˆฐไบ†ไธ€็‚นๅฐ็Šถๅ†ต/i, // ModelScope/Qwen transient error + /ๆŠฑๆญ‰.*?ๆ•ๆ„Ÿๅ†…ๅฎน.*?่ฏทๆฃ€ๆŸฅ/i, // ModelScope/Qwen content moderation with context + /ๅ†…ๅฎน.*?ๆ•ๆ„Ÿ.*?(?:ๆ— ๆณ•|่ฟ‡ๆปค)/i, // Content sensitivity block + /ๆ— ๆณ•ๅ“ๅบ”.*?่ฏทๆฑ‚/i, // "unable to respond to request" + /็จๅŽ้‡่ฏ•/i, // "retry later" in Chinese + /temporary.*error/i, + /transient.*error/i, + /service.*unavailable/i, + /please.*try.*again/i, + // Rate limit errors โ€” some providers return 400 instead of 429 + /\brate.?-?limit.?(?:exceeded|reached|hit)/i, + /too many requests/i, + /่ฏทๆฑ‚่ฟ‡ไบŽ้ข‘็น/i, // Chinese rate limit message + // Tool call function name errors โ€” model-specific, try next combo target + /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i, + /function.*name.*(?:blank|empty|missing)/i, + /tool_call.*name.*(?:blank|empty|missing)/i, ]; // Patterns that signal all accounts for a provider are rate-limited / exhausted. @@ -77,6 +118,7 @@ function isAllAccountsRateLimitedResponse( const MAX_COMBO_DEPTH = 3; const MAX_FALLBACK_WAIT_MS = 5000; +const MAX_GLOBAL_ATTEMPTS = 30; function comboModelNotFoundResponse(message: string) { return errorResponse(404, message); @@ -1451,6 +1493,7 @@ export async function handleComboChat({ let earliestRetryAfter = null; let lastStatus = null; const startTime = Date.now(); + globalAttempts = 0; let fallbackCount = 0; let recordedAttempts = 0; @@ -1484,6 +1527,14 @@ export async function handleComboChat({ // Retry loop for transient errors for (let retry = 0; retry <= maxRetries; retry++) { + globalAttempts++; + if (globalAttempts > MAX_GLOBAL_ATTEMPTS) { + log.warn( + "COMBO", + `Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded across all targets and fallbacks. Terminating loop to prevent runaway background requests.` + ); + return errorResponse(503, "Maximum combo retry limit reached"); + } if (retry > 0) { log.info( "COMBO", @@ -1799,6 +1850,7 @@ async function handleRoundRobinCombo({ let lastError = null; let lastStatus = null; let earliestRetryAfter = null; + let globalAttempts = 0; let fallbackCount = 0; let recordedAttempts = 0; @@ -1852,6 +1904,14 @@ async function handleRoundRobinCombo({ // Retry loop within this model try { for (let retry = 0; retry <= maxRetries; retry++) { + globalAttempts++; + if (globalAttempts > MAX_GLOBAL_ATTEMPTS) { + log.warn( + "COMBO-RR", + `Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded. Terminating loop to prevent runaway requests.` + ); + return errorResponse(503, "Maximum combo retry limit reached"); + } if (retry > 0) { log.info( "COMBO-RR", diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index 39d513ffb..4520c2a6e 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -1,6 +1,7 @@ import { isAccountDeactivated, isCreditsExhausted, + isDailyQuotaExhausted, isOAuthInvalidToken, } from "./accountFallback.ts"; @@ -100,7 +101,11 @@ export function classifyProviderError(statusCode: number, responseBody: unknown) return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; } + // 429: Check if it's a daily quota exhaustion (lock until tomorrow) vs regular rate limit if (statusCode === 429) { + if (isDailyQuotaExhausted(bodyStr)) { + return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; + } return PROVIDER_ERROR_TYPES.RATE_LIMITED; } diff --git a/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx b/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx index 0a22d20e0..55013eab4 100644 --- a/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx +++ b/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx @@ -11,6 +11,7 @@ import { extractIntelligentHealthState, normalizeIntelligentRoutingConfig, } from "@/lib/combos/intelligentRouting"; +import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge"; function getI18nOrFallback(t: any, key: string, fallback: string) { if (typeof t?.has === "function" && t.has(key)) return t(key); @@ -283,6 +284,8 @@ export default function IntelligentComboPanel({

{entry.model}

+ {/* Model status badge - shows cooldown/error state */} + {percentage}% diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index 2b9d0abfc..9164e0571 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -53,6 +53,7 @@ import { getCodexRequestDefaults as _getCodexRequestDefaults, } from "@/lib/providers/requestDefaults"; import { resolveDashboardProviderInfo } from "../providerPageUtils"; +import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge"; type CompatByProtocolMap = Partial< Record< @@ -326,6 +327,7 @@ function anyUpstreamHeadersBadge( interface ModelRowProps { model: { id: string; name?: string; source?: string; isHidden?: boolean }; fullModel: string; + provider: string; copied?: string; onCopy: (text: string, key: string) => void; t: (key: string, values?: Record) => string; @@ -2460,6 +2462,7 @@ export default function ProviderDetailPage() { key={model.id} model={model} fullModel={`${providerDisplayAlias}/${model.id}`} + provider={providerId} copied={copied} onCopy={copy} t={t} @@ -3329,6 +3332,7 @@ export default function ProviderDetailPage() { function ModelRow({ model, fullModel, + provider, copied, onCopy, t, @@ -3358,6 +3362,7 @@ function ModelRow({ {fullModel} +