feat: add ModelScope provider with circuit breaker and daily quota lock (#1430)

Integrated into release/v3.7.0. Thanks @clousky2020 for this massive and important contribution! 🎉 We've translated the deprecation comments to English for consistency, and it is now officially merged into the release branch. Great work on the ModelScope integration and Circuit Breaker!
This commit is contained in:
clousky2020 2026-04-20 08:13:45 +08:00 committed by GitHub
parent 77f326e0dd
commit d6cbdc1580
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 1129 additions and 20 deletions

View file

@ -28,6 +28,7 @@ const nextConfig = {
"./playwright-report/**/*",
"./app.__qa_backup/**/*",
"./tests/**/*",
"./logs/**/*",
],
},
serverExternalPackages: [

View file

@ -1400,6 +1400,25 @@ export const REGISTRY: Record<string, RegistryEntry> = {
passthroughModels: true,
},
modelscope: {
id: "modelscope",
alias: "ms",
format: "openai",
executor: "default",
baseUrl: "https://api-inference.modelscope.cn/v1/chat/completions",
authType: "apikey",
authHeader: "bearer",
// ModelScope uses per-model quotas. Setting passthroughModels: true ensures 429/404
// only locks the specific model, not the entire connection. This allows fallback
// to other models on the same API key.
passthroughModels: true,
models: [
{ id: "moonshotai/Kimi-K2.5", name: "Kimi K2.5" },
{ id: "ZhipuAI/GLM-5", name: "GLM-5" },
{ id: "stepfun-ai/Step-3.5-Flash", name: "Step-3.5-Flash" },
],
},
// ── New Free Providers (2026) ─────────────────────────────────────────────
longcat: {
@ -2128,8 +2147,14 @@ export function getUnsupportedParams(provider: string, modelId: string): readonl
const cached = _unsupportedParamsMap.get(modelId);
if (cached) return cached;
// 3. Handle prefixed model IDs (e.g., "openai/o3" → "o3")
// 3. Handle prefixed model IDs (e.g., "openai/o3" → "o3", "moonshotai/Kimi-K2.5" → "moonshotai/Kimi-K2.5")
// ModelScope models have slash in ID, check both full ID and bare ID
if (modelId.includes("/")) {
// First check full model ID with provider prefix (e.g., "moonshotai/Kimi-K2.5")
const cachedWithPrefix = _unsupportedParamsMap.get(modelId);
if (cachedWithPrefix) return cachedWithPrefix;
// Fall back to bare ID (e.g., "Kimi-K2.5")
const bareId = modelId.split("/").pop() || "";
const bare = _unsupportedParamsMap.get(bareId);
if (bare) return bare;
@ -2154,6 +2179,7 @@ export function getProviderCategory(provider: string): "oauth" | "apikey" {
* Derive the latest opus/sonnet/haiku model IDs from the `claude` registry entry.
* Picks the first model whose ID matches each family pattern registry order
* determines precedence, so newer models should be listed first.
* @deprecated This function will be removed in v4.0, please use REGISTRY.claude?.models directly
*/
export function getClaudeCodeDefaultModels(): {
opus: string;

View file

@ -735,18 +735,45 @@ export class AntigravityExecutor extends BaseExecutor {
// consuming the stream. The client receives the unmodified SSE data.
if (response.body) {
let sseBuffer = "";
const decoder = new TextDecoder(); // Singleton for correct streaming decode
const MAX_BUFFER_SIZE = 16 * 1024; // Limit to prevent OOM on large streams
const passThrough = new TransformStream({
transform(chunk, controller) {
controller.enqueue(chunk);
// Accumulate text to scan for remainingCredits
try {
const text = new TextDecoder().decode(chunk, { stream: true });
const text = decoder.decode(chunk, { stream: true });
sseBuffer += text;
// Limit buffer size to prevent unbounded growth
// Truncate only after a complete newline to avoid splitting SSE lines mid-payload
if (sseBuffer.length > MAX_BUFFER_SIZE) {
const lastNewline = sseBuffer.lastIndexOf(
"\n",
sseBuffer.length - MAX_BUFFER_SIZE
);
if (lastNewline !== -1) {
sseBuffer = sseBuffer.slice(lastNewline + 1);
} else {
// No newline found in discard region — buffer contains an incomplete SSE line.
// Discard it entirely to avoid returning malformed data; the remainingCredits
// parser won't find valid data in a truncated line anyway.
sseBuffer = "";
}
}
} catch {
/* decoding best-effort */
}
},
flush() {
// Final decode for any remaining bytes
try {
const text = decoder.decode(); // Flush pending bytes
sseBuffer += text;
} catch {
/* decoding best-effort */
}
// Parse the accumulated SSE data for remainingCredits
try {
const lines = sseBuffer.split("\n");

View file

@ -271,6 +271,8 @@ function convertSystemToDeveloperRole(body: Record<string, unknown>): void {
* server-generated prefix (rs_, fc_, resp_, msg_) so the content is
* preserved but the backend won't try to look it up
* 4. Always deletes previous_response_id (endpoint doesn't persist responses)
*
* @deprecated This function will be removed in v4.0, Codex executor has updated processing logic
*/
function stripStoredItemReferences(body: Record<string, unknown>): void {
// Always strip previous_response_id — the /codex/responses endpoint does not

View file

@ -14,7 +14,12 @@ import { createRequestLogger } from "../utils/requestLogger.ts";
import { getModelTargetFormat, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.ts";
import { resolveModelAlias } from "../services/modelDeprecation.ts";
import { getUnsupportedParams } from "../config/providerRegistry.ts";
import { hasPerModelQuota, lockModelIfPerModelQuota } from "../services/accountFallback.ts";
import {
hasPerModelQuota,
lockModelIfPerModelQuota,
isDailyQuotaExhausted,
getMsUntilTomorrow,
} from "../services/accountFallback.ts";
import { COOLDOWN_MS } from "../config/constants.ts";
import {
buildErrorBody,
@ -2109,17 +2114,22 @@ export async function handleChatCore({
}
} else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) {
// Providers with per-model quotas — lock the model only, not the connection
// Daily quota exhausted: lock until tomorrow; otherwise use default cooldown
const isDailyQuota = isDailyQuotaExhausted(message);
const quotaCooldownMs = isDailyQuota
? getMsUntilTomorrow()
: retryAfterMs || COOLDOWN_MS.rateLimit;
if (
lockModelIfPerModelQuota(
provider,
connectionId,
model,
"quota_exhausted",
retryAfterMs || COOLDOWN_MS.rateLimit
isDailyQuota ? "daily_quota_exhausted" : "quota_exhausted",
quotaCooldownMs
)
) {
console.warn(
`[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil((retryAfterMs || COOLDOWN_MS.rateLimit) / 1000)}s (connection stays active)`
`[provider] Node ${connectionId} ${isDailyQuota ? "daily " : ""}quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)`
);
} else {
await updateProviderConnection(connectionId, {

View file

@ -50,6 +50,9 @@ export {
isAccountUnavailable,
getUnavailableUntil,
filterAvailableAccounts,
isProviderInCooldown,
getProviderCooldownRemainingMs,
getProvidersInCooldown,
} from "./services/accountFallback.ts";
export {

View file

@ -24,6 +24,30 @@ type ModelFailureState = {
resetAfterMs: number;
};
// Provider-level failure tracking for circuit breaker behavior
type ProviderFailureEntry = {
failureCount: number;
lastFailureAt: number;
resetAfterMs: number;
cooldownUntil: number | null;
};
// Error codes that count toward provider-level failure threshold
const PROVIDER_FAILURE_ERROR_CODES = new Set([429, 408, 500, 502, 503, 504]);
// Configuration for provider-level failure tracking
const PROVIDER_FAILURE_THRESHOLD = 5;
const PROVIDER_FAILURE_WINDOW_MS = 20 * 60 * 1000; // 20 minutes
const PROVIDER_COOLDOWN_MS = 10 * 60 * 1000; // 10 minutes cooling
// Provider-level failure state map: providerId -> failure entry
const providerFailureState = new Map<string, ProviderFailureEntry>();
// Guard against synchronous re-entrant calls within the same event-loop tick.
// NOT a true mutex — Node.js is single-threaded, so different SSE streams
// can interleave across ticks. This Set prevents a single call from recursively
// re-entering recordProviderFailure within the same synchronous call stack.
const providerFailureLocks = new Set<string>();
// T06 (sub2api PR #1037): Signals that indicate permanent account deactivation.
// When a 401 body contains these strings, the account is permanently dead
// and should NOT be retried after token refresh.
@ -82,8 +106,12 @@ const CONTEXT_OVERFLOW_PATTERNS = [
const MALFORMED_REQUEST_PATTERNS = [
/\bimproperly formed request\b/i,
/\binvalid.*message.*format/i,
/\bmessages must alternate/i,
/\bempty (message|content)/i,
/\bmessages must alternate\b/i,
/\bempty (message|content)\b/i,
// Tool call function name errors
/\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i,
/function.*name.*(?:blank|empty|missing)/i,
/tool_call.*name.*(?:blank|empty|missing)/i,
];
/**
@ -355,11 +383,21 @@ export function clearModelLock(provider, connectionId, model) {
* Compatible and passthrough providers multiplex multiple upstream models behind one
* connection, so transient 404/429 responses should stay model-scoped instead of
* poisoning the whole connection.
*
* @param provider - Provider ID
* @param _model - Model ID (reserved for future use)
* @param connectionPassthroughModels - Optional per-connection override from providerSpecificData.
* When provided, takes precedence over registry/provider-level logic.
*/
export function hasPerModelQuota(
provider: string | null | undefined,
_model: string | null | undefined = null
_model: string | null | undefined = null,
connectionPassthroughModels?: boolean
): boolean {
// Connection-level override takes precedence (e.g., user-configured ModelScope)
if (typeof connectionPassthroughModels === "boolean") {
return connectionPassthroughModels;
}
if (!provider) return false;
if (provider === "gemini") return true;
if (getPassthroughProviders().has(provider)) return true;
@ -376,18 +414,23 @@ export function lockModelIfPerModelQuota(
connectionId: string,
model: string | null,
reason: string,
cooldownMs: number
cooldownMs: number,
connectionPassthroughModels?: boolean
): boolean {
if (!hasPerModelQuota(provider, model) || !model) return false;
if (!hasPerModelQuota(provider, model, connectionPassthroughModels) || !model) return false;
// Skip model-level lock if the entire provider is in circuit-breaker cooldown.
// The provider cooldown already prevents all requests, so a model lock is redundant.
if (isProviderInCooldown(provider)) return false;
lockModel(provider, connectionId, model, reason, cooldownMs);
return true;
}
export function shouldMarkAccountExhaustedFrom429(
provider: string | null | undefined,
model: string | null | undefined = null
model: string | null | undefined = null,
connectionPassthroughModels?: boolean
): boolean {
return !hasPerModelQuota(provider, model);
return !hasPerModelQuota(provider, model, connectionPassthroughModels);
}
/**
@ -442,6 +485,145 @@ export function getAllModelLockouts() {
return active;
}
// ─── Provider-Level Failure Tracking ─────────────────────────────────────────
// Track failures at provider level: when a provider has too many transient failures
// across all its connections, cooldown the entire provider temporarily.
/**
* Check if a provider is currently in cooldown due to too many failures
*/
export function isProviderInCooldown(provider: string | null | undefined): boolean {
if (!provider) return false;
const entry = providerFailureState.get(provider);
if (!entry) return false;
// If in cooldown, check if it has expired
if (entry.cooldownUntil !== null && Date.now() >= entry.cooldownUntil) {
providerFailureState.delete(provider);
return false;
}
return entry.cooldownUntil !== null;
}
/**
* Get remaining cooldown time for a provider
*/
export function getProviderCooldownRemainingMs(provider: string | null | undefined): number | null {
if (!provider) return null;
const entry = providerFailureState.get(provider);
if (!entry || entry.cooldownUntil === null) return null;
const remaining = entry.cooldownUntil - Date.now();
return remaining > 0 ? remaining : null;
}
/**
* Record a failure for a provider. When threshold is reached within the window,
* the provider enters cooldown.
*/
export function recordProviderFailure(
provider: string | null | undefined,
log?: { warn?: (...args: unknown[]) => void }
): void {
if (!provider) return;
// Guard against concurrent re-entrant calls within the same tick
if (providerFailureLocks.has(provider)) return;
providerFailureLocks.add(provider);
try {
const now = Date.now();
const entry = providerFailureState.get(provider);
// Check if we're in cooldown period
if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) {
return; // Already in cooldown, don't record
}
// Check if failure window has expired
if (entry && now - entry.lastFailureAt > entry.resetAfterMs) {
// Window expired, reset count
providerFailureState.set(provider, {
failureCount: 1,
lastFailureAt: now,
resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
cooldownUntil: null,
});
return;
}
// Increment failure count
const newCount = entry ? entry.failureCount + 1 : 1;
if (newCount >= PROVIDER_FAILURE_THRESHOLD) {
// Threshold reached, enter cooldown
const cooldownUntil = now + PROVIDER_COOLDOWN_MS;
providerFailureState.set(provider, {
failureCount: newCount,
lastFailureAt: now,
resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
cooldownUntil,
});
log?.warn?.(
`[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s — entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown`
);
} else {
// Just increment counter
providerFailureState.set(provider, {
failureCount: newCount,
lastFailureAt: now,
resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
cooldownUntil: null,
});
}
} finally {
providerFailureLocks.delete(provider);
}
}
/**
* Clear provider failure state (e.g., after successful request)
*/
export function clearProviderFailure(provider: string | null | undefined): void {
if (!provider) return;
providerFailureState.delete(provider);
}
/**
* Get all providers currently in cooldown (for debugging/dashboard)
*/
export function getProvidersInCooldown(): Array<{
provider: string;
failureCount: number;
cooldownRemainingMs: number | null;
lastFailureAt: number;
}> {
const result = [];
for (const [provider, entry] of providerFailureState) {
if (entry.cooldownUntil === null) continue;
const remaining = entry.cooldownUntil - Date.now();
if (remaining <= 0) {
providerFailureState.delete(provider);
continue;
}
result.push({
provider,
failureCount: entry.failureCount,
cooldownRemainingMs: remaining,
lastFailureAt: entry.lastFailureAt,
});
}
return result;
}
/**
* Check if a status code should be counted toward provider failure threshold
*/
export function isProviderFailureCode(status: number): boolean {
return PROVIDER_FAILURE_ERROR_CODES.has(status);
}
// ─── Retry-After Parsing ────────────────────────────────────────────────────
/**
@ -625,6 +807,40 @@ export function classifyError(status, errorText) {
return RateLimitReason.UNKNOWN;
}
// ─── Daily Quota Helpers ────────────────────────────────────────────────────
/**
* Calculate milliseconds from now until tomorrow at midnight (00:00:00).
* Used to lock a model until the next day when daily quota is exhausted.
* @returns {number} Milliseconds until tomorrow
*/
export function getMsUntilTomorrow(): number {
const now = new Date();
const tomorrow = new Date(now);
tomorrow.setDate(tomorrow.getDate() + 1);
tomorrow.setHours(0, 0, 0, 0);
const ms = tomorrow.getTime() - now.getTime();
// Guard against DST edge cases: if ms is negative (shouldn't happen) or
// unreasonably large (>25h due to spring-forward), cap at 24 hours.
return ms > 0 && ms <= 25 * 60 * 60 * 1000 ? ms : 24 * 60 * 60 * 1000;
}
/**
* Check if error text indicates daily quota exhaustion (as opposed to rate limiting).
* Daily quota errors typically mention "today's quota" or "try again tomorrow".
* @param {string} errorText - Error message text
* @returns {boolean} True if daily quota is exhausted
*/
export function isDailyQuotaExhausted(errorText: string): boolean {
if (!errorText) return false;
const lower = errorText.toLowerCase();
return (
lower.includes("today's quota") ||
lower.includes("daily quota") ||
lower.includes("try again tomorrow")
);
}
// ─── Configurable Backoff ───────────────────────────────────────────────────
/**
@ -681,6 +897,12 @@ export function checkFallbackError(
const errorStr = (errorText || "").toString();
const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null);
// Track provider-level failures for circuit breaker behavior
// Only count transient errors that are likely to recover
if (isProviderFailureCode(status)) {
recordProviderFailure(provider);
}
function parseResetFromHeaders(headers, errorStr = "") {
if (!headers) return null;
@ -737,6 +959,19 @@ export function checkFallbackError(
};
}
// Daily quota exhausted — lock model until tomorrow
if (isDailyQuotaExhausted(errorStr)) {
const msUntilTomorrow = getMsUntilTomorrow();
// Cap at 24 hours to handle timezone edge cases
const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000);
return {
shouldFallback: true,
cooldownMs,
reason: RateLimitReason.QUOTA_EXHAUSTED,
dailyQuotaExhausted: true,
};
}
if (lowerError.includes("no credentials")) {
return {
shouldFallback: true,

View file

@ -23,6 +23,9 @@ function stripGeminiThinkingConfig(value: unknown): unknown {
return next;
}
/**
* @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest
*/
export function shouldStripCloudCodeThinking(provider: string, model: string): boolean {
if (!provider || !model) return false;
const normalizedModel = normalizeCloudCodeModel(model);
@ -33,6 +36,9 @@ export function shouldStripCloudCodeThinking(provider: string, model: string): b
return CLOUD_CODE_REASONING_UNSUPPORTED_PATTERNS.some((pattern) => pattern.test(normalizedModel));
}
/**
* @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest
*/
export function stripCloudCodeThinkingConfig(
body: Record<string, unknown>
): Record<string, unknown> {

View file

@ -59,6 +59,47 @@ const COMBO_BAD_REQUEST_FALLBACK_PATTERNS = [
/unsupported content part type/i,
/tool(?:_call|_use)? .* not (?:available|found)/i,
/third-party apps/i,
// Context overflow — model-specific, may succeed on a model with larger context window
/context overflow/i,
/context length exceeded/i,
/prompt too large/i,
/token limit/i,
/too many tokens/i,
/exceeds? context/i,
/maximum context/i,
/input too long/i,
/messages? exceed/i,
// Model not supported/found — permanent model-level error, try next combo target
/no provider supported/i,
/model not found/i,
/model not available/i,
/unsupported model/i,
/model.*has no provider/i,
// Function calling format error — model doesn't support this capability
/function\.?arguments.*(must be|should be|必须).*(json|JSON)/i,
/tool.*arguments.*invalid/i,
/function.*parameter.*(invalid|format)/i,
// Input length range error — model-specific context limit
/range of input length/i,
/input length should be/i,
// Transient 400 errors from upstream — should fallback to next combo target
/服务遇到了一点小状况/i, // ModelScope/Qwen transient error
/抱歉.*?敏感内容.*?请检查/i, // ModelScope/Qwen content moderation with context
/内容.*?敏感.*?(?:无法|过滤)/i, // Content sensitivity block
/无法响应.*?请求/i, // "unable to respond to request"
/稍后重试/i, // "retry later" in Chinese
/temporary.*error/i,
/transient.*error/i,
/service.*unavailable/i,
/please.*try.*again/i,
// Rate limit errors — some providers return 400 instead of 429
/\brate.?-?limit.?(?:exceeded|reached|hit)/i,
/too many requests/i,
/请求过于频繁/i, // Chinese rate limit message
// Tool call function name errors — model-specific, try next combo target
/\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i,
/function.*name.*(?:blank|empty|missing)/i,
/tool_call.*name.*(?:blank|empty|missing)/i,
];
// Patterns that signal all accounts for a provider are rate-limited / exhausted.
@ -77,6 +118,7 @@ function isAllAccountsRateLimitedResponse(
const MAX_COMBO_DEPTH = 3;
const MAX_FALLBACK_WAIT_MS = 5000;
const MAX_GLOBAL_ATTEMPTS = 30;
function comboModelNotFoundResponse(message: string) {
return errorResponse(404, message);
@ -1451,6 +1493,7 @@ export async function handleComboChat({
let earliestRetryAfter = null;
let lastStatus = null;
const startTime = Date.now();
globalAttempts = 0;
let fallbackCount = 0;
let recordedAttempts = 0;
@ -1484,6 +1527,14 @@ export async function handleComboChat({
// Retry loop for transient errors
for (let retry = 0; retry <= maxRetries; retry++) {
globalAttempts++;
if (globalAttempts > MAX_GLOBAL_ATTEMPTS) {
log.warn(
"COMBO",
`Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded across all targets and fallbacks. Terminating loop to prevent runaway background requests.`
);
return errorResponse(503, "Maximum combo retry limit reached");
}
if (retry > 0) {
log.info(
"COMBO",
@ -1799,6 +1850,7 @@ async function handleRoundRobinCombo({
let lastError = null;
let lastStatus = null;
let earliestRetryAfter = null;
let globalAttempts = 0;
let fallbackCount = 0;
let recordedAttempts = 0;
@ -1852,6 +1904,14 @@ async function handleRoundRobinCombo({
// Retry loop within this model
try {
for (let retry = 0; retry <= maxRetries; retry++) {
globalAttempts++;
if (globalAttempts > MAX_GLOBAL_ATTEMPTS) {
log.warn(
"COMBO-RR",
`Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded. Terminating loop to prevent runaway requests.`
);
return errorResponse(503, "Maximum combo retry limit reached");
}
if (retry > 0) {
log.info(
"COMBO-RR",

View file

@ -1,6 +1,7 @@
import {
isAccountDeactivated,
isCreditsExhausted,
isDailyQuotaExhausted,
isOAuthInvalidToken,
} from "./accountFallback.ts";
@ -100,7 +101,11 @@ export function classifyProviderError(statusCode: number, responseBody: unknown)
return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
}
// 429: Check if it's a daily quota exhaustion (lock until tomorrow) vs regular rate limit
if (statusCode === 429) {
if (isDailyQuotaExhausted(bodyStr)) {
return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
}
return PROVIDER_ERROR_TYPES.RATE_LIMITED;
}

View file

@ -11,6 +11,7 @@ import {
extractIntelligentHealthState,
normalizeIntelligentRoutingConfig,
} from "@/lib/combos/intelligentRouting";
import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge";
function getI18nOrFallback(t: any, key: string, fallback: string) {
if (typeof t?.has === "function" && t.has(key)) return t(key);
@ -283,6 +284,8 @@ export default function IntelligentComboPanel({
</p>
<p className="text-[11px] text-text-muted mt-0.5">{entry.model}</p>
</div>
{/* Model status badge - shows cooldown/error state */}
<ModelStatusBadge provider={entry.provider} model={entry.model} size="sm" />
<span className="rounded-full bg-primary/10 px-2 py-1 text-[11px] font-semibold text-primary">
{percentage}%
</span>

View file

@ -53,6 +53,7 @@ import {
getCodexRequestDefaults as _getCodexRequestDefaults,
} from "@/lib/providers/requestDefaults";
import { resolveDashboardProviderInfo } from "../providerPageUtils";
import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge";
type CompatByProtocolMap = Partial<
Record<
@ -326,6 +327,7 @@ function anyUpstreamHeadersBadge(
interface ModelRowProps {
model: { id: string; name?: string; source?: string; isHidden?: boolean };
fullModel: string;
provider: string;
copied?: string;
onCopy: (text: string, key: string) => void;
t: (key: string, values?: Record<string, unknown>) => string;
@ -2460,6 +2462,7 @@ export default function ProviderDetailPage() {
key={model.id}
model={model}
fullModel={`${providerDisplayAlias}/${model.id}`}
provider={providerId}
copied={copied}
onCopy={copy}
t={t}
@ -3329,6 +3332,7 @@ export default function ProviderDetailPage() {
function ModelRow({
model,
fullModel,
provider,
copied,
onCopy,
t,
@ -3358,6 +3362,7 @@ function ModelRow({
<code className="rounded bg-sidebar px-1.5 py-0.5 font-mono text-xs text-text-muted">
{fullModel}
</code>
<ModelStatusBadge provider={provider} model={model.id} size="sm" />
<ModelSourceBadge source={model.source} />
<button
onClick={() => onCopy(fullModel, `model-${model.id}`)}
@ -3407,6 +3412,7 @@ ModelRow.propTypes = {
id: PropTypes.string.isRequired,
}).isRequired,
fullModel: PropTypes.string.isRequired,
provider: PropTypes.string.isRequired,
copied: PropTypes.string,
onCopy: PropTypes.func.isRequired,
t: PropTypes.func,
@ -5402,6 +5408,7 @@ function AddApiKeyModal({
accountId: "",
consoleApiKey: "",
ccCompatibleContext1m: false,
passthroughModels: false,
});
const [validating, setValidating] = useState(false);
const [validationResult, setValidationResult] = useState(null);
@ -5495,6 +5502,9 @@ function AddApiKeyModal({
if (formData.excludedModels.trim()) {
providerSpecificData.excludedModels = parseExcludedModelsInput(formData.excludedModels);
}
if (formData.passthroughModels) {
providerSpecificData.passthroughModels = true;
}
if (provider === "bailian-coding-plan" && formData.consoleApiKey.trim()) {
providerSpecificData.consoleApiKey = formData.consoleApiKey.trim();
}
@ -5685,6 +5695,13 @@ function AddApiKeyModal({
placeholder="gpt-5*, claude-opus-*, gemini-*-pro*"
hint="Comma-separated wildcard patterns. This connection will never serve matching models."
/>
<Toggle
size="sm"
checked={formData.passthroughModels}
onChange={(checked) => setFormData({ ...formData, passthroughModels: checked })}
label={t("perModelQuotaLabel")}
description={t("perModelQuotaDescription")}
/>
{provider === "bailian-coding-plan" && (
<Input
label="Console API Key (Oracle)"
@ -5824,6 +5841,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
codexOpenaiStoreEnabled: false,
consoleApiKey: "",
ccCompatibleContext1m: false,
passthroughModels: connection.providerSpecificData?.passthroughModels === true,
});
const [testing, setTesting] = useState(false);
const [testResult, setTestResult] = useState(null);
@ -5891,6 +5909,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
codexOpenaiStoreEnabled: connection.providerSpecificData?.openaiStoreEnabled === true,
consoleApiKey: existingConsoleApiKey,
ccCompatibleContext1m: ccRequestDefaults.context1m,
passthroughModels: connection.providerSpecificData?.passthroughModels === true,
});
// Load existing extra keys from providerSpecificData
const existing = connection.providerSpecificData?.extraApiKeys;
@ -6031,6 +6050,8 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
tags: parseRoutingTagsInput(formData.routingTags),
excludedModels: parseExcludedModelsInput(formData.excludedModels),
customUserAgent: formData.customUserAgent.trim(),
// Only write when explicitly enabled; omit to let registry default take effect
...(formData.passthroughModels ? { passthroughModels: true } : {}),
};
if (connection.provider === "bailian-coding-plan") {
if (formData.consoleApiKey.trim()) {
@ -6288,6 +6309,13 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
placeholder="my-app/1.0"
hint="Optional override sent upstream as the User-Agent header for this connection"
/>
<Toggle
size="sm"
checked={formData.passthroughModels}
onChange={(checked) => setFormData({ ...formData, passthroughModels: checked })}
label={t("perModelQuotaLabel")}
description={t("perModelQuotaDescription")}
/>
{connection.provider === "bailian-coding-plan" && (
<Input
label="Console API Key (Oracle)"

View file

@ -0,0 +1,162 @@
"use client";
/**
* ModelStatusBadge compact single-model status indicator
*
* Shows a small badge with status icon (cooldown/unavailable/error)
* with a tooltip containing additional details like remaining cooldown time
* or last error message.
* Only renders for non-available models to keep the UI clean.
*
* Uses shared polling via ModelStatusContext to avoid redundant API calls.
*/
import { useState, useEffect, useRef } from "react";
import { useTranslations } from "next-intl";
import Tooltip from "@/shared/components/Tooltip";
import { useModelStatus } from "./ModelStatusContext";
interface ModelStatusBadgeProps {
provider: string;
model: string;
size?: "sm" | "md";
className?: string;
}
function formatRemainingTime(ms: number): string {
if (ms <= 0) return "";
const seconds = Math.ceil(ms / 1000);
if (seconds < 60) return `${seconds}s`;
const minutes = Math.floor(seconds / 60);
const remainingSeconds = seconds % 60;
if (minutes < 60) {
return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`;
}
const hours = Math.floor(minutes / 60);
const remainingMinutes = minutes % 60;
return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`;
}
export default function ModelStatusBadge({
provider,
model,
size = "sm",
className = "",
}: ModelStatusBadgeProps) {
const t = useTranslations("providers");
const status = useModelStatus(provider, model);
// Store the latest remaining time calculated by the interval
const [displayRemainingMs, setDisplayRemainingMs] = useState<number | null>(null);
// Use ref to track server-provided initial cooldown duration for countdown calculation
const initialCooldownMsRef = useRef<number | null>(null);
// Track when we started counting down
const countdownStartRef = useRef<number | null>(null);
// Set up countdown timer when cooldown begins
useEffect(() => {
if (status?.status !== "cooldown" || !status.remainingMs) {
// Reset refs when not in cooldown (no setState here)
initialCooldownMsRef.current = null;
countdownStartRef.current = null;
return;
}
// Initialize timing refs (no setState here)
initialCooldownMsRef.current = status.remainingMs;
countdownStartRef.current = Date.now();
// Update countdown every second via interval callback (setState allowed here)
const interval = setInterval(() => {
if (!initialCooldownMsRef.current || !countdownStartRef.current) {
return;
}
const elapsed = Date.now() - countdownStartRef.current;
const newRemaining = Math.max(0, initialCooldownMsRef.current - elapsed);
setDisplayRemainingMs(newRemaining);
// Stop updating when countdown reaches zero
if (newRemaining === 0) {
clearInterval(interval);
}
}, 1000);
return () => clearInterval(interval);
}, [status?.status, status?.remainingMs]);
// Derive the displayed remaining time: use countdown value if active, else use status value
const remainingMs =
status?.status === "cooldown" ? (displayRemainingMs ?? status.remainingMs ?? null) : null;
// Don't render badge for available models (keep UI clean)
if (!status || status.status === "available" || status.status === "unknown") {
return null;
}
const getStatusColor = () => {
switch (status.status) {
case "cooldown":
return "#f59e0b";
case "unavailable":
case "error":
return "#ef4444";
default:
return "#6b7280";
}
};
const getStatusIcon = () => {
switch (status.status) {
case "cooldown":
return "schedule";
case "unavailable":
case "error":
return "error";
default:
return "help";
}
};
const getTooltipText = () => {
switch (status.status) {
case "cooldown": {
const remaining = remainingMs !== null ? formatRemainingTime(remainingMs) : "";
const reason = status.reason ? ` (${status.reason})` : "";
const remainingText = remaining ? ` - ${remaining}` : "";
return `${t("cooldown")}${reason}${remainingText}`;
}
case "unavailable":
return `${t("unavailable")}${status.reason ? `: ${status.reason}` : ""}`;
case "error":
return `${t("error")}${status.lastError ? `: ${status.lastError}` : ""}`;
default:
return "";
}
};
const color = getStatusColor();
const sizeClasses = size === "sm" ? "px-1.5 py-0.5" : "px-2 py-1";
const iconSize = size === "sm" ? "text-[12px]" : "text-[14px]";
return (
<Tooltip content={getTooltipText()} position="top" delayMs={200}>
<span
className={`inline-flex items-center gap-1 rounded-full text-[10px] font-semibold ${sizeClasses} ${className}`}
style={{
backgroundColor: `${color}15`,
color: color,
}}
>
<span className={`material-symbols-outlined ${iconSize}`} aria-hidden="true">
{getStatusIcon()}
</span>
{status.status === "cooldown" && remainingMs !== null && (
<span>{formatRemainingTime(remainingMs)}</span>
)}
</span>
</Tooltip>
);
}

View file

@ -0,0 +1,186 @@
"use client";
/**
* ModelStatusContext shared polling for model availability
*
* Prevents redundant API calls by having all ModelStatusBadge components
* share a single polling interval. Only one request is made every 15 seconds
* regardless of how many badges are on the page.
*/
import React, { createContext, useContext, useEffect, useRef, useCallback, useMemo } from "react";
export interface ModelStatus {
status: "available" | "cooldown" | "unavailable" | "error" | "unknown";
reason?: string;
remainingMs?: number;
lastError?: string;
}
interface ModelStatusContextValue {
getStatus: (provider: string, model: string) => ModelStatus | null;
registerModel: (key: string, provider: string, model: string) => void;
unregisterModel: (key: string) => void;
}
const ModelStatusContext = createContext<ModelStatusContextValue | null>(null);
// Global map of model key -> status
let modelStatusMap = new Map<string, ModelStatus>();
// Global set of registered model keys
let registeredModels = new Set<string>();
// Polling interval ref (singleton)
let pollIntervalRef: NodeJS.Timeout | null = null;
function getModelKey(provider: string, model: string): string {
return `${provider}/${model}`;
}
async function fetchModelStatus(): Promise<void> {
try {
const res = await fetch("/api/models/availability");
if (!res.ok) return;
const json = await res.json();
const models = json?.models || [];
// Update all registered models with fresh data
const now = Date.now();
registeredModels.forEach((key) => {
const [provider, model] = key.split("/");
// Use exact matching first to avoid gpt-4 matching gpt-4-turbo incorrectly
const modelEntry =
models.find((m: any) => m.provider === provider && m.model === model) ||
models.find(
// Fallback to prefix matching only for models that contain the registered key
// This handles cases like "gpt-4o" matching badge for "gpt-4"
(m: any) =>
m.provider === provider &&
m.model &&
model &&
(m.model.startsWith(model + "-") || model.startsWith(m.model + "-"))
);
if (modelEntry) {
const newStatus: ModelStatus = {
status: modelEntry.status || "unknown",
reason: modelEntry.reason,
remainingMs: modelEntry.remainingMs,
lastError: modelEntry.lastError,
};
// For cooldown status, calculate remaining time based on server-provided value
if (modelEntry.status === "cooldown" && modelEntry.remainingMs) {
newStatus.remainingMs = modelEntry.remainingMs;
}
modelStatusMap.set(key, newStatus);
} else {
modelStatusMap.set(key, { status: "available" });
}
});
// Trigger re-render by dispatching custom event
window.dispatchEvent(new CustomEvent("model-status-update"));
} catch {
// Best-effort polling
}
}
function ensurePolling(): void {
if (pollIntervalRef) return;
// Initial fetch
fetchModelStatus();
// Poll every 15 seconds
pollIntervalRef = setInterval(fetchModelStatus, 15000);
}
function stopPolling(): void {
if (pollIntervalRef) {
clearInterval(pollIntervalRef);
pollIntervalRef = null;
}
}
export function ModelStatusProvider({ children }: { children: React.ReactNode }) {
const [, forceUpdate] = React.useState(0);
// Listen for status updates from the global poller
useEffect(() => {
const handleUpdate = () => forceUpdate((n) => n + 1);
window.addEventListener("model-status-update", handleUpdate);
return () => window.removeEventListener("model-status-update", handleUpdate);
}, []);
// Cleanup on unmount — stop polling only when no models remain registered
useEffect(() => {
return () => {
if (registeredModels.size === 0) {
stopPolling();
}
};
}, []);
const getStatus = useCallback((provider: string, model: string): ModelStatus | null => {
const key = getModelKey(provider, model);
return modelStatusMap.get(key) || null;
}, []);
const registerModel = useCallback((key: string, provider: string, model: string): void => {
const wasEmpty = registeredModels.size === 0;
registeredModels.add(key);
// Start polling when first model registers
if (wasEmpty) {
ensurePolling();
}
// Immediately fetch if no data yet
if (!modelStatusMap.has(key)) {
fetchModelStatus();
}
}, []);
const unregisterModel = useCallback((key: string): void => {
registeredModels.delete(key);
modelStatusMap.delete(key);
// Stop polling when last model unregisters
if (registeredModels.size === 0) {
stopPolling();
}
}, []);
const value = useMemo(
() => ({
getStatus,
registerModel,
unregisterModel,
}),
[getStatus, registerModel, unregisterModel]
);
return <ModelStatusContext.Provider value={value}>{children}</ModelStatusContext.Provider>;
}
export function useModelStatus(provider: string, model: string): ModelStatus | null {
const context = useContext(ModelStatusContext);
if (!context) {
throw new Error("useModelStatus must be used within a ModelStatusProvider");
}
const key = useMemo(() => getModelKey(provider, model), [provider, model]);
// Register/unregister on mount/unmount
useEffect(() => {
context.registerModel(key, provider, model);
return () => context.unregisterModel(key);
}, [context, key, provider, model]);
return context.getStatus(provider, model);
}
export default ModelStatusContext;

View file

@ -1,5 +1,10 @@
import { DashboardLayout } from "@/shared/components";
import { ModelStatusProvider } from "./dashboard/providers/components/ModelStatusContext";
export default function DashboardRootLayout({ children }) {
return <DashboardLayout>{children}</DashboardLayout>;
return (
<ModelStatusProvider>
<DashboardLayout>{children}</DashboardLayout>
</ModelStatusProvider>
);
}

View file

@ -282,6 +282,10 @@
"evals": "Evals",
"utilization": "Utilization",
"utilizationDescription": "Provider quota usage trends and rate limit tracking",
"modelStatus": "Model Status",
"modelStatusCooldown": "Cooldown",
"modelStatusUnavailable": "Unavailable",
"modelStatusError": "Error",
"comboHealth": "Combo Health",
"comboHealthDescription": "Combo-level quota, usage distribution, and performance metrics"
},
@ -644,7 +648,8 @@
"kiro": "Use when integrating Kiro and controlling model routing centrally from OmniRoute.",
"windsurf": "Use when you want an AI-first IDE with Codeium/Windsurf models routed through OmniRoute.",
"antigravity": "Use when Antigravity/Kiro traffic must be intercepted through MITM and routed to OmniRoute.",
"copilot": "Use when you want Copilot chat style UX while enforcing OmniRoute keys and routing rules."
"copilot": "Use when you want Copilot chat style UX while enforcing OmniRoute keys and routing rules.",
"qwen": "Use when you need Alibaba Qwen Code CLI for coding tasks."
},
"toolDescriptions": {
"antigravity": "Google Antigravity IDE with MITM",
@ -1108,6 +1113,7 @@
"builderAccount": "Account",
"builderPreview": "Preview",
"builderAddStep": "Add step",
"builderDuplicateExact": "This exact provider/model/account step is already in the combo.",
"builderComboRef": "Combo Ref",
"builderAddComboRef": "Add combo reference",
"builderComboRefStep": "Add combo reference",
@ -1865,6 +1871,9 @@
"compatUpstreamAddRow": "Add header",
"compatUpstreamRemoveRow": "Remove row",
"compatBadgeUpstreamHeaders": "Headers",
"perModelQuotaLabel": "Per-Model Quota",
"perModelQuotaDescription": "When enabled, 429/404 errors only lock the specific model, not the entire connection. Use for providers with per-model rate limits (e.g., ModelScope).",
"perModelQuotaToggle": "Per-Model Quota Toggle",
"modelId": "Model ID",
"customModelPlaceholder": "e.g. gpt-4.5-turbo",
"loading": "Loading...",

View file

@ -282,6 +282,10 @@
"evals": "评测",
"utilization": "利用率",
"utilizationDescription": "提供商配额使用趋势和速率限制跟踪",
"modelStatus": "模型状态",
"modelStatusCooldown": "冷却中",
"modelStatusUnavailable": "不可用",
"modelStatusError": "错误",
"comboHealth": "组合健康状况",
"comboHealthDescription": "组合级别配额、使用分布和性能指标"
},
@ -644,7 +648,8 @@
"kiro": "在集成 Kiro 并从 OmniRoute 集中控制模型路由时使用。",
"windsurf": "当您需要 Windsurf AI IDE 并通过 OmniRoute 路由模型时使用。",
"antigravity": "当必须通过 MITM 拦截 Antigravity/Kiro 流量并将其路由到 OmniRoute 时使用。",
"copilot": "当您想要 Copilot 聊天风格的 UX 同时强制执行 OmniRoute 键和路由规则时使用。"
"copilot": "当您想要 Copilot 聊天风格的 UX 同时强制执行 OmniRoute 键和路由规则时使用。",
"qwen": "当您需要使用阿里云 Qwen Code CLI 进行编码任务时使用。"
},
"toolDescriptions": {
"antigravity": "带 MITM 的 Google Antigravity IDE",
@ -1022,7 +1027,8 @@
"selectAccount": "选择账户",
"autoSelectAccount": "运行时自动选择账户",
"previewNextStep": "选择提供商和模型以预览下一步。",
"addStep": "添加步骤",
"builderAddStep": "添加步骤",
"builderDuplicateExact": "该提供商/模型/账户步骤已存在于组合中。",
"comboReference": "组合引用",
"selectComboToReference": "选择要引用的现有组合",
"addComboReference": "添加组合引用",
@ -1911,6 +1917,9 @@
"compatUpstreamAddRow": "添加请求头",
"compatUpstreamRemoveRow": "删除此行",
"compatBadgeUpstreamHeaders": "请求头",
"perModelQuotaLabel": "按模型配额",
"perModelQuotaDescription": "启用后429/404错误只会锁定特定模型而不是整个连接。适用于具有按模型速率限制的提供商例如ModelScope。",
"perModelQuotaToggle": "按模型配额开关",
"modelId": "模型 ID",
"customModelPlaceholder": "例如gpt-4.5-turbo",
"loading": "正在加载...",

View file

@ -841,7 +841,11 @@ async function handleSingleModelChat(
// 6. Mark account as quota-exhausted on 429 response
// For providers that route quota/cooldown at model scope, a 429 on one model
// does not mean the whole connection is exhausted.
if (result.status === 429 && shouldMarkAccountExhaustedFrom429(provider, model)) {
const passthroughModels = credentials.providerSpecificData?.passthroughModels;
if (
result.status === 429 &&
shouldMarkAccountExhaustedFrom429(provider, model, passthroughModels)
) {
markAccountExhaustedFrom429(credentials.connectionId, provider);
}

View file

@ -27,7 +27,10 @@ import { getCircuitBreaker, CircuitBreakerOpenError } from "../../shared/utils/c
import { getModelCooldownInfo, isModelAvailable } from "../../domain/modelAvailability";
import { logProxyEvent } from "../../lib/proxyLogger";
import { logTranslationEvent } from "../../lib/translatorEvents";
import { getRuntimeProviderProfile } from "@omniroute/open-sse/services/accountFallback.ts";
import {
getRuntimeProviderProfile,
clearProviderFailure,
} from "@omniroute/open-sse/services/accountFallback.ts";
export async function resolveModelOrError(modelStr: string, body: any, endpointPath: string = "") {
const modelInfo = await getModelInfo(modelStr);
@ -170,6 +173,8 @@ export async function executeChatWithBreaker({
},
onRequestSuccess: async () => {
await clearAccountError(credentials.connectionId, credentials);
// Clear provider-level failure state on successful request
clearProviderFailure(provider);
},
})
);

View file

@ -19,6 +19,8 @@ import {
hasPerModelQuota,
getRuntimeProviderProfile,
recordModelLockoutFailure,
isProviderInCooldown,
getProviderCooldownRemainingMs,
} from "@omniroute/open-sse/services/accountFallback.ts";
import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts";
import { COOLDOWN_MS } from "@omniroute/open-sse/config/constants.ts";
@ -682,6 +684,22 @@ export async function getProviderCredentials(
return true;
});
// Check if the entire provider is in cooldown (too many transient failures)
if (isProviderInCooldown(provider)) {
const cooldownRemaining = getProviderCooldownRemainingMs(provider);
log.warn(
"AUTH",
`${provider} | provider in cooldown for ${Math.ceil((cooldownRemaining || 0) / 1000)}s (${availableConnections.length} connections bypassed)`
);
return {
allRateLimited: true,
retryAfter: new Date(Date.now() + (cooldownRemaining || 0)).toISOString(),
retryAfterHuman: formatRetryAfter(
new Date(Date.now() + (cooldownRemaining || 0)).toISOString()
),
};
}
log.debug(
"AUTH",
`${provider} | available: ${availableConnections.length}/${connections.length}`
@ -1189,7 +1207,13 @@ export async function markAccountUnavailable(
const effectiveProviderProfile =
providerProfile || (provider ? await getRuntimeProviderProfile(provider) : null);
const isPerModelQuotaProvider = hasPerModelQuota(provider, model);
// Read passthroughModels from connection config (user-configured per-model quota)
const connProviderSpecificData = (conn?.providerSpecificData as Record<string, unknown>) || {};
const connectionPassthroughModels = connProviderSpecificData.passthroughModels as
| boolean
| undefined;
const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels);
if (isPerModelQuotaProvider && provider && model && (status === 404 || status === 429)) {
const reason = status === 404 ? "not_found" : "rate_limited";
const fallbackCooldown =

View file

@ -26,6 +26,7 @@ const { BaseExecutor } = await import("../../open-sse/executors/base.ts");
const { resetAllAvailability, setModelUnavailable } =
await import("../../src/domain/modelAvailability.ts");
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
const { clearProviderFailure } = await import("../../open-sse/services/accountFallback.ts");
const originalFetch = globalThis.fetch;
const originalRetryDelayMs = BaseExecutor.RETRY_CONFIG.delayMs;
@ -1017,6 +1018,8 @@ test("chat pipeline maps upstream timeouts to 504 responses", async () => {
});
test("chat pipeline injects memory context before sending the upstream request", async () => {
// Reset provider failure state to avoid circuit breaker interference
clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-memory" });
const apiKey = await seedApiKey();
await settingsDb.updateSettings({
@ -1057,6 +1060,8 @@ test("chat pipeline injects memory context before sending the upstream request",
});
test("chat pipeline injects skills into tools and intercepts tool calls with skill output", async () => {
// Reset provider failure state to avoid circuit breaker interference
clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-skills" });
const apiKey = await seedApiKey();
await settingsDb.updateSettings({ skillsEnabled: true });
@ -1118,6 +1123,8 @@ test("chat pipeline injects skills into tools and intercepts tool calls with ski
});
test("chat pipeline falls back to the next account after a provider failure", async () => {
// Reset provider failure state to avoid circuit breaker interference
clearProviderFailure("openai");
await seedConnection("openai", {
name: "openai-primary",
apiKey: "sk-openai-primary-fallback",
@ -1162,6 +1169,9 @@ test("chat pipeline falls back to the next account after a provider failure", as
});
test("chat pipeline falls back across combo models when the first provider fails", async () => {
// Reset provider failure state to avoid circuit breaker interference
clearProviderFailure("openai");
clearProviderFailure("claude");
await seedConnection("openai", { apiKey: "sk-openai-combo-fail" });
await seedConnection("claude", { apiKey: "sk-claude-combo-fail" });
await combosDb.createCombo({
@ -1206,6 +1216,8 @@ test("chat pipeline falls back across combo models when the first provider fails
});
test("chat pipeline deduplicates concurrent identical non-stream requests", async () => {
// Reset provider failure state to avoid circuit breaker interference
clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-dedup" });
let fetchCount = 0;

View file

@ -22,6 +22,12 @@ const {
recordModelLockoutFailure,
clearModelLock,
shouldMarkAccountExhaustedFrom429,
recordProviderFailure,
isProviderInCooldown,
getProviderCooldownRemainingMs,
clearProviderFailure,
isProviderFailureCode,
getProvidersInCooldown,
} = accountFallback;
const { selectAccount } = accountSelector;
@ -290,3 +296,215 @@ test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and re
clearModelLock("openai-compatible-custom-node", "conn-compatible", "custom-model-a");
}
});
// Provider-level failure circuit breaker tests
test("isProviderFailureCode correctly identifies transient error codes", () => {
assert.equal(isProviderFailureCode(429), true);
assert.equal(isProviderFailureCode(408), true);
assert.equal(isProviderFailureCode(500), true);
assert.equal(isProviderFailureCode(502), true);
assert.equal(isProviderFailureCode(503), true);
assert.equal(isProviderFailureCode(504), true);
assert.equal(isProviderFailureCode(401), false);
assert.equal(isProviderFailureCode(403), false);
assert.equal(isProviderFailureCode(400), false);
assert.equal(isProviderFailureCode(404), false);
assert.equal(isProviderFailureCode(200), false);
});
test("recordProviderFailure tracks failures and triggers cooldown after threshold", () => {
const originalNow = Date.now;
let now = 1_700_000_000_000;
Date.now = () => now;
try {
const provider = "test-provider";
// Clear any existing state
clearProviderFailure(provider);
assert.equal(isProviderInCooldown(provider), false);
assert.equal(getProviderCooldownRemainingMs(provider), null);
// Record 4 failures - should not trigger cooldown yet
for (let i = 0; i < 4; i++) {
recordProviderFailure(provider);
now += 1000; // 1 second between failures
}
assert.equal(isProviderInCooldown(provider), false);
// 5th failure - should trigger cooldown
recordProviderFailure(provider);
assert.equal(isProviderInCooldown(provider), true);
const remaining = getProviderCooldownRemainingMs(provider);
assert.ok(remaining !== null);
assert.ok(remaining > 0);
assert.ok(remaining <= 10 * 60 * 1000); // 10 minutes max
// Check getProvidersInCooldown returns the provider
const inCooldown = getProvidersInCooldown();
assert.ok(inCooldown.some((p) => p.provider === provider));
assert.equal(inCooldown.find((p) => p.provider === provider)?.failureCount, 5);
// Simulate cooldown expiration
now += 11 * 60 * 1000; // 11 minutes later
assert.equal(isProviderInCooldown(provider), false);
assert.equal(getProviderCooldownRemainingMs(provider), null);
assert.equal(
getProvidersInCooldown().some((p) => p.provider === provider),
false
);
} finally {
Date.now = originalNow;
clearProviderFailure("test-provider");
}
});
test("recordProviderFailure resets counter after failure window expires", () => {
const originalNow = Date.now;
let now = 1_700_000_000_000;
Date.now = () => now;
try {
const provider = "test-provider-window";
clearProviderFailure(provider);
// Record 3 failures
for (let i = 0; i < 3; i++) {
recordProviderFailure(provider);
now += 1000;
}
assert.equal(isProviderInCooldown(provider), false);
// Wait for failure window to expire (20 minutes + 1 second)
now += 20 * 60 * 1000 + 1000;
// Next failure should reset counter, not trigger cooldown
recordProviderFailure(provider);
assert.equal(isProviderInCooldown(provider), false);
// Need 4 more failures to trigger cooldown
for (let i = 0; i < 4; i++) {
recordProviderFailure(provider);
now += 1000;
}
assert.equal(isProviderInCooldown(provider), true);
} finally {
Date.now = originalNow;
clearProviderFailure("test-provider-window");
}
});
test("checkFallbackError records provider failure for transient errors", () => {
const originalNow = Date.now;
let now = 1_700_000_000_000;
Date.now = () => now;
try {
const provider = "test-provider-check";
clearProviderFailure(provider);
// Simulate 5 transient errors through checkFallbackError
for (let i = 0; i < 5; i++) {
checkFallbackError(429, "rate limited", 0, null, provider);
now += 1000;
}
// Provider should now be in cooldown
assert.equal(isProviderInCooldown(provider), true);
} finally {
Date.now = originalNow;
clearProviderFailure("test-provider-check");
}
});
test("checkFallbackError does not record provider failure for non-transient errors", () => {
const originalNow = Date.now;
let now = 1_700_000_000_000;
Date.now = () => now;
try {
const provider = "test-provider-no-record";
clearProviderFailure(provider);
// Simulate 5 auth errors (401) - should NOT trigger provider cooldown
for (let i = 0; i < 5; i++) {
checkFallbackError(401, "unauthorized", 0, null, provider);
now += 1000;
}
// Provider should NOT be in cooldown
assert.equal(isProviderInCooldown(provider), false);
} finally {
Date.now = originalNow;
clearProviderFailure("test-provider-no-record");
}
});
test("clearProviderFailure removes provider from cooldown", () => {
const originalNow = Date.now;
let now = 1_700_000_000_000;
Date.now = () => now;
try {
const provider = "test-provider-clear";
clearProviderFailure(provider);
// Trigger cooldown
for (let i = 0; i < 5; i++) {
recordProviderFailure(provider);
now += 1000;
}
assert.equal(isProviderInCooldown(provider), true);
// Clear the failure state
clearProviderFailure(provider);
assert.equal(isProviderInCooldown(provider), false);
assert.equal(getProviderCooldownRemainingMs(provider), null);
} finally {
Date.now = originalNow;
clearProviderFailure("test-provider-clear");
}
});
// Daily quota exhausted detection tests
test("isDailyQuotaExhausted detects today's quota errors", () => {
const { isDailyQuotaExhausted } = accountFallback;
assert.equal(isDailyQuotaExhausted("You have exceeded today's quota for model X"), true);
assert.equal(isDailyQuotaExhausted("exceeded your daily quota"), true);
assert.equal(isDailyQuotaExhausted("Please try again tomorrow"), true);
assert.equal(isDailyQuotaExhausted("rate limit exceeded"), false);
assert.equal(isDailyQuotaExhausted(""), false);
assert.equal(isDailyQuotaExhausted(null), false);
});
test("getMsUntilTomorrow returns positive value less than 24 hours", () => {
const { getMsUntilTomorrow } = accountFallback;
const ms = getMsUntilTomorrow();
assert.ok(ms > 0, "should be positive");
assert.ok(ms <= 24 * 60 * 60 * 1000, "should be <= 24 hours");
});
test("checkFallbackError locks model until tomorrow for daily quota exhaustion", () => {
const result = checkFallbackError(
429,
"You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow"
);
assert.equal(result.shouldFallback, true);
assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
assert.equal(result.dailyQuotaExhausted, true);
assert.ok(result.cooldownMs > 0, "cooldown should be positive");
assert.ok(result.cooldownMs <= 24 * 60 * 60 * 1000, "cooldown should be <= 24 hours");
});
test("checkFallbackError detects daily quota with 'try again tomorrow'", () => {
const result = checkFallbackError(429, "Please try again tomorrow");
assert.equal(result.shouldFallback, true);
assert.equal(result.dailyQuotaExhausted, true);
});
test("checkFallbackError detects daily quota with 'daily quota'", () => {
const result = checkFallbackError(429, "You have exceeded your daily quota");
assert.equal(result.shouldFallback, true);
assert.equal(result.dailyQuotaExhausted, true);
});

View file

@ -21,6 +21,7 @@ const {
const { getCircuitBreaker, STATE } = await import("../../src/shared/utils/circuitBreaker.ts");
const { clearModelUnavailability } = await import("../../src/domain/modelAvailability.ts");
const { clearProviderFailure } = await import("../../open-sse/services/accountFallback.ts");
const { getDefaultTaskModelMap, resetTaskRoutingStats, setTaskRoutingConfig } =
await import("../../open-sse/services/taskAwareRouter.ts");
@ -409,6 +410,8 @@ test("handleChat maps upstream timeouts to HTTP 504", async () => {
});
test("handleChat uses the emergency fallback model on budget exhaustion", async () => {
// Reset provider failure state to avoid circuit breaker interference
clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-billing" });
await seedConnection("nvidia", { apiKey: "sk-nvidia-fallback" });
const seenBodies = [];
@ -448,6 +451,8 @@ test("handleChat uses the emergency fallback model on budget exhaustion", async
});
test("handleChat returns the primary budget error when emergency fallback also fails", async () => {
// Reset provider failure state to avoid circuit breaker interference
clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-billing-fail" });
await seedConnection("nvidia", { apiKey: "sk-nvidia-fallback-fail" });
const seenModels = [];

View file

@ -740,6 +740,52 @@ test("shouldFallbackComboBadRequest only flags known provider-scoped 400 pattern
assert.equal(shouldFallbackComboBadRequest(429, "prohibited_content"), false);
assert.equal(shouldFallbackComboBadRequest(400, null), false);
assert.equal(shouldFallbackComboBadRequest(400, "generic bad request"), false);
// Chinese transient errors (ModelScope/Qwen)
assert.equal(
shouldFallbackComboBadRequest(400, "[400]: 抱歉,服务遇到了一点小状况,请您稍后重试。"),
true
);
assert.equal(shouldFallbackComboBadRequest(400, "服务遇到了一点小状况"), true);
assert.equal(shouldFallbackComboBadRequest(400, "请稍后重试"), true);
// Model not supported errors
assert.equal(
shouldFallbackComboBadRequest(
400,
"Model id : XiaomiMiMo/MiMo-V2-Flash , has no provider supported"
),
true
);
assert.equal(shouldFallbackComboBadRequest(400, "no provider supported"), true);
assert.equal(shouldFallbackComboBadRequest(400, "model not found"), true);
assert.equal(shouldFallbackComboBadRequest(400, "model not available"), true);
// Function calling format errors
assert.equal(
shouldFallbackComboBadRequest(400, "function.arguments parameter must be in JSON format"),
true
);
assert.equal(
shouldFallbackComboBadRequest(
400,
'[400]: <400> InternalError.Algo.InvalidParameter: The "function.arguments" parameter of the code model must be in JSON format.'
),
true
);
assert.equal(shouldFallbackComboBadRequest(400, "tool arguments invalid format"), true);
// Input length range errors
assert.equal(
shouldFallbackComboBadRequest(400, "Range of input length should be [1, 98304]"),
true
);
assert.equal(shouldFallbackComboBadRequest(400, "input length should be"), true);
// Content moderation errors (should fallback to next model)
assert.equal(
shouldFallbackComboBadRequest(400, "抱歉,您的内容包含敏感内容,请检查后重试"),
true
);
assert.equal(shouldFallbackComboBadRequest(400, "内容存在敏感信息,无法响应"), true);
assert.equal(shouldFallbackComboBadRequest(400, "无法响应该请求"), true);
// Generic "please check" should NOT match (was too broad before)
assert.equal(shouldFallbackComboBadRequest(400, "请检查您的参数"), false);
});
test("handleComboChat accepts binary and Responses-style 200 bodies but falls through malformed success payloads", async () => {

View file

@ -65,3 +65,21 @@ test("classifyProviderError: 403 with project string as plain string body => PRO
const result = classifyProviderError(403, body);
assert.equal(result, PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR);
});
test("classifyProviderError: 429 with daily quota signal => QUOTA_EXHAUSTED", () => {
const body = JSON.stringify({
error: {
message:
"You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow",
},
});
const result = classifyProviderError(429, body);
assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);
});
test("classifyProviderError: 429 with 'daily quota' signal => QUOTA_EXHAUSTED", () => {
const result = classifyProviderError(429, {
error: { message: "You have reached your daily quota limit" },
});
assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);
});