fix(sse): skip emergency fallback when its target leaves the live catalog (#16376)

Thanks @maxmad64bis — emergency fallback skips a target that has left the live catalog. `src/sse/handlers/chat.ts` nets down and the header helper lives in `src/sse/handlers/emergencyFallbackHeader.ts`, covered by `tests/unit/services/emergency-fallback.test.ts`.

Boarded on release/v3.8.52 at e23fb0f8e8. Focused tests: 87 passed, 0 failed. Changelog integrity passed. File-size kept the inherited executeTargetAttempt.ts overage (1288 > 1284). This pull request does not touch that file.

⚠️ base-red inherited: #15306

changelog.d/fixes/16376-emergency-fallback-unserved-target.md

@@ -0,0 +1 @@
+- **fix(sse):** skip the emergency fallback redirect when its target is missing from the live catalog ([#16376](https://github.com/diegosouzapw/OmniRoute/pull/16376)) — thanks @maxmad64bis

src/sse/handlers/chat.ts

@@ -85,6 +85,7 @@ import { dispatchChatWithAffinityEviction } from "./chatDispatch";
 import { getCachedSettings, getCombosCacheVersion } from "@/lib/db/readCache";
 import { comboCheckProvider, ghComboGate } from "./chat/githubLiveCatalogFilter.ts";
 import { markEmergencyFallback } from "./emergencyFallbackHeader.ts";
+import { resolveEmergencyFallbackDecision } from "./emergencyFallbackHeader.ts";
 import { evaluateComboTargetPreflight } from "./chat/comboTargetKeyPolicy.ts";
 import * as resolvedPolicy from "./chat/resolvedModelPolicy.ts";
 import { recordGateRejection, recordQuotaParkedSkip } from "./quotaParkedSkipUsage";
@@ -181,10 +182,7 @@ import {
   isSessionRegisteredForKey,
 } from "@omniroute/open-sse/services/sessionManager.ts";
 import { startQuotaMonitor } from "@omniroute/open-sse/services/quotaMonitor.ts";
-import {
-  isFallbackDecision,
-  shouldUseFallback,
-} from "@omniroute/open-sse/services/emergencyFallback.ts";
+import { isFallbackDecision } from "@omniroute/open-sse/services/emergencyFallback.ts";
 import {
   registerCodexConnection,
   registerCodexQuotaFetcher,
@@ -2454,10 +2452,11 @@ async function handleSingleModelChat(
       // (target-level orchestration plus the global fallback #689 after it), and a
       // per-target hop burns extra upstream calls against exhausted providers (#1731).
       if (!runtimeOptions.emergencyFallbackTried && !comboName) {
-        const fallbackDecision = shouldUseFallback(
+        const fallbackDecision = await resolveEmergencyFallbackDecision(
           Number(result.status || 0),
           String(result.error || ""),
-          Array.isArray(body?.tools) && body.tools.length > 0
+          Array.isArray(body?.tools) && body.tools.length > 0,
+          runtimeOptions
         );
 
         if (isFallbackDecision(fallbackDecision)) {

src/sse/handlers/emergencyFallbackHeader.ts

@@ -1,4 +1,16 @@
 import { inheritTrustedLocalRateLimitResponse } from "@omniroute/open-sse/services/rateLimitManager/errors.ts";
+import {
+  EMERGENCY_FALLBACK_CONFIG,
+  isFallbackDecision,
+  shouldUseFallback,
+  type EmergencyFallbackConfig,
+  type FallbackResult,
+} from "@omniroute/open-sse/services/emergencyFallback.ts";
+import {
+  catalogContainsModel,
+  getActiveSyncedCatalog,
+  type ActiveSyncedCatalog,
+} from "@/lib/db/models/activeSyncedCatalog";
 import { OMNIROUTE_RESPONSE_HEADERS } from "@/shared/constants/headers";
 import * as log from "../utils/logger";
 
@@ -47,3 +59,74 @@ export function markEmergencyFallback(
   log.warn("EMERGENCY_FALLBACK", `Served by emergency fallback: ${fromModel} -> ${toModel}`);
   return withEmergencyFallbackHeader(response, fromModel, toModel);
 }
+
+type CatalogLoader = (providerId: string) => Promise<ActiveSyncedCatalog>;
+
+const catalogByScope = new WeakMap<object, Map<string, Promise<ActiveSyncedCatalog>>>();
+const warnedAbsentTargets = new Set<string>();
+const warnedUnknownTargets = new Set<string>();
+
+function loadScopedCatalog(
+  scope: object,
+  providerId: string,
+  loadCatalog: CatalogLoader
+): Promise<ActiveSyncedCatalog> {
+  let byProvider = catalogByScope.get(scope);
+  if (!byProvider) {
+    byProvider = new Map();
+    catalogByScope.set(scope, byProvider);
+  }
+  let pending = byProvider.get(providerId);
+  if (!pending) {
+    pending = loadCatalog(providerId);
+    byProvider.set(providerId, pending);
+  }
+  return pending;
+}
+
+/** Test-only reset for the process-once warning sets. */
+export function resetEmergencyFallbackWarnForTest(): void {
+  warnedAbsentTargets.clear();
+  warnedUnknownTargets.clear();
+}
+
+/**
+ * Live-catalog guard for the budget-exhaustion emergency fallback. The
+ * synchronous trigger runs unchanged (no catalog read without a 402 or budget
+ * keyword). The catalog check only runs on a confirmed trigger and keeps the
+ * redirect only when the configured target is in the provider's authoritative
+ * live catalog. An unknown catalog (no sync yet, no active connection, stale
+ * snapshot, or a read failure) keeps the redirect and warns once per process,
+ * mirroring the model-resolution fail-open path.
+ */
+export async function resolveEmergencyFallbackDecision(
+  status: number,
+  errorBody: string,
+  requestHasTools: boolean,
+  scope: object,
+  loadCatalog: CatalogLoader = getActiveSyncedCatalog,
+  config: EmergencyFallbackConfig = EMERGENCY_FALLBACK_CONFIG
+): Promise<FallbackResult> {
+  const trigger = shouldUseFallback(status, errorBody, requestHasTools, config);
+  if (!isFallbackDecision(trigger)) return trigger;
+  const targetKey = `${config.provider}/${config.model}`;
+  const catalog = await loadScopedCatalog(scope, config.provider, loadCatalog);
+  const served = catalogContainsModel(catalog, config.model);
+  if (served === true) return trigger;
+  if (served === false) {
+    const reason = `emergency fallback target ${targetKey} not in the live catalog; keeping original routing`;
+    if (warnedAbsentTargets.has(targetKey)) {
+      log.debug("EMERGENCY_FALLBACK", reason);
+    } else {
+      warnedAbsentTargets.add(targetKey);
+      log.warn("EMERGENCY_FALLBACK", reason);
+    }
+    return { shouldFallback: false, reason };
+  }
+  const unknownReason = `emergency fallback target ${targetKey} has no authoritative live catalog; keeping redirect`;
+  if (!warnedUnknownTargets.has(targetKey)) {
+    warnedUnknownTargets.add(targetKey);
+    log.warn("EMERGENCY_FALLBACK", unknownReason);
+  }
+  return trigger;
+}

tests/unit/services/emergency-fallback.test.ts

@@ -20,6 +20,11 @@ const {
   resetEmergencyFallbackEnvCache,
   setEmergencyFallbackFeatureFlagResolverForTest,
 } = await import("../../../open-sse/services/emergencyFallback.ts");
+const { resolveEmergencyFallbackDecision, resetEmergencyFallbackWarnForTest } =
+  await import("../../../src/sse/handlers/emergencyFallbackHeader.ts");
+const { getActiveSyncedCatalog } =
+  await import("../../../src/lib/db/models/activeSyncedCatalog.ts");
+const { replaceSyncedAvailableModelsForConnection } = await import("../../../src/lib/db/models.ts");
 
 function restoreEnv(name: string, value: string | undefined) {
   if (value === undefined) {
@@ -36,6 +41,30 @@ function resetTestState() {
   delete process.env.OMNIROUTE_EMERGENCY_FALLBACK;
   resetEmergencyFallbackEnvCache();
   setEmergencyFallbackFeatureFlagResolverForTest(null);
+  resetEmergencyFallbackWarnForTest();
+}
+
+const FALLBACK_PROVIDER = EMERGENCY_FALLBACK_CONFIG.provider;
+const FALLBACK_MODEL = EMERGENCY_FALLBACK_CONFIG.model;
+const FALLBACK_CONNECTION = "emergency-fallback-live-catalog";
+
+async function seedLiveCatalog(modelIds: string[]) {
+  const db = core.getDbInstance();
+  const now = new Date().toISOString();
+  db.prepare(
+    `INSERT OR REPLACE INTO provider_connections
+       (id, provider, is_active, created_at, updated_at)
+     VALUES (?, ?, ?, ?, ?)`
+  ).run(FALLBACK_CONNECTION, FALLBACK_PROVIDER, 1, now, now);
+  await replaceSyncedAvailableModelsForConnection(
+    FALLBACK_PROVIDER,
+    FALLBACK_CONNECTION,
+    modelIds.map((id) => ({ id, name: id, source: "imported" as const }))
+  );
+}
+
+function freshScope(): object {
+  return {};
 }
 
 test.beforeEach(() => {
@@ -45,6 +74,7 @@ test.beforeEach(() => {
 test.afterEach(() => {
   resetEmergencyFallbackEnvCache();
   setEmergencyFallbackFeatureFlagResolverForTest(null);
+  resetEmergencyFallbackWarnForTest();
   delete process.env.OMNIROUTE_EMERGENCY_FALLBACK;
 });
 
@@ -187,3 +217,136 @@ test("tool-bearing requests are still skipped regardless of env switch", () => {
     assert.match(decision.reason, /tools/);
   });
 });
+
+test("402 with the target in the live catalog keeps the redirect", async () => {
+  await seedLiveCatalog([FALLBACK_MODEL]);
+  const catalog = await getActiveSyncedCatalog(FALLBACK_PROVIDER);
+  assert.equal(catalog.authoritative, true);
+  const decision = await resolveEmergencyFallbackDecision(402, "", false, freshScope());
+  assert.equal(decision.shouldFallback, true);
+});
+
+test("402 with the target missing from the live catalog keeps original routing", async () => {
+  await seedLiveCatalog(["openai/gpt-oss-20b"]);
+  const catalog = await getActiveSyncedCatalog(FALLBACK_PROVIDER);
+  assert.equal(catalog.authoritative, true);
+  const decision = await resolveEmergencyFallbackDecision(402, "", false, freshScope());
+  assert.equal(decision.shouldFallback, false);
+  assert.match(decision.reason, /not in the live catalog; keeping original routing/);
+});
+
+test("second 402 with the target still missing logs no new warning", async () => {
+  await seedLiveCatalog(["openai/gpt-oss-20b"]);
+  const { logger: rootLogger } = await import("../../../src/shared/utils/logger.ts");
+  const warnCalls: unknown[][] = [];
+  const debugCalls: unknown[][] = [];
+  const previousWarn = rootLogger.warn.bind(rootLogger);
+  const previousDebug = rootLogger.debug.bind(rootLogger);
+  rootLogger.warn = ((...args: unknown[]) => {
+    warnCalls.push(args);
+    return rootLogger;
+  }) as typeof rootLogger.warn;
+  rootLogger.debug = ((...args: unknown[]) => {
+    debugCalls.push(args);
+    return rootLogger;
+  }) as typeof rootLogger.debug;
+  try {
+    const first = await resolveEmergencyFallbackDecision(402, "", false, freshScope());
+    assert.equal(first.shouldFallback, false);
+    const second = await resolveEmergencyFallbackDecision(402, "", false, freshScope());
+    assert.equal(second.shouldFallback, false);
+    const fallbackWarns = warnCalls.filter((args) =>
+      args.some(
+        (arg) =>
+          (typeof arg === "string" && arg.includes("EMERGENCY_FALLBACK")) ||
+          (typeof arg === "object" &&
+            arg !== null &&
+            (arg as Record<string, unknown>).tag === "EMERGENCY_FALLBACK")
+      )
+    );
+    assert.equal(fallbackWarns.length, 1);
+    assert.ok(debugCalls.length >= 1);
+  } finally {
+    rootLogger.warn = previousWarn;
+    rootLogger.debug = previousDebug;
+  }
+});
+
+test("disabled switch keeps original routing without reading the catalog", async () => {
+  await seedLiveCatalog([FALLBACK_MODEL]);
+  let reads = 0;
+  const countingLoad = async (providerId: string) => {
+    reads += 1;
+    return getActiveSyncedCatalog(providerId);
+  };
+  const previousEnabled = EMERGENCY_FALLBACK_CONFIG.enabled;
+  EMERGENCY_FALLBACK_CONFIG.enabled = false;
+  try {
+    const decision = await resolveEmergencyFallbackDecision(
+      402,
+      "",
+      false,
+      freshScope(),
+      countingLoad
+    );
+    assert.equal(decision.shouldFallback, false);
+    assert.match(decision.reason, /emergency fallback disabled/);
+    assert.equal(reads, 0);
+  } finally {
+    EMERGENCY_FALLBACK_CONFIG.enabled = previousEnabled;
+    resetEmergencyFallbackWarnForTest();
+  }
+});
+
+test("unknown catalog keeps the redirect and warns once per process", async () => {
+  const { logger: rootLogger } = await import("../../../src/shared/utils/logger.ts");
+  const warnCalls: unknown[][] = [];
+  const previousWarn = rootLogger.warn.bind(rootLogger);
+  rootLogger.warn = ((...args: unknown[]) => {
+    warnCalls.push(args);
+    return rootLogger;
+  }) as typeof rootLogger.warn;
+  try {
+    const unknownLoad = async () => ({ authoritative: false, models: [] });
+    const first = await resolveEmergencyFallbackDecision(402, "", false, freshScope(), unknownLoad);
+    assert.equal(first.shouldFallback, true);
+    const second = await resolveEmergencyFallbackDecision(
+      402,
+      "",
+      false,
+      freshScope(),
+      unknownLoad
+    );
+    assert.equal(second.shouldFallback, true);
+    const fallbackWarns = warnCalls.filter((args) =>
+      args.some(
+        (arg) =>
+          (typeof arg === "string" && arg.includes("EMERGENCY_FALLBACK")) ||
+          (typeof arg === "object" &&
+            arg !== null &&
+            (arg as Record<string, unknown>).tag === "EMERGENCY_FALLBACK")
+      )
+    );
+    assert.equal(fallbackWarns.length, 1);
+  } finally {
+    rootLogger.warn = previousWarn;
+  }
+});
+
+test("non-budget failures never read the catalog", async () => {
+  let reads = 0;
+  const countingLoad = async (providerId: string) => {
+    reads += 1;
+    return getActiveSyncedCatalog(providerId);
+  };
+  const scope = freshScope();
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    const decision = await resolveEmergencyFallbackDecision(500, "", false, scope, countingLoad);
+    assert.equal(decision.shouldFallback, false);
+    assert.match(decision.reason, /no budget error detected/);
+  }
+  assert.equal(reads, 0);
+  const triggered = await resolveEmergencyFallbackDecision(402, "", false, scope, countingLoad);
+  assert.equal(triggered.shouldFallback, true);
+  assert.equal(reads, 1);
+});