From 31e4e46ef9d8a0198356bb9bde697c4b2b040348 Mon Sep 17 00:00:00 2001 From: Randi <55005611+rdself@users.noreply.github.com> Date: Sun, 14 Jun 2026 07:45:02 -0400 Subject: [PATCH] Expose Arena ELO sync in feature flags (#3821) Adds ARENA_ELO_SYNC_ENABLED to the Dashboard Feature Flags registry (DB-overridable), routes Arena ELO startup/status checks through the shared feature-flag resolver while preserving the existing env fallback, and refreshes env docs (adds the missing STREAM_READINESS_TIMEOUT_MS example) so env/doc sync stays green. Integrated into release/v3.8.25. Co-authored-by: R.D. --- .env.example | 7 ++- docs/reference/ENVIRONMENT.md | 8 ++-- src/i18n/messages/en.json | 3 +- src/instrumentation-node.ts | 20 ++++----- src/lib/arenaEloSync.ts | 37 ++++++++++++---- src/server-init.ts | 14 +++--- .../constants/featureFlagDefinitions.ts | 13 +++++- src/shared/utils/featureFlags.ts | 4 ++ tests/integration/integration-wiring.test.ts | 5 ++- tests/unit/arena-elo-sync.test.ts | 43 +++++++++++++++++++ tests/unit/feature-flags-settings.test.ts | 30 ++++++++++--- 11 files changed, 143 insertions(+), 41 deletions(-) diff --git a/.env.example b/.env.example index b0fbf5ae5..a8faa0278 100644 --- a/.env.example +++ b/.env.example @@ -829,7 +829,8 @@ GEMINI_CLI_USER_AGENT="google-api-nodejs-client/10.3.0" # Used by: src/shared/utils/runtimeTimeouts.ts — centralized timeout resolution. # # Hierarchy: REQUEST_TIMEOUT_MS acts as a global override. -# If set, it becomes the default for FETCH_TIMEOUT_MS and STREAM_IDLE_TIMEOUT_MS. +# If set, it becomes the default for FETCH_TIMEOUT_MS, STREAM_IDLE_TIMEOUT_MS, +# and STREAM_READINESS_TIMEOUT_MS. # The fine-grained variables below override their respective defaults only when set. # ── Global shortcut ── @@ -899,6 +900,7 @@ GEMINI_CLI_USER_AGENT="google-api-nodejs-client/10.3.0" # ── Stream idle detection ── # STREAM_IDLE_TIMEOUT_MS=600000 # Max silence between SSE chunks (default: 600000) # # Extended-thinking models rarely pause >90s. +# STREAM_READINESS_TIMEOUT_MS=80000 # Time to receive the first non-ping SSE event # ── TLS client (wreq-js fingerprint proxy) ── # TLS_CLIENT_TIMEOUT_MS=600000 # Inherits from FETCH_TIMEOUT_MS by default @@ -1089,7 +1091,8 @@ APP_LOG_TO_FILE=true # Auto-update model intelligence from Arena AI leaderboard ELO scores (powers the # Free Provider Rankings page). ON by default — fetches from api.wulong.dev on startup # (non-blocking, never fatal). Set to false to opt out of the outbound sync. -# Used by: src/lib/arenaEloSync.ts +# Also configurable from Dashboard > Settings > Feature Flags. +# Used by: src/shared/constants/featureFlagDefinitions.ts, src/lib/arenaEloSync.ts # ARENA_ELO_SYNC_ENABLED=true # Sync interval in seconds. Default: 86400 (24 hours). diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 4f26b3953..7e9f66bbf 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -699,10 +699,10 @@ Automatic model pricing data synchronization from external sources. ## Arena ELO Sync -| Variable | Default | Source File | Description | -| ------------------------- | ------------- | ------------------------- | --------------------------------------------------------------------------- | -| `ARENA_ELO_SYNC_ENABLED` | `true` | `src/lib/arenaEloSync.ts` | Periodic Arena AI leaderboard ELO sync (on by default; `false` to opt out). | -| `ARENA_ELO_SYNC_INTERVAL` | `86400` (24h) | `src/lib/arenaEloSync.ts` | Sync interval in seconds. | +| Variable | Default | Source File | Description | +| ------------------------- | ------------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | +| `ARENA_ELO_SYNC_ENABLED` | `true` | `src/shared/constants/featureFlagDefinitions.ts` | Periodic Arena AI leaderboard ELO sync, configurable from Dashboard Feature Flags or with `false` to opt out. | +| `ARENA_ELO_SYNC_INTERVAL` | `86400` (24h) | `src/lib/arenaEloSync.ts` | Sync interval in seconds. | --- diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index ca7539d85..afc92dbc1 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -843,6 +843,7 @@ "batchConceptRetentionNote": "Results and error files are retained for 30 days (Anthropic: 29 days)" }, "featureFlagOmnirouteEmergencyFallbackDescription": "Route budget-exhausted requests to the emergency free fallback provider/model.", + "featureFlagArenaEloSyncEnabledDescription": "Enable periodic Arena AI leaderboard ELO sync for model intelligence rankings.", "sidebar": { "home": "Home", "dashboard": "Dashboard", @@ -8720,7 +8721,7 @@ "subtitle": "Best free providers ranked by model ELO scores from Arena AI leaderboards", "loading": "Loading rankings...", "errorLoading": "Failed to load rankings", - "emptyState": "No rankings available yet. Arena ELO data syncs on startup (daily) — check back shortly, or trigger a manual sync. Set ARENA_ELO_SYNC_ENABLED=false to opt out.", + "emptyState": "No rankings available yet. Arena ELO data syncs on startup (daily) — check back shortly, or trigger a manual sync. Disable Arena ELO Sync in Feature Flags or set ARENA_ELO_SYNC_ENABLED=false to opt out.", "bestModel": "Best", "allCategories": "All Categories", "categoryDefault": "Default", diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index 6805c051a..f1f42d9c1 100755 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -265,18 +265,18 @@ export async function registerNodejs(): Promise { console.warn("[STARTUP] Auto-refresh daemon failed to start (non-fatal):", msg); } - // Arena ELO sync: model intelligence from the Arena AI leaderboard, powering the - // Free Provider Rankings page. On by default (opt out with ARENA_ELO_SYNC_ENABLED=false). - // Non-blocking — the initial sync is fire-and-forget and never fatal. - if (process.env.ARENA_ELO_SYNC_ENABLED !== "false") { - try { - const { initArenaEloSync } = await import("@/lib/arenaEloSync"); - await initArenaEloSync(); + try { + // Arena ELO sync: model intelligence from the Arena AI leaderboard, powering the + // Free Provider Rankings page. On by default; configurable from Dashboard Feature Flags. + // Non-blocking — the initial sync is fire-and-forget and never fatal. + const { initArenaEloSync } = await import("@/lib/arenaEloSync"); + const started = await initArenaEloSync(); + if (started) { console.log("[STARTUP] Arena ELO sync initialized"); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - console.warn("[STARTUP] Arena ELO sync failed to start (non-fatal):", msg); } + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.warn("[STARTUP] Arena ELO sync failed to start (non-fatal):", msg); } // Pricing sync: opt-in external pricing data (self-gated by PRICING_SYNC_ENABLED inside diff --git a/src/lib/arenaEloSync.ts b/src/lib/arenaEloSync.ts index dc6b9b2bd..c57b1106e 100644 --- a/src/lib/arenaEloSync.ts +++ b/src/lib/arenaEloSync.ts @@ -7,9 +7,11 @@ * * Resolution order: user overrides > synced arena ELO > defaults * - * On by default; opt out via ARENA_ELO_SYNC_ENABLED=false. + * On by default; opt out via Dashboard Feature Flags or ARENA_ELO_SYNC_ENABLED=false. */ +import { isArenaEloSyncEnabled } from "@/shared/utils/featureFlags"; + import { backupDbFile } from "./db/backup"; import { bulkUpsertModelIntelligence, @@ -176,6 +178,23 @@ let activeSyncIntervalMs = SYNC_INTERVAL_MS; let firstSyncDone = false; let syncInProgress = false; +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function getEffectiveArenaEloSyncEnabled(): boolean { + try { + return isArenaEloSyncEnabled(); + } catch (error) { + console.warn( + `[ARENA_ELO_SYNC] Failed to resolve ARENA_ELO_SYNC_ENABLED feature flag: ${getErrorMessage( + error + )}` + ); + return process.env.ARENA_ELO_SYNC_ENABLED !== "false"; + } +} + // ─── Model name normalization ──────────────────────────── /** @@ -507,7 +526,7 @@ export function stopArenaEloSync(): void { * next scheduled sync time, interval, and active sources. */ export function getArenaEloSyncStatus(): SyncStatus { - const enabled = process.env.ARENA_ELO_SYNC_ENABLED !== "false"; + const enabled = getEffectiveArenaEloSyncEnabled(); return { enabled, lastSync: lastSyncTime, @@ -524,21 +543,23 @@ export function getArenaEloSyncStatus(): SyncStatus { // ─── Init (called from server-init.ts) ─────────────────── /** - * Initialize Arena ELO sync if enabled via environment variable. + * Initialize Arena ELO sync if enabled via feature flag configuration. * - * Reads `ARENA_ELO_SYNC_ENABLED` (default: true; set to `false` to opt out). + * Reads `ARENA_ELO_SYNC_ENABLED` (default: true; set to `false` to opt out) + * through the feature flag resolver, so DB overrides from the dashboard apply. * When enabled, starts periodic sync with the interval from `ARENA_ELO_SYNC_INTERVAL` * (default: 86400 seconds / daily). * * All errors during initialization or the initial sync are caught and logged * — initialization is never fatal. */ -export async function initArenaEloSync(): Promise { - if (process.env.ARENA_ELO_SYNC_ENABLED === "false") { +export async function initArenaEloSync(): Promise { + if (!getEffectiveArenaEloSyncEnabled()) { console.log( - "[ARENA_ELO_SYNC] Disabled (ARENA_ELO_SYNC_ENABLED=false). Unset or =true to enable." + "[ARENA_ELO_SYNC] Disabled by the effective ARENA_ELO_SYNC_ENABLED feature flag. Enable it from Dashboard Feature Flags, unset the env var, or set it to true to enable." ); - return; + return false; } startPeriodicSync(); + return true; } diff --git a/src/server-init.ts b/src/server-init.ts index 8321226eb..0fd6070bc 100644 --- a/src/server-init.ts +++ b/src/server-init.ts @@ -133,14 +133,12 @@ async function startServer() { } // Arena ELO sync: model intelligence from leaderboard data (non-blocking, never fatal). - // On by default; opt out with ARENA_ELO_SYNC_ENABLED=false. - if (process.env.ARENA_ELO_SYNC_ENABLED !== "false") { - try { - const { initArenaEloSync } = await import("./lib/arenaEloSync"); - await initArenaEloSync(); - } catch (err) { - startupLog.warn({ error: getErrorMessage(err) }, "Arena ELO sync could not initialize"); - } + // On by default; opt out with Dashboard Feature Flags or ARENA_ELO_SYNC_ENABLED=false. + try { + const { initArenaEloSync } = await import("./lib/arenaEloSync"); + await initArenaEloSync(); + } catch (err) { + startupLog.warn({ error: getErrorMessage(err) }, "Arena ELO sync could not initialize"); } } diff --git a/src/shared/constants/featureFlagDefinitions.ts b/src/shared/constants/featureFlagDefinitions.ts index 06e0efe34..111d53e89 100644 --- a/src/shared/constants/featureFlagDefinitions.ts +++ b/src/shared/constants/featureFlagDefinitions.ts @@ -199,7 +199,7 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ warningLevel: "info", }, - // ──────────────── Runtime (9) ──────────────── + // ──────────────── Runtime (10) ──────────────── { key: "OMNIROUTE_MCP_ENFORCE_SCOPES", label: "MCP Enforce Scopes", @@ -302,6 +302,17 @@ export const FEATURE_FLAG_DEFINITIONS: FeatureFlagDefinition[] = [ requiresRestart: false, warningLevel: "info", }, + { + key: "ARENA_ELO_SYNC_ENABLED", + label: "Arena ELO Sync", + description: "Enable periodic Arena AI leaderboard ELO sync for model intelligence rankings.", + descriptionI18nKey: "featureFlagArenaEloSyncEnabledDescription", + category: "runtime", + defaultValue: "true", + type: "boolean", + requiresRestart: false, + warningLevel: "info", + }, // ──────────────── CLI (3) ──────────────── { diff --git a/src/shared/utils/featureFlags.ts b/src/shared/utils/featureFlags.ts index 65d90a3a2..56ae586f4 100644 --- a/src/shared/utils/featureFlags.ts +++ b/src/shared/utils/featureFlags.ts @@ -75,3 +75,7 @@ export function isCcCompatibleProviderEnabled(): boolean { export function isModelCatalogNamesEnabled(): boolean { return isFeatureFlagEnabled("MODEL_CATALOG_INCLUDE_NAMES"); } + +export function isArenaEloSyncEnabled(): boolean { + return isFeatureFlagEnabled("ARENA_ELO_SYNC_ENABLED"); +} diff --git a/tests/integration/integration-wiring.test.ts b/tests/integration/integration-wiring.test.ts index 21e9d4211..63f2c7a2b 100644 --- a/tests/integration/integration-wiring.test.ts +++ b/tests/integration/integration-wiring.test.ts @@ -88,9 +88,10 @@ describe("Pipeline Wiring — instrumentation-node.ts", () => { it("should initialize Arena ELO sync on the live startup path (on by default, opt-out)", () => { // The Next standalone runtime boots through instrumentation-node, NOT server-init.ts. // The Arena ELO sync (which feeds the Free Provider Rankings page) must be wired here, - // or it never runs in production regardless of ARENA_ELO_SYNC_ENABLED. + // or it never runs in production. initArenaEloSync self-gates through the feature flag + // resolver so ARENA_ELO_SYNC_ENABLED and dashboard overrides still apply. assert.match(src, /initArenaEloSync/); - assert.match(src, /ARENA_ELO_SYNC_ENABLED !== "false"/); + assert.match(src, /const started = await initArenaEloSync\(\)/); }); it("should initialize pricing + models.dev sync on the live startup path (self-gated, opt-in)", () => { diff --git a/tests/unit/arena-elo-sync.test.ts b/tests/unit/arena-elo-sync.test.ts index 0611c384e..717963458 100644 --- a/tests/unit/arena-elo-sync.test.ts +++ b/tests/unit/arena-elo-sync.test.ts @@ -35,8 +35,11 @@ const { fetchArenaLeaderboards, syncArenaElo, getArenaEloSyncStatus, + initArenaEloSync, stopArenaEloSync, } = await import("../../src/lib/arenaEloSync.ts"); +const { setFeatureFlagOverride, removeFeatureFlagOverride } = + await import("../../src/lib/db/featureFlags.ts"); import type { ArenaLeaderboardData, @@ -102,6 +105,14 @@ function createTestAdapter(): SqliteAdapter { "\n synced_at TEXT NOT NULL" ); const adapter = tryOpenSync(":memory:")!; + adapter.exec(` + CREATE TABLE IF NOT EXISTS key_value ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (namespace, key) + ); + `); adapter.exec(patchedSql); return adapter; } @@ -124,12 +135,14 @@ beforeEach(() => { testAdapter = createTestAdapter(); globalThis.__omnirouteDb = testAdapter as never; stopArenaEloSync(); + delete process.env.ARENA_ELO_SYNC_ENABLED; }); afterEach(() => { restoreFetch(); stopArenaEloSync(); delete globalThis.__omnirouteDb; + delete process.env.ARENA_ELO_SYNC_ENABLED; }); // ═══════════════════════════════════════════════════════════ @@ -767,6 +780,26 @@ describe("getArenaEloSyncStatus()", () => { delete process.env.ARENA_ELO_SYNC_ENABLED; } }); + + it("reflects dashboard feature flag DB overrides before env values", () => { + process.env.ARENA_ELO_SYNC_ENABLED = "true"; + try { + setFeatureFlagOverride("ARENA_ELO_SYNC_ENABLED", "false"); + + const status = getArenaEloSyncStatus(); + assert.strictEqual(status.enabled, false); + } finally { + removeFeatureFlagOverride("ARENA_ELO_SYNC_ENABLED"); + } + }); + + it("falls back to the env value if the feature flag store is unavailable", () => { + testAdapter.exec("DROP TABLE key_value"); + process.env.ARENA_ELO_SYNC_ENABLED = "false"; + + const status = getArenaEloSyncStatus(); + assert.strictEqual(status.enabled, false); + }); }); // ═══════════════════════════════════════════════════════════ @@ -774,6 +807,16 @@ describe("getArenaEloSyncStatus()", () => { // ═══════════════════════════════════════════════════════════ describe("stopArenaEloSync()", () => { + it("initArenaEloSync returns false when disabled by feature flag", async () => { + try { + setFeatureFlagOverride("ARENA_ELO_SYNC_ENABLED", "false"); + const started = await initArenaEloSync(); + assert.strictEqual(started, false); + } finally { + removeFeatureFlagOverride("ARENA_ELO_SYNC_ENABLED"); + } + }); + it("does not throw when no timer is running", () => { assert.doesNotThrow(() => stopArenaEloSync()); }); diff --git a/tests/unit/feature-flags-settings.test.ts b/tests/unit/feature-flags-settings.test.ts index 83ed22788..98268fad1 100644 --- a/tests/unit/feature-flags-settings.test.ts +++ b/tests/unit/feature-flags-settings.test.ts @@ -26,19 +26,20 @@ const { isRequireApiKeyEnabled, isCcCompatibleProviderEnabled, isModelCatalogNamesEnabled, + isArenaEloSyncEnabled, } = await import("../../src/shared/utils/featureFlags.ts"); // ────────────────────────────────────────────────────── // Test group 1 — Flag definitions registry // ────────────────────────────────────────────────────── describe("featureFlagDefinitions", () => { - it("has exactly 31 flag definitions", () => { - assert.strictEqual(FEATURE_FLAG_DEFINITIONS.length, 31); + it("has exactly 32 flag definitions", () => { + assert.strictEqual(FEATURE_FLAG_DEFINITIONS.length, 32); }); it("has unique keys for all flags", () => { const keys = FEATURE_FLAG_DEFINITIONS.map((d) => d.key); - assert.strictEqual(new Set(keys).size, 31); + assert.strictEqual(new Set(keys).size, 32); }); it("has valid categories for all flags", () => { @@ -96,6 +97,15 @@ describe("featureFlagDefinitions", () => { assert.strictEqual(def.requiresRestart, false); }); + it("defines Arena ELO sync as a runtime boolean flag enabled by default", () => { + const def = FEATURE_FLAG_DEFINITIONS.find((d) => d.key === "ARENA_ELO_SYNC_ENABLED"); + assert.ok(def, "ARENA_ELO_SYNC_ENABLED should exist"); + assert.strictEqual(def.category, "runtime"); + assert.strictEqual(def.type, "boolean"); + assert.strictEqual(def.defaultValue, "true"); + assert.strictEqual(def.requiresRestart, false); + }); + it("defines emergency fallback as a runtime boolean flag enabled by default", () => { const def = FEATURE_FLAG_DEFINITIONS.find((d) => d.key === "OMNIROUTE_EMERGENCY_FALLBACK"); assert.ok(def, "OMNIROUTE_EMERGENCY_FALLBACK should exist"); @@ -241,9 +251,9 @@ describe("resolveFeatureFlag", () => { }); describe("resolveAllFeatureFlags", () => { - it("returns all 31 flags", () => { + it("returns all 32 flags", () => { const all = resolveAllFeatureFlags(); - assert.strictEqual(all.length, 31); + assert.strictEqual(all.length, 32); }); it("marks DB-overridden flags with source 'db'", () => { @@ -306,6 +316,16 @@ describe("resolveFeatureFlag", () => { removeFeatureFlagOverride("MODEL_CATALOG_INCLUDE_NAMES"); } }); + + it("isArenaEloSyncEnabled defaults on and follows DB overrides", () => { + assert.strictEqual(isArenaEloSyncEnabled(), true); + try { + setFeatureFlagOverride("ARENA_ELO_SYNC_ENABLED", "false"); + assert.strictEqual(isArenaEloSyncEnabled(), false); + } finally { + removeFeatureFlagOverride("ARENA_ELO_SYNC_ENABLED"); + } + }); }); });