From 9fdd56da21067057fa649e9c0a417bcd1ecf5911 Mon Sep 17 00:00:00 2001 From: Josh Avant <830519+joshavant@users.noreply.github.com> Date: Mon, 8 Jun 2026 22:55:06 -0500 Subject: [PATCH] fix(openai): require api-key auth for realtime voice (#91567) * fix(openai): require api-key auth for realtime voice * test(plugin-sdk): avoid auth profile store shadowing --- docs/providers/openai.md | 34 ++- docs/web/control-ui.md | 2 +- .../openai/realtime-voice-provider.test.ts | 212 +++++++--------- extensions/openai/realtime-voice-provider.ts | 232 ++++++------------ src/plugin-sdk/provider-auth.test.ts | 83 +++++++ src/plugin-sdk/provider-auth.ts | 27 +- 6 files changed, 287 insertions(+), 303 deletions(-) diff --git a/docs/providers/openai.md b/docs/providers/openai.md index e395df1cab1..081cdbc048e 100644 --- a/docs/providers/openai.md +++ b/docs/providers/openai.md @@ -101,21 +101,19 @@ explicit runtime config. Control UI Talk with `talk.realtime.provider: "openai"`) goes through the public **OpenAI Platform Realtime API**, which is billed against OpenAI Platform credits rather than Codex/ChatGPT subscription quota. An account - with healthy OpenAI OAuth that runs Codex-backed chat models without - issue can still hit `insufficient_quota` / "You exceeded your current - quota" on the first Realtime turn if the same OpenAI organization has no - Platform billing set up. + with healthy OpenAI OAuth that runs Codex-backed chat models without issue + still needs an OpenAI API-key auth profile or a Platform API key with funded + Platform billing for Realtime voice. Fix: top up Platform credits at [platform.openai.com/account/billing](https://platform.openai.com/account/billing) -for the organization backing your realtime credentials. Realtime accepts -either a Platform `OPENAI_API_KEY` (configured via `talk.realtime.providers.openai.apiKey` -for Control UI Talk, or `plugins.entries.voice-call.config.realtime.providers.openai.apiKey` -for Voice Call) or an `openai` OAuth profile whose underlying -organization has Platform billing — both routes mint Realtime client secrets -through the Platform API, so either way the org needs funded Platform -credits. For chat turns you can still use Codex-backed `openai/*` models against the same -OpenClaw install; Realtime is the one route that needs Platform billing. +for the organization backing your realtime credentials. Realtime voice accepts +the `openai` API-key auth profile created by `openclaw onboard --auth-choice openai-api-key`, +a Platform `OPENAI_API_KEY` configured via `talk.realtime.providers.openai.apiKey` +for Control UI Talk, `plugins.entries.voice-call.config.realtime.providers.openai.apiKey` +for Voice Call, or the `OPENAI_API_KEY` environment variable. OpenAI OAuth +profiles can still run Codex-backed `openai/*` chat models in the same +OpenClaw install, but they do not configure Realtime voice. ## Memory embeddings @@ -646,7 +644,7 @@ Legacy `plugins.entries.openai.config.personality` is still read as a compatibil ``` - Set `OPENAI_TTS_BASE_URL` to override the TTS base URL without affecting the chat API endpoint. OpenAI TTS is still configured through an API key; for OAuth-only live talk-back, use the Realtime voice path instead of agent-mode STT -> TTS speech. + Set `OPENAI_TTS_BASE_URL` to override the TTS base URL without affecting the chat API endpoint. OpenAI TTS and Realtime voice are both configured through an OpenAI Platform API key; OAuth-only installs can still use Codex-backed chat models, but not OpenAI live talk-back. @@ -717,7 +715,7 @@ Legacy `plugins.entries.openai.config.personality` is still read as a compatibil | Silence duration | `...openai.silenceDurationMs` | `500` | | Prefix padding | `...openai.prefixPaddingMs` | `300` | | Reasoning effort | `...openai.reasoningEffort` | (unset) | - | Auth | `...openai.apiKey`, `OPENAI_API_KEY`, or `openai` OAuth | Browser Talk and non-Azure backend bridges can use OpenAI OAuth | + | Auth | `openai` API-key auth profile, `...openai.apiKey`, or `OPENAI_API_KEY` | OpenAI Platform API key required; OpenAI OAuth does not configure Realtime voice | Available built-in Realtime voices for `gpt-realtime-2`: `alloy`, `ash`, `ballad`, `coral`, `echo`, `sage`, `shimmer`, `verse`, `marin`, `cedar`. @@ -739,10 +737,10 @@ Legacy `plugins.entries.openai.config.personality` is still read as a compatibil Control UI Talk uses OpenAI browser realtime sessions with a Gateway-minted ephemeral client secret and a direct browser WebRTC SDP exchange against the - OpenAI Realtime API. When no direct OpenAI API key is configured, the - Gateway can mint that client secret with the selected `openai` OAuth - profile. Gateway relay and Voice Call backend realtime WebSocket bridges use - the same OAuth fallback for native OpenAI endpoints. Maintainer live + OpenAI Realtime API. The Gateway mints that client secret with the selected + `openai` API-key auth profile or configured OpenAI Platform API key. Gateway + relay and Voice Call backend realtime WebSocket bridges use the same + API-key-only auth path for native OpenAI endpoints. Maintainer live verification is available with `OPENAI_API_KEY=... GEMINI_API_KEY=... node --import tsx scripts/dev/realtime-talk-live-smoke.ts`; the OpenAI legs verify both the backend WebSocket bridge and the browser diff --git a/docs/web/control-ui.md b/docs/web/control-ui.md index ff11a9871b3..63273955aaf 100644 --- a/docs/web/control-ui.md +++ b/docs/web/control-ui.md @@ -201,7 +201,7 @@ Activity entries keep only sanitized summaries and redacted, truncated output pr - Talk mode uses a registered realtime voice provider. Configure OpenAI with `talk.realtime.provider: "openai"` plus either `talk.realtime.providers.openai.apiKey`, `OPENAI_API_KEY`, or an `openai` OAuth profile; configure Google with `talk.realtime.provider: "google"` plus `talk.realtime.providers.google.apiKey`. For hosted GPT realtime models, OpenClaw prefers the `openai` OAuth profile before `OPENAI_API_KEY`; an explicit OpenAI realtime `apiKey` remains the advanced override. The browser never receives a standard provider API key. OpenAI receives an ephemeral Realtime client secret for WebRTC. Google Live receives a one-use constrained Live API auth token for a browser WebSocket session, with instructions and tool declarations locked into the token by the Gateway. Providers that only expose a backend realtime bridge run through the Gateway relay transport, so credentials and vendor sockets stay server-side while browser audio moves through authenticated Gateway RPCs. The Realtime session prompt is assembled by the Gateway; `talk.client.create` does not accept caller-provided instruction overrides. + Talk mode uses a registered realtime voice provider. Configure OpenAI with `talk.realtime.provider: "openai"` plus an `openai` API-key auth profile, `talk.realtime.providers.openai.apiKey`, or `OPENAI_API_KEY`; OpenAI OAuth profiles do not configure Realtime voice. Configure Google with `talk.realtime.provider: "google"` plus `talk.realtime.providers.google.apiKey`. The browser never receives a standard provider API key. OpenAI receives an ephemeral Realtime client secret for WebRTC. Google Live receives a one-use constrained Live API auth token for a browser WebSocket session, with instructions and tool declarations locked into the token by the Gateway. Providers that only expose a backend realtime bridge run through the Gateway relay transport, so credentials and vendor sockets stay server-side while browser audio moves through authenticated Gateway RPCs. The Realtime session prompt is assembled by the Gateway; `talk.client.create` does not accept caller-provided instruction overrides. The Chat composer includes a Talk options button next to the Talk start/stop button. The options apply to the next Talk session and can override provider, transport, model, voice, reasoning effort, VAD threshold, silence duration, and prefix padding. When an option is blank, the Gateway uses configured defaults where available or the provider default. Selecting Gateway relay forces the backend relay path; selecting WebRTC keeps the session client-owned and fails instead of silently falling back to relay if the provider cannot create a browser session. diff --git a/extensions/openai/realtime-voice-provider.test.ts b/extensions/openai/realtime-voice-provider.test.ts index 3204b272e56..af8a23cfc9d 100644 --- a/extensions/openai/realtime-voice-provider.test.ts +++ b/extensions/openai/realtime-voice-provider.test.ts @@ -280,14 +280,45 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { expect(options?.headers).not.toHaveProperty("OpenAI-Beta"); }); - it("mints an ephemeral Realtime secret for native websocket bridges when using Codex OAuth", async () => { - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("oauth-token"); - fetchWithSsrFGuardMock.mockResolvedValueOnce({ - response: createJsonResponse({ - client_secret: { value: "ephemeral-realtime-secret" }, - }), - release: vi.fn(async () => undefined), + it("requires a Platform API key for native realtime websocket bridges", async () => { + const provider = buildOpenAIRealtimeVoiceProvider(); + const bridge = provider.createBridge({ + cfg: {} as never, + providerConfig: { model: "gpt-realtime-2" }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), }); + + await expect(bridge.connect()).rejects.toThrow( + "OpenAI Realtime voice requires an OpenAI Platform API key", + ); + + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + expect(FakeWebSocket.instances).toHaveLength(0); + }); + + it("uses OPENAI_API_KEY for default GPT realtime bridges", async () => { + vi.stubEnv("OPENAI_API_KEY", "sk-env"); // pragma: allowlist secret + const provider = buildOpenAIRealtimeVoiceProvider(); + const bridge = provider.createBridge({ + cfg: {} as never, + providerConfig: { model: "gpt-realtime-2" }, + onAudio: vi.fn(), + onClearAudio: vi.fn(), + }); + + void bridge.connect(); + await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1)); + bridge.close(); + + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + const socket = FakeWebSocket.instances[0]; + const options = socket?.args[1] as { headers?: Record } | undefined; + expect(options?.headers?.Authorization).toBe("Bearer sk-env"); + }); + + it("uses OpenAI API-key auth profiles for default GPT realtime bridges", async () => { + resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("sk-profile"); // pragma: allowlist secret const provider = buildOpenAIRealtimeVoiceProvider(); const bridge = provider.createBridge({ cfg: {} as never, @@ -303,70 +334,17 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { expect(resolveProviderAuthProfileApiKeyMock).toHaveBeenCalledWith({ provider: "openai", cfg: {}, - includeExternalCliAuth: true, + profileTypes: ["api_key"], }); - const request = requireFetchRequest(); - expectRecordFields(request, "fetch request", { - url: "https://api.openai.com/v1/realtime/client_secrets", - auditContext: "openai-realtime-bridge-session", - }); - expectRecordFields(requireFetchInit(), "fetch init", { method: "POST" }); - expectRecordFields(requireFetchHeaders(), "fetch headers", { - Authorization: "Bearer oauth-token", // pragma: allowlist secret - "Content-Type": "application/json", - }); - const body = requireFetchJsonBody(); - const bodySession = requireRecord(body.session, "fetch session"); - expectRecordFields(bodySession, "fetch session", { - type: "realtime", - model: "gpt-realtime-2", - }); - expectRecordFields( - requireNestedRecord(bodySession, ["audio", "output"]), - "fetch session output", - { - voice: "alloy", - }, - ); + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); const socket = FakeWebSocket.instances[0]; const options = socket?.args[1] as { headers?: Record } | undefined; - expect(options?.headers?.Authorization).toBe("Bearer ephemeral-realtime-secret"); - expect(options?.headers).not.toHaveProperty("OpenAI-Beta"); - }); - - it("prefers Codex OAuth over OPENAI_API_KEY for default GPT realtime bridges", async () => { - vi.stubEnv("OPENAI_API_KEY", "sk-env"); // pragma: allowlist secret - isProviderAuthProfileConfiguredMock.mockReturnValue(true); - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("oauth-token"); - fetchWithSsrFGuardMock.mockResolvedValueOnce({ - response: createJsonResponse({ - client_secret: { value: "ephemeral-realtime-secret" }, - }), - release: vi.fn(async () => undefined), - }); - const provider = buildOpenAIRealtimeVoiceProvider(); - const bridge = provider.createBridge({ - cfg: {} as never, - providerConfig: { model: "gpt-realtime-2" }, - onAudio: vi.fn(), - onClearAudio: vi.fn(), - }); - - void bridge.connect(); - await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(1)); - bridge.close(); - - expectRecordFields(requireFetchHeaders(), "fetch headers", { - Authorization: "Bearer oauth-token", // pragma: allowlist secret - }); - const socket = FakeWebSocket.instances[0]; - const options = socket?.args[1] as { headers?: Record } | undefined; - expect(options?.headers?.Authorization).toBe("Bearer ephemeral-realtime-secret"); + expect(options?.headers?.Authorization).toBe("Bearer sk-profile"); }); it("keeps explicit OpenAI realtime API keys as the advanced override", () => { vi.stubEnv("OPENAI_API_KEY", "sk-env"); // pragma: allowlist secret - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("oauth-token"); + resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("sk-profile"); // pragma: allowlist secret const provider = buildOpenAIRealtimeVoiceProvider(); const bridge = provider.createBridge({ cfg: {} as never, @@ -387,8 +365,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { expect(options?.headers?.Authorization).toBe("Bearer sk-configured"); }); - it("does not fall back to Codex OAuth for custom realtime endpoints", async () => { - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("oauth-token"); + it("requires an API key for custom realtime endpoints", async () => { const provider = buildOpenAIRealtimeVoiceProvider(); const bridge = provider.createBridge({ cfg: {} as never, @@ -400,51 +377,12 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { onClearAudio: vi.fn(), }); - await expect(bridge.connect()).rejects.toThrow("OpenAI API key missing"); + await expect(bridge.connect()).rejects.toThrow("OpenAI Realtime voice requires an API key"); - expect(resolveProviderAuthProfileApiKeyMock).not.toHaveBeenCalled(); expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); expect(FakeWebSocket.instances).toHaveLength(0); }); - it("does not open a native websocket after slow OAuth resolution times out", async () => { - vi.useFakeTimers(); - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("oauth-token"); - let resolveClientSecret: (value: { - response: Response; - release: () => Promise; - }) => void = () => {}; - fetchWithSsrFGuardMock.mockReturnValueOnce( - new Promise((resolve) => { - resolveClientSecret = resolve; - }), - ); - const provider = buildOpenAIRealtimeVoiceProvider(); - const bridge = provider.createBridge({ - cfg: {} as never, - providerConfig: { model: "gpt-realtime-2" }, - onAudio: vi.fn(), - onClearAudio: vi.fn(), - }); - - const connecting = expect(bridge.connect()).rejects.toThrow( - "OpenAI realtime connection timeout", - ); - await vi.advanceTimersByTimeAsync(10_000); - await connecting; - - resolveClientSecret({ - response: createJsonResponse({ - client_secret: { value: "ephemeral-realtime-secret" }, - }), - release: vi.fn(async () => undefined), - }); - await vi.runAllTimersAsync(); - - expect(FakeWebSocket.instances).toHaveLength(0); - bridge.close(); - }); - it("returns browser-safe OpenClaw attribution headers for native WebRTC offers", async () => { vi.stubEnv("OPENCLAW_VERSION", "2026.3.22"); fetchWithSsrFGuardMock.mockResolvedValueOnce({ @@ -546,7 +484,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { }); }); - it("resolves and caches keychain OPENAI_API_KEY refs before creating bridges", () => { + it("resolves and caches keychain OPENAI_API_KEY refs before creating bridges", async () => { vi.stubEnv("OPENAI_API_KEY", "keychain:openclaw:OPENAI_REALTIME_BRIDGE_TEST"); execFileSyncMock.mockReturnValue("sk-bridge-env\n"); // pragma: allowlist secret const provider = buildOpenAIRealtimeVoiceProvider(); @@ -563,6 +501,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { }); void first.connect(); void second.connect(); + await vi.waitFor(() => expect(FakeWebSocket.instances.length).toBe(2)); first.close(); second.close(); @@ -583,7 +522,19 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { expect(execFileSyncMock).not.toHaveBeenCalled(); }); - it("treats OpenAI Codex OAuth profiles as configured for browser realtime sessions", () => { + it("does not treat OpenAI OAuth profiles as configured for browser realtime sessions", () => { + const provider = buildOpenAIRealtimeVoiceProvider(); + const cfg = { agents: { defaults: {} } } as never; + + expect(provider.isConfigured({ cfg, providerConfig: {} })).toBe(false); + expect(isProviderAuthProfileConfiguredMock).toHaveBeenCalledWith({ + provider: "openai", + cfg, + profileTypes: ["api_key"], + }); + }); + + it("treats OpenAI API-key auth profiles as configured for browser realtime sessions", () => { isProviderAuthProfileConfiguredMock.mockReturnValue(true); const provider = buildOpenAIRealtimeVoiceProvider(); const cfg = { agents: { defaults: {} } } as never; @@ -592,12 +543,11 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { expect(isProviderAuthProfileConfiguredMock).toHaveBeenCalledWith({ provider: "openai", cfg, - includeExternalCliAuth: true, + profileTypes: ["api_key"], }); }); - it("does not use Codex OAuth to configure Azure realtime sessions", () => { - isProviderAuthProfileConfiguredMock.mockReturnValue(true); + it("does not configure Azure realtime sessions without a Platform API key", () => { const provider = buildOpenAIRealtimeVoiceProvider(); const cfg = { agents: { defaults: {} } } as never; @@ -610,11 +560,27 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { }, }), ).toBe(false); - expect(isProviderAuthProfileConfiguredMock).not.toHaveBeenCalled(); }); - it("uses OpenAI Codex OAuth to mint browser realtime client secrets when no API key is set", async () => { - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("oauth-realtime-token"); + it("requires a Platform API key before minting browser realtime client secrets", async () => { + const provider = buildOpenAIRealtimeVoiceProvider(); + if (!provider.createBrowserSession) { + throw new Error("expected OpenAI realtime provider to support browser sessions"); + } + const cfg = { agents: { defaults: {} } } as never; + + await expect( + provider.createBrowserSession({ + cfg, + providerConfig: {}, + instructions: "Be concise.", + }), + ).rejects.toThrow("OpenAI Realtime voice requires an OpenAI Platform API key"); + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + }); + + it("uses OpenAI API-key auth profiles to mint browser realtime client secrets", async () => { + resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("sk-profile"); // pragma: allowlist secret fetchWithSsrFGuardMock.mockResolvedValueOnce({ response: createJsonResponse({ client_secret: { value: "client-secret-123" }, @@ -636,16 +602,15 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { expect(resolveProviderAuthProfileApiKeyMock).toHaveBeenCalledWith({ provider: "openai", cfg, - includeExternalCliAuth: true, + profileTypes: ["api_key"], }); expectRecordFields(requireFetchHeaders(), "fetch headers", { - Authorization: "Bearer oauth-realtime-token", // pragma: allowlist secret + Authorization: "Bearer sk-profile", // pragma: allowlist secret }); }); - it("prefers Codex OAuth over OPENAI_API_KEY for default GPT browser sessions", async () => { + it("uses OPENAI_API_KEY for default GPT browser sessions", async () => { vi.stubEnv("OPENAI_API_KEY", "sk-env"); // pragma: allowlist secret - resolveProviderAuthProfileApiKeyMock.mockResolvedValueOnce("oauth-realtime-token"); fetchWithSsrFGuardMock.mockResolvedValueOnce({ response: createJsonResponse({ client_secret: { value: "client-secret-123" }, @@ -665,13 +630,8 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { instructions: "Be concise.", }); - expect(resolveProviderAuthProfileApiKeyMock).toHaveBeenCalledWith({ - provider: "openai", - cfg, - includeExternalCliAuth: true, - }); expectRecordFields(requireFetchHeaders(), "fetch headers", { - Authorization: "Bearer oauth-realtime-token", // pragma: allowlist secret + Authorization: "Bearer sk-env", // pragma: allowlist secret }); }); @@ -688,7 +648,9 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { onClearAudio: vi.fn(), }); - await expect(bridge.connect()).rejects.toThrow("OpenAI API key or Codex OAuth missing"); + await expect(bridge.connect()).rejects.toThrow( + "OpenAI Realtime voice requires an OpenAI Platform API key", + ); }); it("normalizes provider-owned voice settings from raw provider config", () => { diff --git a/extensions/openai/realtime-voice-provider.ts b/extensions/openai/realtime-voice-provider.ts index 498c1ff960d..03254cd52e0 100644 --- a/extensions/openai/realtime-voice-provider.ts +++ b/extensions/openai/realtime-voice-provider.ts @@ -242,10 +242,9 @@ type OpenAIRealtimeApiKeyResolution = | { status: "available"; value: string } | { status: "missing" }; -type OpenAIRealtimeAuthResolution = - | { status: "available"; kind: "api-key" | "codex-oauth"; value: string } - | { status: "missing" }; - +const OPENAI_REALTIME_PLATFORM_API_KEY_REQUIRED = + "OpenAI Realtime voice requires an OpenAI Platform API key"; +const OPENAI_REALTIME_API_KEY_REQUIRED = "OpenAI Realtime voice requires an API key"; const KEYCHAIN_SECRET_REF_RE = /^keychain:([^:]+):([^:]+)$/; const KEYCHAIN_LOOKUP_TIMEOUT_MS = 5000; const resolvedKeychainSecretRefCache = new Map(); @@ -315,12 +314,15 @@ function resolveOpenAIRealtimeApiKey( return resolveOpenAIRealtimeEnvApiKey(); } -function requireOpenAIRealtimeApiKey(configuredApiKey: string | undefined): string { +function requireOpenAIRealtimeApiKey( + configuredApiKey: string | undefined, + errorMessage = OPENAI_REALTIME_API_KEY_REQUIRED, +): string { const resolved = resolveOpenAIRealtimeApiKey(configuredApiKey); if (resolved.status === "available") { return resolved.value; } - throw new Error("OpenAI API key missing"); + throw new Error(errorMessage); } function hasOpenAIRealtimeConfiguredApiKeyInput(configuredApiKey: string | undefined): boolean { @@ -334,8 +336,58 @@ function hasOpenAIRealtimeApiKeyInput(configuredApiKey: string | undefined): boo ); } -function prefersCodexOAuthForRealtimeModel(model: string | undefined): boolean { - return (model ?? OPENAI_REALTIME_DEFAULT_MODEL).trim().toLowerCase().startsWith("gpt-"); +async function resolveOpenAIRealtimePlatformApiKey(params: { + configuredApiKey: string | undefined; + cfg: RealtimeVoiceBrowserSessionCreateRequest["cfg"] | undefined; +}): Promise { + const configured = resolveOpenAIRealtimeSecretInput(params.configuredApiKey); + if ( + configured.status === "available" || + hasOpenAIRealtimeConfiguredApiKeyInput(params.configuredApiKey) + ) { + return configured; + } + + const profileApiKey = await resolveProviderAuthProfileApiKey({ + provider: "openai", + cfg: params.cfg, + profileTypes: ["api_key"], + }); + if (profileApiKey) { + return { status: "available", value: profileApiKey }; + } + + return resolveOpenAIRealtimeEnvApiKey(); +} + +async function requireOpenAIRealtimePlatformApiKey(params: { + configuredApiKey: string | undefined; + cfg: RealtimeVoiceBrowserSessionCreateRequest["cfg"] | undefined; +}): Promise { + const resolved = await resolveOpenAIRealtimePlatformApiKey(params); + if (resolved.status === "available") { + return resolved.value; + } + throw new Error(OPENAI_REALTIME_PLATFORM_API_KEY_REQUIRED); +} + +function hasOpenAIRealtimePlatformApiKeyInput(params: { + configuredApiKey: string | undefined; + cfg: RealtimeVoiceBrowserSessionCreateRequest["cfg"] | undefined; +}): boolean { + if (hasOpenAIRealtimeConfiguredApiKeyInput(params.configuredApiKey)) { + return true; + } + if ( + isProviderAuthProfileConfigured({ + provider: "openai", + cfg: params.cfg, + profileTypes: ["api_key"], + }) + ) { + return true; + } + return hasOpenAIRealtimeApiKeyInput(undefined); } function isOpenAIRealtimeMaxSessionDurationError(detail: string): boolean { @@ -346,71 +398,6 @@ function isOpenAIRealtimeMaxSessionDurationError(detail: string): boolean { ); } -async function resolveOpenAIRealtimeDefaultAuth(params: { - configuredApiKey: string | undefined; - cfg: RealtimeVoiceBrowserSessionCreateRequest["cfg"] | undefined; - model: string | undefined; -}): Promise { - const configured = resolveOpenAIRealtimeSecretInput(params.configuredApiKey); - if ( - configured.status === "available" || - hasOpenAIRealtimeConfiguredApiKeyInput(params.configuredApiKey) - ) { - return configured.status === "available" - ? { status: "available", kind: "api-key", value: configured.value } - : { status: "missing" }; - } - - if (prefersCodexOAuthForRealtimeModel(params.model)) { - const codexToken = await resolveProviderAuthProfileApiKey({ - provider: "openai", - cfg: params.cfg, - includeExternalCliAuth: true, - }); - if (codexToken) { - return { status: "available", kind: "codex-oauth", value: codexToken }; - } - } - - const env = resolveOpenAIRealtimeEnvApiKey(); - if (env.status === "available") { - return { status: "available", kind: "api-key", value: env.value }; - } - return { status: "missing" }; -} - -async function requireOpenAIRealtimeDefaultAuth(params: { - configuredApiKey: string | undefined; - cfg: RealtimeVoiceBrowserSessionCreateRequest["cfg"] | undefined; - model: string | undefined; -}): Promise { - const auth = await resolveOpenAIRealtimeDefaultAuth(params); - if (auth.status === "available") { - return auth; - } - throw new Error("OpenAI API key or Codex OAuth missing"); -} - -function hasOpenAIRealtimeBrowserAuthInput(params: { - configuredApiKey: string | undefined; - cfg: RealtimeVoiceBrowserSessionCreateRequest["cfg"] | undefined; - model: string | undefined; -}): boolean { - if (hasOpenAIRealtimeConfiguredApiKeyInput(params.configuredApiKey)) { - return true; - } - if (prefersCodexOAuthForRealtimeModel(params.model)) { - return ( - isProviderAuthProfileConfigured({ - provider: "openai", - cfg: params.cfg, - includeExternalCliAuth: true, - }) || hasOpenAIRealtimeApiKeyInput(undefined) - ); - } - return hasOpenAIRealtimeApiKeyInput(undefined); -} - function base64ToBuffer(b64: string): Buffer { return Buffer.from(b64, "base64"); } @@ -721,7 +708,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { if (hasOpenAIRealtimeConfiguredApiKeyInput(cfg.apiKey)) { const directApiKey = resolveOpenAIRealtimeSecretInput(cfg.apiKey); if (directApiKey.status === "missing") { - throw new Error("OpenAI API key missing"); + throw new Error(OPENAI_REALTIME_PLATFORM_API_KEY_REQUIRED); } return this.resolveApiKeyConnectionParams(directApiKey.value, model); } @@ -729,35 +716,25 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { if (cfg.azureEndpoint) { const directApiKey = resolveOpenAIRealtimeEnvApiKey(); if (directApiKey.status === "missing") { - throw new Error("OpenAI API key missing"); + throw new Error(OPENAI_REALTIME_API_KEY_REQUIRED); } return this.resolveApiKeyConnectionParams(directApiKey.value, model); } - if (!prefersCodexOAuthForRealtimeModel(model)) { - const directApiKey = resolveOpenAIRealtimeEnvApiKey(); - if (directApiKey.status === "missing") { - throw new Error("OpenAI API key missing"); - } - return this.resolveApiKeyConnectionParams(directApiKey.value, model); - } - - if ( - !isProviderAuthProfileConfigured({ - provider: "openai", - cfg: cfg.cfg, - includeExternalCliAuth: true, - }) - ) { - const directApiKey = resolveOpenAIRealtimeEnvApiKey(); - if (directApiKey.status === "available") { - return this.resolveApiKeyConnectionParams(directApiKey.value, model); - } - } - return this.resolveDefaultConnectionParams(model); } + private async resolveDefaultConnectionParams(model: string): Promise<{ + url: string; + headers: Record; + }> { + const apiKey = await requireOpenAIRealtimePlatformApiKey({ + configuredApiKey: this.config.apiKey, + cfg: this.config.cfg, + }); + return this.resolveApiKeyConnectionParams(apiKey, model); + } + private resolveApiKeyConnectionParams( apiKey: string, model: string, @@ -797,61 +774,6 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { }; } - private async resolveDefaultConnectionParams(model: string): Promise<{ - url: string; - headers: Record; - }> { - const cfg = this.config; - const auth = await requireOpenAIRealtimeDefaultAuth({ - configuredApiKey: cfg.apiKey, - cfg: cfg.cfg, - model, - }); - if (auth.kind === "api-key") { - return this.resolveApiKeyConnectionParams(auth.value, model); - } - if (cfg.azureEndpoint) { - throw new Error("OpenAI API key missing"); - } - return this.resolveOAuthConnectionParams(model, auth.value); - } - - private async resolveOAuthConnectionParams( - model: string, - authToken: string, - ): Promise<{ - url: string; - headers: Record; - }> { - const cfg = this.config; - const clientSecret = await createOpenAIRealtimeClientSecret({ - authToken, - auditContext: "openai-realtime-bridge-session", - session: { - type: "realtime", - model, - audio: { - output: { voice: cfg.voice ?? "alloy" }, - }, - }, - }); - const url = `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`; - return { - url, - headers: resolveProviderRequestHeaders({ - provider: "openai", - baseUrl: url, - capability: "audio", - transport: "websocket", - defaultHeaders: { - Authorization: `Bearer ${clientSecret.value}`, - }, - }) ?? { - Authorization: `Bearer ${clientSecret.value}`, - }, - }; - } - private async attemptReconnect(reason: string): Promise { if (this.intentionallyClosed) { return; @@ -1370,10 +1292,9 @@ async function createOpenAIRealtimeBrowserSession( } const model = req.model ?? config.model ?? OPENAI_REALTIME_DEFAULT_MODEL; - const auth = await requireOpenAIRealtimeDefaultAuth({ + const authToken = await requireOpenAIRealtimePlatformApiKey({ configuredApiKey: config.apiKey, cfg: req.cfg, - model, }); const voice = normalizeOpenAIRealtimeVoice(req.voice) ?? config.voice ?? "alloy"; const session: Record = { @@ -1412,7 +1333,7 @@ async function createOpenAIRealtimeBrowserSession( } const clientSecret = await createOpenAIRealtimeClientSecret({ - authToken: auth.value, + authToken, auditContext: "openai-realtime-browser-session", session, }); @@ -1455,10 +1376,9 @@ export function buildOpenAIRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin if (config.azureEndpoint || config.azureDeployment) { return hasOpenAIRealtimeApiKeyInput(config.apiKey); } - return hasOpenAIRealtimeBrowserAuthInput({ + return hasOpenAIRealtimePlatformApiKeyInput({ configuredApiKey: config.apiKey, cfg, - model: config.model, }); }, createBridge: (req) => { diff --git a/src/plugin-sdk/provider-auth.test.ts b/src/plugin-sdk/provider-auth.test.ts index 5aaef0ea0cc..e825b78bca9 100644 --- a/src/plugin-sdk/provider-auth.test.ts +++ b/src/plugin-sdk/provider-auth.test.ts @@ -101,6 +101,89 @@ describe("provider auth profile helpers", () => { ]); }); + it("filters auth profile API-key resolution by credential type", async () => { + vi.resetModules(); + + const store: AuthProfileStore = { + version: 1, + profiles: { + "openai:oauth": { + type: "oauth", + provider: "openai", + access: "oauth-access", + refresh: "oauth-refresh", + expires: Date.now() + 60_000, + }, + "openai:key": { + type: "api_key", + provider: "openai", + key: "sk-profile", + }, + }, + }; + const resolveApiKeyForProfile = vi.fn( + async (params: { store: AuthProfileStore; profileId: string }) => { + const profile = params.store.profiles[params.profileId]; + if (profile?.type === "oauth") { + return { + apiKey: profile.access, + provider: profile.provider, + profileId: params.profileId, + profileType: profile.type, + }; + } + if (profile?.type === "api_key" && profile.key) { + return { + apiKey: profile.key, + provider: profile.provider, + profileId: params.profileId, + profileType: profile.type, + }; + } + return null; + }, + ); + + vi.doMock("../agents/agent-scope-config.js", () => ({ + resolveDefaultAgentDir: () => "/tmp/openclaw-agent", + })); + vi.doMock("../agents/auth-profiles/oauth.js", () => ({ + resolveApiKeyForProfile, + })); + vi.doMock("../agents/auth-profiles/order.js", () => ({ + resolveAuthProfileOrder: ({ + provider, + store: profileStore, + }: { + provider: string; + store: AuthProfileStore; + }) => + Object.entries(profileStore.profiles) + .filter(([, profile]) => profile.provider === provider) + .map(([profileId]) => profileId), + })); + vi.doMock("../agents/auth-profiles/store.js", () => ({ + ensureAuthProfileStore: vi.fn(() => store), + ensureAuthProfileStoreForLocalUpdate: vi.fn(() => store), + loadAuthProfileStoreForSecretsRuntime: vi.fn(() => store), + loadAuthProfileStoreWithoutExternalProfiles: vi.fn(() => ({ version: 1, profiles: {} })), + updateAuthProfileStoreWithLock: vi.fn(), + })); + + const { resolveProviderAuthProfileApiKey } = await import("./provider-auth.js"); + + await expect( + resolveProviderAuthProfileApiKey({ + provider: "openai", + profileTypes: ["api_key"], + }), + ).resolves.toBe("sk-profile"); + expect(resolveApiKeyForProfile).toHaveBeenCalledTimes(1); + expect(resolveApiKeyForProfile).toHaveBeenCalledWith( + expect.objectContaining({ profileId: "openai:key" }), + ); + }); + it("only discovers external CLI auth when provider resolution opts in", async () => { vi.resetModules(); diff --git a/src/plugin-sdk/provider-auth.ts b/src/plugin-sdk/provider-auth.ts index 2b37d99ef71..53835f3a67a 100644 --- a/src/plugin-sdk/provider-auth.ts +++ b/src/plugin-sdk/provider-auth.ts @@ -362,14 +362,16 @@ export function listUsableProviderAuthProfileIds(params: { cfg?: OpenClawConfig; /** Agent directory containing auth profiles. */ agentDir?: string; + /** Optional allowed profile credential types. */ + profileTypes?: readonly AuthProfileCredential["type"][]; /** Whether profile store reads may prompt for keychain-backed credentials. */ allowKeychainPrompt?: boolean; /** Whether external CLI auth profiles may be discovered and included. */ includeExternalCliAuth?: boolean; }): { agentDir: string; profileIds: string[] } { try { - const { agentDir, profileIds } = resolveUsableProviderAuthProfiles(params); - return { agentDir, profileIds }; + const { agentDir, profileIds, store } = resolveUsableProviderAuthProfiles(params); + return { agentDir, profileIds: filterAuthProfileIdsByType(store, profileIds, params) }; } catch { return { agentDir: "", profileIds: [] }; } @@ -385,6 +387,8 @@ export function isProviderAuthProfileConfigured(params: { cfg?: OpenClawConfig; /** Agent directory containing auth profiles. */ agentDir?: string; + /** Optional allowed profile credential types. */ + profileTypes?: readonly AuthProfileCredential["type"][]; /** Whether profile store reads may prompt for keychain-backed credentials. */ allowKeychainPrompt?: boolean; /** Whether external CLI auth profiles may be discovered and included. */ @@ -403,6 +407,8 @@ export async function resolveProviderAuthProfileApiKey(params: { cfg?: OpenClawConfig; /** Agent directory containing auth profiles. */ agentDir?: string; + /** Optional allowed profile credential types. */ + profileTypes?: readonly AuthProfileCredential["type"][]; /** Whether profile store reads may prompt for keychain-backed credentials. */ allowKeychainPrompt?: boolean; /** Whether external CLI auth profiles may be discovered and included. */ @@ -412,7 +418,7 @@ export async function resolveProviderAuthProfileApiKey(params: { if (!agentDir || profileIds.length === 0) { return undefined; } - for (const profileId of profileIds) { + for (const profileId of filterAuthProfileIdsByType(store, profileIds, params)) { const resolved = await resolveApiKeyForProfile({ cfg: params.cfg, store, @@ -466,3 +472,18 @@ function resolveUsableProviderAuthProfiles(params: { store: fallbackStore, }; } + +function filterAuthProfileIdsByType( + store: AuthProfileStore, + profileIds: readonly string[], + params: { profileTypes?: readonly AuthProfileCredential["type"][] }, +): string[] { + if (!params.profileTypes?.length) { + return [...profileIds]; + } + const allowedTypes = new Set(params.profileTypes); + return profileIds.filter((profileId) => { + const type = store.profiles[profileId]?.type; + return type !== undefined && allowedTypes.has(type); + }); +}