diff --git a/docs/channels/discord/voice-follow.md b/docs/channels/discord/voice-follow.md index f4fe45bee637..f857924e4c42 100644 --- a/docs/channels/discord/voice-follow.md +++ b/docs/channels/discord/voice-follow.md @@ -177,11 +177,11 @@ room disconnects; `voice.realtime.speakerVoice` remains the default for future calls. This requires an owner-authorized voice turn, agent delegation, and access to the `talk_voice` tool. -In-call changes also require a provider voice catalog and a transport that either -delegates directly to the agent or supports suppressed tool results, so pending -answers can move to the replacement voice without duplicate speech. Google Live -currently requires setting `voice.realtime.speakerVoice` and rejoining the voice -channel; `talk_voice` reports that in-call changes are unavailable on that connection. +In-call changes require a provider voice catalog. Google Live supports the same +`talk_voice` flow: OpenClaw opens a fresh connection with the selected voice and +keeps unfinished agent work with the Discord call. Answers waiting to be sent +follow the replacement. A response already submitted to the previous connection +may be interrupted by the switch and is not replayed, avoiding duplicate speech. While a delegated OpenClaw run is active, command-authorized Discord conversation transcripts are treated as live run control before starting another agent turn. Phrases such as "status", "cancel that", "use the smaller fix", or "when you're done also check tests" are classified as status, cancel, steering, or follow-up input for the active session. Status, cancel, accepted steering, and follow-up outcomes are spoken back into the voice channel so the caller knows whether OpenClaw handled the request. diff --git a/docs/providers/google.md b/docs/providers/google.md index fc014ff191cc..4e34824bbe28 100644 --- a/docs/providers/google.md +++ b/docs/providers/google.md @@ -364,6 +364,11 @@ provider. This is not the separate Cloud Text-to-Speech API path. The bundled `google` plugin registers a realtime voice provider backed by the Gemini Live API for backend audio bridges such as Voice Call and Google Meet. +Talk and Discord expose Google's prebuilt voices in their voice catalogs. During +an active Talk or Discord call, use `talk_voice` to select a new voice. OpenClaw +reconnects with that voice while preserving the conversation and unfinished agent +work; saved voice defaults stay unchanged. See [Discord voice changes](/channels/discord/voice-follow). + | Setting | Config path | Default | | --------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | | Model | `plugins.entries.voice-call.config.realtime.providers.google.model` | `gemini-3.1-flash-live-preview` | diff --git a/extensions/discord/src/voice/agent-control.ts b/extensions/discord/src/voice/agent-control.ts index 4b78aedd3213..ec8d4a813723 100644 --- a/extensions/discord/src/voice/agent-control.ts +++ b/extensions/discord/src/voice/agent-control.ts @@ -1,12 +1,18 @@ +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { controlRealtimeVoiceAgentRun, + parseRealtimeVoiceAgentControlToolArgs, shouldAutoControlRealtimeVoiceAgentText, type RealtimeVoiceAgentControlResult, + type RealtimeVoiceBridgeSession, } from "openclaw/plugin-sdk/realtime-voice"; +import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env"; import { getDiscordRuntime } from "../runtime.js"; import type { DiscordVoiceIngressContext } from "./ingress.js"; import type { VoiceSessionEntry } from "./session.js"; +const logger = createSubsystemLogger("discord/voice"); + type DiscordVoiceAgentControlOutcome = | { handled: true; result: RealtimeVoiceAgentControlResult; speakText?: string } | { handled: false; result?: RealtimeVoiceAgentControlResult }; @@ -22,7 +28,7 @@ type DiscordVoiceAgentControlParams = { isCurrent: () => boolean; }; -export async function controlDiscordVoiceAgentRun(params: DiscordVoiceAgentControlParams) { +async function controlDiscordVoiceAgentRun(params: DiscordVoiceAgentControlParams) { const context = params.resolveContext ? await params.resolveContext() : params.context; const assertCurrent = () => { if ( @@ -84,3 +90,42 @@ export async function maybeControlDiscordVoiceAgentRun( ...(result.speak && !result.suppress ? { speakText: result.message } : {}), }; } + +export function logDiscordVoiceAgentControlResult( + entry: VoiceSessionEntry, + result: RealtimeVoiceAgentControlResult, +): void { + logger.info( + `discord voice: realtime active-run control handled mode=${result.mode} ok=${result.ok} active=${result.active} reason=${result.reason ?? "none"} voiceSession=${entry.voiceSessionKey} supervisorSession=${entry.route.sessionKey} agent=${entry.route.agentId}`, + ); +} + +export async function handleDiscordVoiceAgentControlToolCall(params: { + args: unknown; + session: RealtimeVoiceBridgeSession; + callId: string; + getControlParams: () => Omit; + isCurrent: () => boolean; + entry: VoiceSessionEntry; +}): Promise { + let result: RealtimeVoiceAgentControlResult; + try { + const parsed = parseRealtimeVoiceAgentControlToolArgs(params.args); + result = await controlDiscordVoiceAgentRun({ + ...params.getControlParams(), + text: parsed.text, + mode: parsed.mode, + }); + } catch (error) { + if (!params.isCurrent()) { + return; + } + await params.session.submitToolResult(params.callId, { error: formatErrorMessage(error) }); + return; + } + if (!params.isCurrent()) { + return; + } + logDiscordVoiceAgentControlResult(params.entry, result); + await params.session.submitToolResult(params.callId, result); +} diff --git a/extensions/discord/src/voice/realtime-consults.ts b/extensions/discord/src/voice/realtime-consults.ts index 67a2aa5d270b..77953fd090d6 100644 --- a/extensions/discord/src/voice/realtime-consults.ts +++ b/extensions/discord/src/voice/realtime-consults.ts @@ -5,12 +5,10 @@ import { classifyRealtimeVoiceConsultToolCall, classifySkippableRealtimeVoiceConsultTranscript, createRealtimeVoiceAgentTalkbackQueue, - parseRealtimeVoiceAgentControlToolArgs, REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME, type RealtimeVoiceAgentConsultToolPolicy, type RealtimeVoiceAgentConsultRunner, - type RealtimeVoiceAgentControlResult, type RealtimeVoiceAgentTalkbackQueue, type RealtimeVoiceBridgeSession, type RealtimeVoiceForcedConsultHandle, @@ -19,7 +17,11 @@ import { type RealtimeVoiceWakeNamePolicy, } from "openclaw/plugin-sdk/realtime-voice"; import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env"; -import { controlDiscordVoiceAgentRun, maybeControlDiscordVoiceAgentRun } from "./agent-control.js"; +import { + handleDiscordVoiceAgentControlToolCall, + logDiscordVoiceAgentControlResult, + maybeControlDiscordVoiceAgentRun, +} from "./agent-control.js"; import type { DiscordVoiceIngressContext } from "./ingress.js"; import { formatVoiceLogPreview } from "./log-preview.js"; import { formatVoiceIngressPrompt } from "./prompt.js"; @@ -45,12 +47,20 @@ type AgentProxyConsultResult = | { text: string } | ReturnType; -type AgentProxyProviderDelivery = ReturnType>; +type AgentProxyProviderDelivery = ReturnType> & { + submissionStarted: boolean; +}; export type AgentProxyConsultState = { speaker: DiscordRealtimeSpeakerContext; providerEpoch: number; - delivery: "provider" | "forced-pending" | "playback" | "detached" | "retired"; + delivery: + | "provider-pending" + | "provider" + | "forced-pending" + | "playback" + | "detached" + | "retired"; promise?: Promise; result?: AgentProxyConsultResult; }; @@ -153,7 +163,14 @@ export class DiscordRealtimeConsults { return; } if (event.name === REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME) { - await this.handleAgentControlToolCall(event, session, callId, providerEpoch); + await handleDiscordVoiceAgentControlToolCall({ + args: event.args, + session, + callId, + getControlParams: () => this.controlParams(providerEpoch), + isCurrent: () => !this.params.stopped() && providerEpoch === this.params.providerEpoch(), + entry: this.params.entry, + }); return; } if (event.name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { @@ -230,28 +247,39 @@ export class DiscordRealtimeConsults { }); } } - if (!context) { + const state = recent?.context; + if (!context || !state) { logger.warn( `discord voice: realtime consult has no speaker context call=${callId || "unknown"}`, ); await session.submitToolResult(callId, { error: "No Discord speaker context available" }); return; } - const result = await this.trackAgentProxyConsult( - recent, - this.runAgentTurn({ context, message: consultMessage }), - ); - if (providerEpoch !== this.params.providerEpoch()) { - return; - } - if ("text" in result) { - logger.info( - `discord voice: realtime consult answer (${result.text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(result.text)}`, + state.delivery = "provider-pending"; + const delivery = this.beginProviderDelivery(state); + try { + const result = await this.trackAgentProxyConsult( + recent, + this.runAgentTurn({ context, message: consultMessage, deliveryOwner: "consult" }), ); - } else if ("error" in result) { - logger.warn(`discord voice: realtime consult failed call=${callId}: ${result.error}`); + if (this.params.stopped() || providerEpoch !== this.params.providerEpoch()) { + return; + } + if ("text" in result) { + logger.info( + `discord voice: realtime consult answer (${result.text.length} chars) voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId} speaker=${context.speakerLabel} owner=${context.senderIsOwner}: ${formatVoiceLogPreview(result.text)}`, + ); + } else if ("error" in result) { + logger.warn(`discord voice: realtime consult failed call=${callId}: ${result.error}`); + } + delivery.submissionStarted = true; + await this.submitAgentProxyConsultResult(callId, session, result); + if (state.delivery === "provider-pending") { + state.delivery = "provider"; + } + } finally { + this.settleProviderDelivery(state, delivery); } - await this.submitAgentProxyConsultResult(callId, session, result); } async handleAcceptedTranscript( @@ -299,7 +327,7 @@ export class DiscordRealtimeConsults { if (pendingForcedConsult) { this.params.harness.forcedConsults.remove(pendingForcedConsult); } - this.logAgentControlResult(control.result); + logDiscordVoiceAgentControlResult(this.params.entry, control.result); if (control.speakText) { this.params.playback.speakControlResult(control.speakText); } @@ -344,34 +372,6 @@ export class DiscordRealtimeConsults { }); } - private async handleAgentControlToolCall( - event: RealtimeVoiceToolCallEvent, - session: RealtimeVoiceBridgeSession, - callId: string, - providerEpoch: number, - ): Promise { - let result: RealtimeVoiceAgentControlResult; - try { - const parsed = parseRealtimeVoiceAgentControlToolArgs(event.args); - result = await controlDiscordVoiceAgentRun({ - ...this.controlParams(providerEpoch), - text: parsed.text, - mode: parsed.mode, - }); - } catch (error) { - if (this.params.stopped() || providerEpoch !== this.params.providerEpoch()) { - return; - } - await session.submitToolResult(callId, { error: formatErrorMessage(error) }); - return; - } - if (this.params.stopped() || providerEpoch !== this.params.providerEpoch()) { - return; - } - this.logAgentControlResult(result); - await session.submitToolResult(callId, result); - } - private controlParams(providerEpoch: number) { const context = this.params.turns.speakerContext(); if (!context) { @@ -409,12 +409,6 @@ export class DiscordRealtimeConsults { }); } - private logAgentControlResult(result: RealtimeVoiceAgentControlResult): void { - logger.info( - `discord voice: realtime active-run control handled mode=${result.mode} ok=${result.ok} active=${result.active} reason=${result.reason ?? "none"} voiceSession=${this.params.entry.voiceSessionKey} supervisorSession=${this.params.entry.route.sessionKey} agent=${this.params.entry.route.agentId}`, - ); - } - private prepareForcedAgentProxyConsult( transcript: string, speakerContext?: DiscordRealtimeSpeakerContext, @@ -551,7 +545,7 @@ export class DiscordRealtimeConsults { state.result = result; if ( this.detachedProviderEpoch === state.providerEpoch && - state.delivery === "forced-pending" + (state.delivery === "forced-pending" || state.delivery === "provider-pending") ) { state.delivery = "detached"; } @@ -610,31 +604,28 @@ export class DiscordRealtimeConsults { return false; } const providerDelivery = - state.delivery === "forced-pending" && - !state.result && - session.bridge.supportsToolResultSuppression === false - ? createDeferred() + state.delivery === "provider-pending" || + (state.delivery === "forced-pending" && + !state.result && + session.bridge.supportsToolResultSuppression === false) + ? this.beginProviderDelivery(state) : undefined; - if (providerDelivery) { - // Keep every native attempt until settlement: one failed joiner cannot hand - // playback back while another may still accept the answer. - const deliveries = - this.providerDeliveries.get(state) ?? new Set(); - deliveries.add(providerDelivery); - this.providerDeliveries.set(state, deliveries); - } logger.info( `discord voice: realtime consult ${state.result ? "reused recent" : "joined in-flight"} agent result call=${callId} speaker=${state.speaker.speakerLabel} owner=${state.speaker.senderIsOwner}`, ); try { const result = await pendingResult; if ( + this.params.stopped() || state.providerEpoch !== this.params.providerEpoch() || state.delivery === "detached" || state.delivery === "retired" ) { return true; } + if (providerDelivery) { + providerDelivery.submissionStarted = true; + } await this.submitAgentProxyConsultResult( callId, session, @@ -645,23 +636,38 @@ export class DiscordRealtimeConsults { providerDelivery && !("status" in result) && state.providerEpoch === this.params.providerEpoch() && - state.delivery === "forced-pending" + (state.delivery === "forced-pending" || state.delivery === "provider-pending") ) { state.delivery = "provider"; } } finally { if (providerDelivery) { - providerDelivery.resolve(); - const deliveries = this.providerDeliveries.get(state); - deliveries?.delete(providerDelivery); - if (deliveries?.size === 0) { - this.providerDeliveries.delete(state); - } + this.settleProviderDelivery(state, providerDelivery); } } return true; } + private beginProviderDelivery(state: AgentProxyConsultState): AgentProxyProviderDelivery { + const delivery = { ...createDeferred(), submissionStarted: false }; + const deliveries = this.providerDeliveries.get(state) ?? new Set(); + deliveries.add(delivery); + this.providerDeliveries.set(state, deliveries); + return delivery; + } + + private settleProviderDelivery( + state: AgentProxyConsultState, + delivery: AgentProxyProviderDelivery, + ): void { + delivery.resolve(); + const deliveries = this.providerDeliveries.get(state); + deliveries?.delete(delivery); + if (deliveries?.size === 0) { + this.providerDeliveries.delete(state); + } + } + private async submitAgentProxyConsultResult( callId: string, session: RealtimeVoiceBridgeSession, @@ -701,7 +707,16 @@ export class DiscordRealtimeConsults { ...this.providerDeliveries.keys(), ]); for (const state of states) { - if (state.delivery === "forced-pending") { + if (state.delivery === "forced-pending" || state.delivery === "provider-pending") { + const submissions = this.providerDeliveries.get(state); + if ( + preservePendingSpeech && + Array.from(submissions ?? []).some((delivery) => delivery.submissionStarted) + ) { + // A submission may already have reached the old provider; replay could duplicate speech. + state.delivery = "provider"; + continue; + } state.delivery = preservePendingSpeech && state.providerEpoch === this.params.providerEpoch() ? "detached" diff --git a/extensions/discord/src/voice/realtime-speaker-session.ts b/extensions/discord/src/voice/realtime-speaker-session.ts index edb2ec155a41..b0a281dbd343 100644 --- a/extensions/discord/src/voice/realtime-speaker-session.ts +++ b/extensions/discord/src/voice/realtime-speaker-session.ts @@ -540,9 +540,6 @@ export class DiscordRealtimeSpeakerSession implements VoiceRealtimeSession { return { ...this.selection, voices: [...this.selection.voices], - canChange: - this.selection.canChange && - (this.handlesAgentConsult || this.bridge?.bridge.supportsToolResultSuppression !== false), }; } diff --git a/extensions/discord/src/voice/realtime-voice-selection.test.ts b/extensions/discord/src/voice/realtime-voice-selection.test.ts index ebfb24eef34d..13272df94a6c 100644 --- a/extensions/discord/src/voice/realtime-voice-selection.test.ts +++ b/extensions/discord/src/voice/realtime-voice-selection.test.ts @@ -35,6 +35,26 @@ defineDiscordVoiceTests( return session; }); }; + const useGoogleVoices = (voices = ["Puck", "Kore"]) => { + resolveConfiguredRealtimeVoiceProviderMock.mockImplementation((params) => ({ + provider: { id: "google" }, + capabilities: { + supportsActivationNameGating: false, + ...(voices.length > 0 ? { voices } : {}), + }, + providerConfig: { + model: "gemini-3.1-flash-live-preview", + voice: "Puck", + ...params?.providerConfigOverrides, + }, + })); + createRealtimeVoiceBridgeSessionMock.mockImplementation(() => { + const session = createRealtimeSessionMock(); + session.bridge.supportsToolResultSuppression = false; + session.bridge.supportsToolResultContinuation = false; + return session; + }); + }; const selectionOwner = () => registerRealtimeVoiceSelectionMock.mock.calls.at(-1)![0]; const selectionHandle = () => { const result = registerRealtimeVoiceSelectionMock.mock.results.at(-1); @@ -44,44 +64,188 @@ defineDiscordVoiceTests( return result.value; }; - it.each([false, true])( - "rejects voice replacement for an unsuppressed function-response bridge (catalog=%s)", - async (advertisesVoices) => { - resolveConfiguredRealtimeVoiceProviderMock.mockImplementation((params) => ({ - provider: { id: "google" }, - capabilities: { - supportsActivationNameGating: false, - ...(advertisesVoices ? { voices: ["Puck", "Kore"] } : {}), - }, - providerConfig: { - model: "gemini-3.1-flash-live-preview", - voice: "Puck", - ...params?.providerConfigOverrides, - }, - })); - createRealtimeVoiceBridgeSessionMock.mockImplementation(() => { - const session = createRealtimeSessionMock(); - session.bridge.supportsToolResultSuppression = false; - return session; + it("rejects voice replacement when the provider has no voice catalog", async () => { + useGoogleVoices([]); + const { entry, manager } = await createJoinedAgentProxyFixture(); + try { + beginSpeakerTurn(entry).close(); + const original = lastRealtimeBridge(); + const connections = createRealtimeVoiceBridgeSessionMock.mock.calls.length; + expect(selectionOwner().read()).toMatchObject({ + provider: "google", + voice: "Puck", + canChange: false, }); - const { entry, manager } = await createJoinedAgentProxyFixture(); + await expect( + selectionOwner().changeVoice("Kore", { assertCurrent: () => {} }), + ).rejects.toThrow("cannot change voices"); + expect(original.session.close).not.toHaveBeenCalled(); + expect(createRealtimeVoiceBridgeSessionMock).toHaveBeenCalledTimes(connections); + expect(selectionOwner().read()).toMatchObject({ voice: "Puck" }); + expect(manager.status()).toHaveLength(1); + } finally { + await manager.destroy(); + } + }); + + it.each([false, true])( + "switches Gemini from its own function call and speaks the late answer once (transcript first=%s)", + async (transcriptFirst) => { + useGoogleVoices(); + const answer = createDeferred<{ payloads: Array<{ text: string }> }>(); + const { entry, manager } = await createJoinedAgentProxyFixture({ + config: { voice: { realtime: { requireWakeName: false, consultPolicy: "always" } } }, + }); + let submission: Promise | undefined; try { beginSpeakerTurn(entry).close(); const original = lastRealtimeBridge(); - const connections = createRealtimeVoiceBridgeSessionMock.mock.calls.length; - expect.soft(selectionOwner().read()).toMatchObject({ - provider: "google", - voice: "Puck", - canChange: false, + agentCommandMock.mockImplementationOnce(async () => { + await selectionOwner().changeVoice("Kore", { assertCurrent: () => {} }); + return await answer.promise; }); - await expect - .soft(selectionOwner().changeVoice("Kore", { assertCurrent: () => {} })) - .rejects.toThrow("cannot change voices"); - expect(original.session.close).not.toHaveBeenCalled(); - expect(createRealtimeVoiceBridgeSessionMock).toHaveBeenCalledTimes(connections); - expect(selectionOwner().read()).toMatchObject({ voice: "Puck" }); - expect(manager.status()).toHaveLength(1); + if (transcriptFirst) { + vi.useFakeTimers(); + original.bridgeParams.onTranscript?.( + "user", + "Switch to Kore and check the agenda.", + true, + ); + await vi.advanceTimersByTimeAsync(0); + expect(agentCommandMock).not.toHaveBeenCalled(); + } + submission = Promise.resolve( + original.bridgeParams.onToolCall?.( + { + itemId: "gemini-switch-item", + callId: "gemini-switch-call", + name: "openclaw_agent_consult", + args: { question: "Switch to Kore and check the agenda." }, + }, + original.session, + ), + ); + await vi.waitFor(() => expect(selectionOwner().read()).toMatchObject({ voice: "Kore" })); + const replacement = lastRealtimeBridge(); + expect(replacement.session).not.toBe(original.session); + expect(original.session.close).toHaveBeenCalledExactlyOnceWith({ disposition: "detach" }); + expect(sentUserMessages(replacement.session)).toHaveLength(0); + const signal = lastAgentCommandArgs().abortSignal; + expect(signal).toBeInstanceOf(AbortSignal); + if (signal instanceof AbortSignal) { + expect(signal.aborted).toBe(false); + } + answer.resolve({ payloads: [{ text: "The agenda starts with the budget review." }] }); + await submission; + expect(sentUserMessages(replacement.session)).toEqual([ + expect.stringContaining("The agenda starts with the budget review."), + ]); + expect(sentUserMessages(original.session)).toHaveLength(0); + expect(original.session.submitToolResult).not.toHaveBeenCalled(); + expect(replacement.session.submitToolResult).not.toHaveBeenCalled(); + expect(agentCommandMock).toHaveBeenCalledOnce(); } finally { + answer.resolve({ payloads: [] }); + await submission; + vi.useRealTimers(); + await manager.destroy(); + } + }, + ); + + it("hands a pending Gemini forced consult joined by a function call to the new voice", async () => { + useGoogleVoices(); + const answer = createDeferred<{ payloads: Array<{ text: string }> }>(); + agentCommandMock.mockReturnValueOnce(answer.promise); + const { entry, manager } = await createJoinedAgentProxyFixture({ + config: { voice: { realtime: { requireWakeName: false } } }, + }); + let submission: Promise | undefined; + try { + beginSpeakerTurn(entry).close(); + const original = lastRealtimeBridge(); + await emitFinalRealtimeUserTranscript(original.bridgeParams, "Finish the agenda task."); + expect(agentCommandMock).toHaveBeenCalledOnce(); + submission = Promise.resolve( + original.bridgeParams.onToolCall?.( + { + itemId: "gemini-join-item", + callId: "gemini-join-call", + name: "openclaw_agent_consult", + args: { question: "Finish the agenda task." }, + }, + original.session, + ), + ); + await selectionOwner().changeVoice("Kore", { assertCurrent: () => {} }); + const replacement = lastRealtimeBridge(); + answer.resolve({ payloads: [{ text: "The agenda task is finished." }] }); + await submission; + expect(sentUserMessages(replacement.session)).toEqual([ + expect.stringContaining("The agenda task is finished."), + ]); + expect(sentUserMessages(original.session)).toHaveLength(0); + expect(original.session.submitToolResult).not.toHaveBeenCalled(); + expect(agentCommandMock).toHaveBeenCalledOnce(); + } finally { + answer.resolve({ payloads: [] }); + await submission; + await manager.destroy(); + } + }); + + it.each(["accepted", "pending"] as const)( + "does not replay a Gemini answer after provider submission starts (%s)", + async (submissionState) => { + useGoogleVoices(); + const answer = createDeferred<{ payloads: Array<{ text: string }> }>(); + const providerSubmission = createDeferred(); + const submissionStarted = createDeferred(); + agentCommandMock.mockReturnValueOnce(answer.promise); + const { entry, manager } = await createJoinedAgentProxyFixture({ + config: { voice: { realtime: { requireWakeName: false } } }, + }); + let submission: Promise | undefined; + try { + beginSpeakerTurn(entry).close(); + const original = lastRealtimeBridge(); + await emitFinalRealtimeUserTranscript(original.bridgeParams, "Read the agenda result."); + original.session.submitToolResult.mockImplementationOnce(() => { + submissionStarted.resolve(); + return submissionState === "pending" ? providerSubmission.promise : undefined; + }); + submission = Promise.resolve( + original.bridgeParams.onToolCall?.( + { + itemId: "gemini-submitted-item", + callId: "gemini-submitted-call", + name: "openclaw_agent_consult", + args: { question: "Read the agenda result." }, + }, + original.session, + ), + ); + answer.resolve({ payloads: [{ text: "The agenda is ready." }] }); + await submissionStarted.promise; + if (submissionState === "accepted") { + await submission; + } + await selectionOwner().changeVoice("Kore", { assertCurrent: () => {} }); + const replacement = lastRealtimeBridge(); + expect(sentUserMessages(replacement.session)).toHaveLength(0); + providerSubmission.resolve(); + await submission; + expect(original.session.submitToolResult.mock.calls).toEqual([ + ["gemini-submitted-call", { text: "The agenda is ready." }], + ]); + expect(sentUserMessages(original.session)).toHaveLength(0); + expect(sentUserMessages(replacement.session)).toHaveLength(0); + expect(replacement.session.submitToolResult).not.toHaveBeenCalled(); + expect(agentCommandMock).toHaveBeenCalledOnce(); + } finally { + answer.resolve({ payloads: [] }); + providerSubmission.resolve(); + await submission; await manager.destroy(); } }, diff --git a/extensions/google/realtime-voice-lazy.test.ts b/extensions/google/realtime-voice-lazy.test.ts index 4760dcf06d95..98916e551be3 100644 --- a/extensions/google/realtime-voice-lazy.test.ts +++ b/extensions/google/realtime-voice-lazy.test.ts @@ -14,6 +14,13 @@ vi.mock("./realtime-voice-provider.js", () => ({ })); beforeEach(() => createRealtimeBridgeMock.mockReset()); +it("lists the default model and prebuilt voices before connecting", () => { + const provider = createLazyGoogleRealtimeVoiceProvider(); + expect(provider.defaultModel).toBe("gemini-3.1-flash-live-preview"); + expect(provider.voices).toEqual(expect.arrayContaining(["Kore", "Puck"])); + expect(createRealtimeBridgeMock).not.toHaveBeenCalled(); +}); + it("fences all nonterminal callbacks from a closed Google provider generation", async () => { const first = createMockRealtimeBridge(); const replacement = createMockRealtimeBridge(); diff --git a/extensions/google/realtime-voice-lazy.ts b/extensions/google/realtime-voice-lazy.ts index 07702ba592be..d3e05074b7aa 100644 --- a/extensions/google/realtime-voice-lazy.ts +++ b/extensions/google/realtime-voice-lazy.ts @@ -11,6 +11,7 @@ import { asOptionalRecord, normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { GOOGLE_REALTIME_VOICE_METADATA } from "./realtime-voice-metadata.js"; const loadGoogleRealtimeVoiceProvider = createLazyRuntimeSurface( () => import("./realtime-voice-provider.js"), @@ -417,9 +418,7 @@ function createLazyGoogleRealtimeVoiceBridge( export function createLazyGoogleRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin { return { - id: "google", - label: "Google Live Voice", - autoSelectOrder: 20, + ...GOOGLE_REALTIME_VOICE_METADATA, resolveConfig: ({ cfg, rawConfig }) => resolveGoogleRealtimeProviderConfig(rawConfig, cfg), isConfigured: ({ cfg, providerConfig }) => Boolean( diff --git a/extensions/google/realtime-voice-metadata.ts b/extensions/google/realtime-voice-metadata.ts new file mode 100644 index 000000000000..c7bb31e42675 --- /dev/null +++ b/extensions/google/realtime-voice-metadata.ts @@ -0,0 +1,15 @@ +import type { RealtimeVoiceProviderPlugin } from "openclaw/plugin-sdk/realtime-voice"; +import { GOOGLE_PREBUILT_VOICES } from "./voice-catalog.js"; + +export const GOOGLE_REALTIME_DEFAULT_MODEL = "gemini-3.1-flash-live-preview"; + +export const GOOGLE_REALTIME_VOICE_METADATA = { + id: "google", + label: "Google Live Voice", + defaultModel: GOOGLE_REALTIME_DEFAULT_MODEL, + voices: GOOGLE_PREBUILT_VOICES, + autoSelectOrder: 20, +} satisfies Pick< + RealtimeVoiceProviderPlugin, + "id" | "label" | "defaultModel" | "voices" | "autoSelectOrder" +>; diff --git a/extensions/google/realtime-voice-provider.ts b/extensions/google/realtime-voice-provider.ts index fe2cadd359be..9fac51c30bcb 100644 --- a/extensions/google/realtime-voice-provider.ts +++ b/extensions/google/realtime-voice-provider.ts @@ -1,4 +1,3 @@ -// Google provider module implements model/runtime integration. import { randomUUID } from "node:crypto"; import { ActivityHandling, @@ -57,9 +56,12 @@ import { } from "openclaw/plugin-sdk/string-coerce-runtime"; import { canonicalizeGoogleProviderBase64 } from "./base64.js"; import { createGoogleGenAI } from "./google-genai-runtime.js"; +import { + GOOGLE_REALTIME_DEFAULT_MODEL, + GOOGLE_REALTIME_VOICE_METADATA, +} from "./realtime-voice-metadata.js"; import { resolveGoogleGemini3ThinkingLevel } from "./thinking-api.js"; -const GOOGLE_REALTIME_DEFAULT_MODEL = "gemini-3.1-flash-live-preview"; const GOOGLE_REALTIME_DEFAULT_VOICE = "Kore"; const GOOGLE_REALTIME_DEFAULT_API_VERSION = "v1beta"; const GOOGLE_REALTIME_INPUT_SAMPLE_RATE = 16_000; @@ -1390,10 +1392,7 @@ async function createGoogleRealtimeBrowserSession( export function buildGoogleRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin { return { - id: "google", - label: "Google Live Voice", - defaultModel: GOOGLE_REALTIME_DEFAULT_MODEL, - autoSelectOrder: 20, + ...GOOGLE_REALTIME_VOICE_METADATA, capabilities: { transports: ["provider-websocket", "gateway-relay"], inputAudioFormats: [ diff --git a/extensions/google/speech-provider.ts b/extensions/google/speech-provider.ts index 0ea0b6743a96..bb03d247393c 100644 --- a/extensions/google/speech-provider.ts +++ b/extensions/google/speech-provider.ts @@ -1,4 +1,3 @@ -// Google provider module implements model/runtime integration. import type { sanitizeConfiguredModelProviderRequest } from "openclaw/plugin-sdk/provider-http"; import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard"; import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input"; @@ -15,6 +14,7 @@ import { normalizeOptionalString, normalizeOptionalString as trimToUndefined, } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { GOOGLE_PREBUILT_VOICES } from "./voice-catalog.js"; const DEFAULT_GOOGLE_TTS_MODEL = "gemini-3.1-flash-tts-preview"; const DEFAULT_GOOGLE_TTS_VOICE = "Kore"; @@ -29,39 +29,6 @@ const GOOGLE_TTS_MODELS = [ "gemini-2.5-pro-preview-tts", ] as const; -const GOOGLE_TTS_VOICES = [ - "Zephyr", - "Puck", - "Charon", - "Kore", - "Fenrir", - "Leda", - "Orus", - "Aoede", - "Callirrhoe", - "Autonoe", - "Enceladus", - "Iapetus", - "Umbriel", - "Algieba", - "Despina", - "Erinome", - "Algenib", - "Rasalgethi", - "Laomedeia", - "Achernar", - "Alnilam", - "Schedar", - "Gacrux", - "Pulcherrima", - "Achird", - "Zubenelgenubi", - "Vindemiatrix", - "Sadachbia", - "Sadaltager", - "Sulafat", -] as const; - type GoogleTtsProviderConfig = { apiKey?: string; baseUrl?: string; @@ -532,7 +499,7 @@ export function buildGoogleSpeechProvider(): SpeechProviderPlugin { autoSelectOrder: 50, defaultModel: DEFAULT_GOOGLE_TTS_MODEL, models: GOOGLE_TTS_MODELS, - voices: GOOGLE_TTS_VOICES, + voices: GOOGLE_PREBUILT_VOICES, resolveConfig: ({ rawConfig }) => normalizeGoogleTtsProviderConfig(rawConfig), parseDirectiveToken, resolveTalkConfig: ({ baseTtsConfig, talkProviderConfig }) => { @@ -566,7 +533,7 @@ export function buildGoogleSpeechProvider(): SpeechProviderPlugin { ? {} : { model: normalizeGoogleTtsModel(params.modelId) }), }), - listVoices: async () => GOOGLE_TTS_VOICES.map((voice) => ({ id: voice, name: voice })), + listVoices: async () => GOOGLE_PREBUILT_VOICES.map((voice) => ({ id: voice, name: voice })), isConfigured: ({ cfg, providerConfig }) => Boolean(resolveGoogleTtsApiKey({ cfg, providerConfig })), prepareSynthesis: (ctx) => { diff --git a/extensions/google/voice-catalog.ts b/extensions/google/voice-catalog.ts new file mode 100644 index 000000000000..f5184106d89b --- /dev/null +++ b/extensions/google/voice-catalog.ts @@ -0,0 +1,32 @@ +export const GOOGLE_PREBUILT_VOICES = [ + "Zephyr", + "Puck", + "Charon", + "Kore", + "Fenrir", + "Leda", + "Orus", + "Aoede", + "Callirrhoe", + "Autonoe", + "Enceladus", + "Iapetus", + "Umbriel", + "Algieba", + "Despina", + "Erinome", + "Algenib", + "Rasalgethi", + "Laomedeia", + "Achernar", + "Alnilam", + "Schedar", + "Gacrux", + "Pulcherrima", + "Achird", + "Zubenelgenubi", + "Vindemiatrix", + "Sadachbia", + "Sadaltager", + "Sulafat", +] as const;