mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
feat(providers): add Qwen 3.6 Flash and Gemini 3.1 Live (#102791)
* feat(providers): add current Qwen and Gemini Live models * fix(google): align live thinking enum type * fix(google): align live thinking enum type
This commit is contained in:
Peter Steinberger
GitHub
parent
c8c1f37d2f
commit
07a7d596ff
17 changed files with 387 additions and 78 deletions
|
|
@ -736,7 +736,7 @@ An `openclaw doctor --fix` migration exists for the legacy `realtime.provider: "
|
|||
provider: "openai",
|
||||
transcriptionProvider: "openai",
|
||||
voiceProvider: "google",
|
||||
model: "gemini-2.5-flash-native-audio-preview-12-2025",
|
||||
model: "gemini-3.1-flash-live-preview",
|
||||
agentId: "jay",
|
||||
toolPolicy: "owner",
|
||||
introMessage: "Say exactly: I'm here.",
|
||||
|
|
|
|||
|
|
@ -328,7 +328,7 @@ for tool work, current information, memory lookups, or workspace state.
|
|||
<Tabs>
|
||||
<Tab title="Google Gemini Live">
|
||||
Defaults: API key from `realtime.providers.google.apiKey`, `GEMINI_API_KEY`,
|
||||
or `GOOGLE_API_KEY`; model `gemini-2.5-flash-native-audio-preview-12-2025`;
|
||||
or `GOOGLE_API_KEY`; model `gemini-3.1-flash-live-preview`;
|
||||
voice `Kore`. `sessionResumption` and `contextWindowCompression` default on
|
||||
for longer, reconnectable calls. Use `silenceDurationMs`,
|
||||
`startSensitivity`, and `endSensitivity` to tune faster turn-taking on
|
||||
|
|
@ -355,7 +355,7 @@ for tool work, current information, memory lookups, or workspace state.
|
|||
providers: {
|
||||
google: {
|
||||
apiKey: "${GEMINI_API_KEY}",
|
||||
model: "gemini-2.5-flash-native-audio-preview-12-2025",
|
||||
model: "gemini-3.1-flash-live-preview",
|
||||
speakerVoice: "Kore",
|
||||
silenceDurationMs: 500,
|
||||
startSensitivity: "high",
|
||||
|
|
|
|||
|
|
@ -342,14 +342,14 @@ Gemini Live API for backend audio bridges such as Voice Call and Google Meet.
|
|||
|
||||
| Setting | Config path | Default |
|
||||
| --------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------- |
|
||||
| Model | `plugins.entries.voice-call.config.realtime.providers.google.model` | `gemini-2.5-flash-native-audio-preview-12-2025` |
|
||||
| Model | `plugins.entries.voice-call.config.realtime.providers.google.model` | `gemini-3.1-flash-live-preview` |
|
||||
| Voice | `...google.voice` | `Kore` |
|
||||
| Temperature | `...google.temperature` | (unset) |
|
||||
| VAD start sensitivity | `...google.startSensitivity` | (unset) |
|
||||
| VAD end sensitivity | `...google.endSensitivity` | (unset) |
|
||||
| Silence duration | `...google.silenceDurationMs` | (unset) |
|
||||
| Activity handling | `...google.activityHandling` | Google default, `start-of-activity-interrupts` |
|
||||
| Turn coverage | `...google.turnCoverage` | Google default, `only-activity` |
|
||||
| Turn coverage | `...google.turnCoverage` | Google default, `audio-activity-and-all-video` |
|
||||
| Disable auto VAD | `...google.automaticActivityDetectionDisabled` | `false` |
|
||||
| Session resumption | `...google.sessionResumption` | `true` |
|
||||
| Context compression | `...google.contextWindowCompression` | `true` |
|
||||
|
|
@ -369,10 +369,10 @@ Example Voice Call realtime config:
|
|||
provider: "google",
|
||||
providers: {
|
||||
google: {
|
||||
model: "gemini-2.5-flash-native-audio-preview-12-2025",
|
||||
model: "gemini-3.1-flash-live-preview",
|
||||
speakerVoice: "Kore",
|
||||
activityHandling: "start-of-activity-interrupts",
|
||||
turnCoverage: "only-activity",
|
||||
turnCoverage: "audio-activity-and-all-video",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -393,6 +393,15 @@ Gemini API transcription is enabled without `languageCodes`; the current Google
|
|||
SDK rejects language-code hints on this API path.
|
||||
</Note>
|
||||
|
||||
<Note>
|
||||
Gemini 3.1 Live accepts conversational text through realtime input and uses
|
||||
sequential function calling. OpenClaw omits the older `NON_BLOCKING`, function
|
||||
response scheduling, and affective-dialog fields for this model. Prefer
|
||||
`thinkingLevel`; configured positive `thinkingBudget` values are mapped to the
|
||||
nearest supported level, while `-1` leaves Google's default in place. See the
|
||||
[Gemini Live capability comparison](https://ai.google.dev/gemini-api/docs/live-api/capabilities).
|
||||
</Note>
|
||||
|
||||
<Note>
|
||||
Control UI Talk supports Google Live browser sessions with constrained one-use
|
||||
tokens. Backend-only realtime voice providers can also run through the generic
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ Qwen Cloud is an official external OpenClaw provider plugin with canonical id `q
|
|||
| API style | OpenAI-compatible |
|
||||
|
||||
<Tip>
|
||||
`qwen3.7-plus` and `qwen3.6-plus` work with Coding Plan and Standard endpoints. For `qwen3.7-max`, use a **Standard (pay-as-you-go)** endpoint.
|
||||
`qwen3.7-plus` and `qwen3.6-plus` work with Coding Plan and Standard endpoints.
|
||||
For `qwen3.7-max` or `qwen3.6-flash`, use a **Standard (pay-as-you-go)** endpoint.
|
||||
</Tip>
|
||||
|
||||
## Install plugin
|
||||
|
|
@ -84,7 +85,7 @@ Choose your plan type and follow the setup steps.
|
|||
</Tab>
|
||||
|
||||
<Tab title="Standard (pay-as-you-go)">
|
||||
**Best for:** pay-as-you-go access through the Standard Model Studio endpoint, including `qwen3.7-max`, which is not available on the Coding Plan.
|
||||
**Best for:** pay-as-you-go access through the Standard Model Studio endpoint, including `qwen3.7-max` and `qwen3.6-flash`, which are not available on the Coding Plan.
|
||||
|
||||
<Steps>
|
||||
<Step title="Get your API key">
|
||||
|
|
@ -198,6 +199,7 @@ Plan configs omit models that only work on the Standard endpoint.
|
|||
| Model ref | Input | Context | Notes |
|
||||
| --------------------------- | ----------- | --------- | ----------------------- |
|
||||
| `qwen/qwen3.5-plus` | text, image | 1,000,000 | Default model |
|
||||
| `qwen/qwen3.6-flash` | text, image | 1,000,000 | Standard endpoints only |
|
||||
| `qwen/qwen3.6-plus` | text, image | 1,000,000 | Coding Plan + Standard |
|
||||
| `qwen/qwen3.7-max` | text | 1,000,000 | Standard endpoints only |
|
||||
| `qwen/qwen3.7-plus` | text, image | 1,000,000 | Coding Plan + Standard |
|
||||
|
|
@ -217,7 +219,8 @@ present in the static catalog.
|
|||
|
||||
## Thinking controls
|
||||
|
||||
`qwen3.7-max`, `qwen3.7-plus`, `qwen3.6-plus`, and `qwen/MiniMax-M2.5` are
|
||||
`qwen3.7-max`, `qwen3.7-plus`, `qwen3.6-flash`, `qwen3.6-plus`, and
|
||||
`qwen/MiniMax-M2.5` are
|
||||
reasoning-enabled in the built-in catalog. For reasoning models on the `qwen`
|
||||
family, the provider maps OpenClaw thinking levels to DashScope's top-level
|
||||
`enable_thinking` request flag: disabled thinking sends `enable_thinking: false`,
|
||||
|
|
@ -263,15 +266,15 @@ See [Video generation](/tools/video-generation) for shared tool parameters, prov
|
|||
## Advanced configuration
|
||||
|
||||
<AccordionGroup>
|
||||
<Accordion title="Qwen 3.7 availability">
|
||||
`qwen3.7-plus` is available on Coding Plan and Standard endpoints. `qwen3.7-max` is Standard-only. The Standard (pay-as-you-go) endpoints are:
|
||||
<Accordion title="Qwen 3.6 and 3.7 availability">
|
||||
`qwen3.7-plus` and `qwen3.6-plus` are available on Coding Plan and Standard endpoints. `qwen3.7-max` and `qwen3.6-flash` are Standard-only. The Standard (pay-as-you-go) endpoints are:
|
||||
|
||||
- China: `dashscope.aliyuncs.com/compatible-mode/v1`
|
||||
- Global: `dashscope-intl.aliyuncs.com/compatible-mode/v1`
|
||||
|
||||
OpenClaw omits `qwen3.7-max` from Coding Plan catalogs. If a Coding Plan
|
||||
endpoint returns an "unsupported model" error for it, switch to the matching
|
||||
Standard endpoint and key.
|
||||
OpenClaw omits `qwen3.7-max` and `qwen3.6-flash` from Coding Plan catalogs.
|
||||
If a Coding Plan endpoint returns an "unsupported model" error for either,
|
||||
switch to the matching Standard endpoint and key.
|
||||
|
||||
</Accordion>
|
||||
|
||||
|
|
|
|||
|
|
@ -404,6 +404,7 @@ describe("google provider plugin hooks", () => {
|
|||
if (!bridge) {
|
||||
throw new Error("expected Google realtime bridge");
|
||||
}
|
||||
expect(bridge.supportsToolResultContinuation).toBe(false);
|
||||
expect(bridge.sendAudio(Buffer.alloc(160))).toBeUndefined();
|
||||
expect(bridge.setMediaTimestamp(20)).toBeUndefined();
|
||||
expect(bridge.sendUserMessage?.("hello")).toBeUndefined();
|
||||
|
|
|
|||
|
|
@ -245,7 +245,9 @@ function createLazyGoogleRealtimeVoiceBridge(
|
|||
}
|
||||
};
|
||||
return {
|
||||
supportsToolResultContinuation: true,
|
||||
get supportsToolResultContinuation() {
|
||||
return bridge?.supportsToolResultContinuation ?? false;
|
||||
},
|
||||
connect: async () => {
|
||||
const loadedBridge = await loadBridge();
|
||||
if (closed) {
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
it("declares realtime Talk capabilities for catalog selection", () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
|
||||
expect(provider.defaultModel).toBe("gemini-3.1-flash-live-preview");
|
||||
expect(provider.capabilities).toEqual({
|
||||
transports: ["provider-websocket", "gateway-relay"],
|
||||
inputAudioFormats: [
|
||||
|
|
@ -173,6 +174,35 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("uses Gemini 3.1 Live-compatible defaults", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: {
|
||||
apiKey: "gemini-key",
|
||||
enableAffectiveDialog: true,
|
||||
thinkingBudget: 8_193,
|
||||
},
|
||||
tools: [createRealtimeTool("openclaw_agent_consult")],
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
});
|
||||
|
||||
expect(bridge.supportsToolResultContinuation).toBe(false);
|
||||
await bridge.connect();
|
||||
|
||||
const params = lastConnectParams();
|
||||
expect(params.model).toBe("gemini-3.1-flash-live-preview");
|
||||
expect(params.config.thinkingConfig).toEqual({ thinkingLevel: "HIGH" });
|
||||
expect(params.config).not.toHaveProperty("enableAffectiveDialog");
|
||||
const config = params.config as {
|
||||
tools?: Array<{ functionDeclarations?: Array<{ behavior?: string; name?: string }> }>;
|
||||
};
|
||||
expect(config.tools?.[0]?.functionDeclarations?.[0]).toMatchObject({
|
||||
name: "openclaw_agent_consult",
|
||||
});
|
||||
expect(config.tools?.[0]?.functionDeclarations?.[0]).not.toHaveProperty("behavior");
|
||||
});
|
||||
|
||||
it("normalizes provider config and cfg model-provider key fallback", () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const resolved = provider.resolveConfig?.({
|
||||
|
|
@ -416,6 +446,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
const bridge = provider.createBridge({
|
||||
providerConfig: {
|
||||
apiKey: "gemini-key",
|
||||
model: "gemini-live-2.5-flash-preview",
|
||||
thinkingBudget: -1,
|
||||
},
|
||||
onAudio: vi.fn(),
|
||||
|
|
@ -427,6 +458,22 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
expect(lastConnectParams().config.thinkingConfig).toEqual({ thinkingBudget: -1 });
|
||||
});
|
||||
|
||||
it("omits adaptive thinking budgets for Gemini 3.1 Live", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: {
|
||||
apiKey: "gemini-key",
|
||||
thinkingBudget: -1,
|
||||
},
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
});
|
||||
|
||||
await bridge.connect();
|
||||
|
||||
expect(lastConnectParams().config).not.toHaveProperty("thinkingConfig");
|
||||
});
|
||||
|
||||
it("creates constrained browser sessions for Google Live Talk", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
|
||||
|
|
@ -546,6 +593,46 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it("constrains default browser sessions to Gemini 3.1 capabilities", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
|
||||
const sessionLocal = await provider.createBrowserSession?.({
|
||||
providerConfig: {
|
||||
apiKey: "gemini-key",
|
||||
enableAffectiveDialog: true,
|
||||
thinkingLevel: "low",
|
||||
thinkingBudget: 8_193,
|
||||
},
|
||||
tools: [createRealtimeTool("openclaw_agent_consult")],
|
||||
});
|
||||
|
||||
const tokenConfig = requireFirstMockArg(createTokenMock, "Google Live auth token config") as {
|
||||
config?: {
|
||||
liveConnectConstraints?: {
|
||||
config?: {
|
||||
enableAffectiveDialog?: boolean;
|
||||
thinkingConfig?: unknown;
|
||||
tools?: Array<{
|
||||
functionDeclarations?: Array<{ behavior?: string; name?: string }>;
|
||||
}>;
|
||||
};
|
||||
model?: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
const constraints = tokenConfig.config?.liveConnectConstraints;
|
||||
expect(constraints?.model).toBe("gemini-3.1-flash-live-preview");
|
||||
expect(constraints?.config?.thinkingConfig).toEqual({ thinkingLevel: "LOW" });
|
||||
expect(constraints?.config).not.toHaveProperty("enableAffectiveDialog");
|
||||
expect(constraints?.config?.tools?.[0]?.functionDeclarations?.[0]).toMatchObject({
|
||||
name: "openclaw_agent_consult",
|
||||
});
|
||||
expect(constraints?.config?.tools?.[0]?.functionDeclarations?.[0]).not.toHaveProperty(
|
||||
"behavior",
|
||||
);
|
||||
expect(sessionLocal?.model).toBe("gemini-3.1-flash-live-preview");
|
||||
});
|
||||
|
||||
it("rejects browser session expiry outside Date range", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(8_640_000_000_000_001);
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
|
|
@ -907,7 +994,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
expect(config.realtimeInputConfig?.automaticActivityDetection?.disabled).toBe(true);
|
||||
});
|
||||
|
||||
it("sends text prompts as ordered client turns", async () => {
|
||||
it("sends Gemini 3.1 text prompts as realtime input", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: { apiKey: "gemini-key" },
|
||||
|
|
@ -921,6 +1008,27 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
|
||||
bridge.sendUserMessage?.(" Say hello. ");
|
||||
|
||||
expect(session.sendRealtimeInput).toHaveBeenCalledWith({ text: "Say hello." });
|
||||
expect(session.sendClientContent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps ordered client turns for explicit Gemini 2.5 sessions", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: {
|
||||
apiKey: "gemini-key",
|
||||
model: "gemini-live-2.5-flash-preview",
|
||||
},
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
});
|
||||
|
||||
await bridge.connect();
|
||||
lastConnectParams().callbacks.onopen();
|
||||
lastConnectParams().callbacks.onmessage({ setupComplete: { sessionId: "session-1" } });
|
||||
|
||||
bridge.sendUserMessage?.(" Say hello. ");
|
||||
|
||||
expect(session.sendClientContent).toHaveBeenCalledWith({
|
||||
turns: [{ role: "user", parts: [{ text: "Say hello." }] }],
|
||||
turnComplete: true,
|
||||
|
|
@ -1203,7 +1311,10 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
it("keeps Google Live consult calls open after continuing tool responses", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: { apiKey: "gemini-key" },
|
||||
providerConfig: {
|
||||
apiKey: "gemini-key",
|
||||
model: "gemini-live-2.5-flash-preview",
|
||||
},
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
onToolCall: vi.fn(),
|
||||
|
|
@ -1249,6 +1360,46 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("keeps Gemini 3.1 consult calls pending after rejecting continuation", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const onError = vi.fn();
|
||||
const bridge = provider.createBridge({
|
||||
providerConfig: { apiKey: "gemini-key" },
|
||||
onAudio: vi.fn(),
|
||||
onClearAudio: vi.fn(),
|
||||
onToolCall: vi.fn(),
|
||||
onError,
|
||||
});
|
||||
|
||||
await bridge.connect();
|
||||
lastConnectParams().callbacks.onmessage({
|
||||
setupComplete: { sessionId: "session-1" },
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{ id: "consult-call", name: "openclaw_agent_consult", args: { prompt: "hi" } },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
bridge.submitToolResult("consult-call", { status: "working" }, { willContinue: true });
|
||||
expect(session.sendToolResponse).not.toHaveBeenCalled();
|
||||
expect(requireFirstError(onError).message).toContain(
|
||||
"does not support continuing tool responses",
|
||||
);
|
||||
|
||||
bridge.submitToolResult("consult-call", { text: "The meeting starts at 3." });
|
||||
|
||||
expect(session.sendToolResponse).toHaveBeenCalledWith({
|
||||
functionResponses: [
|
||||
{
|
||||
id: "consult-call",
|
||||
name: "openclaw_agent_consult",
|
||||
response: { text: "The meeting starts at 3." },
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("does not send malformed Live API tool responses without a matching call name", async () => {
|
||||
const provider = buildGoogleRealtimeVoiceProvider();
|
||||
const onError = vi.fn();
|
||||
|
|
|
|||
|
|
@ -50,8 +50,9 @@ import {
|
|||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { createGoogleGenAI } from "./google-genai-runtime.js";
|
||||
import { resolveGoogleGemini3ThinkingLevel } from "./thinking.js";
|
||||
|
||||
const GOOGLE_REALTIME_DEFAULT_MODEL = "gemini-2.5-flash-native-audio-preview-12-2025";
|
||||
const GOOGLE_REALTIME_DEFAULT_MODEL = "gemini-3.1-flash-live-preview";
|
||||
const GOOGLE_REALTIME_DEFAULT_VOICE = "Kore";
|
||||
const GOOGLE_REALTIME_DEFAULT_API_VERSION = "v1beta";
|
||||
const GOOGLE_REALTIME_INPUT_SAMPLE_RATE = 16_000;
|
||||
|
|
@ -131,6 +132,7 @@ type GoogleLiveSession = {
|
|||
sendRealtimeInput: (params: {
|
||||
audio?: { data: string; mimeType: string };
|
||||
audioStreamEnd?: boolean;
|
||||
text?: string;
|
||||
}) => void;
|
||||
sendToolResponse: (params: { functionResponses: FunctionResponse[] | FunctionResponse }) => void;
|
||||
close: () => void;
|
||||
|
|
@ -305,9 +307,30 @@ function mapTurnCoverage(value: GoogleRealtimeTurnCoverage | undefined): TurnCov
|
|||
}
|
||||
}
|
||||
|
||||
function buildThinkingConfig(config: GoogleRealtimeLiveConfig): ThinkingConfig | undefined {
|
||||
if (config.thinkingLevel) {
|
||||
return { thinkingLevel: config.thinkingLevel.toUpperCase() as ThinkingConfig["thinkingLevel"] };
|
||||
// Gemini 3.1 Live replaces client-content text and async tools with realtime text
|
||||
// and sequential function responses; explicit older models keep their prior contract.
|
||||
function isGemini31LiveModel(model: string): boolean {
|
||||
const modelId = model.startsWith("models/") ? model.slice("models/".length) : model;
|
||||
return modelId.startsWith("gemini-3.1-") && modelId.includes("-live");
|
||||
}
|
||||
|
||||
function supportsAsyncFunctionCalling(model: string): boolean {
|
||||
return !isGemini31LiveModel(model);
|
||||
}
|
||||
|
||||
function buildThinkingConfig(
|
||||
config: GoogleRealtimeLiveConfig,
|
||||
model: string,
|
||||
): ThinkingConfig | undefined {
|
||||
if (isGemini31LiveModel(model)) {
|
||||
const thinkingLevel = resolveGoogleGemini3ThinkingLevel({
|
||||
modelId: model,
|
||||
thinkingLevel: config.thinkingLevel,
|
||||
thinkingBudget: config.thinkingBudget,
|
||||
});
|
||||
return thinkingLevel
|
||||
? { thinkingLevel: thinkingLevel as ThinkingConfig["thinkingLevel"] }
|
||||
: undefined;
|
||||
}
|
||||
if (typeof config.thinkingBudget === "number") {
|
||||
return { thinkingBudget: config.thinkingBudget };
|
||||
|
|
@ -343,7 +366,10 @@ function buildRealtimeInputConfig(
|
|||
return Object.keys(realtimeInputConfig).length > 0 ? realtimeInputConfig : undefined;
|
||||
}
|
||||
|
||||
function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): FunctionDeclaration[] {
|
||||
function buildFunctionDeclarations(
|
||||
tools: RealtimeVoiceTool[] | undefined,
|
||||
allowNonBlocking: boolean,
|
||||
): FunctionDeclaration[] {
|
||||
const declarations: FunctionDeclaration[] = [];
|
||||
let omitted = 0;
|
||||
for (const tool of tools ?? []) {
|
||||
|
|
@ -360,7 +386,7 @@ function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): Func
|
|||
description: tool.description,
|
||||
parameters: tool.parameters as unknown as FunctionDeclaration["parameters"],
|
||||
};
|
||||
if (name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
|
||||
if (allowNonBlocking && name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
|
||||
declaration.behavior = "NON_BLOCKING" as Behavior;
|
||||
}
|
||||
declarations.push(declaration);
|
||||
|
|
@ -374,10 +400,16 @@ function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): Func
|
|||
return declarations;
|
||||
}
|
||||
|
||||
function buildGoogleLiveConnectConfig(config: GoogleRealtimeLiveConfig): LiveConnectConfig {
|
||||
const functionDeclarations = buildFunctionDeclarations(config.tools);
|
||||
function buildGoogleLiveConnectConfig(
|
||||
config: GoogleRealtimeLiveConfig,
|
||||
model: string,
|
||||
): LiveConnectConfig {
|
||||
const functionDeclarations = buildFunctionDeclarations(
|
||||
config.tools,
|
||||
supportsAsyncFunctionCalling(model),
|
||||
);
|
||||
const realtimeInputConfig = buildRealtimeInputConfig(config);
|
||||
const thinkingConfig = buildThinkingConfig(config);
|
||||
const thinkingConfig = buildThinkingConfig(config, model);
|
||||
return {
|
||||
responseModalities: ["AUDIO" as Modality],
|
||||
...(typeof config.temperature === "number" && config.temperature > 0
|
||||
|
|
@ -395,7 +427,7 @@ function buildGoogleLiveConnectConfig(config: GoogleRealtimeLiveConfig): LiveCon
|
|||
...(realtimeInputConfig ? { realtimeInputConfig } : {}),
|
||||
inputAudioTranscription: {},
|
||||
outputAudioTranscription: {},
|
||||
...(typeof config.enableAffectiveDialog === "boolean"
|
||||
...(!isGemini31LiveModel(model) && typeof config.enableAffectiveDialog === "boolean"
|
||||
? { enableAffectiveDialog: config.enableAffectiveDialog }
|
||||
: {}),
|
||||
...(thinkingConfig ? { thinkingConfig } : {}),
|
||||
|
|
@ -461,7 +493,7 @@ function formatGoogleLiveCloseEvent(
|
|||
}
|
||||
|
||||
class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
||||
readonly supportsToolResultContinuation = true;
|
||||
readonly supportsToolResultContinuation: boolean;
|
||||
|
||||
private session: GoogleLiveSession | null = null;
|
||||
private connected = false;
|
||||
|
|
@ -473,6 +505,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
|||
private audioStreamEnded = false;
|
||||
private pendingFunctionNames = new Map<string, string>();
|
||||
private readonly audioFormat: RealtimeVoiceAudioFormat;
|
||||
private readonly model: string;
|
||||
private resumptionHandle: string | undefined;
|
||||
private reconnectAttempts = 0;
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
|
@ -484,6 +517,8 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
|||
|
||||
constructor(private readonly config: GoogleRealtimeVoiceBridgeConfig) {
|
||||
this.audioFormat = config.audioFormat ?? REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ;
|
||||
this.model = config.model ?? GOOGLE_REALTIME_DEFAULT_MODEL;
|
||||
this.supportsToolResultContinuation = supportsAsyncFunctionCalling(this.model);
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
|
|
@ -508,9 +543,9 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
|||
});
|
||||
|
||||
this.session = (await ai.live.connect({
|
||||
model: this.config.model ?? GOOGLE_REALTIME_DEFAULT_MODEL,
|
||||
model: this.model,
|
||||
config: {
|
||||
...buildGoogleLiveConnectConfig(this.config),
|
||||
...buildGoogleLiveConnectConfig(this.config, this.model),
|
||||
...(this.config.sessionResumption === false
|
||||
? {}
|
||||
: {
|
||||
|
|
@ -611,6 +646,10 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
|||
if (!normalized || !this.session || !this.connected || !this.sessionConfigured) {
|
||||
return;
|
||||
}
|
||||
if (isGemini31LiveModel(this.model)) {
|
||||
this.session.sendRealtimeInput({ text: normalized });
|
||||
return;
|
||||
}
|
||||
this.session.sendClientContent({
|
||||
turns: [{ role: "user", parts: [{ text: normalized }] }],
|
||||
turnComplete: true,
|
||||
|
|
@ -642,6 +681,12 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
|||
}
|
||||
try {
|
||||
const isConsultTool = name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME;
|
||||
if (options?.willContinue === true && !this.supportsToolResultContinuation) {
|
||||
this.config.onError?.(
|
||||
new Error(`Google Live model ${this.model} does not support continuing tool responses`),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const functionResponse: FunctionResponse = {
|
||||
id: callId,
|
||||
name,
|
||||
|
|
@ -650,7 +695,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
|
|||
? (result as Record<string, unknown>)
|
||||
: { output: result },
|
||||
};
|
||||
if (isConsultTool) {
|
||||
if (isConsultTool && this.supportsToolResultContinuation) {
|
||||
functionResponse.scheduling = "WHEN_IDLE" as FunctionResponseScheduling;
|
||||
if (options?.willContinue === true) {
|
||||
functionResponse.willContinue = true;
|
||||
|
|
@ -959,14 +1004,17 @@ async function createGoogleRealtimeBrowserSession(
|
|||
newSessionExpireTime,
|
||||
liveConnectConstraints: {
|
||||
model,
|
||||
config: buildGoogleLiveConnectConfig({
|
||||
...config,
|
||||
apiKey,
|
||||
config: buildGoogleLiveConnectConfig(
|
||||
{
|
||||
...config,
|
||||
apiKey,
|
||||
model,
|
||||
voice,
|
||||
instructions: req.instructions,
|
||||
tools: req.tools,
|
||||
},
|
||||
model,
|
||||
voice,
|
||||
instructions: req.instructions,
|
||||
tools: req.tools,
|
||||
}),
|
||||
),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ export {
|
|||
isQwen36PlusSupportedBaseUrl,
|
||||
isQwenCodingPlanBaseUrl,
|
||||
isQwenStandardOnlyModelId,
|
||||
QWEN_36_FLASH_MODEL_ID,
|
||||
QWEN_36_PLUS_MODEL_ID,
|
||||
QWEN_37_MAX_MODEL_ID,
|
||||
QWEN_37_PLUS_MODEL_ID,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import type { ProviderCatalogResult } from "openclaw/plugin-sdk/provider-catalog
|
|||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
QWEN_36_FLASH_MODEL_ID,
|
||||
QWEN_36_PLUS_MODEL_ID,
|
||||
QWEN_37_MAX_MODEL_ID,
|
||||
QWEN_37_PLUS_MODEL_ID,
|
||||
|
|
@ -41,6 +42,7 @@ describe("qwen provider plugin", () => {
|
|||
baseUrl: QWEN_BASE_URL,
|
||||
models: [
|
||||
{ id: "qwen3.5-plus" },
|
||||
{ id: QWEN_36_FLASH_MODEL_ID },
|
||||
{ id: QWEN_36_PLUS_MODEL_ID },
|
||||
{ id: QWEN_37_MAX_MODEL_ID },
|
||||
{ id: QWEN_37_PLUS_MODEL_ID },
|
||||
|
|
@ -81,6 +83,7 @@ describe("qwen provider plugin", () => {
|
|||
const catalogProvider = requireCatalogProvider(result);
|
||||
expect(catalogProvider.baseUrl).toBe("https://portal.qwen.ai/v1");
|
||||
expect(catalogProvider.models?.map((model) => model.id)).toContain("qwen3.5-plus");
|
||||
expect(catalogProvider.models?.map((model) => model.id)).not.toContain(QWEN_36_FLASH_MODEL_ID);
|
||||
expect(catalogProvider.models?.map((model) => model.id)).not.toContain(QWEN_37_MAX_MODEL_ID);
|
||||
expect(catalogProvider.models?.map((model) => model.id)).not.toContain(QWEN_37_PLUS_MODEL_ID);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ export default defineSingleProviderPluginEntry({
|
|||
"Manage API keys: https://home.qwencloud.com/api-keys",
|
||||
"Docs: https://docs.qwencloud.com/",
|
||||
"Endpoint: dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
"Models: qwen3.7-max, qwen3.7-plus, qwen3.6-plus, qwen3.5-plus, etc.",
|
||||
"Models: qwen3.7-max, qwen3.7-plus, qwen3.6-plus, qwen3.6-flash, qwen3.5-plus, etc.",
|
||||
].join("\n"),
|
||||
noteTitle: "Qwen Cloud Standard (China)",
|
||||
wizard: {
|
||||
|
|
@ -96,7 +96,7 @@ export default defineSingleProviderPluginEntry({
|
|||
"Manage API keys: https://home.qwencloud.com/api-keys",
|
||||
"Docs: https://docs.qwencloud.com/",
|
||||
"Endpoint: dashscope-intl.aliyuncs.com/compatible-mode/v1",
|
||||
"Models: qwen3.7-max, qwen3.7-plus, qwen3.6-plus, qwen3.5-plus, etc.",
|
||||
"Models: qwen3.7-max, qwen3.7-plus, qwen3.6-plus, qwen3.6-flash, qwen3.5-plus, etc.",
|
||||
].join("\n"),
|
||||
noteTitle: "Qwen Cloud Standard (Global/Intl)",
|
||||
wizard: {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ export const QWEN_OAUTH_PROVIDER_ID = "qwen-oauth";
|
|||
export const QWEN_OAUTH_BASE_URL = "https://portal.qwen.ai/v1";
|
||||
|
||||
export const QWEN_DEFAULT_MODEL_ID = "qwen3.5-plus";
|
||||
export const QWEN_36_FLASH_MODEL_ID = "qwen3.6-flash";
|
||||
export const QWEN_36_PLUS_MODEL_ID = "qwen3.6-plus";
|
||||
export const QWEN_37_MAX_MODEL_ID = "qwen3.7-max";
|
||||
export const QWEN_37_PLUS_MODEL_ID = "qwen3.7-plus";
|
||||
|
|
@ -40,6 +41,15 @@ export const QWEN_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
|
|||
contextWindow: 1_000_000,
|
||||
maxTokens: 65_536,
|
||||
},
|
||||
{
|
||||
id: QWEN_36_FLASH_MODEL_ID,
|
||||
name: QWEN_36_FLASH_MODEL_ID,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: QWEN_DEFAULT_COST,
|
||||
contextWindow: 1_000_000,
|
||||
maxTokens: 65_536,
|
||||
},
|
||||
{
|
||||
id: QWEN_36_PLUS_MODEL_ID,
|
||||
name: QWEN_36_PLUS_MODEL_ID,
|
||||
|
|
@ -152,9 +162,13 @@ export function isQwen36PlusSupportedBaseUrl(_baseUrl: string | undefined): bool
|
|||
return true;
|
||||
}
|
||||
|
||||
const QWEN_STANDARD_ONLY_MODEL_IDS = new Set<string>([QWEN_37_MAX_MODEL_ID]);
|
||||
const QWEN_STANDARD_ONLY_MODEL_IDS = new Set<string>([
|
||||
QWEN_36_FLASH_MODEL_ID,
|
||||
QWEN_37_MAX_MODEL_ID,
|
||||
]);
|
||||
|
||||
const QWEN_OAUTH_UNSUPPORTED_MODEL_IDS = new Set<string>([
|
||||
QWEN_36_FLASH_MODEL_ID,
|
||||
QWEN_37_MAX_MODEL_ID,
|
||||
QWEN_37_PLUS_MODEL_ID,
|
||||
]);
|
||||
|
|
|
|||
|
|
@ -60,6 +60,15 @@
|
|||
}
|
||||
},
|
||||
"suppressions": [
|
||||
{
|
||||
"provider": "qwen",
|
||||
"model": "qwen3.6-flash",
|
||||
"reason": "qwen3.6-flash is not supported on the Qwen Coding Plan endpoint; use a Standard pay-as-you-go Qwen endpoint.",
|
||||
"when": {
|
||||
"baseUrlHosts": ["coding.dashscope.aliyuncs.com", "coding-intl.dashscope.aliyuncs.com"],
|
||||
"providerConfigApiIn": ["qwen", "modelstudio"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"provider": "qwen",
|
||||
"model": "qwen3.7-max",
|
||||
|
|
@ -69,6 +78,15 @@
|
|||
"providerConfigApiIn": ["qwen", "modelstudio"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"provider": "modelstudio",
|
||||
"model": "qwen3.6-flash",
|
||||
"reason": "qwen3.6-flash is not supported on the Qwen Coding Plan endpoint; use a Standard pay-as-you-go Qwen endpoint.",
|
||||
"when": {
|
||||
"baseUrlHosts": ["coding.dashscope.aliyuncs.com", "coding-intl.dashscope.aliyuncs.com"],
|
||||
"providerConfigApiIn": ["qwen", "modelstudio"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"provider": "modelstudio",
|
||||
"model": "qwen3.7-max",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
applyQwenNativeStreamingUsageCompat,
|
||||
buildQwenProvider,
|
||||
QWEN_BASE_URL,
|
||||
QWEN_36_FLASH_MODEL_ID,
|
||||
QWEN_37_MAX_MODEL_ID,
|
||||
QWEN_37_PLUS_MODEL_ID,
|
||||
QWEN_STANDARD_GLOBAL_BASE_URL,
|
||||
|
|
@ -27,6 +28,7 @@ describe("qwen provider catalog", () => {
|
|||
const modelIds = getQwenModelIds(provider);
|
||||
expect(modelIds.length).toBeGreaterThan(0);
|
||||
expect(modelIds).toContain(QWEN_DEFAULT_MODEL_ID);
|
||||
expect(modelIds).not.toContain(QWEN_36_FLASH_MODEL_ID);
|
||||
expect(modelIds).toContain("qwen3.6-plus");
|
||||
expect(modelIds).not.toContain(QWEN_37_MAX_MODEL_ID);
|
||||
expect(modelIds).toContain(QWEN_37_PLUS_MODEL_ID);
|
||||
|
|
@ -42,10 +44,18 @@ describe("qwen provider catalog", () => {
|
|||
expect(getQwenModelIds(coding)).toContain("qwen3.6-plus");
|
||||
expect(getQwenModelIds(codingTrailingDot)).toContain("qwen3.6-plus");
|
||||
expect(getQwenModelIds(standard)).toContain("qwen3.6-plus");
|
||||
expect(getQwenModelIds(coding)).not.toContain(QWEN_36_FLASH_MODEL_ID);
|
||||
expect(getQwenModelIds(codingTrailingDot)).not.toContain(QWEN_36_FLASH_MODEL_ID);
|
||||
expect(getQwenModelIds(coding)).not.toContain(QWEN_37_MAX_MODEL_ID);
|
||||
expect(getQwenModelIds(coding)).toContain(QWEN_37_PLUS_MODEL_ID);
|
||||
expect(coding.models.find((model) => model.id === "qwen3.6-plus")?.reasoning).toBe(true);
|
||||
|
||||
expect(standard.models.find((model) => model.id === QWEN_36_FLASH_MODEL_ID)).toMatchObject({
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
contextWindow: 1_000_000,
|
||||
maxTokens: 65_536,
|
||||
});
|
||||
expect(standard.models.find((model) => model.id === QWEN_37_MAX_MODEL_ID)).toMatchObject({
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
|
|
@ -60,13 +70,17 @@ describe("qwen provider catalog", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("keeps Qwen 3.7 out of the portal catalog", () => {
|
||||
it("keeps unsupported Qwen models out of the portal catalog", () => {
|
||||
const portal = buildQwenOAuthProvider();
|
||||
const portalQwen36 = portal.models.find((model) => model.id === "qwen3.6-plus");
|
||||
const manifestQwen36 = manifest.modelCatalog.providers["qwen-oauth"].models.find(
|
||||
(model) => model.id === "qwen3.6-plus",
|
||||
);
|
||||
|
||||
expect(getQwenModelIds(portal)).not.toContain(QWEN_36_FLASH_MODEL_ID);
|
||||
expect(
|
||||
manifest.modelCatalog.providers["qwen-oauth"].models.map((model) => model.id),
|
||||
).not.toContain(QWEN_36_FLASH_MODEL_ID);
|
||||
expect(getQwenModelIds(portal)).not.toContain(QWEN_37_MAX_MODEL_ID);
|
||||
expect(getQwenModelIds(portal)).not.toContain(QWEN_37_PLUS_MODEL_ID);
|
||||
expect(portalQwen36?.reasoning).toBe(true);
|
||||
|
|
|
|||
|
|
@ -16,8 +16,7 @@ const OPENAI_REALTIME_VOICE = process.env.OPENCLAW_REALTIME_OPENAI_VOICE?.trim()
|
|||
const DEFAULT_OPENAI_HTTP_TIMEOUT_MS = 30_000;
|
||||
const OPENAI_HTTP_RESPONSE_MAX_BYTES = 256 * 1024;
|
||||
const GOOGLE_REALTIME_MODEL =
|
||||
process.env.OPENCLAW_REALTIME_GOOGLE_MODEL?.trim() ||
|
||||
"gemini-2.5-flash-native-audio-preview-12-2025";
|
||||
process.env.OPENCLAW_REALTIME_GOOGLE_MODEL?.trim() || "gemini-3.1-flash-live-preview";
|
||||
const GOOGLE_REALTIME_VOICE = process.env.OPENCLAW_REALTIME_GOOGLE_VOICE?.trim() || "Kore";
|
||||
const GOOGLE_LIVE_WS_URL =
|
||||
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
|
||||
|
|
|
|||
|
|
@ -442,6 +442,59 @@ describe("GoogleLiveRealtimeTalkTransport", () => {
|
|||
expect(onStatus).not.toHaveBeenCalledWith("listening");
|
||||
});
|
||||
|
||||
it("submits completed consults without asynchronous scheduling", async () => {
|
||||
const listeners = new Set<(event: { event: string; payload?: unknown }) => void>();
|
||||
const client = {
|
||||
addEventListener: vi.fn((listener: (event: { event: string; payload?: unknown }) => void) => {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
}),
|
||||
request: vi.fn(async (method: string) => {
|
||||
expect(method).toBe("talk.client.toolCall");
|
||||
return { runId: "run-1" };
|
||||
}),
|
||||
} as unknown as RealtimeTalkTransportContext["client"];
|
||||
const transport = createTransport({}, client);
|
||||
await transport.start();
|
||||
const ws = latestWebSocket();
|
||||
|
||||
ws.emitMessage(
|
||||
encodeJsonFrame({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: "call-1",
|
||||
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
|
||||
args: { question: "check the session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
await vi.waitFor(() => expect(listeners.size).toBe(1));
|
||||
for (const listener of listeners) {
|
||||
listener({
|
||||
event: "chat",
|
||||
payload: { runId: "run-1", state: "final", message: { text: "done" } },
|
||||
});
|
||||
}
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({
|
||||
toolResponse: {
|
||||
functionResponses: [
|
||||
{
|
||||
id: "call-1",
|
||||
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
|
||||
response: { result: "done" },
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
transport.stop();
|
||||
});
|
||||
|
||||
it("sends spoken active-control acknowledgements through Google Live", async () => {
|
||||
const client = createClient();
|
||||
vi.mocked(client["request"]).mockImplementation(async (method) => {
|
||||
|
|
@ -507,18 +560,8 @@ describe("GoogleLiveRealtimeTalkTransport", () => {
|
|||
expect(createdSources[0]?.stop).toHaveBeenCalledTimes(1);
|
||||
const sent = ws.sent.map((payload) => JSON.parse(payload));
|
||||
expect(sent).toContainEqual({
|
||||
clientContent: {
|
||||
turns: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
text: expect.stringContaining('Status: "OpenClaw is working in read (running)."'),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
turnComplete: true,
|
||||
realtimeInput: {
|
||||
text: expect.stringContaining('Status: "OpenClaw is working in read (running)."'),
|
||||
},
|
||||
});
|
||||
transport.stop();
|
||||
|
|
@ -590,18 +633,8 @@ describe("GoogleLiveRealtimeTalkTransport", () => {
|
|||
expect(createdSources[0]?.stop).toHaveBeenCalledTimes(1);
|
||||
const sent = ws.sent.map((payload) => JSON.parse(payload));
|
||||
expect(sent).toContainEqual({
|
||||
clientContent: {
|
||||
turns: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
text: expect.stringContaining('Status: "Got it. I steered the active run."'),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
turnComplete: true,
|
||||
realtimeInput: {
|
||||
text: expect.stringContaining('Status: "Got it. I steered the active run."'),
|
||||
},
|
||||
});
|
||||
transport.stop();
|
||||
|
|
|
|||
|
|
@ -53,6 +53,16 @@ const GOOGLE_LIVE_WEBSOCKET_HOST = "generativelanguage.googleapis.com";
|
|||
const GOOGLE_LIVE_WEBSOCKET_PATH =
|
||||
/^\/ws\/google\.ai\.generativelanguage\.v[0-9a-z]+\.GenerativeService\.BidiGenerateContent(?:Constrained)?$/;
|
||||
|
||||
// Browser sessions can still pin a 2.5 model, whose text and tool-response wire
|
||||
// contract differs from the 3.1 default carried in new session metadata.
|
||||
function isGemini31LiveModel(model: string | undefined): boolean {
|
||||
if (!model) {
|
||||
return true;
|
||||
}
|
||||
const modelId = model.startsWith("models/") ? model.slice("models/".length) : model;
|
||||
return modelId.startsWith("gemini-3.1-") && modelId.includes("-live");
|
||||
}
|
||||
|
||||
export function buildGoogleLiveUrl(session: RealtimeTalkJsonPcmWebSocketSessionResult): string {
|
||||
let url: URL;
|
||||
try {
|
||||
|
|
@ -392,7 +402,7 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
{
|
||||
id: callId,
|
||||
name: pending.name,
|
||||
scheduling: "WHEN_IDLE",
|
||||
...(!isGemini31LiveModel(this.session.model) ? { scheduling: "WHEN_IDLE" } : {}),
|
||||
response:
|
||||
result && typeof result === "object" && !Array.isArray(result)
|
||||
? result
|
||||
|
|
@ -405,15 +415,18 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
|
||||
private sendControlSpeechMessage(message: string): void {
|
||||
this.stopOutput();
|
||||
if (!isGemini31LiveModel(this.session.model)) {
|
||||
this.send({
|
||||
clientContent: {
|
||||
turns: [{ role: "user", parts: [{ text: message }] }],
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
this.send({
|
||||
clientContent: {
|
||||
turns: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ text: message }],
|
||||
},
|
||||
],
|
||||
turnComplete: true,
|
||||
realtimeInput: {
|
||||
text: message,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue