mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
feat(ui): show microphone activity in Talk mode (#102559)
This commit is contained in:
Peter Steinberger
GitHub
parent
f984209f54
commit
efe705b889
19 changed files with 982 additions and 58 deletions
|
|
@ -18,7 +18,18 @@ let server: ControlUiE2eServer;
|
|||
|
||||
async function installTalkBrowserFixtures(page: Page) {
|
||||
await page.addInitScript(() => {
|
||||
const state = { audioContextsClosed: 0, tracksStopped: 0, constraints: [] as unknown[] };
|
||||
type InputProcessor = {
|
||||
onaudioprocess:
|
||||
| ((event: { inputBuffer: { getChannelData: () => Float32Array } }) => void)
|
||||
| null;
|
||||
};
|
||||
const state = {
|
||||
audioContextsClosed: 0,
|
||||
tracksStopped: 0,
|
||||
constraints: [] as unknown[],
|
||||
inputProcessor: null as InputProcessor | null,
|
||||
meterLevel: 0,
|
||||
};
|
||||
const track = { stop: () => (state.tracksStopped += 1) };
|
||||
Object.defineProperty(navigator, "mediaDevices", {
|
||||
configurable: true,
|
||||
|
|
@ -49,7 +60,20 @@ async function installTalkBrowserFixtures(page: Page) {
|
|||
}
|
||||
|
||||
createScriptProcessor() {
|
||||
return { connect() {}, disconnect() {}, onaudioprocess: null };
|
||||
const processor = { connect() {}, disconnect() {}, onaudioprocess: null };
|
||||
state.inputProcessor = processor;
|
||||
return processor;
|
||||
}
|
||||
|
||||
createAnalyser() {
|
||||
return {
|
||||
fftSize: 0,
|
||||
smoothingTimeConstant: 0,
|
||||
disconnect() {},
|
||||
getFloatTimeDomainData(samples: Float32Array) {
|
||||
samples.fill(state.meterLevel);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async close() {
|
||||
|
|
@ -116,6 +140,7 @@ describeControlUiE2e("Control UI browser Talk", () => {
|
|||
await installTalkBrowserFixtures(page);
|
||||
|
||||
try {
|
||||
await page.emulateMedia({ reducedMotion: "reduce" });
|
||||
await page.goto(`${server.baseUrl}chat`);
|
||||
await page.setViewportSize({ width: 320, height: 720 });
|
||||
await expect
|
||||
|
|
@ -157,18 +182,77 @@ describeControlUiE2e("Control UI browser Talk", () => {
|
|||
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained?access_token=auth_tokens%2Fbrowser-talk-e2e",
|
||||
]);
|
||||
|
||||
await expect
|
||||
.poll(() => page.locator('.agent-chat__voice-activity[data-status="connecting"]').count())
|
||||
.toBe(1);
|
||||
await expect
|
||||
.poll(() =>
|
||||
page
|
||||
.locator(".agent-chat__voice-activity-bar")
|
||||
.first()
|
||||
.evaluate((element) => {
|
||||
return getComputedStyle(element).animationName;
|
||||
}),
|
||||
)
|
||||
.toBe("none");
|
||||
const connectingReducedMotionTransform = await page
|
||||
.locator(".agent-chat__voice-activity-bar")
|
||||
.first()
|
||||
.evaluate((element) => getComputedStyle(element).transform);
|
||||
|
||||
await gateway.deliverLatest({ setupComplete: {} });
|
||||
await expect
|
||||
.poll(async () =>
|
||||
(await page.locator(".agent-chat__talk-status-text").textContent())?.trim(),
|
||||
)
|
||||
.poll(() => page.locator('.agent-chat__voice-activity[data-status="listening"]').count())
|
||||
.toBe(1);
|
||||
await expect.poll(() => page.locator(".agent-chat__talk-status-text").count()).toBe(0);
|
||||
await expect
|
||||
.poll(() => page.locator('[role="status"] .agent-chat__sr-only').textContent())
|
||||
.toBe("Listening...");
|
||||
const reducedMotionTransform = await page
|
||||
.locator(".agent-chat__voice-activity-bar")
|
||||
.first()
|
||||
.evaluate((element) => getComputedStyle(element).transform);
|
||||
expect(reducedMotionTransform).not.toBe(connectingReducedMotionTransform);
|
||||
|
||||
await page.evaluate(() => {
|
||||
const state = (
|
||||
window as Window & {
|
||||
openclawTalkE2eState?: {
|
||||
inputProcessor?: {
|
||||
onaudioprocess?: (event: {
|
||||
inputBuffer: { getChannelData: () => Float32Array };
|
||||
}) => void;
|
||||
};
|
||||
meterLevel?: number;
|
||||
};
|
||||
}
|
||||
).openclawTalkE2eState;
|
||||
if (state) {
|
||||
state.meterLevel = 0.25;
|
||||
}
|
||||
state?.inputProcessor?.onaudioprocess?.({
|
||||
inputBuffer: { getChannelData: () => new Float32Array(4096).fill(0.25) },
|
||||
});
|
||||
});
|
||||
await expect
|
||||
.poll(async () =>
|
||||
Number(await page.locator(".agent-chat__voice-activity").getAttribute("data-level")),
|
||||
)
|
||||
.toBeGreaterThan(0);
|
||||
await expect
|
||||
.poll(() =>
|
||||
page
|
||||
.locator(".agent-chat__voice-activity-bar")
|
||||
.first()
|
||||
.evaluate((element) => getComputedStyle(element).transform),
|
||||
)
|
||||
.toBe(reducedMotionTransform);
|
||||
|
||||
await page.getByRole("button", { name: "Stop voice input" }).click();
|
||||
await expect
|
||||
.poll(() => page.getByRole("button", { name: "Start voice input" }).isVisible())
|
||||
.toBe(true);
|
||||
await expect.poll(() => page.locator(".agent-chat__talk-status-text").count()).toBe(0);
|
||||
await expect.poll(() => page.locator(".agent-chat__voice-activity").count()).toBe(0);
|
||||
await expect
|
||||
.poll(() =>
|
||||
page.evaluate(() => {
|
||||
|
|
|
|||
|
|
@ -798,6 +798,7 @@ class ChatPane extends LitElement {
|
|||
state.realtimeTalkSession = null;
|
||||
state.realtimeTalkActive = false;
|
||||
state.realtimeTalkStatus = "idle";
|
||||
state.realtimeTalkInputLevel.set(0);
|
||||
state.resetToolStream();
|
||||
state.requestUpdate?.();
|
||||
return;
|
||||
|
|
@ -895,6 +896,7 @@ class ChatPane extends LitElement {
|
|||
realtimeTalkActive: state.realtimeTalkActive,
|
||||
realtimeTalkStatus: state.realtimeTalkStatus,
|
||||
realtimeTalkDetail: state.realtimeTalkDetail,
|
||||
realtimeTalkInputLevel: state.realtimeTalkInputLevel,
|
||||
realtimeTalkConversation: state.realtimeTalkConversation,
|
||||
connected: state.connected,
|
||||
canSend: state.connected && !selectedSessionArchived,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {
|
|||
createInitialChatRealtimeState,
|
||||
type ChatRealtimeState,
|
||||
} from "./chat-realtime.ts";
|
||||
import type { RealtimeTalkCallbacks } from "./realtime-talk.ts";
|
||||
|
||||
function mediaDevice(kind: MediaDeviceKind, deviceId: string, label: string): MediaDeviceInfo {
|
||||
return { kind, deviceId, label, groupId: "", toJSON: () => ({}) } as MediaDeviceInfo;
|
||||
|
|
@ -44,7 +45,8 @@ describe("chat realtime microphone selection", () => {
|
|||
beforeEach(() => {
|
||||
localStorage.clear();
|
||||
realtimeTalkSessionCtor.mockClear();
|
||||
sessionStart.mockClear();
|
||||
sessionStart.mockReset();
|
||||
sessionStart.mockResolvedValue(undefined);
|
||||
sessionStop.mockClear();
|
||||
});
|
||||
|
||||
|
|
@ -75,6 +77,63 @@ describe("chat realtime microphone selection", () => {
|
|||
expect(sessionStart).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("propagates normalized microphone levels and resets them on error", async () => {
|
||||
const state = createState();
|
||||
await state.toggleRealtimeTalk();
|
||||
const constructorCalls = realtimeTalkSessionCtor.mock.calls as unknown[][];
|
||||
const callbacks = constructorCalls[0]?.[2] as RealtimeTalkCallbacks | undefined;
|
||||
if (!callbacks) {
|
||||
throw new Error("expected realtime callbacks");
|
||||
}
|
||||
|
||||
const updatesBeforeLevels = vi.mocked(state.requestUpdate).mock.calls.length;
|
||||
callbacks.onInputLevel?.(0.456);
|
||||
expect(state.realtimeTalkInputLevel.value).toBe(0.46);
|
||||
|
||||
callbacks.onInputLevel?.(2);
|
||||
expect(state.realtimeTalkInputLevel.value).toBe(1);
|
||||
expect(state.requestUpdate).toHaveBeenCalledTimes(updatesBeforeLevels);
|
||||
|
||||
callbacks.onStatus?.("error", "capture failed");
|
||||
expect(state.realtimeTalkInputLevel.value).toBe(0);
|
||||
});
|
||||
|
||||
it("ignores a stopped session that rejects after its replacement starts", async () => {
|
||||
let rejectFirstStart: (error: Error) => void = () => undefined;
|
||||
sessionStart.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<undefined>((_resolve, reject) => {
|
||||
rejectFirstStart = reject;
|
||||
}),
|
||||
);
|
||||
const state = createState();
|
||||
|
||||
const firstStart = state.toggleRealtimeTalk();
|
||||
await vi.waitFor(() => expect(realtimeTalkSessionCtor).toHaveBeenCalledTimes(1));
|
||||
const firstCallbacks = (realtimeTalkSessionCtor.mock.calls as unknown[][])[0]?.[2] as
|
||||
| RealtimeTalkCallbacks
|
||||
| undefined;
|
||||
await state.toggleRealtimeTalk();
|
||||
await state.toggleRealtimeTalk();
|
||||
const secondSession = realtimeTalkSessionCtor.mock.results[1]?.value;
|
||||
const secondCallbacks = (realtimeTalkSessionCtor.mock.calls as unknown[][])[1]?.[2] as
|
||||
| RealtimeTalkCallbacks
|
||||
| undefined;
|
||||
secondCallbacks?.onStatus?.("listening");
|
||||
|
||||
rejectFirstStart(new Error("late setup failure"));
|
||||
await firstStart;
|
||||
firstCallbacks?.onInputLevel?.(0.9);
|
||||
firstCallbacks?.onTranscript?.({ role: "user", text: "stale", final: true });
|
||||
firstCallbacks?.onStatus?.("error", "stale failure");
|
||||
|
||||
expect(state.realtimeTalkSession).toBe(secondSession);
|
||||
expect(state.realtimeTalkActive).toBe(true);
|
||||
expect(state.realtimeTalkStatus).toBe("listening");
|
||||
expect(state.realtimeTalkInputLevel.value).toBe(0);
|
||||
expect(state.realtimeTalkConversation).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not reject a persisted input from incomplete passive discovery", async () => {
|
||||
vi.stubGlobal("navigator", {
|
||||
mediaDevices: {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
type RealtimeTalkConversationState,
|
||||
} from "./realtime-talk-conversation.ts";
|
||||
import { discoverRealtimeTalkInputs, type RealtimeTalkInputDevice } from "./realtime-talk-input.ts";
|
||||
import { RealtimeTalkLevelSignal } from "./realtime-talk-level.ts";
|
||||
import {
|
||||
RealtimeTalkSession,
|
||||
type RealtimeTalkLaunchOptions,
|
||||
|
|
@ -41,6 +42,7 @@ export type ChatRealtimeState = {
|
|||
realtimeTalkActive: boolean;
|
||||
realtimeTalkStatus: RealtimeTalkStatus;
|
||||
realtimeTalkDetail: string | null;
|
||||
realtimeTalkInputLevel: RealtimeTalkLevelSignal;
|
||||
realtimeTalkConversation: RealtimeTalkConversationEntry[];
|
||||
realtimeTalkOptions: RealtimeTalkOptions;
|
||||
realtimeTalkInputDevices: RealtimeTalkInputDevice[];
|
||||
|
|
@ -71,6 +73,7 @@ export function createInitialChatRealtimeState(inputDeviceId = "") {
|
|||
realtimeTalkActive: false,
|
||||
realtimeTalkStatus: "idle" as RealtimeTalkStatus,
|
||||
realtimeTalkDetail: null,
|
||||
realtimeTalkInputLevel: new RealtimeTalkLevelSignal(),
|
||||
realtimeTalkConversation: [],
|
||||
realtimeTalkOptions: createDefaultRealtimeTalkOptions(),
|
||||
realtimeTalkInputDevices: [] as RealtimeTalkInputDevice[],
|
||||
|
|
@ -97,6 +100,7 @@ export function dismissRealtimeTalkError(state: ChatRealtimeState) {
|
|||
state.realtimeTalkActive = false;
|
||||
state.realtimeTalkStatus = "idle";
|
||||
state.realtimeTalkDetail = null;
|
||||
state.realtimeTalkInputLevel.set(0);
|
||||
state.resetRealtimeTalkConversation();
|
||||
}
|
||||
|
||||
|
|
@ -167,6 +171,7 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) {
|
|||
state.realtimeTalkActive = false;
|
||||
state.realtimeTalkStatus = "idle";
|
||||
state.realtimeTalkDetail = null;
|
||||
state.realtimeTalkInputLevel.set(0);
|
||||
state.resetRealtimeTalkConversation();
|
||||
state.requestUpdate();
|
||||
return;
|
||||
|
|
@ -188,18 +193,34 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) {
|
|||
state.realtimeTalkActive = true;
|
||||
state.realtimeTalkStatus = "connecting";
|
||||
state.realtimeTalkDetail = null;
|
||||
state.realtimeTalkInputLevel.set(0);
|
||||
state.resetRealtimeTalkConversation();
|
||||
const session = new RealtimeTalkSession(
|
||||
state.client,
|
||||
state.sessionKey,
|
||||
{
|
||||
onStatus: (status, detail) => {
|
||||
if (state.realtimeTalkSession !== session) {
|
||||
return;
|
||||
}
|
||||
state.realtimeTalkStatus = status;
|
||||
state.realtimeTalkDetail = detail ?? null;
|
||||
state.realtimeTalkActive = status !== "idle";
|
||||
if (status === "idle" || status === "error") {
|
||||
state.realtimeTalkInputLevel.set(0);
|
||||
}
|
||||
state.requestUpdate();
|
||||
},
|
||||
onInputLevel: (level) => {
|
||||
if (state.realtimeTalkSession !== session) {
|
||||
return;
|
||||
}
|
||||
state.realtimeTalkInputLevel.set(level);
|
||||
},
|
||||
onTranscript: (entry) => {
|
||||
if (state.realtimeTalkSession !== session) {
|
||||
return;
|
||||
}
|
||||
state.realtimeTalkConversationState = updateRealtimeTalkConversation(
|
||||
state.realtimeTalkConversationState,
|
||||
entry,
|
||||
|
|
@ -215,11 +236,15 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) {
|
|||
try {
|
||||
await session.start();
|
||||
} catch (error) {
|
||||
if (state.realtimeTalkSession !== session) {
|
||||
return;
|
||||
}
|
||||
session.stop();
|
||||
state.realtimeTalkSession = null;
|
||||
state.realtimeTalkActive = false;
|
||||
state.realtimeTalkStatus = "error";
|
||||
state.realtimeTalkDetail = error instanceof Error ? error.message : String(error);
|
||||
state.realtimeTalkInputLevel.set(0);
|
||||
state.requestUpdate();
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ import {
|
|||
toggleChatThreadSearch,
|
||||
} from "./components/chat-thread.ts";
|
||||
import { renderWelcomeState } from "./components/chat-welcome.ts";
|
||||
import { RealtimeTalkLevelSignal } from "./realtime-talk-level.ts";
|
||||
|
||||
const refreshVisibleToolsEffectiveForCurrentSessionMock = vi.hoisted(() =>
|
||||
vi.fn(async (state: ChatHeaderTestState) => {
|
||||
|
|
@ -1771,6 +1772,73 @@ describe("chat voice controls", () => {
|
|||
expect(onSend).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["connecting", "Connecting voice input..."],
|
||||
["listening", "Listening..."],
|
||||
["thinking", "Asking OpenClaw..."],
|
||||
] as const)("renders %s voice activity without visible status copy", (status, label) => {
|
||||
const inputLevel = new RealtimeTalkLevelSignal();
|
||||
inputLevel.set(0.64);
|
||||
const container = renderChatView({
|
||||
realtimeTalkActive: true,
|
||||
realtimeTalkStatus: status,
|
||||
realtimeTalkInputLevel: inputLevel,
|
||||
});
|
||||
|
||||
const statusRegion = container.querySelector('[role="status"].agent-chat__talk-status');
|
||||
const visualizer = statusRegion?.querySelector<HTMLElement>(
|
||||
`.agent-chat__voice-activity[data-status="${status}"]`,
|
||||
);
|
||||
expect(visualizer?.getAttribute("data-level")).toBe("0.64");
|
||||
expect(visualizer?.getAttribute("data-source")).toBe("microphone");
|
||||
expect(visualizer?.getAttribute("role")).toBe("img");
|
||||
expect(visualizer?.getAttribute("aria-label")).toBe(t("chat.composer.microphoneInput"));
|
||||
expect(visualizer?.querySelectorAll(".agent-chat__voice-activity-bar")).toHaveLength(7);
|
||||
expect(statusRegion?.getAttribute("aria-live")).toBe("polite");
|
||||
expect(statusRegion?.getAttribute("aria-atomic")).toBe("true");
|
||||
expect(statusRegion?.querySelector(".agent-chat__sr-only")?.textContent?.trim()).toBe(label);
|
||||
expect(statusRegion?.querySelector(".agent-chat__talk-status-text")).toBeNull();
|
||||
});
|
||||
|
||||
it("clamps the rendered microphone level", () => {
|
||||
const inputLevel = new RealtimeTalkLevelSignal();
|
||||
inputLevel.set(4);
|
||||
const container = renderChatView({
|
||||
realtimeTalkActive: true,
|
||||
realtimeTalkStatus: "listening",
|
||||
realtimeTalkInputLevel: inputLevel,
|
||||
});
|
||||
|
||||
expect(container.querySelector(".agent-chat__voice-activity")?.getAttribute("data-level")).toBe(
|
||||
"1",
|
||||
);
|
||||
});
|
||||
|
||||
it("updates microphone bars without rerendering the chat", () => {
|
||||
const inputLevel = new RealtimeTalkLevelSignal();
|
||||
inputLevel.set(0.2);
|
||||
const container = renderChatView({
|
||||
realtimeTalkActive: true,
|
||||
realtimeTalkStatus: "listening",
|
||||
realtimeTalkInputLevel: inputLevel,
|
||||
});
|
||||
document.body.append(container);
|
||||
try {
|
||||
const visualizer = container.querySelector<HTMLElement>(".agent-chat__voice-activity");
|
||||
const centerBar = visualizer?.querySelector<HTMLElement>(
|
||||
".agent-chat__voice-activity-bar:nth-child(4)",
|
||||
);
|
||||
const initialScale = centerBar?.style.getPropertyValue("--talk-bar-scale");
|
||||
|
||||
inputLevel.set(0.8);
|
||||
|
||||
expect(visualizer?.getAttribute("data-level")).toBe("0.8");
|
||||
expect(centerBar?.style.getPropertyValue("--talk-bar-scale")).not.toBe(initialScale);
|
||||
} finally {
|
||||
container.remove();
|
||||
}
|
||||
});
|
||||
|
||||
it("shows every available microphone in Voice settings", () => {
|
||||
const onRealtimeTalkInputSelect = vi.fn();
|
||||
const container = renderVoiceOptions({
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import {
|
|||
} from "./components/chat-thread.ts";
|
||||
import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "./input-history.ts";
|
||||
import type { RealtimeTalkConversationEntry } from "./realtime-talk-conversation.ts";
|
||||
import type { RealtimeTalkLevelSignal } from "./realtime-talk-level.ts";
|
||||
import type { RealtimeTalkStatus } from "./realtime-talk.ts";
|
||||
import type { ChatRunUiStatus } from "./run-lifecycle.ts";
|
||||
import type { CompactionStatus, FallbackStatus } from "./tool-stream.ts";
|
||||
|
|
@ -72,6 +73,7 @@ export type ChatProps = {
|
|||
realtimeTalkActive?: boolean;
|
||||
realtimeTalkStatus?: RealtimeTalkStatus;
|
||||
realtimeTalkDetail?: string | null;
|
||||
realtimeTalkInputLevel?: RealtimeTalkLevelSignal;
|
||||
realtimeTalkConversation?: RealtimeTalkConversationEntry[];
|
||||
connected: boolean;
|
||||
canSend: boolean;
|
||||
|
|
@ -235,6 +237,7 @@ export function renderChat(props: ChatProps) {
|
|||
realtimeTalkActive: props.realtimeTalkActive,
|
||||
realtimeTalkStatus: props.realtimeTalkStatus,
|
||||
realtimeTalkDetail: props.realtimeTalkDetail,
|
||||
realtimeTalkInputLevel: props.realtimeTalkInputLevel,
|
||||
realtimeTalkConversation: props.realtimeTalkConversation,
|
||||
composerControls: props.composerControls,
|
||||
getDraft: props.getDraft,
|
||||
|
|
|
|||
|
|
@ -41,9 +41,11 @@ import {
|
|||
import { exportChatMarkdown } from "../export.ts";
|
||||
import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "../input-history.ts";
|
||||
import type { RealtimeTalkConversationEntry } from "../realtime-talk-conversation.ts";
|
||||
import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts";
|
||||
import type { RealtimeTalkStatus } from "../realtime-talk.ts";
|
||||
import { CHAT_RUN_STATUS_TOAST_DURATION_MS, type ChatRunUiStatus } from "../run-lifecycle.ts";
|
||||
import type { CompactionStatus, FallbackStatus } from "../tool-stream.ts";
|
||||
import { renderChatVoiceActivity } from "./chat-voice-activity.ts";
|
||||
|
||||
const COMPACTION_TOAST_DURATION_MS = 5000;
|
||||
const FALLBACK_TOAST_DURATION_MS = 8000;
|
||||
|
|
@ -92,6 +94,7 @@ type ChatComposerProps = {
|
|||
realtimeTalkActive?: boolean;
|
||||
realtimeTalkStatus?: RealtimeTalkStatus;
|
||||
realtimeTalkDetail?: string | null;
|
||||
realtimeTalkInputLevel?: RealtimeTalkLevelSignal;
|
||||
realtimeTalkConversation?: RealtimeTalkConversationEntry[];
|
||||
composerControls?: TemplateResult | typeof nothing;
|
||||
getDraft?: () => string;
|
||||
|
|
@ -2286,37 +2289,13 @@ export function renderChatComposer(props: ChatComposerProps) {
|
|||
}}
|
||||
/>
|
||||
|
||||
${props.realtimeTalkActive || props.realtimeTalkDetail
|
||||
? html`
|
||||
<div
|
||||
class="agent-chat__stt-interim agent-chat__talk-status"
|
||||
role=${props.realtimeTalkStatus === "error" ? "alert" : nothing}
|
||||
>
|
||||
<span class="agent-chat__talk-status-text">
|
||||
${props.realtimeTalkDetail ??
|
||||
(props.realtimeTalkStatus === "thinking"
|
||||
? "Asking OpenClaw..."
|
||||
: props.realtimeTalkStatus === "connecting"
|
||||
? "Connecting voice input..."
|
||||
: "Listening...")}
|
||||
</span>
|
||||
${props.realtimeTalkStatus === "error" && props.onDismissRealtimeTalkError
|
||||
? html`
|
||||
<openclaw-tooltip .content=${t("chat.composer.dismissVoiceInputError")}>
|
||||
<button
|
||||
class="callout__dismiss"
|
||||
type="button"
|
||||
@click=${props.onDismissRealtimeTalkError}
|
||||
aria-label=${t("chat.composer.dismissVoiceInputError")}
|
||||
>
|
||||
${icons.x}
|
||||
</button>
|
||||
</openclaw-tooltip>
|
||||
`
|
||||
: nothing}
|
||||
</div>
|
||||
`
|
||||
: nothing}
|
||||
${renderChatVoiceActivity({
|
||||
active: props.realtimeTalkActive,
|
||||
status: props.realtimeTalkStatus,
|
||||
detail: props.realtimeTalkDetail,
|
||||
inputLevel: props.realtimeTalkInputLevel,
|
||||
onDismissError: props.onDismissRealtimeTalkError,
|
||||
})}
|
||||
|
||||
<div class="agent-chat__composer-input-row">
|
||||
<details class="agent-chat__attach-menu">
|
||||
|
|
|
|||
148
ui/src/pages/chat/components/chat-voice-activity.ts
Normal file
148
ui/src/pages/chat/components/chat-voice-activity.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import { html, nothing, type TemplateResult } from "lit";
|
||||
import { icons } from "../../../components/icons.ts";
|
||||
import "../../../components/tooltip.ts";
|
||||
import { t } from "../../../i18n/index.ts";
|
||||
import { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts";
|
||||
import type { RealtimeTalkStatus } from "../realtime-talk.ts";
|
||||
|
||||
const BAR_GAINS = [0.38, 0.62, 0.84, 1, 0.84, 0.62, 0.38];
|
||||
const MICROPHONE_ACTIVITY_TAG = "openclaw-microphone-activity";
|
||||
const EMPTY_LEVEL_SIGNAL = new RealtimeTalkLevelSignal();
|
||||
|
||||
class MicrophoneActivityElement extends HTMLElement {
|
||||
private levelSignal: RealtimeTalkLevelSignal | undefined;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
set signal(signal: RealtimeTalkLevelSignal | undefined) {
|
||||
if (signal === this.levelSignal) {
|
||||
return;
|
||||
}
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
this.levelSignal = signal;
|
||||
this.ensureBars();
|
||||
this.renderLevel(signal?.value ?? 0);
|
||||
if (this.isConnected) {
|
||||
this.subscribe();
|
||||
}
|
||||
}
|
||||
|
||||
connectedCallback(): void {
|
||||
this.ensureBars();
|
||||
this.subscribe();
|
||||
}
|
||||
|
||||
disconnectedCallback(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
}
|
||||
|
||||
private ensureBars(): void {
|
||||
if (!this.firstElementChild) {
|
||||
for (const [index] of BAR_GAINS.entries()) {
|
||||
const bar = document.createElement("span");
|
||||
bar.className = "agent-chat__voice-activity-bar";
|
||||
bar.style.setProperty("--talk-bar-delay", `${index * -70}ms`);
|
||||
this.append(bar);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private subscribe(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = this.levelSignal?.subscribe((level) => this.renderLevel(level)) ?? null;
|
||||
}
|
||||
|
||||
private renderLevel(level: number): void {
|
||||
this.dataset.level = String(level);
|
||||
for (const [index, bar] of [...this.children].entries()) {
|
||||
const gain = BAR_GAINS[index] ?? 1;
|
||||
(bar as HTMLElement).style.setProperty(
|
||||
"--talk-bar-scale",
|
||||
String(0.18 + level * gain * 0.82),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!customElements.get(MICROPHONE_ACTIVITY_TAG)) {
|
||||
customElements.define(MICROPHONE_ACTIVITY_TAG, MicrophoneActivityElement);
|
||||
}
|
||||
|
||||
type ChatVoiceActivityProps = {
|
||||
active?: boolean;
|
||||
status?: RealtimeTalkStatus;
|
||||
detail?: string | null;
|
||||
inputLevel?: RealtimeTalkLevelSignal;
|
||||
onDismissError?: () => void;
|
||||
};
|
||||
|
||||
function activeStatus(
|
||||
status: RealtimeTalkStatus | undefined,
|
||||
): "connecting" | "listening" | "thinking" {
|
||||
return status === "connecting" || status === "thinking" ? status : "listening";
|
||||
}
|
||||
|
||||
function statusLabel(status: RealtimeTalkStatus | undefined, detail: string | null | undefined) {
|
||||
const explicitDetail = detail?.trim();
|
||||
if (explicitDetail) {
|
||||
return explicitDetail;
|
||||
}
|
||||
if (status === "thinking") {
|
||||
return "Asking OpenClaw...";
|
||||
}
|
||||
if (status === "connecting") {
|
||||
return "Connecting voice input...";
|
||||
}
|
||||
return "Listening...";
|
||||
}
|
||||
|
||||
export function renderChatVoiceActivity(
|
||||
props: ChatVoiceActivityProps,
|
||||
): TemplateResult | typeof nothing {
|
||||
if (props.status === "error" && props.detail) {
|
||||
return html`
|
||||
<div class="agent-chat__stt-interim agent-chat__talk-status" role="alert">
|
||||
<span class="agent-chat__talk-status-text">${props.detail}</span>
|
||||
${props.onDismissError
|
||||
? html`
|
||||
<openclaw-tooltip .content=${t("chat.composer.dismissVoiceInputError")}>
|
||||
<button
|
||||
class="callout__dismiss"
|
||||
type="button"
|
||||
@click=${props.onDismissError}
|
||||
aria-label=${t("chat.composer.dismissVoiceInputError")}
|
||||
>
|
||||
${icons.x}
|
||||
</button>
|
||||
</openclaw-tooltip>
|
||||
`
|
||||
: nothing}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
if (!props.active) {
|
||||
return nothing;
|
||||
}
|
||||
|
||||
const status = activeStatus(props.status);
|
||||
return html`
|
||||
<div
|
||||
class="agent-chat__stt-interim agent-chat__talk-status agent-chat__talk-status--active"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
aria-atomic="true"
|
||||
>
|
||||
<openclaw-microphone-activity
|
||||
class="agent-chat__voice-activity"
|
||||
data-status=${status}
|
||||
data-source="microphone"
|
||||
role="img"
|
||||
aria-label=${t("chat.composer.microphoneInput")}
|
||||
.signal=${props.inputLevel ?? EMPTY_LEVEL_SIGNAL}
|
||||
>
|
||||
</openclaw-microphone-activity>
|
||||
<span class="agent-chat__sr-only">${statusLabel(props.status, props.detail)}</span>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
115
ui/src/pages/chat/realtime-talk-audio.test.ts
Normal file
115
ui/src/pages/chat/realtime-talk-audio.test.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
// @vitest-environment jsdom
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
measureRealtimeTalkAudioFrame,
|
||||
RealtimeTalkAudioLevelMeter,
|
||||
RealtimeTalkMediaStreamMeter,
|
||||
} from "./realtime-talk-audio.ts";
|
||||
|
||||
describe("RealtimeTalkAudioLevelMeter", () => {
|
||||
it("keeps silence flat and makes louder speech more visible", () => {
|
||||
const silentFrame = measureRealtimeTalkAudioFrame(new Float32Array(512));
|
||||
const silent = new RealtimeTalkAudioLevelMeter().sample(new Float32Array(512));
|
||||
const quiet = new RealtimeTalkAudioLevelMeter().sample(new Float32Array(512).fill(0.03));
|
||||
const loud = new RealtimeTalkAudioLevelMeter().sample(new Float32Array(512).fill(0.25));
|
||||
|
||||
expect(silentFrame).toEqual({ peak: 0, rms: 0 });
|
||||
expect(silent).toBe(0);
|
||||
expect(quiet).toBeGreaterThan(0);
|
||||
expect(loud).toBeGreaterThan(quiet);
|
||||
expect(loud).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("ignores invalid samples and releases smoothly after speech", () => {
|
||||
const meter = new RealtimeTalkAudioLevelMeter();
|
||||
const invalid = meter.sample(new Float32Array([Number.NaN, Number.POSITIVE_INFINITY]));
|
||||
const speech = meter.sample(new Float32Array(512).fill(0.4));
|
||||
const firstSilence = meter.sample(new Float32Array(512));
|
||||
let settled = firstSilence;
|
||||
for (let index = 0; index < 12; index += 1) {
|
||||
settled = meter.sample(new Float32Array(512));
|
||||
}
|
||||
|
||||
expect(invalid).toBe(0);
|
||||
expect(firstSilence).toBeLessThan(speech);
|
||||
expect(firstSilence).toBeGreaterThan(0);
|
||||
expect(settled).toBeLessThan(firstSilence);
|
||||
});
|
||||
|
||||
it("settles steady room noise while keeping a speech burst visible", () => {
|
||||
const meter = new RealtimeTalkAudioLevelMeter();
|
||||
let noiseLevel = 0;
|
||||
for (let index = 0; index < 80; index += 1) {
|
||||
noiseLevel = meter.sample(new Float32Array(512).fill(0.015));
|
||||
}
|
||||
const speechLevel = meter.sample(new Float32Array(512).fill(0.2));
|
||||
|
||||
expect(noiseLevel).toBeLessThan(0.08);
|
||||
expect(speechLevel).toBeGreaterThan(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("RealtimeTalkMediaStreamMeter", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("samples a WebRTC input stream and resets its level when stopped", () => {
|
||||
vi.useFakeTimers();
|
||||
const close = vi.fn(async () => undefined);
|
||||
const disconnectSource = vi.fn();
|
||||
const disconnectAnalyser = vi.fn();
|
||||
const analyser = {
|
||||
fftSize: 0,
|
||||
smoothingTimeConstant: 0,
|
||||
disconnect: disconnectAnalyser,
|
||||
getFloatTimeDomainData: vi
|
||||
.fn()
|
||||
.mockImplementationOnce((samples: Float32Array) => samples.fill(0.2))
|
||||
.mockImplementation((samples: Float32Array) => samples.fill(0)),
|
||||
};
|
||||
class MockAudioContext {
|
||||
readonly close = close;
|
||||
createMediaStreamSource() {
|
||||
return { connect: vi.fn(), disconnect: disconnectSource };
|
||||
}
|
||||
createAnalyser() {
|
||||
return analyser;
|
||||
}
|
||||
}
|
||||
vi.stubGlobal("AudioContext", MockAudioContext);
|
||||
const onLevel = vi.fn();
|
||||
const meter = new RealtimeTalkMediaStreamMeter(onLevel);
|
||||
|
||||
meter.start({} as MediaStream);
|
||||
vi.advanceTimersByTime(3_000);
|
||||
|
||||
expect(onLevel.mock.calls.some(([level]) => level > 0)).toBe(true);
|
||||
expect(onLevel).toHaveBeenLastCalledWith(0);
|
||||
meter.stop();
|
||||
|
||||
expect(analyser.fftSize).toBe(512);
|
||||
expect(onLevel).toHaveBeenLastCalledWith(0);
|
||||
expect(disconnectSource).toHaveBeenCalledOnce();
|
||||
expect(disconnectAnalyser).toHaveBeenCalledOnce();
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("closes an owned AudioContext when analyser setup fails", () => {
|
||||
const close = vi.fn(async () => undefined);
|
||||
class MockAudioContext {
|
||||
readonly close = close;
|
||||
createMediaStreamSource() {
|
||||
throw new Error("source unavailable");
|
||||
}
|
||||
}
|
||||
vi.stubGlobal("AudioContext", MockAudioContext);
|
||||
const onLevel = vi.fn();
|
||||
|
||||
new RealtimeTalkMediaStreamMeter(onLevel).start({} as MediaStream);
|
||||
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
expect(onLevel).toHaveBeenLastCalledWith(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -28,6 +28,122 @@ export function floatToPcm16(samples: Float32Array): Uint8Array {
|
|||
return bytes;
|
||||
}
|
||||
|
||||
export type RealtimeTalkAudioFrame = {
|
||||
peak: number;
|
||||
rms: number;
|
||||
};
|
||||
|
||||
export function measureRealtimeTalkAudioFrame(samples: Float32Array): RealtimeTalkAudioFrame {
|
||||
let peak = 0;
|
||||
let sumSquares = 0;
|
||||
for (const rawSample of samples) {
|
||||
const sample = Number.isFinite(rawSample) ? rawSample : 0;
|
||||
const absolute = Math.abs(sample);
|
||||
peak = Math.max(peak, absolute);
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
return {
|
||||
peak,
|
||||
rms: samples.length > 0 ? Math.sqrt(sumSquares / samples.length) : 0,
|
||||
};
|
||||
}
|
||||
|
||||
export class RealtimeTalkAudioLevelMeter {
|
||||
private level = 0;
|
||||
private noiseFloor = 0.01;
|
||||
|
||||
sample(samples: Float32Array): number {
|
||||
const frame = measureRealtimeTalkAudioFrame(samples);
|
||||
const signal = 0.65 * frame.rms + 0.35 * frame.peak;
|
||||
if (signal <= Math.max(0.02, this.noiseFloor * 2)) {
|
||||
this.noiseFloor = 0.95 * this.noiseFloor + 0.05 * signal;
|
||||
}
|
||||
const gatedSignal = Math.max(0, signal - this.noiseFloor * 1.5);
|
||||
const target = Math.min(1, Math.sqrt(gatedSignal / 0.18));
|
||||
const smoothing = target > this.level ? 0.65 : 0.18;
|
||||
this.level = smoothing * target + (1 - smoothing) * this.level;
|
||||
if (this.level < 0.01) {
|
||||
this.level = 0;
|
||||
}
|
||||
return this.level;
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.level = 0;
|
||||
this.noiseFloor = 0.01;
|
||||
}
|
||||
}
|
||||
|
||||
export class RealtimeTalkMediaStreamMeter {
|
||||
private context: AudioContext | null = null;
|
||||
private source: MediaStreamAudioSourceNode | null = null;
|
||||
private analyser: AnalyserNode | null = null;
|
||||
private timer: ReturnType<typeof globalThis.setInterval> | null = null;
|
||||
private ownsContext = false;
|
||||
private readonly levelMeter = new RealtimeTalkAudioLevelMeter();
|
||||
private readonly samples = new Float32Array(512);
|
||||
private lastLevel = -1;
|
||||
|
||||
constructor(private readonly onLevel: (level: number) => void) {}
|
||||
|
||||
start(media: MediaStream, sharedContext?: AudioContext): void {
|
||||
this.stop(false);
|
||||
try {
|
||||
const context = sharedContext ?? new AudioContext();
|
||||
this.context = context;
|
||||
this.ownsContext = !sharedContext;
|
||||
const source = context.createMediaStreamSource(media);
|
||||
this.source = source;
|
||||
const analyser = context.createAnalyser();
|
||||
this.analyser = analyser;
|
||||
analyser.fftSize = this.samples.length;
|
||||
analyser.smoothingTimeConstant = 0;
|
||||
source.connect(analyser);
|
||||
this.publishCurrentLevel();
|
||||
this.timer = globalThis.setInterval(() => this.publishCurrentLevel(), 100);
|
||||
} catch {
|
||||
// Metering is feedback only; capture must still work if Web Audio analysis
|
||||
// is unavailable in an otherwise functional WebRTC browser.
|
||||
this.stop();
|
||||
}
|
||||
}
|
||||
|
||||
stop(notify = true): void {
|
||||
if (this.timer !== null) {
|
||||
globalThis.clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
this.source?.disconnect();
|
||||
this.source = null;
|
||||
this.analyser?.disconnect();
|
||||
this.analyser = null;
|
||||
if (this.ownsContext) {
|
||||
void this.context?.close();
|
||||
}
|
||||
this.context = null;
|
||||
this.ownsContext = false;
|
||||
this.levelMeter.reset();
|
||||
this.lastLevel = -1;
|
||||
if (notify) {
|
||||
this.onLevel(0);
|
||||
}
|
||||
}
|
||||
|
||||
private publishCurrentLevel(): void {
|
||||
if (!this.analyser) {
|
||||
return;
|
||||
}
|
||||
this.samples.fill(0);
|
||||
this.analyser.getFloatTimeDomainData(this.samples);
|
||||
const level = Math.round(this.levelMeter.sample(this.samples) * 100) / 100;
|
||||
if (level === this.lastLevel) {
|
||||
return;
|
||||
}
|
||||
this.lastLevel = level;
|
||||
this.onLevel(level);
|
||||
}
|
||||
}
|
||||
|
||||
function pcm16ToFloat(bytes: Uint8Array): Float32Array {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const samples = new Float32Array(Math.floor(bytes.byteLength / 2));
|
||||
|
|
|
|||
|
|
@ -44,6 +44,15 @@ class MockAudioContext {
|
|||
return processor;
|
||||
}
|
||||
|
||||
createAnalyser() {
|
||||
return {
|
||||
fftSize: 0,
|
||||
smoothingTimeConstant: 0,
|
||||
disconnect: vi.fn(),
|
||||
getFloatTimeDomainData: (samples: Float32Array) => samples.fill(0.25),
|
||||
};
|
||||
}
|
||||
|
||||
createBuffer(_channels: number, length: number, sampleRate: number) {
|
||||
const channel = new Float32Array(length);
|
||||
return {
|
||||
|
|
@ -155,6 +164,30 @@ describe("GatewayRelayRealtimeTalkTransport", () => {
|
|||
transport.stop();
|
||||
});
|
||||
|
||||
it("releases microphone access that resolves after stop", async () => {
|
||||
let resolveMedia: (media: MediaStream) => void = () => undefined;
|
||||
const pendingMedia = new Promise<MediaStream>((resolve) => {
|
||||
resolveMedia = resolve;
|
||||
});
|
||||
getUserMedia.mockReturnValue(pendingMedia);
|
||||
const stopTrack = vi.fn();
|
||||
const onInputLevel = vi.fn();
|
||||
const transport = new GatewayRelayRealtimeTalkTransport(createSession(), {
|
||||
callbacks: { onInputLevel },
|
||||
client: createClient(),
|
||||
sessionKey: "main",
|
||||
});
|
||||
|
||||
const start = transport.start();
|
||||
transport.stop();
|
||||
resolveMedia({ getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream);
|
||||
await start;
|
||||
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(processors).toHaveLength(0);
|
||||
expect(onInputLevel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("forwards common Talk events from Gateway relay frames", async () => {
|
||||
const onTalkEvent = vi.fn();
|
||||
const transport = new GatewayRelayRealtimeTalkTransport(createSession(), {
|
||||
|
|
@ -247,6 +280,23 @@ describe("GatewayRelayRealtimeTalkTransport", () => {
|
|||
transport.stop();
|
||||
});
|
||||
|
||||
it("reports microphone activity and resets it when stopped", async () => {
|
||||
const onInputLevel = vi.fn();
|
||||
const transport = new GatewayRelayRealtimeTalkTransport(createSession(), {
|
||||
callbacks: { onInputLevel },
|
||||
client: createClient(),
|
||||
sessionKey: "main",
|
||||
});
|
||||
|
||||
await transport.start();
|
||||
pumpMicrophone(new Float32Array(4096));
|
||||
pumpMicrophone(new Float32Array(4096).fill(0.25));
|
||||
transport.stop();
|
||||
|
||||
expect(onInputLevel.mock.calls.some(([level]) => level > 0)).toBe(true);
|
||||
expect(onInputLevel).toHaveBeenLastCalledWith(0);
|
||||
});
|
||||
|
||||
it("stops microphone pumping when the relay rejects appended audio", async () => {
|
||||
const onStatus = vi.fn();
|
||||
const client = createClient();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
// Control UI chat module implements realtime talk gateway relay behavior.
|
||||
import { bytesToBase64, floatToPcm16, RealtimeTalkPcmOutputQueue } from "./realtime-talk-audio.ts";
|
||||
import {
|
||||
bytesToBase64,
|
||||
floatToPcm16,
|
||||
measureRealtimeTalkAudioFrame,
|
||||
RealtimeTalkMediaStreamMeter,
|
||||
RealtimeTalkPcmOutputQueue,
|
||||
type RealtimeTalkAudioFrame,
|
||||
} from "./realtime-talk-audio.ts";
|
||||
import { openRealtimeTalkInput } from "./realtime-talk-input.ts";
|
||||
import {
|
||||
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
|
||||
|
|
@ -46,6 +53,7 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport
|
|||
private media: MediaStream | null = null;
|
||||
private inputContext: AudioContext | null = null;
|
||||
private outputContext: AudioContext | null = null;
|
||||
private inputMeter: RealtimeTalkMediaStreamMeter | null = null;
|
||||
private inputSource: MediaStreamAudioSourceNode | null = null;
|
||||
private inputProcessor: ScriptProcessorNode | null = null;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
|
@ -79,13 +87,30 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport
|
|||
}
|
||||
this.handleRelayEvent(evt.payload as GatewayRelayEvent);
|
||||
});
|
||||
this.media = await openRealtimeTalkInput(this.ctx.inputDeviceId, {
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
});
|
||||
let media: MediaStream;
|
||||
try {
|
||||
media = await openRealtimeTalkInput(this.ctx.inputDeviceId, {
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
});
|
||||
} catch (error) {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (this.closed) {
|
||||
media.getTracks().forEach((track) => track.stop());
|
||||
return;
|
||||
}
|
||||
this.media = media;
|
||||
this.inputContext = new AudioContext({ sampleRate: this.session.audio.inputSampleRateHz });
|
||||
this.outputContext = new AudioContext({ sampleRate: this.session.audio.outputSampleRateHz });
|
||||
if (this.ctx.callbacks.onInputLevel) {
|
||||
this.inputMeter = new RealtimeTalkMediaStreamMeter(this.ctx.callbacks.onInputLevel);
|
||||
this.inputMeter.start(this.media, this.inputContext);
|
||||
}
|
||||
this.startMicrophonePump();
|
||||
}
|
||||
|
||||
|
|
@ -109,6 +134,8 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport
|
|||
this.inputProcessor = null;
|
||||
this.inputSource?.disconnect();
|
||||
this.inputSource = null;
|
||||
this.inputMeter?.stop();
|
||||
this.inputMeter = null;
|
||||
this.abortConsults();
|
||||
this.media?.getTracks().forEach((track) => track.stop());
|
||||
this.media = null;
|
||||
|
|
@ -339,20 +366,12 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport
|
|||
}
|
||||
|
||||
private detectBargeInSpeech(samples: Float32Array): boolean {
|
||||
if (!this.outputQueue.isPlaying || this.cancelRequestedForPlayback || samples.length === 0) {
|
||||
if (!this.outputQueue.isPlaying || this.cancelRequestedForPlayback) {
|
||||
this.speechFramesDuringPlayback = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
let sumSquares = 0;
|
||||
let peak = 0;
|
||||
for (const sample of samples) {
|
||||
const abs = Math.abs(sample);
|
||||
peak = Math.max(peak, abs);
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / samples.length);
|
||||
if (rms >= BARGE_IN_RMS_THRESHOLD && peak >= BARGE_IN_PEAK_THRESHOLD) {
|
||||
const frame: RealtimeTalkAudioFrame = measureRealtimeTalkAudioFrame(samples);
|
||||
if (frame.rms >= BARGE_IN_RMS_THRESHOLD && frame.peak >= BARGE_IN_PEAK_THRESHOLD) {
|
||||
this.speechFramesDuringPlayback += 1;
|
||||
} else {
|
||||
this.speechFramesDuringPlayback = 0;
|
||||
|
|
|
|||
|
|
@ -21,6 +21,9 @@ type MockWebSocketEventType = "close" | "error" | "message" | "open";
|
|||
|
||||
const wsInstances: MockGoogleLiveWebSocket[] = [];
|
||||
const createdSources: MockAudioBufferSource[] = [];
|
||||
const inputProcessors: Array<{
|
||||
onaudioprocess: ((event: { inputBuffer: { getChannelData: () => Float32Array } }) => void) | null;
|
||||
}> = [];
|
||||
let getUserMedia: ReturnType<typeof vi.fn>;
|
||||
|
||||
async function flushMicrotasks(): Promise<void> {
|
||||
|
|
@ -96,11 +99,22 @@ class MockAudioContext {
|
|||
}
|
||||
|
||||
createScriptProcessor() {
|
||||
return {
|
||||
const processor = {
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
onaudioprocess: null,
|
||||
};
|
||||
inputProcessors.push(processor);
|
||||
return processor;
|
||||
}
|
||||
|
||||
createAnalyser() {
|
||||
return {
|
||||
fftSize: 0,
|
||||
smoothingTimeConstant: 0,
|
||||
disconnect: vi.fn(),
|
||||
getFloatTimeDomainData: (samples: Float32Array) => samples.fill(0.25),
|
||||
};
|
||||
}
|
||||
|
||||
createBuffer(_channels: number, length: number, sampleRate: number) {
|
||||
|
|
@ -175,6 +189,14 @@ function latestWebSocket(): MockGoogleLiveWebSocket {
|
|||
return ws;
|
||||
}
|
||||
|
||||
function pumpMicrophone(samples: Float32Array): void {
|
||||
const processor = inputProcessors.at(-1);
|
||||
if (!processor) {
|
||||
throw new Error("missing microphone processor");
|
||||
}
|
||||
processor.onaudioprocess?.({ inputBuffer: { getChannelData: () => samples } });
|
||||
}
|
||||
|
||||
function requireFirstTalkEvent(onTalkEvent: ReturnType<typeof vi.fn>): Record<string, unknown> {
|
||||
const [call] = onTalkEvent.mock.calls;
|
||||
if (!call) {
|
||||
|
|
@ -191,6 +213,7 @@ describe("GoogleLiveRealtimeTalkTransport", () => {
|
|||
beforeEach(() => {
|
||||
wsInstances.length = 0;
|
||||
createdSources.length = 0;
|
||||
inputProcessors.length = 0;
|
||||
vi.stubGlobal("WebSocket", MockGoogleLiveWebSocket);
|
||||
vi.stubGlobal("AudioContext", MockAudioContext);
|
||||
getUserMedia = vi.fn(async () => ({
|
||||
|
|
@ -218,6 +241,27 @@ describe("GoogleLiveRealtimeTalkTransport", () => {
|
|||
transport.stop();
|
||||
});
|
||||
|
||||
it("releases microphone access that resolves after stop", async () => {
|
||||
let resolveMedia: (media: MediaStream) => void = () => undefined;
|
||||
const pendingMedia = new Promise<MediaStream>((resolve) => {
|
||||
resolveMedia = resolve;
|
||||
});
|
||||
getUserMedia.mockReturnValue(pendingMedia);
|
||||
const stopTrack = vi.fn();
|
||||
const onInputLevel = vi.fn();
|
||||
const transport = createTransport({ onInputLevel });
|
||||
|
||||
const start = transport.start();
|
||||
transport.stop();
|
||||
resolveMedia({ getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream);
|
||||
await start;
|
||||
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(inputProcessors).toHaveLength(0);
|
||||
expect(wsInstances).toHaveLength(0);
|
||||
expect(onInputLevel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("requests ArrayBuffer frames and decodes binary setup messages", async () => {
|
||||
const onStatus = vi.fn();
|
||||
const onTalkEvent = vi.fn();
|
||||
|
|
@ -235,6 +279,20 @@ describe("GoogleLiveRealtimeTalkTransport", () => {
|
|||
expect(readyEvent.transport).toBe("provider-websocket");
|
||||
});
|
||||
|
||||
it("reports microphone activity and resets it when stopped", async () => {
|
||||
const onInputLevel = vi.fn();
|
||||
const transport = createTransport({ onInputLevel });
|
||||
|
||||
await transport.start();
|
||||
latestWebSocket().emitOpen();
|
||||
pumpMicrophone(new Float32Array(4096));
|
||||
pumpMicrophone(new Float32Array(4096).fill(0.25));
|
||||
transport.stop();
|
||||
|
||||
expect(onInputLevel.mock.calls.some(([level]) => level > 0)).toBe(true);
|
||||
expect(onInputLevel).toHaveBeenLastCalledWith(0);
|
||||
});
|
||||
|
||||
it("decodes Blob setup messages", async () => {
|
||||
const onStatus = vi.fn();
|
||||
const transport = createTransport({ onStatus });
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import {
|
|||
base64ToBytes,
|
||||
bytesToBase64,
|
||||
floatToPcm16,
|
||||
RealtimeTalkMediaStreamMeter,
|
||||
RealtimeTalkPcmOutputQueue,
|
||||
} from "./realtime-talk-audio.ts";
|
||||
import { openRealtimeTalkInput } from "./realtime-talk-input.ts";
|
||||
|
|
@ -81,6 +82,7 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
private media: MediaStream | null = null;
|
||||
private inputContext: AudioContext | null = null;
|
||||
private outputContext: AudioContext | null = null;
|
||||
private inputMeter: RealtimeTalkMediaStreamMeter | null = null;
|
||||
private inputSource: MediaStreamAudioSourceNode | null = null;
|
||||
private inputProcessor: ScriptProcessorNode | null = null;
|
||||
private closed = false;
|
||||
|
|
@ -105,9 +107,26 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
}
|
||||
const wsUrl = buildGoogleLiveUrl(this.session);
|
||||
this.closed = false;
|
||||
this.media = await openRealtimeTalkInput(this.ctx.inputDeviceId);
|
||||
let media: MediaStream;
|
||||
try {
|
||||
media = await openRealtimeTalkInput(this.ctx.inputDeviceId);
|
||||
} catch (error) {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (this.closed) {
|
||||
media.getTracks().forEach((track) => track.stop());
|
||||
return;
|
||||
}
|
||||
this.media = media;
|
||||
this.inputContext = new AudioContext({ sampleRate: this.session.audio.inputSampleRateHz });
|
||||
this.outputContext = new AudioContext({ sampleRate: this.session.audio.outputSampleRateHz });
|
||||
if (this.ctx.callbacks.onInputLevel) {
|
||||
this.inputMeter = new RealtimeTalkMediaStreamMeter(this.ctx.callbacks.onInputLevel);
|
||||
this.inputMeter.start(this.media, this.inputContext);
|
||||
}
|
||||
this.ws = new WebSocket(wsUrl);
|
||||
this.ws.binaryType = "arraybuffer";
|
||||
this.ws.addEventListener("open", () => {
|
||||
|
|
@ -146,6 +165,8 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
this.inputProcessor = null;
|
||||
this.inputSource?.disconnect();
|
||||
this.inputSource = null;
|
||||
this.inputMeter?.stop();
|
||||
this.inputMeter = null;
|
||||
this.media?.getTracks().forEach((track) => track.stop());
|
||||
this.media = null;
|
||||
this.stopOutput();
|
||||
|
|
@ -167,7 +188,8 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
if (this.ws?.readyState !== WebSocket.OPEN) {
|
||||
return;
|
||||
}
|
||||
const pcm = floatToPcm16(event.inputBuffer.getChannelData(0));
|
||||
const samples = event.inputBuffer.getChannelData(0);
|
||||
const pcm = floatToPcm16(samples);
|
||||
this.send({
|
||||
realtimeInput: {
|
||||
audio: {
|
||||
|
|
|
|||
28
ui/src/pages/chat/realtime-talk-level.ts
Normal file
28
ui/src/pages/chat/realtime-talk-level.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
export type RealtimeTalkLevelListener = (level: number) => void;
|
||||
|
||||
export class RealtimeTalkLevelSignal {
|
||||
private listeners = new Set<RealtimeTalkLevelListener>();
|
||||
private currentLevel = 0;
|
||||
|
||||
get value(): number {
|
||||
return this.currentLevel;
|
||||
}
|
||||
|
||||
set(rawLevel: number): void {
|
||||
const boundedLevel = Number.isFinite(rawLevel) ? Math.min(1, Math.max(0, rawLevel)) : 0;
|
||||
const level = Math.round(boundedLevel * 100) / 100;
|
||||
if (level === this.currentLevel) {
|
||||
return;
|
||||
}
|
||||
this.currentLevel = level;
|
||||
for (const listener of this.listeners) {
|
||||
listener(level);
|
||||
}
|
||||
}
|
||||
|
||||
subscribe(listener: RealtimeTalkLevelListener): () => void {
|
||||
this.listeners.add(listener);
|
||||
listener(this.currentLevel);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
}
|
||||
|
|
@ -16,6 +16,7 @@ export type RealtimeTalkEvent = TalkEvent;
|
|||
|
||||
export type RealtimeTalkCallbacks = {
|
||||
onStatus?: (status: RealtimeTalkStatus, detail?: string) => void;
|
||||
onInputLevel?: (level: number) => void;
|
||||
onTranscript?: (entry: { role: "user" | "assistant"; text: string; final: boolean }) => void;
|
||||
onTalkEvent?: (event: RealtimeTalkEvent) => void;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -205,6 +205,35 @@ describe("WebRtcSdpRealtimeTalkTransport", () => {
|
|||
transport.stop();
|
||||
});
|
||||
|
||||
it("reports microphone activity and resets it when stopped", async () => {
|
||||
stubAnswerSdpFetch();
|
||||
const close = vi.fn(async () => undefined);
|
||||
class MockAudioContext {
|
||||
readonly close = close;
|
||||
createMediaStreamSource() {
|
||||
return { connect: vi.fn(), disconnect: vi.fn() };
|
||||
}
|
||||
createAnalyser() {
|
||||
return {
|
||||
fftSize: 0,
|
||||
smoothingTimeConstant: 0,
|
||||
disconnect: vi.fn(),
|
||||
getFloatTimeDomainData: (samples: Float32Array) => samples.fill(0.25),
|
||||
};
|
||||
}
|
||||
}
|
||||
vi.stubGlobal("AudioContext", MockAudioContext);
|
||||
const onInputLevel = vi.fn();
|
||||
const transport = createOpenAiTransport({}, { onInputLevel });
|
||||
|
||||
await transport.start();
|
||||
transport.stop();
|
||||
|
||||
expect(onInputLevel.mock.calls.some(([level]) => level > 0)).toBe(true);
|
||||
expect(onInputLevel).toHaveBeenLastCalledWith(0);
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("does not continue WebRTC setup when stopped while microphone access is pending", async () => {
|
||||
const fetchMock = vi.fn(async () => new Response("answer-sdp"));
|
||||
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
// Control UI chat module implements realtime talk webrtc behavior.
|
||||
import { RealtimeTalkMediaStreamMeter } from "./realtime-talk-audio.ts";
|
||||
import { openRealtimeTalkInput } from "./realtime-talk-input.ts";
|
||||
import type { RealtimeTalkWebRtcSdpSessionResult } from "./realtime-talk-shared.ts";
|
||||
import {
|
||||
|
|
@ -42,6 +43,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
private channel: RTCDataChannel | null = null;
|
||||
private media: MediaStream | null = null;
|
||||
private audio: HTMLAudioElement | null = null;
|
||||
private inputMeter: RealtimeTalkMediaStreamMeter | null = null;
|
||||
private closed = false;
|
||||
private responseActive = false;
|
||||
private responseCreateInFlight = false;
|
||||
|
|
@ -82,6 +84,10 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
return;
|
||||
}
|
||||
this.media = media;
|
||||
if (this.ctx.callbacks.onInputLevel) {
|
||||
this.inputMeter = new RealtimeTalkMediaStreamMeter(this.ctx.callbacks.onInputLevel);
|
||||
this.inputMeter.start(media);
|
||||
}
|
||||
for (const track of media.getAudioTracks()) {
|
||||
peer.addTrack(track, media);
|
||||
}
|
||||
|
|
@ -185,6 +191,8 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
|
|||
this.peer = null;
|
||||
this.media?.getTracks().forEach((track) => track.stop());
|
||||
this.media = null;
|
||||
this.inputMeter?.stop();
|
||||
this.inputMeter = null;
|
||||
this.audio?.remove();
|
||||
this.audio = null;
|
||||
for (const controller of this.consultAbortControllers) {
|
||||
|
|
|
|||
|
|
@ -1521,11 +1521,121 @@ openclaw-chat-page {
|
|||
color: var(--text);
|
||||
}
|
||||
|
||||
.agent-chat__talk-status--active {
|
||||
justify-content: center;
|
||||
min-height: 30px;
|
||||
padding-top: 8px;
|
||||
}
|
||||
|
||||
.agent-chat__talk-status-text {
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.agent-chat__voice-activity {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 4px;
|
||||
width: 64px;
|
||||
height: 24px;
|
||||
overflow: hidden;
|
||||
border: 1px solid color-mix(in srgb, var(--accent) 18%, var(--border));
|
||||
border-radius: 999px;
|
||||
background: color-mix(in srgb, var(--accent) 7%, var(--card));
|
||||
box-shadow: 0 0 16px color-mix(in srgb, var(--accent) 12%, transparent);
|
||||
}
|
||||
|
||||
.agent-chat__voice-activity::before {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
border-radius: inherit;
|
||||
background: radial-gradient(
|
||||
circle at 50% 50%,
|
||||
color-mix(in srgb, var(--accent) 12%, transparent),
|
||||
transparent 72%
|
||||
);
|
||||
content: "";
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.agent-chat__voice-activity-bar {
|
||||
position: relative;
|
||||
width: 3px;
|
||||
height: 16px;
|
||||
flex: 0 0 3px;
|
||||
border-radius: 999px;
|
||||
background: color-mix(in srgb, var(--accent) 88%, var(--text));
|
||||
box-shadow: 0 0 5px color-mix(in srgb, var(--accent) 30%, transparent);
|
||||
opacity: 0.9;
|
||||
transform: scaleY(var(--talk-bar-scale, 0.18));
|
||||
transform-origin: center;
|
||||
transition:
|
||||
transform 90ms linear,
|
||||
opacity var(--duration-fast) ease;
|
||||
}
|
||||
|
||||
.agent-chat__voice-activity[data-status="connecting"] .agent-chat__voice-activity-bar {
|
||||
animation: agent-chat-voice-connecting 1.05s ease-in-out infinite;
|
||||
animation-delay: var(--talk-bar-delay);
|
||||
}
|
||||
|
||||
.agent-chat__voice-activity[data-status="thinking"] .agent-chat__voice-activity-bar {
|
||||
animation: agent-chat-voice-thinking 0.9s ease-in-out infinite;
|
||||
animation-delay: var(--talk-bar-delay);
|
||||
}
|
||||
|
||||
@keyframes agent-chat-voice-connecting {
|
||||
0%,
|
||||
100% {
|
||||
opacity: 0.42;
|
||||
transform: scaleY(0.2);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.9;
|
||||
transform: scaleY(0.58);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes agent-chat-voice-thinking {
|
||||
0%,
|
||||
100% {
|
||||
opacity: 0.4;
|
||||
transform: scaleY(0.24);
|
||||
}
|
||||
45% {
|
||||
opacity: 1;
|
||||
transform: scaleY(0.92);
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.agent-chat__voice-activity-bar {
|
||||
animation: none !important;
|
||||
opacity: 0.78;
|
||||
transform: scaleY(0.38);
|
||||
transition: none !important;
|
||||
}
|
||||
|
||||
.agent-chat__voice-activity[data-status="connecting"] .agent-chat__voice-activity-bar {
|
||||
opacity: 0.48;
|
||||
transform: scaleY(0.18);
|
||||
}
|
||||
|
||||
.agent-chat__voice-activity[data-status="connecting"]
|
||||
.agent-chat__voice-activity-bar:nth-child(4) {
|
||||
opacity: 0.9;
|
||||
transform: scaleY(0.56);
|
||||
}
|
||||
|
||||
.agent-chat__voice-activity[data-status="thinking"] .agent-chat__voice-activity-bar:nth-child(3),
|
||||
.agent-chat__voice-activity[data-status="thinking"] .agent-chat__voice-activity-bar:nth-child(4),
|
||||
.agent-chat__voice-activity[data-status="thinking"] .agent-chat__voice-activity-bar:nth-child(5) {
|
||||
transform: scaleY(0.72);
|
||||
}
|
||||
}
|
||||
|
||||
.agent-chat__voice-turns {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue