diff --git a/ui/src/e2e/browser-talk-start-stop.e2e.test.ts b/ui/src/e2e/browser-talk-start-stop.e2e.test.ts index 28ee32d8a89..2316a1d3008 100644 --- a/ui/src/e2e/browser-talk-start-stop.e2e.test.ts +++ b/ui/src/e2e/browser-talk-start-stop.e2e.test.ts @@ -18,7 +18,18 @@ let server: ControlUiE2eServer; async function installTalkBrowserFixtures(page: Page) { await page.addInitScript(() => { - const state = { audioContextsClosed: 0, tracksStopped: 0, constraints: [] as unknown[] }; + type InputProcessor = { + onaudioprocess: + | ((event: { inputBuffer: { getChannelData: () => Float32Array } }) => void) + | null; + }; + const state = { + audioContextsClosed: 0, + tracksStopped: 0, + constraints: [] as unknown[], + inputProcessor: null as InputProcessor | null, + meterLevel: 0, + }; const track = { stop: () => (state.tracksStopped += 1) }; Object.defineProperty(navigator, "mediaDevices", { configurable: true, @@ -49,7 +60,20 @@ async function installTalkBrowserFixtures(page: Page) { } createScriptProcessor() { - return { connect() {}, disconnect() {}, onaudioprocess: null }; + const processor = { connect() {}, disconnect() {}, onaudioprocess: null }; + state.inputProcessor = processor; + return processor; + } + + createAnalyser() { + return { + fftSize: 0, + smoothingTimeConstant: 0, + disconnect() {}, + getFloatTimeDomainData(samples: Float32Array) { + samples.fill(state.meterLevel); + }, + }; } async close() { @@ -116,6 +140,7 @@ describeControlUiE2e("Control UI browser Talk", () => { await installTalkBrowserFixtures(page); try { + await page.emulateMedia({ reducedMotion: "reduce" }); await page.goto(`${server.baseUrl}chat`); await page.setViewportSize({ width: 320, height: 720 }); await expect @@ -157,18 +182,77 @@ describeControlUiE2e("Control UI browser Talk", () => { "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained?access_token=auth_tokens%2Fbrowser-talk-e2e", ]); + await expect + .poll(() => page.locator('.agent-chat__voice-activity[data-status="connecting"]').count()) + .toBe(1); + await expect + .poll(() => + page + .locator(".agent-chat__voice-activity-bar") + .first() + .evaluate((element) => { + return getComputedStyle(element).animationName; + }), + ) + .toBe("none"); + const connectingReducedMotionTransform = await page + .locator(".agent-chat__voice-activity-bar") + .first() + .evaluate((element) => getComputedStyle(element).transform); + await gateway.deliverLatest({ setupComplete: {} }); await expect - .poll(async () => - (await page.locator(".agent-chat__talk-status-text").textContent())?.trim(), - ) + .poll(() => page.locator('.agent-chat__voice-activity[data-status="listening"]').count()) + .toBe(1); + await expect.poll(() => page.locator(".agent-chat__talk-status-text").count()).toBe(0); + await expect + .poll(() => page.locator('[role="status"] .agent-chat__sr-only').textContent()) .toBe("Listening..."); + const reducedMotionTransform = await page + .locator(".agent-chat__voice-activity-bar") + .first() + .evaluate((element) => getComputedStyle(element).transform); + expect(reducedMotionTransform).not.toBe(connectingReducedMotionTransform); + + await page.evaluate(() => { + const state = ( + window as Window & { + openclawTalkE2eState?: { + inputProcessor?: { + onaudioprocess?: (event: { + inputBuffer: { getChannelData: () => Float32Array }; + }) => void; + }; + meterLevel?: number; + }; + } + ).openclawTalkE2eState; + if (state) { + state.meterLevel = 0.25; + } + state?.inputProcessor?.onaudioprocess?.({ + inputBuffer: { getChannelData: () => new Float32Array(4096).fill(0.25) }, + }); + }); + await expect + .poll(async () => + Number(await page.locator(".agent-chat__voice-activity").getAttribute("data-level")), + ) + .toBeGreaterThan(0); + await expect + .poll(() => + page + .locator(".agent-chat__voice-activity-bar") + .first() + .evaluate((element) => getComputedStyle(element).transform), + ) + .toBe(reducedMotionTransform); await page.getByRole("button", { name: "Stop voice input" }).click(); await expect .poll(() => page.getByRole("button", { name: "Start voice input" }).isVisible()) .toBe(true); - await expect.poll(() => page.locator(".agent-chat__talk-status-text").count()).toBe(0); + await expect.poll(() => page.locator(".agent-chat__voice-activity").count()).toBe(0); await expect .poll(() => page.evaluate(() => { diff --git a/ui/src/pages/chat/chat-pane.ts b/ui/src/pages/chat/chat-pane.ts index 72e773cf702..15959cd528f 100644 --- a/ui/src/pages/chat/chat-pane.ts +++ b/ui/src/pages/chat/chat-pane.ts @@ -798,6 +798,7 @@ class ChatPane extends LitElement { state.realtimeTalkSession = null; state.realtimeTalkActive = false; state.realtimeTalkStatus = "idle"; + state.realtimeTalkInputLevel.set(0); state.resetToolStream(); state.requestUpdate?.(); return; @@ -895,6 +896,7 @@ class ChatPane extends LitElement { realtimeTalkActive: state.realtimeTalkActive, realtimeTalkStatus: state.realtimeTalkStatus, realtimeTalkDetail: state.realtimeTalkDetail, + realtimeTalkInputLevel: state.realtimeTalkInputLevel, realtimeTalkConversation: state.realtimeTalkConversation, connected: state.connected, canSend: state.connected && !selectedSessionArchived, diff --git a/ui/src/pages/chat/chat-realtime.test.ts b/ui/src/pages/chat/chat-realtime.test.ts index a4717b5fc28..881ed3faef9 100644 --- a/ui/src/pages/chat/chat-realtime.test.ts +++ b/ui/src/pages/chat/chat-realtime.test.ts @@ -19,6 +19,7 @@ import { createInitialChatRealtimeState, type ChatRealtimeState, } from "./chat-realtime.ts"; +import type { RealtimeTalkCallbacks } from "./realtime-talk.ts"; function mediaDevice(kind: MediaDeviceKind, deviceId: string, label: string): MediaDeviceInfo { return { kind, deviceId, label, groupId: "", toJSON: () => ({}) } as MediaDeviceInfo; @@ -44,7 +45,8 @@ describe("chat realtime microphone selection", () => { beforeEach(() => { localStorage.clear(); realtimeTalkSessionCtor.mockClear(); - sessionStart.mockClear(); + sessionStart.mockReset(); + sessionStart.mockResolvedValue(undefined); sessionStop.mockClear(); }); @@ -75,6 +77,63 @@ describe("chat realtime microphone selection", () => { expect(sessionStart).toHaveBeenCalledOnce(); }); + it("propagates normalized microphone levels and resets them on error", async () => { + const state = createState(); + await state.toggleRealtimeTalk(); + const constructorCalls = realtimeTalkSessionCtor.mock.calls as unknown[][]; + const callbacks = constructorCalls[0]?.[2] as RealtimeTalkCallbacks | undefined; + if (!callbacks) { + throw new Error("expected realtime callbacks"); + } + + const updatesBeforeLevels = vi.mocked(state.requestUpdate).mock.calls.length; + callbacks.onInputLevel?.(0.456); + expect(state.realtimeTalkInputLevel.value).toBe(0.46); + + callbacks.onInputLevel?.(2); + expect(state.realtimeTalkInputLevel.value).toBe(1); + expect(state.requestUpdate).toHaveBeenCalledTimes(updatesBeforeLevels); + + callbacks.onStatus?.("error", "capture failed"); + expect(state.realtimeTalkInputLevel.value).toBe(0); + }); + + it("ignores a stopped session that rejects after its replacement starts", async () => { + let rejectFirstStart: (error: Error) => void = () => undefined; + sessionStart.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectFirstStart = reject; + }), + ); + const state = createState(); + + const firstStart = state.toggleRealtimeTalk(); + await vi.waitFor(() => expect(realtimeTalkSessionCtor).toHaveBeenCalledTimes(1)); + const firstCallbacks = (realtimeTalkSessionCtor.mock.calls as unknown[][])[0]?.[2] as + | RealtimeTalkCallbacks + | undefined; + await state.toggleRealtimeTalk(); + await state.toggleRealtimeTalk(); + const secondSession = realtimeTalkSessionCtor.mock.results[1]?.value; + const secondCallbacks = (realtimeTalkSessionCtor.mock.calls as unknown[][])[1]?.[2] as + | RealtimeTalkCallbacks + | undefined; + secondCallbacks?.onStatus?.("listening"); + + rejectFirstStart(new Error("late setup failure")); + await firstStart; + firstCallbacks?.onInputLevel?.(0.9); + firstCallbacks?.onTranscript?.({ role: "user", text: "stale", final: true }); + firstCallbacks?.onStatus?.("error", "stale failure"); + + expect(state.realtimeTalkSession).toBe(secondSession); + expect(state.realtimeTalkActive).toBe(true); + expect(state.realtimeTalkStatus).toBe("listening"); + expect(state.realtimeTalkInputLevel.value).toBe(0); + expect(state.realtimeTalkConversation).toEqual([]); + }); + it("does not reject a persisted input from incomplete passive discovery", async () => { vi.stubGlobal("navigator", { mediaDevices: { diff --git a/ui/src/pages/chat/chat-realtime.ts b/ui/src/pages/chat/chat-realtime.ts index 14109dc9a39..2d39509fc2b 100644 --- a/ui/src/pages/chat/chat-realtime.ts +++ b/ui/src/pages/chat/chat-realtime.ts @@ -9,6 +9,7 @@ import { type RealtimeTalkConversationState, } from "./realtime-talk-conversation.ts"; import { discoverRealtimeTalkInputs, type RealtimeTalkInputDevice } from "./realtime-talk-input.ts"; +import { RealtimeTalkLevelSignal } from "./realtime-talk-level.ts"; import { RealtimeTalkSession, type RealtimeTalkLaunchOptions, @@ -41,6 +42,7 @@ export type ChatRealtimeState = { realtimeTalkActive: boolean; realtimeTalkStatus: RealtimeTalkStatus; realtimeTalkDetail: string | null; + realtimeTalkInputLevel: RealtimeTalkLevelSignal; realtimeTalkConversation: RealtimeTalkConversationEntry[]; realtimeTalkOptions: RealtimeTalkOptions; realtimeTalkInputDevices: RealtimeTalkInputDevice[]; @@ -71,6 +73,7 @@ export function createInitialChatRealtimeState(inputDeviceId = "") { realtimeTalkActive: false, realtimeTalkStatus: "idle" as RealtimeTalkStatus, realtimeTalkDetail: null, + realtimeTalkInputLevel: new RealtimeTalkLevelSignal(), realtimeTalkConversation: [], realtimeTalkOptions: createDefaultRealtimeTalkOptions(), realtimeTalkInputDevices: [] as RealtimeTalkInputDevice[], @@ -97,6 +100,7 @@ export function dismissRealtimeTalkError(state: ChatRealtimeState) { state.realtimeTalkActive = false; state.realtimeTalkStatus = "idle"; state.realtimeTalkDetail = null; + state.realtimeTalkInputLevel.set(0); state.resetRealtimeTalkConversation(); } @@ -167,6 +171,7 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) { state.realtimeTalkActive = false; state.realtimeTalkStatus = "idle"; state.realtimeTalkDetail = null; + state.realtimeTalkInputLevel.set(0); state.resetRealtimeTalkConversation(); state.requestUpdate(); return; @@ -188,18 +193,34 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) { state.realtimeTalkActive = true; state.realtimeTalkStatus = "connecting"; state.realtimeTalkDetail = null; + state.realtimeTalkInputLevel.set(0); state.resetRealtimeTalkConversation(); const session = new RealtimeTalkSession( state.client, state.sessionKey, { onStatus: (status, detail) => { + if (state.realtimeTalkSession !== session) { + return; + } state.realtimeTalkStatus = status; state.realtimeTalkDetail = detail ?? null; state.realtimeTalkActive = status !== "idle"; + if (status === "idle" || status === "error") { + state.realtimeTalkInputLevel.set(0); + } state.requestUpdate(); }, + onInputLevel: (level) => { + if (state.realtimeTalkSession !== session) { + return; + } + state.realtimeTalkInputLevel.set(level); + }, onTranscript: (entry) => { + if (state.realtimeTalkSession !== session) { + return; + } state.realtimeTalkConversationState = updateRealtimeTalkConversation( state.realtimeTalkConversationState, entry, @@ -215,11 +236,15 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) { try { await session.start(); } catch (error) { + if (state.realtimeTalkSession !== session) { + return; + } session.stop(); state.realtimeTalkSession = null; state.realtimeTalkActive = false; state.realtimeTalkStatus = "error"; state.realtimeTalkDetail = error instanceof Error ? error.message : String(error); + state.realtimeTalkInputLevel.set(0); state.requestUpdate(); } }; diff --git a/ui/src/pages/chat/chat-view.test.ts b/ui/src/pages/chat/chat-view.test.ts index 5392d6e1f5f..3388e55580b 100644 --- a/ui/src/pages/chat/chat-view.test.ts +++ b/ui/src/pages/chat/chat-view.test.ts @@ -45,6 +45,7 @@ import { toggleChatThreadSearch, } from "./components/chat-thread.ts"; import { renderWelcomeState } from "./components/chat-welcome.ts"; +import { RealtimeTalkLevelSignal } from "./realtime-talk-level.ts"; const refreshVisibleToolsEffectiveForCurrentSessionMock = vi.hoisted(() => vi.fn(async (state: ChatHeaderTestState) => { @@ -1771,6 +1772,73 @@ describe("chat voice controls", () => { expect(onSend).not.toHaveBeenCalled(); }); + it.each([ + ["connecting", "Connecting voice input..."], + ["listening", "Listening..."], + ["thinking", "Asking OpenClaw..."], + ] as const)("renders %s voice activity without visible status copy", (status, label) => { + const inputLevel = new RealtimeTalkLevelSignal(); + inputLevel.set(0.64); + const container = renderChatView({ + realtimeTalkActive: true, + realtimeTalkStatus: status, + realtimeTalkInputLevel: inputLevel, + }); + + const statusRegion = container.querySelector('[role="status"].agent-chat__talk-status'); + const visualizer = statusRegion?.querySelector( + `.agent-chat__voice-activity[data-status="${status}"]`, + ); + expect(visualizer?.getAttribute("data-level")).toBe("0.64"); + expect(visualizer?.getAttribute("data-source")).toBe("microphone"); + expect(visualizer?.getAttribute("role")).toBe("img"); + expect(visualizer?.getAttribute("aria-label")).toBe(t("chat.composer.microphoneInput")); + expect(visualizer?.querySelectorAll(".agent-chat__voice-activity-bar")).toHaveLength(7); + expect(statusRegion?.getAttribute("aria-live")).toBe("polite"); + expect(statusRegion?.getAttribute("aria-atomic")).toBe("true"); + expect(statusRegion?.querySelector(".agent-chat__sr-only")?.textContent?.trim()).toBe(label); + expect(statusRegion?.querySelector(".agent-chat__talk-status-text")).toBeNull(); + }); + + it("clamps the rendered microphone level", () => { + const inputLevel = new RealtimeTalkLevelSignal(); + inputLevel.set(4); + const container = renderChatView({ + realtimeTalkActive: true, + realtimeTalkStatus: "listening", + realtimeTalkInputLevel: inputLevel, + }); + + expect(container.querySelector(".agent-chat__voice-activity")?.getAttribute("data-level")).toBe( + "1", + ); + }); + + it("updates microphone bars without rerendering the chat", () => { + const inputLevel = new RealtimeTalkLevelSignal(); + inputLevel.set(0.2); + const container = renderChatView({ + realtimeTalkActive: true, + realtimeTalkStatus: "listening", + realtimeTalkInputLevel: inputLevel, + }); + document.body.append(container); + try { + const visualizer = container.querySelector(".agent-chat__voice-activity"); + const centerBar = visualizer?.querySelector( + ".agent-chat__voice-activity-bar:nth-child(4)", + ); + const initialScale = centerBar?.style.getPropertyValue("--talk-bar-scale"); + + inputLevel.set(0.8); + + expect(visualizer?.getAttribute("data-level")).toBe("0.8"); + expect(centerBar?.style.getPropertyValue("--talk-bar-scale")).not.toBe(initialScale); + } finally { + container.remove(); + } + }); + it("shows every available microphone in Voice settings", () => { const onRealtimeTalkInputSelect = vi.fn(); const container = renderVoiceOptions({ diff --git a/ui/src/pages/chat/chat-view.ts b/ui/src/pages/chat/chat-view.ts index 7f8d352d6f2..3b294c1143e 100644 --- a/ui/src/pages/chat/chat-view.ts +++ b/ui/src/pages/chat/chat-view.ts @@ -42,6 +42,7 @@ import { } from "./components/chat-thread.ts"; import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "./input-history.ts"; import type { RealtimeTalkConversationEntry } from "./realtime-talk-conversation.ts"; +import type { RealtimeTalkLevelSignal } from "./realtime-talk-level.ts"; import type { RealtimeTalkStatus } from "./realtime-talk.ts"; import type { ChatRunUiStatus } from "./run-lifecycle.ts"; import type { CompactionStatus, FallbackStatus } from "./tool-stream.ts"; @@ -72,6 +73,7 @@ export type ChatProps = { realtimeTalkActive?: boolean; realtimeTalkStatus?: RealtimeTalkStatus; realtimeTalkDetail?: string | null; + realtimeTalkInputLevel?: RealtimeTalkLevelSignal; realtimeTalkConversation?: RealtimeTalkConversationEntry[]; connected: boolean; canSend: boolean; @@ -235,6 +237,7 @@ export function renderChat(props: ChatProps) { realtimeTalkActive: props.realtimeTalkActive, realtimeTalkStatus: props.realtimeTalkStatus, realtimeTalkDetail: props.realtimeTalkDetail, + realtimeTalkInputLevel: props.realtimeTalkInputLevel, realtimeTalkConversation: props.realtimeTalkConversation, composerControls: props.composerControls, getDraft: props.getDraft, diff --git a/ui/src/pages/chat/components/chat-composer.ts b/ui/src/pages/chat/components/chat-composer.ts index 553f68f7470..e7a739f8e63 100644 --- a/ui/src/pages/chat/components/chat-composer.ts +++ b/ui/src/pages/chat/components/chat-composer.ts @@ -41,9 +41,11 @@ import { import { exportChatMarkdown } from "../export.ts"; import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "../input-history.ts"; import type { RealtimeTalkConversationEntry } from "../realtime-talk-conversation.ts"; +import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts"; import type { RealtimeTalkStatus } from "../realtime-talk.ts"; import { CHAT_RUN_STATUS_TOAST_DURATION_MS, type ChatRunUiStatus } from "../run-lifecycle.ts"; import type { CompactionStatus, FallbackStatus } from "../tool-stream.ts"; +import { renderChatVoiceActivity } from "./chat-voice-activity.ts"; const COMPACTION_TOAST_DURATION_MS = 5000; const FALLBACK_TOAST_DURATION_MS = 8000; @@ -92,6 +94,7 @@ type ChatComposerProps = { realtimeTalkActive?: boolean; realtimeTalkStatus?: RealtimeTalkStatus; realtimeTalkDetail?: string | null; + realtimeTalkInputLevel?: RealtimeTalkLevelSignal; realtimeTalkConversation?: RealtimeTalkConversationEntry[]; composerControls?: TemplateResult | typeof nothing; getDraft?: () => string; @@ -2286,37 +2289,13 @@ export function renderChatComposer(props: ChatComposerProps) { }} /> - ${props.realtimeTalkActive || props.realtimeTalkDetail - ? html` -
- - ${props.realtimeTalkDetail ?? - (props.realtimeTalkStatus === "thinking" - ? "Asking OpenClaw..." - : props.realtimeTalkStatus === "connecting" - ? "Connecting voice input..." - : "Listening...")} - - ${props.realtimeTalkStatus === "error" && props.onDismissRealtimeTalkError - ? html` - - - - ` - : nothing} -
- ` - : nothing} + ${renderChatVoiceActivity({ + active: props.realtimeTalkActive, + status: props.realtimeTalkStatus, + detail: props.realtimeTalkDetail, + inputLevel: props.realtimeTalkInputLevel, + onDismissError: props.onDismissRealtimeTalkError, + })}
diff --git a/ui/src/pages/chat/components/chat-voice-activity.ts b/ui/src/pages/chat/components/chat-voice-activity.ts new file mode 100644 index 00000000000..0bdbc7415ce --- /dev/null +++ b/ui/src/pages/chat/components/chat-voice-activity.ts @@ -0,0 +1,148 @@ +import { html, nothing, type TemplateResult } from "lit"; +import { icons } from "../../../components/icons.ts"; +import "../../../components/tooltip.ts"; +import { t } from "../../../i18n/index.ts"; +import { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts"; +import type { RealtimeTalkStatus } from "../realtime-talk.ts"; + +const BAR_GAINS = [0.38, 0.62, 0.84, 1, 0.84, 0.62, 0.38]; +const MICROPHONE_ACTIVITY_TAG = "openclaw-microphone-activity"; +const EMPTY_LEVEL_SIGNAL = new RealtimeTalkLevelSignal(); + +class MicrophoneActivityElement extends HTMLElement { + private levelSignal: RealtimeTalkLevelSignal | undefined; + private unsubscribe: (() => void) | null = null; + + set signal(signal: RealtimeTalkLevelSignal | undefined) { + if (signal === this.levelSignal) { + return; + } + this.unsubscribe?.(); + this.unsubscribe = null; + this.levelSignal = signal; + this.ensureBars(); + this.renderLevel(signal?.value ?? 0); + if (this.isConnected) { + this.subscribe(); + } + } + + connectedCallback(): void { + this.ensureBars(); + this.subscribe(); + } + + disconnectedCallback(): void { + this.unsubscribe?.(); + this.unsubscribe = null; + } + + private ensureBars(): void { + if (!this.firstElementChild) { + for (const [index] of BAR_GAINS.entries()) { + const bar = document.createElement("span"); + bar.className = "agent-chat__voice-activity-bar"; + bar.style.setProperty("--talk-bar-delay", `${index * -70}ms`); + this.append(bar); + } + } + } + + private subscribe(): void { + this.unsubscribe?.(); + this.unsubscribe = this.levelSignal?.subscribe((level) => this.renderLevel(level)) ?? null; + } + + private renderLevel(level: number): void { + this.dataset.level = String(level); + for (const [index, bar] of [...this.children].entries()) { + const gain = BAR_GAINS[index] ?? 1; + (bar as HTMLElement).style.setProperty( + "--talk-bar-scale", + String(0.18 + level * gain * 0.82), + ); + } + } +} + +if (!customElements.get(MICROPHONE_ACTIVITY_TAG)) { + customElements.define(MICROPHONE_ACTIVITY_TAG, MicrophoneActivityElement); +} + +type ChatVoiceActivityProps = { + active?: boolean; + status?: RealtimeTalkStatus; + detail?: string | null; + inputLevel?: RealtimeTalkLevelSignal; + onDismissError?: () => void; +}; + +function activeStatus( + status: RealtimeTalkStatus | undefined, +): "connecting" | "listening" | "thinking" { + return status === "connecting" || status === "thinking" ? status : "listening"; +} + +function statusLabel(status: RealtimeTalkStatus | undefined, detail: string | null | undefined) { + const explicitDetail = detail?.trim(); + if (explicitDetail) { + return explicitDetail; + } + if (status === "thinking") { + return "Asking OpenClaw..."; + } + if (status === "connecting") { + return "Connecting voice input..."; + } + return "Listening..."; +} + +export function renderChatVoiceActivity( + props: ChatVoiceActivityProps, +): TemplateResult | typeof nothing { + if (props.status === "error" && props.detail) { + return html` + + `; + } + if (!props.active) { + return nothing; + } + + const status = activeStatus(props.status); + return html` +
+ + + ${statusLabel(props.status, props.detail)} +
+ `; +} diff --git a/ui/src/pages/chat/realtime-talk-audio.test.ts b/ui/src/pages/chat/realtime-talk-audio.test.ts new file mode 100644 index 00000000000..3d038660404 --- /dev/null +++ b/ui/src/pages/chat/realtime-talk-audio.test.ts @@ -0,0 +1,115 @@ +// @vitest-environment jsdom +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + measureRealtimeTalkAudioFrame, + RealtimeTalkAudioLevelMeter, + RealtimeTalkMediaStreamMeter, +} from "./realtime-talk-audio.ts"; + +describe("RealtimeTalkAudioLevelMeter", () => { + it("keeps silence flat and makes louder speech more visible", () => { + const silentFrame = measureRealtimeTalkAudioFrame(new Float32Array(512)); + const silent = new RealtimeTalkAudioLevelMeter().sample(new Float32Array(512)); + const quiet = new RealtimeTalkAudioLevelMeter().sample(new Float32Array(512).fill(0.03)); + const loud = new RealtimeTalkAudioLevelMeter().sample(new Float32Array(512).fill(0.25)); + + expect(silentFrame).toEqual({ peak: 0, rms: 0 }); + expect(silent).toBe(0); + expect(quiet).toBeGreaterThan(0); + expect(loud).toBeGreaterThan(quiet); + expect(loud).toBeLessThanOrEqual(1); + }); + + it("ignores invalid samples and releases smoothly after speech", () => { + const meter = new RealtimeTalkAudioLevelMeter(); + const invalid = meter.sample(new Float32Array([Number.NaN, Number.POSITIVE_INFINITY])); + const speech = meter.sample(new Float32Array(512).fill(0.4)); + const firstSilence = meter.sample(new Float32Array(512)); + let settled = firstSilence; + for (let index = 0; index < 12; index += 1) { + settled = meter.sample(new Float32Array(512)); + } + + expect(invalid).toBe(0); + expect(firstSilence).toBeLessThan(speech); + expect(firstSilence).toBeGreaterThan(0); + expect(settled).toBeLessThan(firstSilence); + }); + + it("settles steady room noise while keeping a speech burst visible", () => { + const meter = new RealtimeTalkAudioLevelMeter(); + let noiseLevel = 0; + for (let index = 0; index < 80; index += 1) { + noiseLevel = meter.sample(new Float32Array(512).fill(0.015)); + } + const speechLevel = meter.sample(new Float32Array(512).fill(0.2)); + + expect(noiseLevel).toBeLessThan(0.08); + expect(speechLevel).toBeGreaterThan(0.5); + }); +}); + +describe("RealtimeTalkMediaStreamMeter", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it("samples a WebRTC input stream and resets its level when stopped", () => { + vi.useFakeTimers(); + const close = vi.fn(async () => undefined); + const disconnectSource = vi.fn(); + const disconnectAnalyser = vi.fn(); + const analyser = { + fftSize: 0, + smoothingTimeConstant: 0, + disconnect: disconnectAnalyser, + getFloatTimeDomainData: vi + .fn() + .mockImplementationOnce((samples: Float32Array) => samples.fill(0.2)) + .mockImplementation((samples: Float32Array) => samples.fill(0)), + }; + class MockAudioContext { + readonly close = close; + createMediaStreamSource() { + return { connect: vi.fn(), disconnect: disconnectSource }; + } + createAnalyser() { + return analyser; + } + } + vi.stubGlobal("AudioContext", MockAudioContext); + const onLevel = vi.fn(); + const meter = new RealtimeTalkMediaStreamMeter(onLevel); + + meter.start({} as MediaStream); + vi.advanceTimersByTime(3_000); + + expect(onLevel.mock.calls.some(([level]) => level > 0)).toBe(true); + expect(onLevel).toHaveBeenLastCalledWith(0); + meter.stop(); + + expect(analyser.fftSize).toBe(512); + expect(onLevel).toHaveBeenLastCalledWith(0); + expect(disconnectSource).toHaveBeenCalledOnce(); + expect(disconnectAnalyser).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it("closes an owned AudioContext when analyser setup fails", () => { + const close = vi.fn(async () => undefined); + class MockAudioContext { + readonly close = close; + createMediaStreamSource() { + throw new Error("source unavailable"); + } + } + vi.stubGlobal("AudioContext", MockAudioContext); + const onLevel = vi.fn(); + + new RealtimeTalkMediaStreamMeter(onLevel).start({} as MediaStream); + + expect(close).toHaveBeenCalledOnce(); + expect(onLevel).toHaveBeenLastCalledWith(0); + }); +}); diff --git a/ui/src/pages/chat/realtime-talk-audio.ts b/ui/src/pages/chat/realtime-talk-audio.ts index 340412603b9..32208032dd7 100644 --- a/ui/src/pages/chat/realtime-talk-audio.ts +++ b/ui/src/pages/chat/realtime-talk-audio.ts @@ -28,6 +28,122 @@ export function floatToPcm16(samples: Float32Array): Uint8Array { return bytes; } +export type RealtimeTalkAudioFrame = { + peak: number; + rms: number; +}; + +export function measureRealtimeTalkAudioFrame(samples: Float32Array): RealtimeTalkAudioFrame { + let peak = 0; + let sumSquares = 0; + for (const rawSample of samples) { + const sample = Number.isFinite(rawSample) ? rawSample : 0; + const absolute = Math.abs(sample); + peak = Math.max(peak, absolute); + sumSquares += sample * sample; + } + return { + peak, + rms: samples.length > 0 ? Math.sqrt(sumSquares / samples.length) : 0, + }; +} + +export class RealtimeTalkAudioLevelMeter { + private level = 0; + private noiseFloor = 0.01; + + sample(samples: Float32Array): number { + const frame = measureRealtimeTalkAudioFrame(samples); + const signal = 0.65 * frame.rms + 0.35 * frame.peak; + if (signal <= Math.max(0.02, this.noiseFloor * 2)) { + this.noiseFloor = 0.95 * this.noiseFloor + 0.05 * signal; + } + const gatedSignal = Math.max(0, signal - this.noiseFloor * 1.5); + const target = Math.min(1, Math.sqrt(gatedSignal / 0.18)); + const smoothing = target > this.level ? 0.65 : 0.18; + this.level = smoothing * target + (1 - smoothing) * this.level; + if (this.level < 0.01) { + this.level = 0; + } + return this.level; + } + + reset(): void { + this.level = 0; + this.noiseFloor = 0.01; + } +} + +export class RealtimeTalkMediaStreamMeter { + private context: AudioContext | null = null; + private source: MediaStreamAudioSourceNode | null = null; + private analyser: AnalyserNode | null = null; + private timer: ReturnType | null = null; + private ownsContext = false; + private readonly levelMeter = new RealtimeTalkAudioLevelMeter(); + private readonly samples = new Float32Array(512); + private lastLevel = -1; + + constructor(private readonly onLevel: (level: number) => void) {} + + start(media: MediaStream, sharedContext?: AudioContext): void { + this.stop(false); + try { + const context = sharedContext ?? new AudioContext(); + this.context = context; + this.ownsContext = !sharedContext; + const source = context.createMediaStreamSource(media); + this.source = source; + const analyser = context.createAnalyser(); + this.analyser = analyser; + analyser.fftSize = this.samples.length; + analyser.smoothingTimeConstant = 0; + source.connect(analyser); + this.publishCurrentLevel(); + this.timer = globalThis.setInterval(() => this.publishCurrentLevel(), 100); + } catch { + // Metering is feedback only; capture must still work if Web Audio analysis + // is unavailable in an otherwise functional WebRTC browser. + this.stop(); + } + } + + stop(notify = true): void { + if (this.timer !== null) { + globalThis.clearInterval(this.timer); + this.timer = null; + } + this.source?.disconnect(); + this.source = null; + this.analyser?.disconnect(); + this.analyser = null; + if (this.ownsContext) { + void this.context?.close(); + } + this.context = null; + this.ownsContext = false; + this.levelMeter.reset(); + this.lastLevel = -1; + if (notify) { + this.onLevel(0); + } + } + + private publishCurrentLevel(): void { + if (!this.analyser) { + return; + } + this.samples.fill(0); + this.analyser.getFloatTimeDomainData(this.samples); + const level = Math.round(this.levelMeter.sample(this.samples) * 100) / 100; + if (level === this.lastLevel) { + return; + } + this.lastLevel = level; + this.onLevel(level); + } +} + function pcm16ToFloat(bytes: Uint8Array): Float32Array { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const samples = new Float32Array(Math.floor(bytes.byteLength / 2)); diff --git a/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts b/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts index 05139fc46bc..1a231b80f35 100644 --- a/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts +++ b/ui/src/pages/chat/realtime-talk-gateway-relay.test.ts @@ -44,6 +44,15 @@ class MockAudioContext { return processor; } + createAnalyser() { + return { + fftSize: 0, + smoothingTimeConstant: 0, + disconnect: vi.fn(), + getFloatTimeDomainData: (samples: Float32Array) => samples.fill(0.25), + }; + } + createBuffer(_channels: number, length: number, sampleRate: number) { const channel = new Float32Array(length); return { @@ -155,6 +164,30 @@ describe("GatewayRelayRealtimeTalkTransport", () => { transport.stop(); }); + it("releases microphone access that resolves after stop", async () => { + let resolveMedia: (media: MediaStream) => void = () => undefined; + const pendingMedia = new Promise((resolve) => { + resolveMedia = resolve; + }); + getUserMedia.mockReturnValue(pendingMedia); + const stopTrack = vi.fn(); + const onInputLevel = vi.fn(); + const transport = new GatewayRelayRealtimeTalkTransport(createSession(), { + callbacks: { onInputLevel }, + client: createClient(), + sessionKey: "main", + }); + + const start = transport.start(); + transport.stop(); + resolveMedia({ getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream); + await start; + + expect(stopTrack).toHaveBeenCalledOnce(); + expect(processors).toHaveLength(0); + expect(onInputLevel).not.toHaveBeenCalled(); + }); + it("forwards common Talk events from Gateway relay frames", async () => { const onTalkEvent = vi.fn(); const transport = new GatewayRelayRealtimeTalkTransport(createSession(), { @@ -247,6 +280,23 @@ describe("GatewayRelayRealtimeTalkTransport", () => { transport.stop(); }); + it("reports microphone activity and resets it when stopped", async () => { + const onInputLevel = vi.fn(); + const transport = new GatewayRelayRealtimeTalkTransport(createSession(), { + callbacks: { onInputLevel }, + client: createClient(), + sessionKey: "main", + }); + + await transport.start(); + pumpMicrophone(new Float32Array(4096)); + pumpMicrophone(new Float32Array(4096).fill(0.25)); + transport.stop(); + + expect(onInputLevel.mock.calls.some(([level]) => level > 0)).toBe(true); + expect(onInputLevel).toHaveBeenLastCalledWith(0); + }); + it("stops microphone pumping when the relay rejects appended audio", async () => { const onStatus = vi.fn(); const client = createClient(); diff --git a/ui/src/pages/chat/realtime-talk-gateway-relay.ts b/ui/src/pages/chat/realtime-talk-gateway-relay.ts index e1b13b3c8ec..90d20c13dbd 100644 --- a/ui/src/pages/chat/realtime-talk-gateway-relay.ts +++ b/ui/src/pages/chat/realtime-talk-gateway-relay.ts @@ -1,5 +1,12 @@ // Control UI chat module implements realtime talk gateway relay behavior. -import { bytesToBase64, floatToPcm16, RealtimeTalkPcmOutputQueue } from "./realtime-talk-audio.ts"; +import { + bytesToBase64, + floatToPcm16, + measureRealtimeTalkAudioFrame, + RealtimeTalkMediaStreamMeter, + RealtimeTalkPcmOutputQueue, + type RealtimeTalkAudioFrame, +} from "./realtime-talk-audio.ts"; import { openRealtimeTalkInput } from "./realtime-talk-input.ts"; import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, @@ -46,6 +53,7 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport private media: MediaStream | null = null; private inputContext: AudioContext | null = null; private outputContext: AudioContext | null = null; + private inputMeter: RealtimeTalkMediaStreamMeter | null = null; private inputSource: MediaStreamAudioSourceNode | null = null; private inputProcessor: ScriptProcessorNode | null = null; private unsubscribe: (() => void) | null = null; @@ -79,13 +87,30 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport } this.handleRelayEvent(evt.payload as GatewayRelayEvent); }); - this.media = await openRealtimeTalkInput(this.ctx.inputDeviceId, { - autoGainControl: true, - echoCancellation: true, - noiseSuppression: true, - }); + let media: MediaStream; + try { + media = await openRealtimeTalkInput(this.ctx.inputDeviceId, { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }); + } catch (error) { + if (this.closed) { + return; + } + throw error; + } + if (this.closed) { + media.getTracks().forEach((track) => track.stop()); + return; + } + this.media = media; this.inputContext = new AudioContext({ sampleRate: this.session.audio.inputSampleRateHz }); this.outputContext = new AudioContext({ sampleRate: this.session.audio.outputSampleRateHz }); + if (this.ctx.callbacks.onInputLevel) { + this.inputMeter = new RealtimeTalkMediaStreamMeter(this.ctx.callbacks.onInputLevel); + this.inputMeter.start(this.media, this.inputContext); + } this.startMicrophonePump(); } @@ -109,6 +134,8 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport this.inputProcessor = null; this.inputSource?.disconnect(); this.inputSource = null; + this.inputMeter?.stop(); + this.inputMeter = null; this.abortConsults(); this.media?.getTracks().forEach((track) => track.stop()); this.media = null; @@ -339,20 +366,12 @@ export class GatewayRelayRealtimeTalkTransport implements RealtimeTalkTransport } private detectBargeInSpeech(samples: Float32Array): boolean { - if (!this.outputQueue.isPlaying || this.cancelRequestedForPlayback || samples.length === 0) { + if (!this.outputQueue.isPlaying || this.cancelRequestedForPlayback) { this.speechFramesDuringPlayback = 0; return false; } - - let sumSquares = 0; - let peak = 0; - for (const sample of samples) { - const abs = Math.abs(sample); - peak = Math.max(peak, abs); - sumSquares += sample * sample; - } - const rms = Math.sqrt(sumSquares / samples.length); - if (rms >= BARGE_IN_RMS_THRESHOLD && peak >= BARGE_IN_PEAK_THRESHOLD) { + const frame: RealtimeTalkAudioFrame = measureRealtimeTalkAudioFrame(samples); + if (frame.rms >= BARGE_IN_RMS_THRESHOLD && frame.peak >= BARGE_IN_PEAK_THRESHOLD) { this.speechFramesDuringPlayback += 1; } else { this.speechFramesDuringPlayback = 0; diff --git a/ui/src/pages/chat/realtime-talk-google-live.test.ts b/ui/src/pages/chat/realtime-talk-google-live.test.ts index df3823888d4..dc345cb4288 100644 --- a/ui/src/pages/chat/realtime-talk-google-live.test.ts +++ b/ui/src/pages/chat/realtime-talk-google-live.test.ts @@ -21,6 +21,9 @@ type MockWebSocketEventType = "close" | "error" | "message" | "open"; const wsInstances: MockGoogleLiveWebSocket[] = []; const createdSources: MockAudioBufferSource[] = []; +const inputProcessors: Array<{ + onaudioprocess: ((event: { inputBuffer: { getChannelData: () => Float32Array } }) => void) | null; +}> = []; let getUserMedia: ReturnType; async function flushMicrotasks(): Promise { @@ -96,11 +99,22 @@ class MockAudioContext { } createScriptProcessor() { - return { + const processor = { connect: vi.fn(), disconnect: vi.fn(), onaudioprocess: null, }; + inputProcessors.push(processor); + return processor; + } + + createAnalyser() { + return { + fftSize: 0, + smoothingTimeConstant: 0, + disconnect: vi.fn(), + getFloatTimeDomainData: (samples: Float32Array) => samples.fill(0.25), + }; } createBuffer(_channels: number, length: number, sampleRate: number) { @@ -175,6 +189,14 @@ function latestWebSocket(): MockGoogleLiveWebSocket { return ws; } +function pumpMicrophone(samples: Float32Array): void { + const processor = inputProcessors.at(-1); + if (!processor) { + throw new Error("missing microphone processor"); + } + processor.onaudioprocess?.({ inputBuffer: { getChannelData: () => samples } }); +} + function requireFirstTalkEvent(onTalkEvent: ReturnType): Record { const [call] = onTalkEvent.mock.calls; if (!call) { @@ -191,6 +213,7 @@ describe("GoogleLiveRealtimeTalkTransport", () => { beforeEach(() => { wsInstances.length = 0; createdSources.length = 0; + inputProcessors.length = 0; vi.stubGlobal("WebSocket", MockGoogleLiveWebSocket); vi.stubGlobal("AudioContext", MockAudioContext); getUserMedia = vi.fn(async () => ({ @@ -218,6 +241,27 @@ describe("GoogleLiveRealtimeTalkTransport", () => { transport.stop(); }); + it("releases microphone access that resolves after stop", async () => { + let resolveMedia: (media: MediaStream) => void = () => undefined; + const pendingMedia = new Promise((resolve) => { + resolveMedia = resolve; + }); + getUserMedia.mockReturnValue(pendingMedia); + const stopTrack = vi.fn(); + const onInputLevel = vi.fn(); + const transport = createTransport({ onInputLevel }); + + const start = transport.start(); + transport.stop(); + resolveMedia({ getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream); + await start; + + expect(stopTrack).toHaveBeenCalledOnce(); + expect(inputProcessors).toHaveLength(0); + expect(wsInstances).toHaveLength(0); + expect(onInputLevel).not.toHaveBeenCalled(); + }); + it("requests ArrayBuffer frames and decodes binary setup messages", async () => { const onStatus = vi.fn(); const onTalkEvent = vi.fn(); @@ -235,6 +279,20 @@ describe("GoogleLiveRealtimeTalkTransport", () => { expect(readyEvent.transport).toBe("provider-websocket"); }); + it("reports microphone activity and resets it when stopped", async () => { + const onInputLevel = vi.fn(); + const transport = createTransport({ onInputLevel }); + + await transport.start(); + latestWebSocket().emitOpen(); + pumpMicrophone(new Float32Array(4096)); + pumpMicrophone(new Float32Array(4096).fill(0.25)); + transport.stop(); + + expect(onInputLevel.mock.calls.some(([level]) => level > 0)).toBe(true); + expect(onInputLevel).toHaveBeenLastCalledWith(0); + }); + it("decodes Blob setup messages", async () => { const onStatus = vi.fn(); const transport = createTransport({ onStatus }); diff --git a/ui/src/pages/chat/realtime-talk-google-live.ts b/ui/src/pages/chat/realtime-talk-google-live.ts index 3fcae2aba06..b1fe7273733 100644 --- a/ui/src/pages/chat/realtime-talk-google-live.ts +++ b/ui/src/pages/chat/realtime-talk-google-live.ts @@ -3,6 +3,7 @@ import { base64ToBytes, bytesToBase64, floatToPcm16, + RealtimeTalkMediaStreamMeter, RealtimeTalkPcmOutputQueue, } from "./realtime-talk-audio.ts"; import { openRealtimeTalkInput } from "./realtime-talk-input.ts"; @@ -81,6 +82,7 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { private media: MediaStream | null = null; private inputContext: AudioContext | null = null; private outputContext: AudioContext | null = null; + private inputMeter: RealtimeTalkMediaStreamMeter | null = null; private inputSource: MediaStreamAudioSourceNode | null = null; private inputProcessor: ScriptProcessorNode | null = null; private closed = false; @@ -105,9 +107,26 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { } const wsUrl = buildGoogleLiveUrl(this.session); this.closed = false; - this.media = await openRealtimeTalkInput(this.ctx.inputDeviceId); + let media: MediaStream; + try { + media = await openRealtimeTalkInput(this.ctx.inputDeviceId); + } catch (error) { + if (this.closed) { + return; + } + throw error; + } + if (this.closed) { + media.getTracks().forEach((track) => track.stop()); + return; + } + this.media = media; this.inputContext = new AudioContext({ sampleRate: this.session.audio.inputSampleRateHz }); this.outputContext = new AudioContext({ sampleRate: this.session.audio.outputSampleRateHz }); + if (this.ctx.callbacks.onInputLevel) { + this.inputMeter = new RealtimeTalkMediaStreamMeter(this.ctx.callbacks.onInputLevel); + this.inputMeter.start(this.media, this.inputContext); + } this.ws = new WebSocket(wsUrl); this.ws.binaryType = "arraybuffer"; this.ws.addEventListener("open", () => { @@ -146,6 +165,8 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { this.inputProcessor = null; this.inputSource?.disconnect(); this.inputSource = null; + this.inputMeter?.stop(); + this.inputMeter = null; this.media?.getTracks().forEach((track) => track.stop()); this.media = null; this.stopOutput(); @@ -167,7 +188,8 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { if (this.ws?.readyState !== WebSocket.OPEN) { return; } - const pcm = floatToPcm16(event.inputBuffer.getChannelData(0)); + const samples = event.inputBuffer.getChannelData(0); + const pcm = floatToPcm16(samples); this.send({ realtimeInput: { audio: { diff --git a/ui/src/pages/chat/realtime-talk-level.ts b/ui/src/pages/chat/realtime-talk-level.ts new file mode 100644 index 00000000000..a722d9f0435 --- /dev/null +++ b/ui/src/pages/chat/realtime-talk-level.ts @@ -0,0 +1,28 @@ +export type RealtimeTalkLevelListener = (level: number) => void; + +export class RealtimeTalkLevelSignal { + private listeners = new Set(); + private currentLevel = 0; + + get value(): number { + return this.currentLevel; + } + + set(rawLevel: number): void { + const boundedLevel = Number.isFinite(rawLevel) ? Math.min(1, Math.max(0, rawLevel)) : 0; + const level = Math.round(boundedLevel * 100) / 100; + if (level === this.currentLevel) { + return; + } + this.currentLevel = level; + for (const listener of this.listeners) { + listener(level); + } + } + + subscribe(listener: RealtimeTalkLevelListener): () => void { + this.listeners.add(listener); + listener(this.currentLevel); + return () => this.listeners.delete(listener); + } +} diff --git a/ui/src/pages/chat/realtime-talk-shared.ts b/ui/src/pages/chat/realtime-talk-shared.ts index aa1efcfccf4..1c5a1cf47d9 100644 --- a/ui/src/pages/chat/realtime-talk-shared.ts +++ b/ui/src/pages/chat/realtime-talk-shared.ts @@ -16,6 +16,7 @@ export type RealtimeTalkEvent = TalkEvent; export type RealtimeTalkCallbacks = { onStatus?: (status: RealtimeTalkStatus, detail?: string) => void; + onInputLevel?: (level: number) => void; onTranscript?: (entry: { role: "user" | "assistant"; text: string; final: boolean }) => void; onTalkEvent?: (event: RealtimeTalkEvent) => void; }; diff --git a/ui/src/pages/chat/realtime-talk-webrtc.test.ts b/ui/src/pages/chat/realtime-talk-webrtc.test.ts index cd4c0dd068b..99dafb43b1c 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.test.ts @@ -205,6 +205,35 @@ describe("WebRtcSdpRealtimeTalkTransport", () => { transport.stop(); }); + it("reports microphone activity and resets it when stopped", async () => { + stubAnswerSdpFetch(); + const close = vi.fn(async () => undefined); + class MockAudioContext { + readonly close = close; + createMediaStreamSource() { + return { connect: vi.fn(), disconnect: vi.fn() }; + } + createAnalyser() { + return { + fftSize: 0, + smoothingTimeConstant: 0, + disconnect: vi.fn(), + getFloatTimeDomainData: (samples: Float32Array) => samples.fill(0.25), + }; + } + } + vi.stubGlobal("AudioContext", MockAudioContext); + const onInputLevel = vi.fn(); + const transport = createOpenAiTransport({}, { onInputLevel }); + + await transport.start(); + transport.stop(); + + expect(onInputLevel.mock.calls.some(([level]) => level > 0)).toBe(true); + expect(onInputLevel).toHaveBeenLastCalledWith(0); + expect(close).toHaveBeenCalledOnce(); + }); + it("does not continue WebRTC setup when stopped while microphone access is pending", async () => { const fetchMock = vi.fn(async () => new Response("answer-sdp")); vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); diff --git a/ui/src/pages/chat/realtime-talk-webrtc.ts b/ui/src/pages/chat/realtime-talk-webrtc.ts index f284e1a70c7..1a644c6a1e6 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.ts @@ -1,4 +1,5 @@ // Control UI chat module implements realtime talk webrtc behavior. +import { RealtimeTalkMediaStreamMeter } from "./realtime-talk-audio.ts"; import { openRealtimeTalkInput } from "./realtime-talk-input.ts"; import type { RealtimeTalkWebRtcSdpSessionResult } from "./realtime-talk-shared.ts"; import { @@ -42,6 +43,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { private channel: RTCDataChannel | null = null; private media: MediaStream | null = null; private audio: HTMLAudioElement | null = null; + private inputMeter: RealtimeTalkMediaStreamMeter | null = null; private closed = false; private responseActive = false; private responseCreateInFlight = false; @@ -82,6 +84,10 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { return; } this.media = media; + if (this.ctx.callbacks.onInputLevel) { + this.inputMeter = new RealtimeTalkMediaStreamMeter(this.ctx.callbacks.onInputLevel); + this.inputMeter.start(media); + } for (const track of media.getAudioTracks()) { peer.addTrack(track, media); } @@ -185,6 +191,8 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { this.peer = null; this.media?.getTracks().forEach((track) => track.stop()); this.media = null; + this.inputMeter?.stop(); + this.inputMeter = null; this.audio?.remove(); this.audio = null; for (const controller of this.consultAbortControllers) { diff --git a/ui/src/styles/chat/layout.css b/ui/src/styles/chat/layout.css index 2ad4bf8485e..a6ec2b3e9c0 100644 --- a/ui/src/styles/chat/layout.css +++ b/ui/src/styles/chat/layout.css @@ -1521,11 +1521,121 @@ openclaw-chat-page { color: var(--text); } +.agent-chat__talk-status--active { + justify-content: center; + min-height: 30px; + padding-top: 8px; +} + .agent-chat__talk-status-text { min-width: 0; flex: 1; } +.agent-chat__voice-activity { + position: relative; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 4px; + width: 64px; + height: 24px; + overflow: hidden; + border: 1px solid color-mix(in srgb, var(--accent) 18%, var(--border)); + border-radius: 999px; + background: color-mix(in srgb, var(--accent) 7%, var(--card)); + box-shadow: 0 0 16px color-mix(in srgb, var(--accent) 12%, transparent); +} + +.agent-chat__voice-activity::before { + position: absolute; + inset: 0; + border-radius: inherit; + background: radial-gradient( + circle at 50% 50%, + color-mix(in srgb, var(--accent) 12%, transparent), + transparent 72% + ); + content: ""; + pointer-events: none; +} + +.agent-chat__voice-activity-bar { + position: relative; + width: 3px; + height: 16px; + flex: 0 0 3px; + border-radius: 999px; + background: color-mix(in srgb, var(--accent) 88%, var(--text)); + box-shadow: 0 0 5px color-mix(in srgb, var(--accent) 30%, transparent); + opacity: 0.9; + transform: scaleY(var(--talk-bar-scale, 0.18)); + transform-origin: center; + transition: + transform 90ms linear, + opacity var(--duration-fast) ease; +} + +.agent-chat__voice-activity[data-status="connecting"] .agent-chat__voice-activity-bar { + animation: agent-chat-voice-connecting 1.05s ease-in-out infinite; + animation-delay: var(--talk-bar-delay); +} + +.agent-chat__voice-activity[data-status="thinking"] .agent-chat__voice-activity-bar { + animation: agent-chat-voice-thinking 0.9s ease-in-out infinite; + animation-delay: var(--talk-bar-delay); +} + +@keyframes agent-chat-voice-connecting { + 0%, + 100% { + opacity: 0.42; + transform: scaleY(0.2); + } + 50% { + opacity: 0.9; + transform: scaleY(0.58); + } +} + +@keyframes agent-chat-voice-thinking { + 0%, + 100% { + opacity: 0.4; + transform: scaleY(0.24); + } + 45% { + opacity: 1; + transform: scaleY(0.92); + } +} + +@media (prefers-reduced-motion: reduce) { + .agent-chat__voice-activity-bar { + animation: none !important; + opacity: 0.78; + transform: scaleY(0.38); + transition: none !important; + } + + .agent-chat__voice-activity[data-status="connecting"] .agent-chat__voice-activity-bar { + opacity: 0.48; + transform: scaleY(0.18); + } + + .agent-chat__voice-activity[data-status="connecting"] + .agent-chat__voice-activity-bar:nth-child(4) { + opacity: 0.9; + transform: scaleY(0.56); + } + + .agent-chat__voice-activity[data-status="thinking"] .agent-chat__voice-activity-bar:nth-child(3), + .agent-chat__voice-activity[data-status="thinking"] .agent-chat__voice-activity-bar:nth-child(4), + .agent-chat__voice-activity[data-status="thinking"] .agent-chat__voice-activity-bar:nth-child(5) { + transform: scaleY(0.72); + } +} + .agent-chat__voice-turns { display: flex; flex-direction: column;