test(ui): synchronize voice and side-chat readiness

Talk and slash commands wait for accepted initial history even though the
composer already permits drafting and dictation. Synchronize those browser
fixtures with their actual readiness signals before starting a call or
submitting a side-chat command. Share the Talk setup with microphone lifecycle
proof; preserve media, transcript, focus, and stop/restart race assertions.

Cancel a held module's unrequested observation when its browser context closes.
Manual release still only opens the response gate, route failures propagate,
and teardown continues joining owned callbacks before reusing fixture state.
The native lifecycle regression fails against the original helper with the
same timeout/unsafe-cleanup cascade and passes with the repair.

Keep the concurrent main fixes 6b9da9dc02 and f03d64f6a5; their sidebar,
membership, effort, and relay-admission changes supersede duplicate work here.

Validation on npl11:
- Affected six E2E files passed three runs (53 cases), with 113 cases including
  siblings in the first run. Integrated main replay: 113/113, 94.45s wall.
- Fixture-owner tests: 21/21; integrated run 30.15s wall.
- UI production/test typechecks, targeted lint/format/ratchets, and full
  check-changed --base HEAD passed on integrated main.
- Fresh independent P2 review: no accepted/actionable findings.

Release-note context: test reliability only; production behavior unchanged.
This commit is contained in:
Peter Steinberger 2026-09-28 09:32:38 -07:00
parent 202336dcd5
commit e256cbef8a
No known key found for this signature in database
6 changed files with 110 additions and 36 deletions

View file

@ -5,7 +5,9 @@ import {
captureMicrophoneLossProof,
installMicrophoneLossWebRtcFixture,
type MicrophoneLossE2eProof,
TALK_READY_HISTORY_MESSAGE,
videoTalkCatalog,
waitForTalkReady,
} from "./browser-talk-start-stop.fixtures.ts";
import { createControlUiE2eSuite } from "./control-ui-e2e-suite.test-support.ts";
@ -20,6 +22,7 @@ suite.define(() => {
it("guides a pending microphone request and clears guidance when voice connects", async () => {
await suite.withPage({ permissions: ["microphone"] }, async ({ page }) => {
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.catalog": videoTalkCatalog("openai"),
"talk.client.create": {
@ -45,6 +48,7 @@ suite.define(() => {
};
});
await page.goto(`${suite.server.baseUrl}chat`);
await waitForTalkReady(page);
await page.getByRole("button", { name: "Start voice input" }).click();
await expect
.poll(() =>
@ -84,6 +88,7 @@ suite.define(() => {
it("surfaces microphone loss and closes native browser call resources", async () => {
await suite.withPage({ permissions: ["microphone"] }, async ({ page }) => {
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.catalog": videoTalkCatalog("openai"),
"talk.client.create": {
@ -96,6 +101,7 @@ suite.define(() => {
});
await installMicrophoneLossWebRtcFixture(page);
await page.goto(`${suite.server.baseUrl}chat`);
await waitForTalkReady(page);
await page.getByRole("button", { name: "Start voice input" }).click();
try {
await expect

View file

@ -11,7 +11,9 @@ import {
installOpenAiTalkFixture,
installTalkBrowserFixtures,
installVideoTalkMediaFixture,
TALK_READY_HISTORY_MESSAGE,
videoTalkCatalog,
waitForTalkReady,
} from "./browser-talk-start-stop.fixtures.ts";
import { createControlUiE2eSuite } from "./control-ui-e2e-suite.test-support.ts";
@ -27,6 +29,7 @@ suite.define(() => {
it("starts a provider WebSocket session and stops browser audio resources", async () => {
await suite.withPage({ permissions: ["microphone"] }, async ({ page }) => {
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.client.create": {
provider: "google",
@ -60,6 +63,7 @@ suite.define(() => {
await microphoneSelect.selectOption("usb");
await page.goto(`${suite.server.baseUrl}chat`);
await page.setViewportSize({ width: 320, height: 720 });
await waitForTalkReady(page);
await page.getByRole("button", { name: "Tap to talk" }).click();
const createRequest = await gateway.waitForRequest("talk.client.create");
@ -205,6 +209,7 @@ suite.define(() => {
await suite.withPage({ permissions: ["microphone"] }, async ({ page }) => {
const gateway = await installMockGateway(page, {
deferredMethods: ["chat.send"],
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.client.create": {
provider: "google",
@ -229,6 +234,7 @@ suite.define(() => {
await page.goto(`${suite.server.baseUrl}chat`);
await page.setViewportSize({ width: 1366, height: 900 });
await waitForTalkReady(page);
await page.getByRole("button", { name: "Start voice input" }).click();
await gateway.waitForRequest("talk.client.create");
await gateway.deliverLatest({ setupComplete: {} });
@ -333,6 +339,7 @@ suite.define(() => {
it("starts OpenAI Talk, enables a fake camera, and submits describe_view", async () => {
await suite.withPage({ permissions: ["camera", "microphone"] }, async ({ page }) => {
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.catalog": videoTalkCatalog("openai"),
"talk.client.create": {
@ -348,6 +355,7 @@ suite.define(() => {
await page.setViewportSize({ width: 1366, height: 900 });
await page.goto(`${suite.server.baseUrl}chat`);
await waitForTalkReady(page);
await captureVideoTalkProof(suite, page, "01-before-video-talk.png");
await page.getByRole("button", { name: "Start voice input" }).click();
@ -499,6 +507,7 @@ suite.define(() => {
it("starts Gemini Live Talk, enables a fake camera, and handles describe_view", async () => {
await suite.withPage({ permissions: ["camera", "microphone"] }, async ({ page }) => {
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.catalog": videoTalkCatalog("google"),
"talk.client.create": {
@ -548,6 +557,7 @@ suite.define(() => {
await page.setViewportSize({ width: 1366, height: 900 });
await page.goto(`${suite.server.baseUrl}chat`);
await waitForTalkReady(page);
await page.getByRole("button", { name: "Start voice input" }).click();
const request = await gateway.waitForRequest("talk.client.create");
expect(request.params).toMatchObject({
@ -624,6 +634,7 @@ suite.define(() => {
it("shows actionable guidance when Video Talk camera permission is blocked", async () => {
await suite.withPage(undefined, async ({ page }) => {
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.catalog": videoTalkCatalog("google"),
"talk.client.create": {
@ -658,6 +669,7 @@ suite.define(() => {
await page.setViewportSize({ width: 1366, height: 900 });
await page.goto(`${suite.server.baseUrl}chat`);
await waitForTalkReady(page);
await page.getByRole("button", { name: "Start voice input" }).click();
await gateway.waitForRequest("talk.client.create");
const turnCameraOn = page.getByRole("button", { name: "Turn camera on" });
@ -689,6 +701,7 @@ suite.define(() => {
const relaySessionId = "relay-e2e-transcript";
const gateway = await installMockGateway(page, {
deferredMethods: ["talk.client.create"],
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.client.create": {
provider: "openai",
@ -709,6 +722,7 @@ suite.define(() => {
await page.goto(`${suite.server.baseUrl}chat`);
await page.setViewportSize({ width: 1366, height: 900 });
await waitForTalkReady(page);
await page.getByRole("button", { name: "Start voice input" }).click();
await gateway.waitForRequest("talk.client.create");
// Microphone acquisition precedes relay admission and cannot prove readiness.
@ -786,6 +800,7 @@ suite.define(() => {
await suite.withPage({ permissions: ["microphone"] }, async ({ page }) => {
const relaySessionId = "relay-e2e-input-backpressure";
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.client.create": {
provider: "openai",
@ -805,6 +820,7 @@ suite.define(() => {
await installTalkBrowserFixtures(page);
await page.goto(`${suite.server.baseUrl}chat`);
await waitForTalkReady(page);
for (let index = 0; index < 4; index += 1) {
await gateway.deferNext("talk.session.appendAudio");
}
@ -861,6 +877,7 @@ suite.define(() => {
const currentRelaySessionId = "relay-current-e2e";
const staleRelaySessionId = "relay-stale-e2e";
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
methodResponses: {
"talk.client.create": {
provider: "openai",
@ -880,6 +897,7 @@ suite.define(() => {
await installTalkBrowserFixtures(page);
await page.goto(`${suite.server.baseUrl}chat`);
await waitForTalkReady(page);
await gateway.deferNext("talk.client.create");
await page.getByRole("button", { name: "Start voice input" }).click();
@ -979,11 +997,14 @@ suite.define(() => {
it("shows actionable guidance when Talk microphone permission is blocked", async () => {
await suite.withPage(undefined, async ({ page }) => {
const gateway = await installMockGateway(page);
const gateway = await installMockGateway(page, {
historyMessages: [TALK_READY_HISTORY_MESSAGE],
});
await installBlockedMicrophoneFixture(page);
await page.setViewportSize({ width: 320, height: 720 });
await page.goto(`${suite.server.baseUrl}chat`);
await waitForTalkReady(page);
await page.getByRole("button", { name: "Tap to talk" }).click();
await expect
.poll(() => page.getByRole("alert").locator(".agent-chat__talk-status-text").textContent())

View file

@ -1,6 +1,17 @@
import path from "node:path";
import type { Page } from "playwright";
export const TALK_READY_HISTORY_MESSAGE = {
role: "assistant",
content: "Talk history is ready.",
};
export async function waitForTalkReady(page: Page): Promise<void> {
// Dictation keeps the microphone enabled before Talk's history/catalog admission.
await page.getByText(TALK_READY_HISTORY_MESSAGE.content, { exact: true }).waitFor();
await page.locator('[data-chat-talk-capability="realtime"]').waitFor({ state: "detached" });
}
export async function dispatchOpenAiTalkEvent(page: Page, event: unknown) {
await page.evaluate((payload) => {
const channel = (

View file

@ -1,4 +1,5 @@
import path from "node:path";
import type { Locator, Page } from "playwright";
import { expect, it } from "vitest";
import { createControlUiE2eArtifactDir } from "../test-helpers/control-ui-e2e-artifacts.ts";
import { installMockGateway } from "../test-helpers/control-ui-e2e.ts";
@ -11,6 +12,14 @@ import { openSessionMenuSubmenu } from "./session-management.test-support.ts";
const suite = createControlUiE2eSuite({ name: "side-chat input focus" });
async function enterSideCommand(scope: Page | Locator, command: string): Promise<void> {
const input = scope.locator(".agent-chat__composer-shell textarea");
await input.fill(command);
// Drafting starts before history admission; slash commands wait for Send readiness.
await scope.locator(".chat-send-btn--send:enabled").waitFor({ state: "visible" });
await input.press("Enter");
}
suite.define(() => {
for (const viewport of [
{ width: 1440, height: 900 },
@ -54,8 +63,7 @@ suite.define(() => {
const input = page.getByRole("textbox", { name: "Ask in side chat", exact: true });
expect(await input.isVisible()).toBe(false);
for (const draft of ["", "Keep this side draft"]) {
await mainInput.fill(command);
await mainInput.press("Enter");
await enterSideCommand(page, command);
await expect.poll(() => input.isVisible()).toBe(true);
await expect
.poll(() => input.evaluate((element) => document.activeElement === element))
@ -83,8 +91,7 @@ suite.define(() => {
await installMockGateway(page);
await page.goto(`${suite.server.baseUrl}chat`);
const mainInput = page.locator(".agent-chat__composer-shell textarea");
await mainInput.fill(command);
await mainInput.press("Enter");
await enterSideCommand(page, command);
await held.request;
const sideInput = page.getByRole("textbox", { name: "Ask in side chat", exact: true });
expect(await sideInput.count()).toBe(0);
@ -118,9 +125,7 @@ suite.define(() => {
try {
await installMockGateway(page);
await page.goto(`${suite.server.baseUrl}chat`);
const mainInput = page.locator(".agent-chat__composer-shell textarea");
await mainInput.fill("/btw");
await mainInput.press("Enter");
await enterSideCommand(page, "/btw");
await held.request;
const sideInput = page.getByRole("textbox", { name: "Ask in side chat", exact: true });
expect(await sideInput.count()).toBe(0);
@ -184,8 +189,7 @@ suite.define(() => {
.toBe(true);
}
const mainInput = page.locator(".agent-chat__composer-shell textarea");
await mainInput.fill("/btw what is this?");
await mainInput.press("Enter");
await enterSideCommand(page, "/btw what is this?");
const request = await gateway.waitForRequest("sessions.companion.ask");
expect(request.params).toMatchObject({ question: "what is this?" });
await page.locator(".chat-session-rail__exchange--pending").waitFor();
@ -219,8 +223,7 @@ suite.define(() => {
await expect.poll(() => panes.count()).toBe(2);
const firstInput = panes.first().locator(".agent-chat__composer-shell textarea");
const secondInput = panes.last().locator(".agent-chat__composer-shell textarea");
await firstInput.fill("/btw what is this?");
await firstInput.press("Enter");
await enterSideCommand(panes.first(), "/btw what is this?");
await gateway.waitForRequest("sessions.companion.ask");
const sideInput = panes
.first()
@ -275,8 +278,7 @@ suite.define(() => {
await page.goto(`${suite.server.baseUrl}chat`);
const mainInput = page.locator(".agent-chat__composer-shell textarea");
const sideInput = page.locator(".chat-session-rail__input");
await mainInput.fill("/btw what is this?");
await mainInput.press("Enter");
await enterSideCommand(page, "/btw what is this?");
const request = await gateway.waitForRequest("sessions.companion.ask");
expect(request.params).toMatchObject({ agentId: "main", sessionKey: "global" });
await page.locator(".chat-session-rail__exchange--pending").waitFor();
@ -333,8 +335,7 @@ suite.define(() => {
});
await page.goto(`${suite.server.baseUrl}chat`);
const mainInput = page.locator(".agent-chat__composer-shell textarea");
await mainInput.fill("/btw what is this?");
await mainInput.press("Enter");
await enterSideCommand(page, "/btw what is this?");
await gateway.waitForRequest("sessions.companion.ask");
const sideInput = page.locator(".chat-session-rail__input");
if (held) {

View file

@ -104,7 +104,7 @@ export function tooltipTitleText(item: Locator) {
type HeldModuleContext = {
closing: boolean;
pages: Map<Page, Array<{ release: () => void; installed: ReturnType<Page["route"]> }>>;
pages: Map<Page, Array<{ close: () => void; installed: ReturnType<Page["route"]> }>>;
};
const heldModuleContexts = new WeakMap<BrowserContext, HeldModuleContext>();
@ -122,29 +122,37 @@ export async function holdModuleResponse(page: Page, module: RegExp) {
if (held.closing) {
throw new Error("Cannot hold a module after browser context cleanup begins");
}
let release!: () => void;
let requested!: (url: string) => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const request = new Promise<string>((resolve) => {
requested = resolve;
});
const gate = createDeferredCore();
const request = createDeferredCore<string>();
// Cleanup can cancel an unused hold; callers still receive the original rejection.
void request.promise.catch(() => {});
const release = () => gate.resolve();
let requests = 0;
const installed = page.route(module, async (route) => {
requests += 1;
const response = await route.fetch();
expect(response.status()).toBe(200);
requested(route.request().url());
await gate;
await route.fulfill({ response });
try {
const response = await route.fetch();
expect(response.status()).toBe(200);
request.resolve(route.request().url());
await gate.promise;
await route.fulfill({ response });
} catch (error) {
request.reject(error);
throw error;
}
});
// Register before awaiting installation so teardown also owns a pending route().
const registrations = held.pages.get(page) ?? [];
registrations.push({ release, installed });
registrations.push({
close: () => {
release();
request.reject(new Error("Browser context cleanup canceled the held module request"));
},
installed,
});
held.pages.set(page, registrations);
await installed;
return { request, release, requests: () => requests };
return { request: request.promise, release, requests: () => requests };
}
class ControlUiE2eAcquisitionClosedError extends Error {}
@ -304,7 +312,7 @@ export function createControlUiE2eSuite(options: ControlUiE2eSuiteOptions): Cont
},
async () => {
for (const registration of registrations) {
registration.release();
registration.close();
}
// Release all gates before joining active page/context callbacks.
await settleControlUiCleanup(registrations.map(({ installed }) => installed));

View file

@ -395,6 +395,31 @@ suite.define(() => {
}
expect(failure).toBe(bodyFault);
});
it("settles unrequested module observation during context cleanup", async () => {
let publishContext;
const acquired = new Promise(resolve => { publishContext = resolve; });
const outcome = suite.withPage({}, async ({ context, page }) => {
const released = await holdModuleResponse(page, /module-unblocked/u);
released.release();
const callback = page.dispatchModule("module-unblocked", "unblocked");
expect(await released.request).toBe("https://fixture.invalid/module-unblocked.js");
expect(await callback).toMatchObject({ status: "fulfilled" });
const held = await holdModuleResponse(page, /module-never-requested/u);
await holdModuleResponse(page, /module-unobserved/u);
publishContext(context);
try {
await held.request;
} finally {
state.events.push("held request settled"); record();
}
}).then(() => undefined, error => error);
await suite.closeBrowserContext(await acquired);
expect(await outcome).toMatchObject({
message: "Browser context cleanup canceled the held module request",
});
state.events.push("held callback joined"); record();
});
} else if (${JSON.stringify(mode)} === "concurrent-close") {
it("joins the first context close", async () => {
const context = await suite.newBrowserContext({});
@ -703,11 +728,11 @@ it.for([
}),
);
it("drains held-module callbacks before closing the context after a body failure", (context) =>
it("drains held modules and cancels unrequested observations during cleanup", (context) =>
runJoinedShutdownTest(context, async () => {
const result = await runFixture("held-route-drain", context.signal);
expect(result.code, result.output).toBe(0);
expect(result.report.numPassedTests, result.output).toBe(1);
expect(result.report.numPassedTests, result.output).toBe(2);
expect(result.report.numFailedTests, result.output).toBe(0);
expect(
result.journal.firstCleanupEvent,
@ -729,8 +754,10 @@ it("drains held-module callbacks before closing the context after a body failure
expect(result.journal.events.indexOf("fulfilled later")).toBeLessThan(
result.journal.events.indexOf("close"),
);
expect(result.journal.events).toContain("held request settled");
expect(result.journal.events.at(-1)).toBe("held callback joined");
expect(result.journal).toMatchObject({
closeCalls: 1,
closeCalls: 2,
heldBodyErrorRetained: true,
browserClosed: true,
serverClosed: true,