From b137df3cf28ec1c0f9e68c8d2828c6ea91978299 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 9 Jul 2026 04:36:30 -0700 Subject: [PATCH] fix(ui): queue chat input while reconnecting --- CHANGELOG.md | 1 + docs/web/control-ui.md | 6 +- ui/src/e2e/chat-flow.e2e.test.ts | 88 +++++++++++++++++++ ui/src/pages/chat/chat-composer.test.ts | 30 +++++++ ui/src/pages/chat/chat-pane.ts | 8 +- ui/src/pages/chat/chat-queue.ts | 13 ++- ui/src/pages/chat/chat-send.test.ts | 56 ++++++++++++ ui/src/pages/chat/chat-send.ts | 79 +++++++++++------ ui/src/pages/chat/chat-view.test.ts | 60 +++++++++++++ ui/src/pages/chat/chat-view.ts | 2 +- ui/src/pages/chat/components/chat-composer.ts | 40 +++++---- .../pages/chat/composer-persistence.test.ts | 18 ++++ ui/src/pages/chat/composer-persistence.ts | 16 +++- ui/src/test-helpers/control-ui-e2e.ts | 50 ++++++++--- 14 files changed, 404 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74d9908736e..2d0dad40dbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- **Control UI offline chat:** keep ordinary text and attachment input available during recoverable Gateway disconnects, persist queued sends in the existing session-scoped store, and resume delivery after reconnect. - **Browser actions on Node 24:** keep browser request cancellation bound to the client and response lifetime instead of Node 24.16+'s prematurely aborted body-stream signal, preventing valid POST actions from failing after JSON parsing. Thanks @obviyus and @vincentkoc. - **SecretRef model credentials:** keep resolved provider secrets behind process-local sentinels through auth storage, stream setup, SDK configuration, and managed local-provider probing, then inject plaintext only at the final network or provider-plugin boundary while retaining exact-value log redaction. (#102008, #102009) - **Lean local model shell access:** keep `exec` directly visible beside the default structured Tool Search controls so coding-tuned local models can use their shell fallback instead of searching for missing domain tools. (#87587) Thanks @vincentkoc. diff --git a/docs/web/control-ui.md b/docs/web/control-ui.md index aaade71d038..386aae6da03 100644 --- a/docs/web/control-ui.md +++ b/docs/web/control-ui.md @@ -286,7 +286,11 @@ The terminal is also available as a full-screen, terminal-only document at `/?vi Once a session is established, a dropped Gateway connection does not log you out. The dashboard stays visible with a floating amber "Gateway connection lost — Reconnecting…" pill under the top bar while the client retries automatically with backoff (800 ms up to 15 s). Live updates and -actions pause until the connection returns; **Retry now** in the pill forces an immediate attempt. +realtime/session actions pause until the connection returns; **Retry now** in the pill forces an +immediate attempt. Chat remains editable: ordinary text and attachment sends are kept in the +current tab's gateway/session-scoped browser storage, shown as waiting for reconnect, and sent +automatically when the Gateway returns. Live controls and slash commands remain unavailable while +offline. When this browser already holds credentials (a configured token/password or an approved device token), first opens and reloads show a small animated OpenClaw mark while the connection is diff --git a/ui/src/e2e/chat-flow.e2e.test.ts b/ui/src/e2e/chat-flow.e2e.test.ts index 8fc6bdda445..6cafba60ef8 100644 --- a/ui/src/e2e/chat-flow.e2e.test.ts +++ b/ui/src/e2e/chat-flow.e2e.test.ts @@ -1894,6 +1894,94 @@ describeControlUiE2e("Control UI mocked Gateway E2E", () => { } }); + it("stores new input while offline and sends it after reconnect", async () => { + const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim(); + const context = await newBrowserContext({ + locale: "en-US", + ...(artifactDir + ? { recordVideo: { dir: artifactDir, size: { height: 900, width: 1280 } } } + : {}), + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + const gateway = await installMockGateway(page); + + try { + await page.goto(`${server.baseUrl}chat`); + const composer = page.locator(".agent-chat__composer-combobox textarea"); + await composer.waitFor({ state: "visible", timeout: 10_000 }); + + await gateway.setOnline(false); + await page.locator("openclaw-connection-banner").waitFor({ timeout: 10_000 }); + + const prompt = "send this when the Gateway returns"; + const composerEnabled = await composer.isEnabled(); + expect(composerEnabled).toBe(true); + await composer.fill(prompt); + const send = page.getByRole("button", { name: "Send message" }); + const sendEnabled = await send.isEnabled(); + expect(sendEnabled).toBe(true); + await send.click(); + + const queue = page.locator(".chat-queue"); + await queue.getByText("Waiting for reconnect").waitFor({ timeout: 10_000 }); + await queue.getByText(prompt).waitFor({ timeout: 10_000 }); + const requestsBeforeReconnect = await gateway.getRequests("chat.send"); + expect(requestsBeforeReconnect).toHaveLength(0); + const readStoredProof = () => + page.evaluate((expectedPrompt) => { + const stored = Object.entries(sessionStorage) + .filter(([key]) => key.startsWith("openclaw.control.chatComposer.v1:")) + .map(([, value]) => value) + .join("\n"); + return { + prompt: stored.includes(expectedPrompt), + waitingReconnect: stored.includes('"sendState":"waiting-reconnect"'), + }; + }, prompt); + await expect.poll(readStoredProof).toEqual({ prompt: true, waitingReconnect: true }); + const storedProof = await readStoredProof(); + if (artifactDir) { + await page.screenshot({ path: `${artifactDir}/01-offline-queued.png`, fullPage: true }); + } + + await expect.poll(() => gateway.getSocketCount()).toBeGreaterThan(1); + await gateway.setOnline(true); + + const request = await gateway.waitForRequest("chat.send"); + const params = requireRecord(request.params); + const runId = requireString(params.idempotencyKey, "offline send idempotency key"); + expect(params.message).toBe(prompt); + await expect.poll(async () => (await gateway.getRequests("chat.send")).length).toBe(1); + const requestsAfterReconnect = await gateway.getRequests("chat.send"); + await gateway.emitChatFinal({ runId, text: "Delivered after reconnect." }); + await queue.waitFor({ state: "detached", timeout: 10_000 }); + await page.locator("openclaw-connection-banner").waitFor({ state: "detached" }); + if (artifactDir) { + await page.screenshot({ path: `${artifactDir}/02-online-delivered.png`, fullPage: true }); + } + if (process.env.OPENCLAW_BEHAVIOR_PROOF === "1") { + process.stdout.write( + `${JSON.stringify({ + proof: "offline-chat-reconnect", + composerEnabled, + sendEnabled, + waitingStateVisible: true, + storedPrompt: storedProof.prompt, + storedWaitingState: storedProof.waitingReconnect, + requestsBeforeReconnect: requestsBeforeReconnect.length, + requestsAfterReconnect: requestsAfterReconnect.length, + idempotencyKeyPresent: runId.length > 0, + queueClearedAfterDelivery: true, + })}\n`, + ); + } + } finally { + await closeBrowserContext(context); + } + }); + it("keeps a session model override selected after switching away and back", async () => { const context = await newBrowserContext({ locale: "en-US", diff --git a/ui/src/pages/chat/chat-composer.test.ts b/ui/src/pages/chat/chat-composer.test.ts index 1ff14867197..fdd2e5104a0 100644 --- a/ui/src/pages/chat/chat-composer.test.ts +++ b/ui/src/pages/chat/chat-composer.test.ts @@ -28,6 +28,7 @@ vi.mock("../../components/markdown.ts", () => ({ function createProps(overrides: Partial = {}): ChatRunControlsProps { return { canAbort: false, + canSend: true, connected: true, draft: "", hasMessages: false, @@ -123,6 +124,35 @@ describe("chat run controls", () => { expect(container.querySelector('button[aria-label="Start voice input"]')).toBeNull(); }); + it("keeps queued sends available offline while disabling live voice input", () => { + const container = document.createElement("div"); + const onSend = vi.fn(); + const onToggleVoice = vi.fn(); + + render( + renderChatRunControls( + createProps({ + connected: false, + draft: "queue this offline", + onSend, + onToggleVoice, + }), + ), + container, + ); + + const sendButton = getButton( + container, + `button[aria-label="${t("chat.runControls.sendMessage")}"]`, + ); + expect(sendButton.disabled).toBe(false); + sendButton.click(); + expect(onSend).toHaveBeenCalledTimes(1); + + render(renderChatRunControls(createProps({ connected: false, onToggleVoice })), container); + expect(getButton(container, 'button[aria-label="Start voice input"]').disabled).toBe(true); + }); + it("keeps voice and generation stop actions available when both are active", () => { const container = document.createElement("div"); const onAbort = vi.fn(); diff --git a/ui/src/pages/chat/chat-pane.ts b/ui/src/pages/chat/chat-pane.ts index 15959cd528f..bbeaf77be31 100644 --- a/ui/src/pages/chat/chat-pane.ts +++ b/ui/src/pages/chat/chat-pane.ts @@ -859,11 +859,7 @@ class ChatPane extends LitElement { state.sessionsResult?.sessions.some( (row) => row.archived === true && areUiSessionKeysEquivalent(row.key, state.sessionKey), ) === true; - const disabledReason = !state.connected - ? t("chat.disconnected") - : selectedSessionArchived - ? t("chat.archivedSessionDisabled") - : null; + const disabledReason = selectedSessionArchived ? t("chat.archivedSessionDisabled") : null; const canOpenRealtimeTalkSettings = hasOperatorAdminAccess( this.context.gateway.snapshot.hello?.auth ?? null, ); @@ -899,7 +895,7 @@ class ChatPane extends LitElement { realtimeTalkInputLevel: state.realtimeTalkInputLevel, realtimeTalkConversation: state.realtimeTalkConversation, connected: state.connected, - canSend: state.connected && !selectedSessionArchived, + canSend: !selectedSessionArchived, disabledReason, error: state.lastError, sessions: state.sessionsResult, diff --git a/ui/src/pages/chat/chat-queue.ts b/ui/src/pages/chat/chat-queue.ts index 6d441bfb57f..879bf79528a 100644 --- a/ui/src/pages/chat/chat-queue.ts +++ b/ui/src/pages/chat/chat-queue.ts @@ -125,8 +125,17 @@ export function updateQueuedMessageForSession( return nextItem; } -export function persistQueuedMessagesForSession(host: ChatQueueSessionHost, sessionKey: string) { - persistStoredChatComposerQueue(host, sessionKey, readChatQueueForSession(host, sessionKey)); +export function persistQueuedMessagesForSession( + host: ChatQueueSessionHost, + sessionKey: string, + requiredQueueItemId?: string, +) { + return persistStoredChatComposerQueue( + host, + sessionKey, + readChatQueueForSession(host, sessionKey), + requiredQueueItemId, + ); } export function removeQueuedMessageWithoutReleasing( diff --git a/ui/src/pages/chat/chat-send.test.ts b/ui/src/pages/chat/chat-send.test.ts index 8062ee13eed..42545b90a6e 100644 --- a/ui/src/pages/chat/chat-send.test.ts +++ b/ui/src/pages/chat/chat-send.test.ts @@ -4,6 +4,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } import type { GatewaySessionRow, SessionsListResult } from "../../api/types.ts"; import type { UiSettings } from "../../app/settings.ts"; import { createSessionCapability } from "../../lib/sessions/index.ts"; +import { createStorageMock } from "../../test-helpers/storage.ts"; import { getChatAttachmentDataUrl, getChatAttachmentPreviewUrl, @@ -1100,6 +1101,7 @@ describe("handleSendChat", () => { beforeEach(() => { executeSlashCommandMock.mockReset(); setLastActiveSessionKeyMock.mockReset(); + vi.stubGlobal("sessionStorage", createStorageMock()); }); afterEach(() => { @@ -2616,6 +2618,60 @@ describe("handleSendChat", () => { ); }); + it("retains offline attachments when browser storage rejects the queue", async () => { + const storage = createStorageMock(); + vi.spyOn(storage, "setItem").mockImplementation(() => { + throw new DOMException("quota exceeded", "QuotaExceededError"); + }); + vi.stubGlobal("sessionStorage", storage); + const attachment = { + id: "offline-attachment", + mimeType: "image/png", + fileName: "offline.png", + sizeBytes: 3, + dataUrl: "data:image/png;base64,AAA", + }; + const host = makeHost({ + client: null, + connected: false, + chatAttachments: [attachment], + }); + + await handleSendChat(host); + + expect(host.chatAttachments).toEqual([attachment]); + expect(host.chatQueue).toStrictEqual([]); + expect(host.lastError).toBe( + "Could not store this message for reconnect. Free browser storage or reconnect before sending.", + ); + }); + + it("consumes a reply target after queueing its turn offline", async () => { + const host = makeHost({ + client: null, + connected: false, + chatMessage: "continue offline", + chatReplyTarget: { + messageId: "reply-source-offline", + text: "quoted body", + senderLabel: "User", + }, + }); + + await handleSendChat(host); + + expect(host.chatQueue[0]).toMatchObject({ + text: "> **User:** quoted body\n\ncontinue offline", + sendState: "waiting-reconnect", + }); + expect(host.chatReplyTarget).toBeNull(); + + host.chatMessage = "next offline turn"; + await handleSendChat(host); + + expect(host.chatQueue[1]?.text).toBe("next offline turn"); + }); + it("replays queued global sends under the originally selected agent", async () => { const request = vi.fn((method: string) => { if (method === "chat.send") { diff --git a/ui/src/pages/chat/chat-send.ts b/ui/src/pages/chat/chat-send.ts index d26fadafcfe..7780581b6e8 100644 --- a/ui/src/pages/chat/chat-send.ts +++ b/ui/src/pages/chat/chat-send.ts @@ -530,6 +530,9 @@ function cancelPendingSendBeforeRequest( type QueuedChatSendResult = "sent" | "pending" | "failed"; +const OFFLINE_QUEUE_STORAGE_ERROR = + "Could not store this message for reconnect. Free browser storage or reconnect before sending."; + function ensureQueuedSendState( host: ChatHost, item: ChatQueueItem, @@ -583,11 +586,21 @@ async function sendQueuedChatMessage( } const sessionKey = prepared.sessionKey ?? host.sessionKey; if (!host.connected || !host.client) { - updateQueuedMessageForSession(host, sessionKey, id, (item) => ({ + const waiting = updateQueuedMessageForSession(host, sessionKey, id, (item) => ({ ...item, sendState: "waiting-reconnect", sendError: undefined, })); + if (!waiting || !persistQueuedMessagesForSession(host, sessionKey, waiting.id)) { + cancelPendingSendBeforeRequest(host, waiting ?? prepared, { + previousDraft: opts?.previousDraft, + previousAttachments: opts?.previousAttachments, + }); + if (visibleSessionMatches(host, sessionKey, prepared.agentId)) { + setChatError(host, OFFLINE_QUEUE_STORAGE_ERROR); + } + return "failed"; + } return "pending"; } @@ -744,13 +757,25 @@ async function sendQueuedChatMessage( } catch (err) { const error = formatConnectError(err); if (isRecoverableChatSendError(err, error)) { - updateQueuedMessageForSession(host, sessionKey, id, (item) => ({ + const waiting = updateQueuedMessageForSession(host, sessionKey, id, (item) => ({ ...item, sendError: error, sendState: "waiting-reconnect", })); + const stored = Boolean( + waiting && persistQueuedMessagesForSession(host, sessionKey, waiting.id), + ); + if (!stored && waiting) { + updateQueuedMessageForSession(host, sessionKey, id, (item) => ({ + ...item, + sendError: OFFLINE_QUEUE_STORAGE_ERROR, + })); + } if (isVisibleSession()) { - setChatError(host, "Message will send when the Gateway reconnects."); + setChatError( + host, + stored ? "Message will send when the Gateway reconnects." : OFFLINE_QUEUE_STORAGE_ERROR, + ); } recordChatSendTiming(host, prepared, "waiting-reconnect", prepared.sendSubmittedAtMs, { error, @@ -786,7 +811,7 @@ async function sendChatMessageNow( refreshSessions?: boolean; submittedAtMs?: number; }, -) { +): Promise { resetToolStream(host as unknown as Parameters[0]); // Reset scroll state before sending to ensure auto-scroll works for the response resetChatScroll(host as unknown as Parameters[0]); @@ -801,15 +826,15 @@ async function sendChatMessageNow( opts?.submittedAtMs, ); if (!queued) { - return false; + return "failed"; } const queuedSessionKey = queued.sessionKey ?? host.sessionKey; const result = await sendQueuedChatMessage(host, queued.id, { previousDraft: opts?.previousDraft, previousAttachments: opts?.previousAttachments, }); - const ok = result === "sent"; - if (ok && host.sessionKey === queuedSessionKey) { + const sent = result === "sent"; + if (sent && host.sessionKey === queuedSessionKey) { setLastActiveSessionKey( host as unknown as Parameters[0], queuedSessionKey, @@ -817,7 +842,7 @@ async function sendChatMessageNow( resetChatInputHistoryNavigation(host); } if ( - ok && + sent && host.sessionKey === queuedSessionKey && opts?.restoreDraft && opts.previousDraft?.trim() @@ -825,7 +850,7 @@ async function sendChatMessageNow( host.chatMessage = opts.previousDraft; } if ( - ok && + sent && host.sessionKey === queuedSessionKey && opts?.restoreAttachments && opts.previousAttachments?.length @@ -836,10 +861,10 @@ async function sendChatMessageNow( if (host.sessionKey === queuedSessionKey) { scheduleChatScroll(host as unknown as Parameters[0], true); } - if (ok && host.sessionKey === queuedSessionKey && !host.chatRunId) { + if (sent && host.sessionKey === queuedSessionKey && !host.chatRunId) { void flushChatQueue(host); } - return ok; + return result; } function attachmentSubmitSignature(attachment: ChatAttachment): string { @@ -1051,11 +1076,12 @@ async function flushChatQueue(host: ChatHost) { }); ok = true; } else { - ok = await sendChatMessageNow(host, next.text, { + const result = await sendChatMessageNow(host, next.text, { queueItemId: next.id, attachments: next.attachments, refreshSessions: next.refreshSessions, }); + ok = result === "sent"; } } catch (err) { setChatError(host, String(err)); @@ -1280,6 +1306,7 @@ export async function handleSendChat( return; } + let sendResult: QueuedChatSendResult; if (isChatBusy(host)) { updateQueuedMessage(host, queued.id, (item) => ({ ...item, @@ -1287,25 +1314,27 @@ export async function handleSendChat( sendState: undefined, })); recordChatSendTiming(host, queued, "queued-busy", submittedAtMs); - return; + sendResult = "pending"; + } else { + sendResult = await sendChatMessageNow(host, effectiveMessage, { + queueItemId: queued.id, + previousDraft: cleared.previousDraft, + restoreDraft: Boolean(messageOverride && opts?.restoreDraft), + attachments: hasAttachments ? attachmentsToSend : undefined, + previousAttachments: cleared.previousAttachments, + restoreAttachments: Boolean(messageOverride && opts?.restoreDraft), + refreshSessions, + submittedAtMs, + }); } - - const accepted = await sendChatMessageNow(host, effectiveMessage, { - queueItemId: queued.id, - previousDraft: cleared.previousDraft, - restoreDraft: Boolean(messageOverride && opts?.restoreDraft), - attachments: hasAttachments ? attachmentsToSend : undefined, - previousAttachments: cleared.previousAttachments, - restoreAttachments: Boolean(messageOverride && opts?.restoreDraft), - refreshSessions, - submittedAtMs, - }); if ( - accepted && + sendResult !== "failed" && replyTarget && host.chatReplyTarget?.messageId === replyTarget.messageId && host.sessionKey === submittedSessionKey ) { + // A reconnect queue owns the quoted turn before the Gateway ACK. Consume + // its reply target so later offline turns cannot reuse stale context. host.chatReplyTarget = null; } }); diff --git a/ui/src/pages/chat/chat-view.test.ts b/ui/src/pages/chat/chat-view.test.ts index 0acf618dde0..1ead00500b6 100644 --- a/ui/src/pages/chat/chat-view.test.ts +++ b/ui/src/pages/chat/chat-view.test.ts @@ -1092,6 +1092,31 @@ describe("chat goal status", () => { }); describe("chat composer workbench", () => { + it("queues ordinary input offline while keeping live commands disabled", () => { + const onSend = vi.fn(); + const container = renderChatView({ + connected: false, + draft: "queue this offline", + getDraft: () => "queue this offline", + onSend, + }); + + expect(container.querySelector("textarea")?.disabled).toBe(false); + expect(container.querySelector(".agent-chat__file-input")?.disabled).toBe( + false, + ); + const send = container.querySelector('button[aria-label="Send message"]'); + expect(send?.disabled).toBe(false); + send?.click(); + expect(onSend).toHaveBeenCalledTimes(1); + + const commandContainer = renderChatView({ connected: false, draft: "/status" }); + expect( + commandContainer.querySelector('button[aria-label="Send message"]') + ?.disabled, + ).toBe(true); + }); + it("renders session controls in the composer and workspace files in the expanded rail", () => { const onToggleCollapsed = vi.fn(); const onRefresh = vi.fn(); @@ -2399,6 +2424,41 @@ describe("chat slash menu accessibility", () => { expect(container.querySelector(".slash-menu")).toBeNull(); }); + it("does not submit a stale slash argument menu after disconnect", () => { + let draft = ""; + const onDraftChange = vi.fn((next: string) => { + draft = next; + }); + const onSend = vi.fn(); + const container = document.createElement("div"); + const renderCurrent = (connected: boolean) => { + render( + renderChat( + createChatProps({ + connected, + draft, + getDraft: () => draft, + onDraftChange, + onSend, + }), + ), + container, + ); + }; + + renderCurrent(true); + inputDraft(container, "/tools "); + renderCurrent(true); + expect(container.querySelector(".slash-menu")).not.toBeNull(); + + renderCurrent(false); + expect(container.querySelector(".slash-menu")).toBeNull(); + keydownComposer(container, "Enter"); + + expect(onSend).not.toHaveBeenCalled(); + expect(draft).toBe("/tools "); + }); + it("clears the visible local draft immediately when send clears the host draft", () => { let draft = ""; const container = document.createElement("div"); diff --git a/ui/src/pages/chat/chat-view.ts b/ui/src/pages/chat/chat-view.ts index 3b294c1143e..f17f088922a 100644 --- a/ui/src/pages/chat/chat-view.ts +++ b/ui/src/pages/chat/chat-view.ts @@ -164,7 +164,7 @@ export function renderChat(props: ChatProps) { const requestUpdate = props.onRequestUpdate ?? (() => {}); const splitRatio = props.splitRatio ?? 0.6; const sidebarOpen = Boolean(props.sidebarOpen && props.onCloseSidebar); - const canCompose = props.connected && props.canSend; + const canCompose = props.canSend; let chatSection: HTMLElement | null = null; const thread = renderChatThread({ diff --git a/ui/src/pages/chat/components/chat-composer.ts b/ui/src/pages/chat/components/chat-composer.ts index e7a739f8e63..95874a7a051 100644 --- a/ui/src/pages/chat/components/chat-composer.ts +++ b/ui/src/pages/chat/components/chat-composer.ts @@ -1731,6 +1731,7 @@ export function renderContextNotice( export type ChatRunControlsProps = { canAbort: boolean; + canSend: boolean; connected: boolean; draft: string; hasAttachments?: boolean; @@ -1794,7 +1795,7 @@ function renderChatPrimaryActions(props: ChatRunControlsProps) {