From 28069db63aad2cc92d367fd441eec6d3e8c77041 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 13:23:59 +0100 Subject: [PATCH] feat(macos): render system chat option cards (#110584) * feat(macos): render system chat option cards * fix(macos): localize question card controls * test(macos): stabilize main actor attachment tests --- apps/.i18n/native-source.json | 32 ++- apps/.i18n/native/ar.json | 10 + apps/.i18n/native/de.json | 10 + apps/.i18n/native/es.json | 10 + apps/.i18n/native/fa.json | 10 + apps/.i18n/native/fr.json | 10 + apps/.i18n/native/hi.json | 10 + apps/.i18n/native/id.json | 10 + apps/.i18n/native/it.json | 10 + apps/.i18n/native/ja-JP.json | 10 + apps/.i18n/native/ko.json | 10 + apps/.i18n/native/nl.json | 10 + apps/.i18n/native/pl.json | 10 + apps/.i18n/native/pt-BR.json | 10 + apps/.i18n/native/ru.json | 10 + apps/.i18n/native/sv.json | 10 + apps/.i18n/native/th.json | 10 + apps/.i18n/native/tr.json | 10 + apps/.i18n/native/uk.json | 10 + apps/.i18n/native/vi.json | 10 + apps/.i18n/native/zh-CN.json | 10 + apps/.i18n/native/zh-TW.json | 10 + .../OpenClaw/OnboardingSystemAgentChat.swift | 186 ++++++++++++++++-- .../OnboardingSystemAgentChatTests.swift | 140 ++++++++++++- .../OpenClawKit/SystemAgentChatQuestion.swift | 63 ++++++ .../ChatViewModelAttachmentTests.swift | 6 +- .../SystemAgentChatQuestionTests.swift | 104 ++++++++++ docs/start/onboarding.md | 5 +- 28 files changed, 716 insertions(+), 30 deletions(-) create mode 100644 apps/shared/OpenClawKit/Sources/OpenClawKit/SystemAgentChatQuestion.swift create mode 100644 apps/shared/OpenClawKit/Tests/OpenClawKitTests/SystemAgentChatQuestionTests.swift diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index 5a9304050e0..9b72511f992 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -32243,7 +32243,7 @@ }, { "kind": "ui-localized-call", - "line": 10, + "line": 11, "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", "source": "Wake up, my friend!", "surface": "apple", @@ -32251,7 +32251,7 @@ }, { "kind": "conditional-branch", - "line": 129, + "line": 229, "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", "source": "", "surface": "apple", @@ -32259,7 +32259,7 @@ }, { "kind": "conditional-branch", - "line": 228, + "line": 288, "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", "source": "OpenClaw was interrupted. Restart to try again.", "surface": "apple", @@ -32267,7 +32267,7 @@ }, { "kind": "conditional-branch", - "line": 229, + "line": 289, "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", "source": "The Gateway connection changed. Restart OpenClaw to reconnect.", "surface": "apple", @@ -32275,7 +32275,7 @@ }, { "kind": "ui-call", - "line": 253, + "line": 331, "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", "source": "OpenClaw is working…", "surface": "apple", @@ -32283,7 +32283,7 @@ }, { "kind": "ui-call", - "line": 278, + "line": 356, "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", "source": "Restart", "surface": "apple", @@ -32291,7 +32291,7 @@ }, { "kind": "ui-call", - "line": 289, + "line": 367, "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", "source": "Enter secret…", "surface": "apple", @@ -32299,12 +32299,28 @@ }, { "kind": "ui-call", - "line": 291, + "line": 369, "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", "source": "Reply to OpenClaw… (yes sets everything up)", "surface": "apple", "id": "native.apple.35a542956f4f5497" }, + { + "kind": "ui-call", + "line": 409, + "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", + "source": "Skip for now", + "surface": "apple", + "id": "native.apple.134b9ae1d8a543df" + }, + { + "kind": "ui-call", + "line": 444, + "path": "apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift", + "source": "Recommended", + "surface": "apple", + "id": "native.apple.5435391fa338f227" + }, { "kind": "ui-call", "line": 10, diff --git a/apps/.i18n/native/ar.json b/apps/.i18n/native/ar.json index 2698e247d00..9c72e30e546 100644 --- a/apps/.i18n/native/ar.json +++ b/apps/.i18n/native/ar.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "رد على OpenClaw… (تؤدي الإجابة بنعم إلى إعداد كل شيء)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "تخطي الآن" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "موصى به" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/de.json b/apps/.i18n/native/de.json index 5f93ae8c189..6936c497922 100644 --- a/apps/.i18n/native/de.json +++ b/apps/.i18n/native/de.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "OpenClaw antworten… („yes“ richtet alles ein)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Vorerst überspringen" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Empfohlen" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/es.json b/apps/.i18n/native/es.json index 3fc69f9e398..6aa37f60f72 100644 --- a/apps/.i18n/native/es.json +++ b/apps/.i18n/native/es.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Responde a OpenClaw… («sí» lo configura todo)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Omitir por ahora" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Recomendado" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/fa.json b/apps/.i18n/native/fa.json index 0d73f0b9ab2..216e7830794 100644 --- a/apps/.i18n/native/fa.json +++ b/apps/.i18n/native/fa.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "به OpenClaw پاسخ دهید… (با «بله» همه‌چیز راه‌اندازی می‌شود)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "فعلاً رد شود" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "پیشنهادشده" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/fr.json b/apps/.i18n/native/fr.json index 578fd210a50..9afa88f674f 100644 --- a/apps/.i18n/native/fr.json +++ b/apps/.i18n/native/fr.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Répondre à OpenClaw… (« yes » configure tout)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Ignorer pour l’instant" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Recommandé" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/hi.json b/apps/.i18n/native/hi.json index 9d0e2e93564..1756e77d771 100644 --- a/apps/.i18n/native/hi.json +++ b/apps/.i18n/native/hi.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "OpenClaw को जवाब दें… (हाँ कहने पर सब कुछ सेट हो जाएगा)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "अभी के लिए छोड़ें" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "अनुशंसित" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/id.json b/apps/.i18n/native/id.json index 6d8c5fae09d..8bd4df8cd41 100644 --- a/apps/.i18n/native/id.json +++ b/apps/.i18n/native/id.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Balas OpenClaw… (yes menyiapkan semuanya)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Lewati untuk sekarang" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Direkomendasikan" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/it.json b/apps/.i18n/native/it.json index 395dff367e6..e9165565695 100644 --- a/apps/.i18n/native/it.json +++ b/apps/.i18n/native/it.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Rispondi a OpenClaw… (sì configura tutto)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Salta per ora" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Consigliato" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/ja-JP.json b/apps/.i18n/native/ja-JP.json index 0f711b64b8c..b9e31674c2c 100644 --- a/apps/.i18n/native/ja-JP.json +++ b/apps/.i18n/native/ja-JP.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "OpenClawに返信…（yesですべてセットアップされます）" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "今はスキップ" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "おすすめ" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/ko.json b/apps/.i18n/native/ko.json index e3aea0e9edb..652b5c3e550 100644 --- a/apps/.i18n/native/ko.json +++ b/apps/.i18n/native/ko.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "OpenClaw에 답장… (yes를 입력하면 모든 설정이 완료됩니다)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "지금은 건너뛰기" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "권장" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/nl.json b/apps/.i18n/native/nl.json index ebeacd69de1..ac55f7b12f3 100644 --- a/apps/.i18n/native/nl.json +++ b/apps/.i18n/native/nl.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Antwoord aan OpenClaw… (met ja wordt alles ingesteld)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Voor nu overslaan" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Aanbevolen" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/pl.json b/apps/.i18n/native/pl.json index f8c52040a94..fc9772e9b76 100644 --- a/apps/.i18n/native/pl.json +++ b/apps/.i18n/native/pl.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Odpowiedz OpenClaw… („yes” skonfiguruje wszystko)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Pomiń na razie" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Zalecane" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/pt-BR.json b/apps/.i18n/native/pt-BR.json index 19157fc44a0..4f46705ec54 100644 --- a/apps/.i18n/native/pt-BR.json +++ b/apps/.i18n/native/pt-BR.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Responda ao OpenClaw… (sim configura tudo)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Pular por enquanto" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Recomendado" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/ru.json b/apps/.i18n/native/ru.json index b85d5701a5c..93b72c61f39 100644 --- a/apps/.i18n/native/ru.json +++ b/apps/.i18n/native/ru.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Ответьте OpenClaw… (yes настроит всё)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Пропустить пока" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Рекомендуется" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/sv.json b/apps/.i18n/native/sv.json index 82875ddfbac..055980b9650 100644 --- a/apps/.i18n/native/sv.json +++ b/apps/.i18n/native/sv.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Svara OpenClaw… (ja konfigurerar allt)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Hoppa över tills vidare" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Rekommenderas" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/th.json b/apps/.i18n/native/th.json index 5bf285935c3..f04d133b26b 100644 --- a/apps/.i18n/native/th.json +++ b/apps/.i18n/native/th.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "ตอบกลับ OpenClaw… (พิมพ์ yes เพื่อตั้งค่าทุกอย่าง)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "ข้ามไปก่อน" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "แนะนำ" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/tr.json b/apps/.i18n/native/tr.json index a34847d4884..3acba5e8713 100644 --- a/apps/.i18n/native/tr.json +++ b/apps/.i18n/native/tr.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "OpenClaw'a yanıt verin… (evet, her şeyi ayarlar)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Şimdilik atla" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Önerilen" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/uk.json b/apps/.i18n/native/uk.json index bbdbab70e4d..afc99d27cdd 100644 --- a/apps/.i18n/native/uk.json +++ b/apps/.i18n/native/uk.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Відповісти OpenClaw… (так налаштує все)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Пропустити зараз" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Рекомендовано" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/vi.json b/apps/.i18n/native/vi.json index 1e4200348a2..9652f177c89 100644 --- a/apps/.i18n/native/vi.json +++ b/apps/.i18n/native/vi.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "Trả lời OpenClaw… (chọn có để thiết lập mọi thứ)" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "Bỏ qua lúc này" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "Được đề xuất" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/zh-CN.json b/apps/.i18n/native/zh-CN.json index 59c85e46326..02ae28184cc 100644 --- a/apps/.i18n/native/zh-CN.json +++ b/apps/.i18n/native/zh-CN.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "回复 OpenClaw…（回复 yes 即可完成所有设置）" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "暂时跳过" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "推荐" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/.i18n/native/zh-TW.json b/apps/.i18n/native/zh-TW.json index 5456d8f418e..7099064c50c 100644 --- a/apps/.i18n/native/zh-TW.json +++ b/apps/.i18n/native/zh-TW.json @@ -20193,6 +20193,16 @@ "source": "Reply to OpenClaw… (yes sets everything up)", "translated": "回覆 OpenClaw…（輸入 yes 即可完成所有設定）" }, + { + "id": "native.apple.134b9ae1d8a543df", + "source": "Skip for now", + "translated": "暫時略過" + }, + { + "id": "native.apple.5435391fa338f227", + "source": "Recommended", + "translated": "建議" + }, { "id": "native.apple.78d1ad73762a7ce8", "source": "Connect your AI", diff --git a/apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift b/apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift index da3412de170..66dff302560 100644 --- a/apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift +++ b/apps/macos/Sources/OpenClaw/OnboardingSystemAgentChat.swift @@ -1,5 +1,6 @@ import Foundation import Observation +import OpenClawKit import SwiftUI enum SystemAgentDraft: String, Decodable { @@ -60,12 +61,21 @@ final class SystemAgentOnboardingChatModel { let id = UUID() let role: Role let text: String + let question: SystemAgentChatQuestion? + + init(role: Role, text: String, question: SystemAgentChatQuestion? = nil) { + self.role = role + self.text = text + self.question = question + } } private(set) var messages: [Message] = [] private(set) var isSending = false private(set) var errorMessage: String? private(set) var expectsSensitiveReply = false + private(set) var dismissedQuestionMessageIDs: Set = [] + private(set) var retiredQuestionMessageIDs: Set = [] var input = "" /// Set when OpenClaw hands off to the normal agent ("talk to agent"). var onAgentHandoff: ((SystemAgentDraft?) -> Void)? @@ -100,6 +110,7 @@ final class SystemAgentOnboardingChatModel { let action: String let sensitive: Bool? let agentDraft: SystemAgentDraft? + let question: AnyCodable? } func startIfNeeded() async { @@ -118,23 +129,42 @@ final class SystemAgentOnboardingChatModel { @discardableResult func send() -> Task? { let text = self.input.trimmingCharacters(in: .whitespacesAndNewlines) - guard let generation = self.requestGeneration, - !text.isEmpty, - !self.isSending, - self.errorMessage == nil + return self.send(message: text) + } + + @discardableResult + func answerQuestion(messageID: UUID, optionLabel: String) -> Task? { + guard let message = self.messages.first(where: { $0.id == messageID }), + let question = message.question, + let option = question.options.first(where: { $0.label == optionLabel }), + self.canAnswerQuestion(message) else { return nil } - self.input = "" - self.messages.append(Message( - role: .user, - text: self.expectsSensitiveReply ? "" : text)) - let task = Task { [weak self] in - guard let self else { return } - await self.requestReply(message: text, generation: generation) - } - self.requestTask = task + // The typed Gateway contract separates the visible label from its canonical reply. + // Keep the transcript human-readable while returning the machine-facing value. + return self.send(message: option.reply ?? option.label, displayText: option.label) + } + + @discardableResult + func skipQuestion(messageID: UUID) -> Task? { + guard let message = self.messages.first(where: { $0.id == messageID }), + self.canAnswerQuestion(message), + let task = self.send(message: "Skip for now", displayText: "Skip for now") + else { return nil } + self.dismissedQuestionMessageIDs.insert(messageID) return task } + func isQuestionVisible(_ message: Message) -> Bool { + message.question != nil && !self.dismissedQuestionMessageIDs.contains(message.id) + } + + func canAnswerQuestion(_ message: Message) -> Bool { + self.isQuestionVisible(message) && + !self.retiredQuestionMessageIDs.contains(message.id) && + !self.isSending && + self.errorMessage == nil + } + @discardableResult func restartAfterError() -> Task? { guard let previousGeneration = self.requestGeneration else { return nil } @@ -145,6 +175,8 @@ final class SystemAgentOnboardingChatModel { self.route = nil self.started = true self.messages.removeAll() + self.dismissedQuestionMessageIDs.removeAll() + self.retiredQuestionMessageIDs.removeAll() self.input = "" self.expectsSensitiveReply = false let task = Task { [weak self] in @@ -184,6 +216,31 @@ final class SystemAgentOnboardingChatModel { return route } + private func send(message: String, displayText: String? = nil) -> Task? { + guard let generation = self.requestGeneration, + !message.isEmpty, + !self.isSending, + self.errorMessage == nil + else { return nil } + self.retireQuestions() + self.input = "" + self.messages.append(Message( + role: .user, + text: displayText ?? (self.expectsSensitiveReply ? "" : message))) + let task = Task { [weak self] in + guard let self else { return } + await self.requestReply(message: message, generation: generation) + } + self.requestTask = task + return task + } + + private func retireQuestions() { + for message in self.messages where message.question != nil { + self.retiredQuestionMessageIDs.insert(message.id) + } + } + private func requestReply(message: String?, generation: UInt64) async { guard self.isCurrentRequest(generation) else { return } self.isSending = true @@ -215,7 +272,10 @@ final class SystemAgentOnboardingChatModel { let result = try JSONDecoder().decode(ChatResult.self, from: data) guard self.isCurrentRequest(generation) else { return } self.expectsSensitiveReply = result.sensitive == true - self.messages.append(Message(role: .assistant, text: result.reply)) + self.messages.append(Message( + role: .assistant, + text: result.reply, + question: SystemAgentChatQuestion.parse(result.question?.dictionaryValue))) self.onReplyReceived?() if result.action == "open-agent" { self.onAgentHandoff?(result.agentDraft) @@ -243,8 +303,26 @@ struct SystemAgentOnboardingChatView: View { ScrollView { LazyVStack(alignment: .leading, spacing: 10) { ForEach(self.model.messages) { message in - SystemAgentChatBubble(message: message) - .id(message.id) + VStack(alignment: .leading, spacing: 8) { + SystemAgentChatBubble(message: message) + if let question = message.question, + self.model.isQuestionVisible(message) + { + SystemAgentChatQuestionCard( + question: question, + isEnabled: self.model.canAnswerQuestion(message), + onSelect: { option in + self.model.answerQuestion( + messageID: message.id, + optionLabel: option.label) + }, + onSkip: { + self.model.skipQuestion(messageID: message.id) + }) + .padding(.trailing, 40) + } + } + .id(message.id) } if self.model.isSending { HStack(spacing: 8) { @@ -312,6 +390,82 @@ struct SystemAgentOnboardingChatView: View { } } +private struct SystemAgentChatQuestionCard: View { + let question: SystemAgentChatQuestion + let isEnabled: Bool + let onSelect: (SystemAgentChatQuestion.Option) -> Void + let onSkip: () -> Void + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + Text(self.question.header.uppercased()) + .font(.caption2.weight(.semibold)) + .foregroundStyle(Color.accentColor) + Text(self.question.question) + .font(.callout.weight(.semibold)) + ForEach(self.question.options, id: \.label) { option in + self.optionButton(option) + } + Button("Skip for now", action: self.onSkip) + .buttonStyle(.plain) + .font(.caption) + .foregroundStyle(.secondary) + .disabled(!self.isEnabled) + } + .padding(12) + .frame(maxWidth: 460, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(Color(NSColor.controlBackgroundColor))) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(.secondary.opacity(0.2))) + .accessibilityElement(children: .contain) + .accessibilityLabel(self.question.question) + } + + private func optionButton(_ option: SystemAgentChatQuestion.Option) -> some View { + Button { + self.onSelect(option) + } label: { + HStack(alignment: .top, spacing: 10) { + VStack(alignment: .leading, spacing: 2) { + Text(option.label) + .font(.callout.weight(.medium)) + .foregroundStyle(.primary) + if let description = option.description { + Text(description) + .font(.caption) + .foregroundStyle(.secondary) + } + } + Spacer(minLength: 8) + if option.recommended { + Text("Recommended") + .font(.caption2.weight(.semibold)) + .foregroundStyle(Color.accentColor) + } + } + .padding(.horizontal, 10) + .padding(.vertical, 8) + .frame(maxWidth: .infinity, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 9, style: .continuous) + .fill(option.recommended + ? Color.accentColor.opacity(0.12) + : Color(NSColor.windowBackgroundColor))) + .overlay( + RoundedRectangle(cornerRadius: 9, style: .continuous) + .stroke(option.recommended + ? Color.accentColor.opacity(0.55) + : Color.secondary.opacity(0.16))) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .disabled(!self.isEnabled) + } +} + private struct SystemAgentChatBubble: View { let message: SystemAgentOnboardingChatModel.Message diff --git a/apps/macos/Tests/OpenClawIPCTests/OnboardingSystemAgentChatTests.swift b/apps/macos/Tests/OpenClawIPCTests/OnboardingSystemAgentChatTests.swift index bebffb435ea..25dcc50a34e 100644 --- a/apps/macos/Tests/OpenClawIPCTests/OnboardingSystemAgentChatTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/OnboardingSystemAgentChatTests.swift @@ -27,6 +27,18 @@ private actor SystemAgentSessionRecorder { } } +private actor SystemAgentMessageRecorder { + private var messages: [String] = [] + + func record(_ message: String) { + self.messages.append(message) + } + + func snapshot() -> [String] { + self.messages + } +} + private actor SystemAgentMethodRecorder { private var methods: [String] = [] @@ -88,6 +100,20 @@ private func systemAgentRequestMethod(from message: URLSessionWebSocketTask.Mess return object["method"] as? String } +private func systemAgentChatMessage(from message: URLSessionWebSocketTask.Message) -> String? { + let data: Data? = switch message { + case let .data(data): data + case let .string(string): string.data(using: .utf8) + @unknown default: nil + } + guard let data, + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + object["method"] as? String == "openclaw.chat", + let params = object["params"] as? [String: Any] + else { return nil } + return params["message"] as? String +} + private func respondToSystemAgentHealth( task: GatewayTestWebSocketTask, id: String, @@ -101,9 +127,11 @@ private func respondToSystemAgentHealth( private func systemAgentResponse( id: String, action: String = "none", - agentDraft: String? = nil) -> Data + agentDraft: String? = nil, + questionJSON: String? = nil) -> Data { let agentDraftField = agentDraft.map { ",\n \"agentDraft\": \"\($0)\"" } ?? "" + let questionField = questionJSON.map { ",\n \"question\": \($0)" } ?? "" return Data( """ { @@ -114,7 +142,7 @@ private func systemAgentResponse( "sessionId": "test-session", "reply": "ready", "action": "\(action)", - "sensitive": false\(agentDraftField) + "sensitive": false\(agentDraftField)\(questionField) } } """.utf8) @@ -482,6 +510,114 @@ struct OnboardingSystemAgentChatTests { #expect(session.latestTask()?.snapshotSendCount() == 2) } + @Test func `typed question sends reply while transcript shows label`() async throws { + let recordedMessages = SystemAgentMessageRecorder() + let questionJSON = + """ + {"id":"next","header":"Next step","question":"What now?","options":[ + {"label":"Talk to my agent","reply":"talk to agent","recommended":true}, + {"label":"Connect WhatsApp","reply":"connect whatsapp","description":"Chat there."} + ],"isOther":true} + """ + let session = GatewayTestWebSocketSession(taskFactory: { + GatewayTestWebSocketTask(sendHook: { task, message, sendIndex in + guard sendIndex > 0, + let id = GatewayWebSocketTestSupport.requestID(from: message) + else { return } + if let message = systemAgentChatMessage(from: message) { + await recordedMessages.record(message) + task.emitReceiveSuccess(.data(systemAgentResponse(id: id))) + } else { + task.emitReceiveSuccess(.data(systemAgentResponse( + id: id, + questionJSON: questionJSON))) + } + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let gateway = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + let chat = SystemAgentOnboardingChatModel(gateway: gateway) + + await chat.startIfNeeded() + let assistant = try #require(chat.messages.first) + let question = try #require(assistant.question) + #expect(question.options.first?.recommended == true) + + let task = try #require(chat.answerQuestion( + messageID: assistant.id, + optionLabel: "Connect WhatsApp")) + await task.value + + #expect(await recordedMessages.snapshot() == ["connect whatsapp"]) + #expect(chat.messages.map(\.text) == ["ready", "Connect WhatsApp", "ready"]) + #expect(!chat.canAnswerQuestion(assistant)) + } + + @Test func `typed question skip sends fixed reply and dismisses cards`() async throws { + let recordedMessages = SystemAgentMessageRecorder() + let questionJSON = + #"{"id":"next","header":"Next step","question":"What now?","options":[{"label":"A"},{"label":"B"}]}"# + let session = GatewayTestWebSocketSession(taskFactory: { + GatewayTestWebSocketTask(sendHook: { task, message, sendIndex in + guard sendIndex > 0, + let id = GatewayWebSocketTestSupport.requestID(from: message) + else { return } + if let message = systemAgentChatMessage(from: message) { + await recordedMessages.record(message) + task.emitReceiveSuccess(.data(systemAgentResponse(id: id))) + } else { + task.emitReceiveSuccess(.data(systemAgentResponse( + id: id, + questionJSON: questionJSON))) + } + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let gateway = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + let chat = SystemAgentOnboardingChatModel(gateway: gateway) + + await chat.startIfNeeded() + let assistant = try #require(chat.messages.first) + let task = try #require(chat.skipQuestion(messageID: assistant.id)) + await task.value + + #expect(await recordedMessages.snapshot() == ["Skip for now"]) + #expect(chat.messages.map(\.text) == ["ready", "Skip for now", "ready"]) + #expect(!chat.isQuestionVisible(assistant)) + } + + @Test(arguments: [ + #"{"id":"dupes","header":"Next step","question":"What now?","options":[{"label":"Same"},{"label":"same"}]}"#, + #""invalid""#, + #"[]"#, + ]) + func `malformed typed question keeps prose reply only`(questionJSON: String) async throws { + let session = GatewayTestWebSocketSession(taskFactory: { + GatewayTestWebSocketTask(sendHook: { task, message, sendIndex in + guard sendIndex > 0, + let id = GatewayWebSocketTestSupport.requestID(from: message) + else { return } + task.emitReceiveSuccess(.data(systemAgentResponse( + id: id, + questionJSON: questionJSON))) + }) + }) + let url = try #require(URL(string: "ws://example.invalid")) + let gateway = GatewayConnection( + configProvider: { (url: url, token: nil, password: nil) }, + sessionBox: WebSocketSessionBox(session: session)) + let chat = SystemAgentOnboardingChatModel(gateway: gateway) + + await chat.startIfNeeded() + + #expect(chat.messages.map(\.text) == ["ready"]) + #expect(chat.messages.first?.question == nil) + } + @Test func `agent handoff carries the hatch draft intent`() async throws { let session = GatewayTestWebSocketSession(taskFactory: { GatewayTestWebSocketTask(sendHook: { task, message, sendIndex in diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/SystemAgentChatQuestion.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/SystemAgentChatQuestion.swift new file mode 100644 index 00000000000..2039af3fa6d --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/SystemAgentChatQuestion.swift @@ -0,0 +1,63 @@ +import Foundation + +public struct SystemAgentChatQuestion: Equatable, Sendable { + public struct Option: Equatable, Sendable { + public let label: String + public let description: String? + public let recommended: Bool + public let reply: String? + } + + public let id: String + public let header: String + public let question: String + public let options: [Option] + public let isOther: Bool + + /// Card-capable clients validate the open payload before turning values into actions. + /// Invalid questions stay prose-only instead of exposing partial or ambiguous choices. + public static func parse(_ value: [String: AnyCodable]?) -> Self? { + guard let value, + let id = nonEmptyString(value["id"]), + let header = nonEmptyString(value["header"]), + let question = nonEmptyString(value["question"]), + let rawOptions = value["options"]?.arrayValue, + (2...4).contains(rawOptions.count) + else { return nil } + + var labels = Set() + var recommendedCount = 0 + var options: [Option] = [] + options.reserveCapacity(rawOptions.count) + + for rawOption in rawOptions { + guard let option = rawOption.dictionaryValue, + let label = Self.nonEmptyString(option["label"]), + labels.insert(label.lowercased()).inserted + else { return nil } + + let recommended = option["recommended"]?.boolValue == true + recommendedCount += recommended ? 1 : 0 + guard recommendedCount <= 1 else { return nil } + options.append(Option( + label: label, + description: Self.nonEmptyString(option["description"]), + recommended: recommended, + reply: Self.nonEmptyString(option["reply"]))) + } + + return Self( + id: id, + header: header, + question: question, + options: options, + isOther: value["isOther"]?.boolValue == true) + } + + private static func nonEmptyString(_ value: AnyCodable?) -> String? { + guard let string = value?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines), + !string.isEmpty + else { return nil } + return string + } +} diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatViewModelAttachmentTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatViewModelAttachmentTests.swift index c411c87e666..57a3d152f8b 100644 --- a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatViewModelAttachmentTests.swift +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatViewModelAttachmentTests.swift @@ -297,7 +297,7 @@ final class ChatViewModelAttachmentTests: XCTestCase { } @MainActor - func testPartialIdentitySyncPreservesTheOtherDeferredComponent() { + func testPartialIdentitySyncPreservesTheOtherDeferredComponent() async { let oldContract = "per-sender|main|main" let newContract = "per-sender|work-main|main" let contractViewModel = OpenClawChatViewModel( @@ -342,7 +342,7 @@ final class ChatViewModelAttachmentTests: XCTestCase { } @MainActor - func testAttachmentStagingPinsSessionAndIdentityUntilItFinishes() { + func testAttachmentStagingPinsSessionAndIdentityUntilItFinishes() async { let viewModel = OpenClawChatViewModel( sessionKey: "main", transport: AttachmentProcessingTransport(), @@ -369,7 +369,7 @@ final class ChatViewModelAttachmentTests: XCTestCase { } @MainActor - func testRecordingPinsSessionAndIdentityUntilItEnds() { + func testRecordingPinsSessionAndIdentityUntilItEnds() async { let ownerActivity = AttachmentOwnerActivity() let viewModel = OpenClawChatViewModel( sessionKey: "main", diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/SystemAgentChatQuestionTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/SystemAgentChatQuestionTests.swift new file mode 100644 index 00000000000..f1159d0d674 --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/SystemAgentChatQuestionTests.swift @@ -0,0 +1,104 @@ +import Foundation +import OpenClawKit +import OpenClawProtocol +import Testing + +struct SystemAgentChatQuestionTests { + @Test + func `valid gateway question keeps display and reply fields`() throws { + let decoded = try Self.parse( + """ + { + "id": " onboarding-next-step ", + "header": " Next step ", + "question": " What would you like to do first? ", + "options": [ + { + "label": " Talk to my agent ", + "reply": " talk to agent ", + "recommended": true + }, + { + "label": "Connect WhatsApp", + "reply": "connect whatsapp", + "description": " Chat there. " + } + ], + "isOther": true + } + """) + let parsed = try #require(decoded) + + #expect(parsed.id == "onboarding-next-step") + #expect(parsed.header == "Next step") + #expect(parsed.question == "What would you like to do first?") + #expect(parsed.isOther) + #expect(parsed.options.count == 2) + #expect(parsed.options[0].label == "Talk to my agent") + #expect(parsed.options[0].reply == "talk to agent") + #expect(parsed.options[0].recommended) + #expect(parsed.options[1].description == "Chat there.") + #expect(!parsed.options[1].recommended) + } + + @Test + func `optional malformed metadata is ignored`() throws { + let decoded = try Self.parse( + """ + { + "id": "channel", + "header": "Channel", + "question": "Which channel?", + "options": [ + {"label": "WhatsApp", "description": 42, "reply": " ", "recommended": "yes"}, + {"label": "Telegram"} + ], + "isOther": "yes" + } + """) + let parsed = try #require(decoded) + + #expect(parsed.options[0].description == nil) + #expect(parsed.options[0].reply == nil) + #expect(!parsed.options[0].recommended) + #expect(!parsed.isOther) + } + + @Test + func `malformed gateway questions degrade to nil`() throws { + let malformedQuestions = [ + #"{"id":"one","header":"H","question":"Q","options":[{"label":"Only"}]}"#, + """ + {"id":"five","header":"H","question":"Q","options":\ + [{"label":"A"},{"label":"B"},{"label":"C"},{"label":"D"},{"label":"E"}]} + """, + #"{"id":"dupes","header":"H","question":"Q","options":[{"label":"Same"},{"label":"same"}]}"#, + """ + {"id":"recommended","header":"H","question":"Q","options":\ + [{"label":"A","recommended":true},{"label":"B","recommended":true}]} + """, + #"{"id":"blank","header":"H","question":"Q","options":[{"label":"A"},{"label":" "}]}"#, + #"{"id":"missing-header","question":"Q","options":[{"label":"A"},{"label":"B"}]}"#, + #"{"id":"bad-option","header":"H","question":"Q","options":[{"label":"A"},"B"]}"#, + ] + + for json in malformedQuestions { + #expect(try Self.parse(json) == nil) + } + } + + private static func parse(_ questionJSON: String) throws -> SystemAgentChatQuestion? { + let result = try JSONDecoder().decode( + SystemAgentChatResult.self, + from: Data( + """ + { + "sessionId": "test-session", + "reply": "The prose reply stands alone.", + "action": "none", + "question": \(questionJSON) + } + """.utf8)) + return SystemAgentChatQuestion.parse(result.question) + } +} diff --git a/docs/start/onboarding.md b/docs/start/onboarding.md index 0fbfe5f2667..758e569bd63 100644 --- a/docs/start/onboarding.md +++ b/docs/start/onboarding.md @@ -103,7 +103,10 @@ the credential with the same live test before storing its auth profile. Next remains locked until one backend has passed, so the first agent chat cannot start without working inference. After that live check passes, OpenClaw becomes available to help configure the remaining workspace, Gateway, channels, and -other optional features; it is also available later under Settings → OpenClaw. +other optional features. When OpenClaw offers a short list of choices, the app +shows native option cards; choosing one sends the selection, and **Skip for +now** always leaves the choice optional. OpenClaw is also available later under +Settings → OpenClaw. For a local Gateway, onboarding checks the Mac for memories from supported AI