From 48e77b6abf5e6c089037b44a6508b308ef134513 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 7 Jul 2026 08:46:31 +0100 Subject: [PATCH] feat(ios): render inline LaTeX math in completed chat prose (#101388) --- apps/ios/Tests/SwiftUIRenderSmokeTests.swift | 6 + .../OpenClawChatUI/ChatInlineMath.swift | 413 ++++++++++++++++++ .../ChatMarkdownBlockViews.swift | 65 --- .../OpenClawChatUI/ChatMarkdownRenderer.swift | 232 +++++++++- .../OpenClawChatUI/ChatMessageViews.swift | 9 + .../ChatInlineMathTests.swift | 161 +++++++ scripts/ios-write-swift-filelist.mjs | 1 + 7 files changed, 811 insertions(+), 76 deletions(-) create mode 100644 apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatInlineMath.swift create mode 100644 apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatInlineMathTests.swift diff --git a/apps/ios/Tests/SwiftUIRenderSmokeTests.swift b/apps/ios/Tests/SwiftUIRenderSmokeTests.swift index 34a8c78a971..6b994115f70 100644 --- a/apps/ios/Tests/SwiftUIRenderSmokeTests.swift +++ b/apps/ios/Tests/SwiftUIRenderSmokeTests.swift @@ -115,6 +115,12 @@ struct SwiftUIRenderSmokeTests { @Test @MainActor func `display math builds valid and fallback view hierarchies`() { for typeSize in [DynamicTypeSize.large, .accessibility2] { let root = VStack { + ChatMarkdownRenderer( + text: #"Inline math \(E = mc^2\) stays inside prose."#, + context: .assistant, + variant: .standard, + font: OpenClawChatTypography.body, + textColor: OpenClawChatTheme.assistantText) ChatMathBlockView(block: ChatMathBlock( latex: #"\frac{-b \pm \sqrt{b^2 - 4ac}}{2a}"#, isComplete: true), textColor: OpenClawChatTheme.assistantText) diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatInlineMath.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatInlineMath.swift new file mode 100644 index 00000000000..fab6d167000 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatInlineMath.swift @@ -0,0 +1,413 @@ +import Foundation +import SwiftMath +import SwiftUI +#if os(macOS) +import AppKit + +typealias ChatInlineMathPlatformImage = NSImage +#else +import UIKit + +typealias ChatInlineMathPlatformImage = UIImage +#endif + +struct ChatInlineMathSpan { + let latex: String + let source: String +} + +enum ChatInlineMathScanner { + enum Piece: Equatable { + case markdown(String) + case math(latex: String, source: String) + case literal(String) + } + + static let maxSpanCount = 16 + static let maxSourceBytes = 200 + + static func pieces(in markdown: String) -> [Piece] { + guard markdown.contains(#"\("#) else { return [.markdown(markdown)] } + + var pieces: [Piece] = [] + var textStart = markdown.startIndex + var cursor = markdown.startIndex + var spanCount = 0 + let codeSpans = self.confirmedCodeSpans(in: markdown) + var codeSpanIndex = 0 + + while cursor < markdown.endIndex { + if codeSpanIndex < codeSpans.count, + cursor == codeSpans[codeSpanIndex].lowerBound + { + cursor = codeSpans[codeSpanIndex].upperBound + codeSpanIndex += 1 + continue + } + + guard markdown[cursor...].hasPrefix(#"\("#), + !self.isEscaped(at: cursor, in: markdown) + else { + cursor = markdown.index(after: cursor) + continue + } + + if textStart < cursor { + pieces.append(.markdown(String(markdown[textStart..]) -> Candidate? + { + var cursor = start + var codeSpanIndex = self.firstCodeSpan(endingAfter: start, in: codeSpans) + var containsNewline = false + while cursor < markdown.endIndex { + if codeSpanIndex < codeSpans.count, + cursor == codeSpans[codeSpanIndex].lowerBound + { + cursor = codeSpans[codeSpanIndex].upperBound + codeSpanIndex += 1 + continue + } + let character = markdown[cursor] + if character == "\n" || character == "\r" { + containsNewline = true + } + if markdown[cursor...].hasPrefix(#"\)"#), + !self.isEscaped(at: cursor, in: markdown) + { + return Candidate( + closeStart: cursor, + end: markdown.index(cursor, offsetBy: 2), + containsNewline: containsNewline) + } + cursor = markdown.index(after: cursor) + } + return nil + } + + private struct BacktickRun { + let start: String.Index + let end: String.Index + let length: Int + let canOpen: Bool + } + + private static func confirmedCodeSpans(in markdown: String) -> [Range] { + var runs: [BacktickRun] = [] + var cursor = markdown.startIndex + while cursor < markdown.endIndex { + guard markdown[cursor] == "`" else { + cursor = markdown.index(after: cursor) + continue + } + let end = self.endOfBacktickRun(at: cursor, in: markdown) + runs.append(BacktickRun( + start: cursor, + end: end, + length: markdown.distance(from: cursor, to: end), + canOpen: !self.isEscaped(at: cursor, in: markdown))) + cursor = end + } + + var nextMatchingRun = [Int?](repeating: nil, count: runs.count) + var nextIndexByLength: [Int: Int] = [:] + for index in runs.indices.reversed() { + nextMatchingRun[index] = nextIndexByLength[runs[index].length] + nextIndexByLength[runs[index].length] = index + } + + var spans: [Range] = [] + var index = 0 + while index < runs.count { + guard runs[index].canOpen, + let closeIndex = nextMatchingRun[index] + else { + index += 1 + continue + } + spans.append(runs[index].start..]) -> Int + { + var lower = 0 + var upper = spans.count + while lower < upper { + let middle = lower + (upper - lower) / 2 + if spans[middle].upperBound <= index { + lower = middle + 1 + } else { + upper = middle + } + } + return lower + } + + private static func endOfBacktickRun(at start: String.Index, in markdown: String) -> String.Index { + var end = start + while end < markdown.endIndex, markdown[end] == "`" { + end = markdown.index(after: end) + } + return end + } + + private static func isEscaped(at index: String.Index, in markdown: String) -> Bool { + var cursor = index + var slashCount = 0 + while cursor > markdown.startIndex { + let previous = markdown.index(before: cursor) + guard markdown[previous] == "\\" else { break } + slashCount += 1 + cursor = previous + } + return slashCount.isMultiple(of: 2) == false + } +} + +/// Parsed math is stable after its delimiter closes. A bounded cache avoids +/// repeating SwiftMath parsing as later streaming deltas rerender old blocks. +@MainActor +enum ChatMathParseCache { + private enum Result { + case parsed(MTMathList) + case invalid + } + + private static var cache: [String: Result] = [:] + private static let capacity = 80 + private static let maxNestingDepth = 64 + private static let maxCommandCount = 128 + private static let unsafeCommands = [#"\color"#, #"\colorbox"#, #"\textcolor"#] + + static func mathList(latex: String) -> MTMathList? { + guard !latex.isEmpty else { return nil } + // SwiftMath silently drops unsupported Unicode instead of reporting a + // parse error. Preserve the source through the raw-text fallback. + guard latex.unicodeScalars.allSatisfy(\.isASCII) else { return nil } + // SwiftMath recursively parses groups. Bound hostile nesting before + // entering the dependency so a short expression cannot exhaust stack. + guard self.isWithinParserLimits(latex) else { return nil } + // SwiftMath 1.7.3 traps while typesetting empty color-command bodies. + // Chat owns the surrounding color, so preserve these as raw source. + guard !self.unsafeCommands.contains(where: latex.contains) else { return nil } + if let hit = self.cache[latex] { + if case let .parsed(mathList) = hit { + return mathList + } + return nil + } + + let result = MTMathListBuilder.build(fromString: latex) + .map(Result.parsed) ?? .invalid + if self.cache.count >= self.capacity { + self.cache.removeAll(keepingCapacity: true) + } + self.cache[latex] = result + if case let .parsed(mathList) = result { + return mathList + } + return nil + } + + private static func isWithinParserLimits(_ latex: String) -> Bool { + var depth = 0 + var commandCount = 0 + var escaped = false + for character in latex { + if escaped { + escaped = false + continue + } + if character == "\\" { + commandCount += 1 + if commandCount > self.maxCommandCount { + return false + } + escaped = true + } else if character == "{" { + depth += 1 + if depth > self.maxNestingDepth { + return false + } + } else if character == "}" { + depth = max(0, depth - 1) + } + } + return true + } +} + +@MainActor +enum ChatInlineMathImageCache { + struct RenderedImage { + let image: ChatInlineMathPlatformImage + let baselineOffset: CGFloat + } + + private struct Key: Hashable { + let latex: String + let fontSize: CGFloat + let colorHash: Int + } + + private static var cache: [Key: RenderedImage] = [:] + private static let capacity = 80 + private static let maxRenderedPixelArea: CGFloat = 262_144 + private static let maxRenderedPixelDimension: CGFloat = 4096 + + static func image( + latex: String, + fontSize: CGFloat, + textColor: Color, + colorScheme: ColorScheme) -> RenderedImage? + { + guard fontSize.isFinite, fontSize > 0, + let mathList = ChatMathParseCache.mathList(latex: latex) + else { return nil } + let platformColor = self.platformColor(textColor, colorScheme: colorScheme) + let key = Key( + latex: latex, + fontSize: fontSize, + colorHash: self.colorHash(platformColor)) + if let cached = self.cache[key] { + return cached + } + + let metricsLabel = MTMathUILabel() + metricsLabel.mathList = mathList + metricsLabel.labelMode = .text + metricsLabel.fontSize = fontSize + metricsLabel.textColor = platformColor + let size: CGSize + #if os(macOS) + size = metricsLabel.fittingSize + metricsLabel.frame = CGRect(origin: .zero, size: size) + metricsLabel.layoutSubtreeIfNeeded() + #else + size = metricsLabel.intrinsicContentSize + metricsLabel.frame = CGRect(origin: .zero, size: size) + metricsLabel.layoutIfNeeded() + #endif + guard self.isSafeImageSize(size, scale: self.imageScale) else { return nil } + + // MTMathImage allocates immediately in asImage(). Measure with the same + // text-mode typesetter first so hostile input cannot request an empty or + // oversized backing bitmap. + let imageRenderer = MTMathImage( + latex: latex, + fontSize: fontSize, + textColor: platformColor, + labelMode: .text, + textAlignment: .left) + guard let image = imageRenderer.asImage().1 else { return nil } + + // Text(Image) places the image bottom on the surrounding baseline. + // SwiftMath exposes its baseline descent through displayList, so lower + // the image by that descent to align the two typographic baselines. + let rendered = RenderedImage( + image: image, + baselineOffset: -(metricsLabel.displayList?.descent ?? 0)) + if self.cache.count >= self.capacity { + self.cache.removeAll(keepingCapacity: true) + } + self.cache[key] = rendered + return rendered + } + + static func isSafeImageSize(_ size: CGSize, scale: CGFloat) -> Bool { + guard size.width.isFinite, size.height.isFinite, scale.isFinite, + size.width > 0, size.height > 0, scale > 0 + else { return false } + let pixelWidth = size.width * scale + let pixelHeight = size.height * scale + return pixelWidth.isFinite && pixelHeight.isFinite && + pixelWidth <= self.maxRenderedPixelDimension && + pixelHeight <= self.maxRenderedPixelDimension && + pixelWidth * pixelHeight <= self.maxRenderedPixelArea + } + + private static var imageScale: CGFloat { + #if os(macOS) + NSScreen.main?.backingScaleFactor ?? 2 + #else + UIScreen.main.scale + #endif + } + + private static func platformColor(_ color: Color, colorScheme: ColorScheme) -> MTColor { + #if os(macOS) + guard let appearance = NSAppearance( + named: colorScheme == .dark ? .darkAqua : .aqua) + else { return NSColor(color) } + var platformColor = NSColor.clear + appearance.performAsCurrentDrawingAppearance { + let resolved = NSColor(color) + platformColor = resolved.usingColorSpace(.deviceRGB) ?? resolved + } + return platformColor + #else + let style: UIUserInterfaceStyle = colorScheme == .dark ? .dark : .light + return UIColor(color).resolvedColor(with: UITraitCollection(userInterfaceStyle: style)) + #endif + } + + private static func colorHash(_ color: MTColor) -> Int { + var red: CGFloat = 0 + var green: CGFloat = 0 + var blue: CGFloat = 0 + var alpha: CGFloat = 0 + color.getRed(&red, green: &green, blue: &blue, alpha: &alpha) + var hasher = Hasher() + hasher.combine(red) + hasher.combine(green) + hasher.combine(blue) + hasher.combine(alpha) + return hasher.finalize() + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockViews.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockViews.swift index 0e60b84e375..0d69eb627e9 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockViews.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockViews.swift @@ -77,71 +77,6 @@ struct ChatMathBlockView: View { } } -/// Parsed math is stable after its delimiter closes. A bounded cache avoids -/// repeating SwiftMath parsing as later streaming deltas rerender old blocks. -@MainActor -private enum ChatMathParseCache { - private enum Result { - case parsed(MTMathList) - case invalid - } - - private static var cache: [String: Result] = [:] - private static let capacity = 80 - private static let maxNestingDepth = 64 - private static let maxCommandCount = 128 - private static let unsafeCommands = [#"\color"#, #"\colorbox"#, #"\textcolor"#] - - static func mathList(latex: String) -> MTMathList? { - guard !latex.isEmpty else { return nil } - // SwiftMath silently drops unsupported Unicode instead of reporting a - // parse error. Preserve the source through the raw-text fallback. - guard latex.unicodeScalars.allSatisfy(\.isASCII) else { return nil } - // SwiftMath recursively parses groups. Bound hostile nesting before - // entering the dependency so a short expression cannot exhaust stack. - guard self.isWithinParserLimits(latex) else { return nil } - // SwiftMath 1.7.3 traps while typesetting empty color-command bodies. - // Chat owns the surrounding color, so preserve these as raw source. - guard !self.unsafeCommands.contains(where: latex.contains) else { return nil } - if let hit = self.cache[latex] { - if case let .parsed(mathList) = hit { return mathList } - return nil - } - - let result = MTMathListBuilder.build(fromString: latex) - .map(Result.parsed) ?? .invalid - if self.cache.count >= self.capacity { - self.cache.removeAll(keepingCapacity: true) - } - self.cache[latex] = result - if case let .parsed(mathList) = result { return mathList } - return nil - } - - private static func isWithinParserLimits(_ latex: String) -> Bool { - var depth = 0 - var commandCount = 0 - var escaped = false - for character in latex { - if escaped { - escaped = false - continue - } - if character == "\\" { - commandCount += 1 - if commandCount > self.maxCommandCount { return false } - escaped = true - } else if character == "{" { - depth += 1 - if depth > self.maxNestingDepth { return false } - } else if character == "}" { - depth = max(0, depth - 1) - } - } - return true - } -} - #if os(macOS) @MainActor private struct ChatMathPlatformView: NSViewRepresentable { diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownRenderer.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownRenderer.swift index eab2e91bb8d..410d4e27fd3 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownRenderer.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownRenderer.swift @@ -14,6 +14,14 @@ struct ChatMarkdownRenderer: View { case assistant } + struct InlineMathTypography { + static let body = Self(size: OpenClawChatTypography.bodySize, relativeTo: .body) + static let callout = Self(size: 16, relativeTo: .callout) + + let size: CGFloat + let relativeTo: Font.TextStyle + } + let snapshot: ChatMarkdownRenderSnapshot let context: Context let variant: ChatMarkdownVariant @@ -21,12 +29,16 @@ struct ChatMarkdownRenderer: View { let textColor: Color var reveal: ChatMarkdownProseReveal? + @ScaledMetric private var inlineMathFontSize: CGFloat + @Environment(\.colorScheme) private var colorScheme + init( text: String, context: Context, variant: ChatMarkdownVariant, font: Font, textColor: Color, + inlineMathTypography: InlineMathTypography = .body, isComplete: Bool = true) { self.init( @@ -34,7 +46,8 @@ struct ChatMarkdownRenderer: View { context: context, variant: variant, font: font, - textColor: textColor) + textColor: textColor, + inlineMathTypography: inlineMathTypography) } init( @@ -43,6 +56,7 @@ struct ChatMarkdownRenderer: View { variant: ChatMarkdownVariant, font: Font, textColor: Color, + inlineMathTypography: InlineMathTypography = .body, reveal: ChatMarkdownProseReveal? = nil) { self.snapshot = snapshot @@ -51,6 +65,9 @@ struct ChatMarkdownRenderer: View { self.font = font self.textColor = textColor self.reveal = reveal + self._inlineMathFontSize = ScaledMetric( + wrappedValue: inlineMathTypography.size, + relativeTo: inlineMathTypography.relativeTo) } var body: some View { @@ -75,6 +92,7 @@ struct ChatMarkdownRenderer: View { .tint(self.linkColor) .textSelection(.enabled) .lineSpacing(self.variant == .compact ? 2 : 4) + .modifier(ChatInlineMathAccessibilityModifier(label: prose.inlineAccessibilityText)) case let .code(code): ChatCodeBlockView(block: code) case let .math(math): @@ -86,7 +104,10 @@ struct ChatMarkdownRenderer: View { private func proseText(_ prose: ChatMarkdownProse, index: Int) -> SwiftUI.Text { guard let reveal = self.reveal, reveal.blockIndex == index else { - return SwiftUI.Text(prose.attributed) + return prose.renderedText( + fontSize: self.inlineMathFontSize, + textColor: self.textColor, + colorScheme: self.colorScheme) } return prose.revealedText( frame: revealedOpacities(state: reveal.state, now: reveal.now), @@ -104,6 +125,7 @@ struct ChatMarkdownProseReveal { let now: TimeInterval } +@MainActor struct ChatMarkdownRenderSnapshot { let blocks: [ChatMarkdownRenderedBlock] let images: [ChatMarkdownPreprocessor.InlineImage] @@ -115,7 +137,10 @@ struct ChatMarkdownRenderSnapshot { isComplete: isComplete).map { block in switch block { case let .prose(markdown): - .prose(ChatMarkdownProse(markdown: markdown, preparesReveal: preparesReveal)) + .prose(ChatMarkdownProse( + markdown: markdown, + isComplete: isComplete, + preparesReveal: preparesReveal)) case let .code(code): .code(code) case let .math(math): @@ -129,7 +154,9 @@ struct ChatMarkdownRenderSnapshot { var lastProseIndex: Int? { self.blocks.lastIndex { - if case .prose = $0 { return true } + if case .prose = $0 { + return true + } return false } } @@ -142,6 +169,7 @@ enum ChatMarkdownRenderedBlock { case table(ChatMarkdownTable) } +@MainActor struct ChatMarkdownProse { struct TailPiece { let attributed: AttributedString @@ -152,14 +180,23 @@ struct ChatMarkdownProse { let plainText: String let prefix: AttributedString let tail: [TailPiece] + let inlineContent: [InlineContent]? - init(markdown: String, preparesReveal: Bool) { - let displayMarkdown = ChatMarkdownDisplayPreprocessor.preserveChatSoftBreaks(in: markdown) - let options = AttributedString.MarkdownParsingOptions( - interpretedSyntax: .full, - failurePolicy: .returnPartiallyParsedIfPossible) - let attributed = (try? AttributedString(markdown: displayMarkdown, options: options)) - ?? AttributedString(displayMarkdown) + enum InlineContent { + case text(AttributedString) + case math(ChatInlineMathSpan) + } + + init(markdown: String, isComplete: Bool, preparesReveal: Bool) { + // preparesReveal belongs only to the streaming snapshot. Keep that path + // on its single AttributedString build; completed snapshots alone split + // prose for inline image interpolation. + let inlineContent = isComplete && !preparesReveal + ? Self.makeInlineContent(markdown: markdown) + : nil + let attributed = inlineContent == nil + ? Self.parseMarkdown(markdown) + : AttributedString() let plainText = preparesReveal ? String(attributed.characters) : "" let wordRanges = preparesReveal ? Array(chatStreamingWordRanges(in: plainText).suffix(24)) @@ -168,6 +205,7 @@ struct ChatMarkdownProse { self.attributed = attributed self.plainText = plainText + self.inlineContent = inlineContent if preparesReveal { self.prefix = Self.slice(attributed, characterRange: 0.. SwiftUI.Text + { + guard let inlineContent else { return SwiftUI.Text(self.attributed) } + return inlineContent.reduce(SwiftUI.Text("")) { text, content in + switch content { + case let .text(attributed): + return text + SwiftUI.Text(attributed) + case let .math(span): + guard let rendered = ChatInlineMathImageCache.image( + latex: span.latex, + fontSize: fontSize, + textColor: textColor, + colorScheme: colorScheme) + else { return text + SwiftUI.Text(span.source) } + #if os(macOS) + let image = Image(nsImage: rendered.image) + #else + let image = Image(uiImage: rendered.image) + #endif + return text + SwiftUI.Text(image).baselineOffset(rendered.baselineOffset) + } + } + } + func revealedText(frame: ChatStreamingRevealFrame, textColor: Color) -> SwiftUI.Text { self.tail.reduce(SwiftUI.Text(self.prefix)) { text, piece in var attributed = piece.attributed @@ -193,6 +279,118 @@ struct ChatMarkdownProse { } } + private static func makeInlineContent(markdown: String) -> [InlineContent]? { + let pieces = ChatInlineMathScanner.pieces(in: markdown) + guard pieces.contains(where: { piece in + if case .markdown = piece { + return false + } + return true + }) else { return nil } + + var substitutedMarkdown = "" + var replacements: [InlineReplacement] = [] + var markerValue: UInt32 = 0xE000 + var occupiedMarkerValues = Set(markdown.unicodeScalars.map(\.value)) + for piece in pieces { + switch piece { + case let .markdown(source): + substitutedMarkdown += source + case let .literal(source): + substitutedMarkdown += Self.markdownEscapedLiteral(source) + case let .math(latex, source): + guard ChatMathParseCache.mathList(latex: latex) != nil else { + substitutedMarkdown += Self.markdownEscapedLiteral(source) + continue + } + let marker = Self.nextMarker( + startingAt: &markerValue, + occupiedValues: &occupiedMarkerValues) + substitutedMarkdown.append(marker) + replacements.append(InlineReplacement( + marker: marker, + span: ChatInlineMathSpan(latex: latex, source: source))) + } + } + + let attributed = self.parseMarkdown(substitutedMarkdown) + var content: [InlineContent] = [] + var cursor = attributed.startIndex + for replacement in replacements { + guard let markerIndex = attributed.characters[cursor...] + .firstIndex(of: replacement.marker) + else { return nil } + if cursor < markerIndex { + content.append(.text(AttributedString(attributed[cursor..) -> Character + { + while occupiedValues.contains(value) { + value += 1 + } + guard let scalar = UnicodeScalar(value) else { + preconditionFailure("inline math marker range exhausted") + } + occupiedValues.insert(value) + let marker = Character(String(scalar)) + value += 1 + return marker + } + + private static func markdownEscapedLiteral(_ source: String) -> String { + source.reduce(into: "") { escaped, character in + if character.unicodeScalars.count == 1, + let scalar = character.unicodeScalars.first, + scalar.isASCII, + (33...47).contains(scalar.value) || + (58...64).contains(scalar.value) || + (91...96).contains(scalar.value) || + (123...126).contains(scalar.value) + { + escaped.append("\\") + } + escaped.append(character) + } + } + + private static func parseMarkdown(_ markdown: String) -> AttributedString { + let displayMarkdown = ChatMarkdownDisplayPreprocessor.preserveChatSoftBreaks(in: markdown) + let options = AttributedString.MarkdownParsingOptions( + interpretedSyntax: .full, + failurePolicy: .returnPartiallyParsedIfPossible) + return (try? AttributedString(markdown: displayMarkdown, options: options)) + ?? AttributedString(displayMarkdown) + } + private static func tailPieces( attributed: AttributedString, textLength: Int, @@ -235,6 +433,18 @@ struct ChatMarkdownProse { } } +private struct ChatInlineMathAccessibilityModifier: ViewModifier { + let label: String? + + func body(content: Content) -> some View { + if let label { + content.accessibilityLabel(Text(label)) + } else { + content + } + } +} + /// Fenced code, display math, and GFM tables are split out by `ChatMarkdownBlockSegmenter` /// before this runs, so prose only needs chat-style soft-break preservation. enum ChatMarkdownDisplayPreprocessor { diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift index 2355c0c1379..8a1be62084e 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift @@ -920,12 +920,16 @@ private struct ChatAssistantTextBody: View { let font = segment.kind == .thinking ? OpenClawChatTypography.callout.italic() : OpenClawChatTypography.body + let inlineMathTypography: ChatMarkdownRenderer.InlineMathTypography = segment.kind == .thinking + ? .callout + : .body ChatMarkdownRenderer( text: segment.text, context: .assistant, variant: self.markdownVariant, font: font, textColor: OpenClawChatTheme.assistantText, + inlineMathTypography: inlineMathTypography, isComplete: self.isComplete) } } @@ -1003,6 +1007,9 @@ private struct ChatStreamingAssistantTextBody: View { let font = segment.kind == .thinking ? OpenClawChatTypography.callout.italic() : OpenClawChatTypography.body + let inlineMathTypography: ChatMarkdownRenderer.InlineMathTypography = segment.kind == .thinking + ? .callout + : .body let reveal = self.reveal( segmentIndex: entry.offset, now: now) @@ -1012,6 +1019,7 @@ private struct ChatStreamingAssistantTextBody: View { variant: self.markdownVariant, font: font, textColor: OpenClawChatTheme.assistantText, + inlineMathTypography: inlineMathTypography, reveal: reveal) } } @@ -1060,6 +1068,7 @@ private struct ChatStreamingAssistantTextBody: View { return deadline } + @MainActor private struct Snapshot { struct Segment { let kind: AssistantTextSegment.Kind diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatInlineMathTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatInlineMathTests.swift new file mode 100644 index 00000000000..007426c636c --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatInlineMathTests.swift @@ -0,0 +1,161 @@ +import CoreGraphics +import Foundation +import Testing +@testable import OpenClawChatUI + +struct ChatInlineMathTests { + @Test func `basic span splits from prose`() { + #expect(ChatInlineMathScanner.pieces(in: #"before \(x + 1\) after"#) == [ + .markdown("before "), + .math(latex: "x + 1", source: #"\(x + 1\)"#), + .markdown(" after"), + ]) + } + + @Test func `multiple spans split independently`() { + let pieces = ChatInlineMathScanner.pieces(in: #"\(x\), then \(y\)"#) + #expect(pieces.compactMap(\.mathLatex) == ["x", "y"]) + } + + @Test func `span inside backticks stays markdown`() { + let source = #"use `\(not math\)` here"# + #expect(ChatInlineMathScanner.pieces(in: source) == [.markdown(source)]) + } + + @Test func `unmatched backtick does not hide later math`() { + #expect(ChatInlineMathScanner.pieces(in: #"stray ` before \(x\)"#) == [ + .markdown("stray ` before "), + .math(latex: "x", source: #"\(x\)"#), + ]) + } + + @Test func `escaped delimiters stay markdown`() { + let source = #"escaped \\(x\\)"# + #expect(ChatInlineMathScanner.pieces(in: source) == [.markdown(source)]) + } + + @Test func `unclosed opener stays literal`() { + #expect(ChatInlineMathScanner.pieces(in: #"before \(x"#) == [ + .markdown("before "), + .literal(#"\(x"#), + ]) + } + + @Test func `newline span stays literal`() { + let source = "before \\(x\ny\\) after" + #expect(ChatInlineMathScanner.pieces(in: source) == [ + .markdown("before "), + .literal("\\(x\ny\\)"), + .markdown(" after"), + ]) + } + + @Test func `only first sixteen spans are candidates`() { + let source = (0..<17).map { #"\(x\#($0)\)"# }.joined(separator: " ") + let pieces = ChatInlineMathScanner.pieces(in: source) + #expect(pieces.compactMap(\.mathLatex).count == ChatInlineMathScanner.maxSpanCount) + #expect(pieces.contains(.literal(#"\(x16\)"#))) + } + + @Test @MainActor func `markdown after excess spans remains parsed`() throws { + let spans = (0..<17).map { #"\(x\#($0)\)"# }.joined(separator: " ") + let snapshot = ChatMarkdownRenderSnapshot( + text: spans + " [docs](https://example.com)", + isComplete: true) + let prose = try self.prose(in: snapshot) + let hasLink = prose.inlineContent?.contains { content in + guard case let .text(attributed) = content else { return false } + return attributed.runs.contains { $0.link != nil } + } + + #expect(hasLink == true) + } + + @Test @MainActor func `math link label stays literal and linked`() throws { + let snapshot = ChatMarkdownRenderSnapshot( + text: #"[\(x\)](https://example.com)"#, + isComplete: true) + let prose = try self.prose(in: snapshot) + let linkedText = prose.inlineContent?.compactMap { content -> AttributedString? in + guard case let .text(attributed) = content, + attributed.runs.contains(where: { $0.link != nil }) + else { return nil } + return attributed + }.first + + #expect(prose.inlineMathLatex.isEmpty) + #expect(try String(#require(linkedText).characters) == #"\(x\)"#) + } + + @Test func `two hundred byte cap is inclusive`() { + let acceptedLatex = String(repeating: "x", count: ChatInlineMathScanner.maxSourceBytes - 4) + let rejectedLatex = acceptedLatex + "x" + #expect(ChatInlineMathScanner.pieces(in: "\\(\(acceptedLatex)\\)").compactMap(\.mathLatex) == [ + acceptedLatex, + ]) + #expect(ChatInlineMathScanner.pieces(in: "\\(\(rejectedLatex)\\)") == [ + .literal("\\(\(rejectedLatex)\\)"), + ]) + } + + @Test @MainActor func `only completed prose prepares inline math`() throws { + let complete = ChatMarkdownRenderSnapshot(text: #"value \(x^2\)"#, isComplete: true) + let streaming = ChatMarkdownRenderSnapshot( + text: #"value \(x^2\)"#, + isComplete: false, + preparesReveal: true) + + #expect(try self.prose(in: complete).inlineMathLatex == ["x^2"]) + #expect(try self.prose(in: streaming).inlineMathLatex.isEmpty) + } + + @Test @MainActor func `display math guards also reject inline math`() throws { + for source in [#"value \(α + β\)"#, #"value \(x\color{red}\)"#] { + let snapshot = ChatMarkdownRenderSnapshot(text: source, isComplete: true) + #expect(try self.prose(in: snapshot).inlineMathLatex.isEmpty) + } + } + + @Test @MainActor func `markdown attributes can cross inline math`() throws { + let snapshot = ChatMarkdownRenderSnapshot(text: #"**value \(x\)**"#, isComplete: true) + let prose = try self.prose(in: snapshot) + + #expect(prose.inlineMathLatex == ["x"]) + #expect(prose.inlineAccessibilityText == "value x") + } + + @Test @MainActor func `image sizing rejects unsafe bitmap bounds`() { + #expect(!ChatInlineMathImageCache.isSafeImageSize(.zero, scale: 2)) + #expect(!ChatInlineMathImageCache.isSafeImageSize( + CGSize(width: 4097, height: 1), + scale: 1)) + #expect(!ChatInlineMathImageCache.isSafeImageSize( + CGSize(width: 1024, height: 1024), + scale: 1)) + #expect(ChatInlineMathImageCache.isSafeImageSize( + CGSize(width: 256, height: 256), + scale: 2)) + } + + @MainActor + private func prose(in snapshot: ChatMarkdownRenderSnapshot) throws -> ChatMarkdownProse { + guard case let .prose(prose) = try #require(snapshot.blocks.first) else { + Issue.record("expected prose block") + throw InlineMathTestError.expectedProse + } + return prose + } +} + +private enum InlineMathTestError: Error { + case expectedProse +} + +extension ChatInlineMathScanner.Piece { + fileprivate var mathLatex: String? { + if case let .math(latex, _) = self { + return latex + } + return nil + } +} diff --git a/scripts/ios-write-swift-filelist.mjs b/scripts/ios-write-swift-filelist.mjs index 2776137fa05..a50fa195e0e 100644 --- a/scripts/ios-write-swift-filelist.mjs +++ b/scripts/ios-write-swift-filelist.mjs @@ -16,6 +16,7 @@ const iosSourceRoots = [ const sharedSwiftFiles = [ "../shared/OpenClawKit/Sources/OpenClawChatUI/ChatComposer.swift", "../shared/OpenClawKit/Sources/OpenClawChatUI/ChatCodeHighlighter.swift", + "../shared/OpenClawKit/Sources/OpenClawChatUI/ChatInlineMath.swift", "../shared/OpenClawKit/Sources/OpenClawChatUI/ChatLinkPreview.swift", "../shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockSegmenter.swift", "../shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockViews.swift",