mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
feat(ios): render inline LaTeX math in completed chat prose (#101388)
This commit is contained in:
Peter Steinberger
GitHub
parent
e72dadbb3b
commit
48e77b6abf
7 changed files with 811 additions and 76 deletions
|
|
@ -115,6 +115,12 @@ struct SwiftUIRenderSmokeTests {
|
|||
@Test @MainActor func `display math builds valid and fallback view hierarchies`() {
|
||||
for typeSize in [DynamicTypeSize.large, .accessibility2] {
|
||||
let root = VStack {
|
||||
ChatMarkdownRenderer(
|
||||
text: #"Inline math \(E = mc^2\) stays inside prose."#,
|
||||
context: .assistant,
|
||||
variant: .standard,
|
||||
font: OpenClawChatTypography.body,
|
||||
textColor: OpenClawChatTheme.assistantText)
|
||||
ChatMathBlockView(block: ChatMathBlock(
|
||||
latex: #"\frac{-b \pm \sqrt{b^2 - 4ac}}{2a}"#,
|
||||
isComplete: true), textColor: OpenClawChatTheme.assistantText)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,413 @@
|
|||
import Foundation
|
||||
import SwiftMath
|
||||
import SwiftUI
|
||||
#if os(macOS)
|
||||
import AppKit
|
||||
|
||||
typealias ChatInlineMathPlatformImage = NSImage
|
||||
#else
|
||||
import UIKit
|
||||
|
||||
typealias ChatInlineMathPlatformImage = UIImage
|
||||
#endif
|
||||
|
||||
struct ChatInlineMathSpan {
|
||||
let latex: String
|
||||
let source: String
|
||||
}
|
||||
|
||||
enum ChatInlineMathScanner {
|
||||
enum Piece: Equatable {
|
||||
case markdown(String)
|
||||
case math(latex: String, source: String)
|
||||
case literal(String)
|
||||
}
|
||||
|
||||
static let maxSpanCount = 16
|
||||
static let maxSourceBytes = 200
|
||||
|
||||
static func pieces(in markdown: String) -> [Piece] {
|
||||
guard markdown.contains(#"\("#) else { return [.markdown(markdown)] }
|
||||
|
||||
var pieces: [Piece] = []
|
||||
var textStart = markdown.startIndex
|
||||
var cursor = markdown.startIndex
|
||||
var spanCount = 0
|
||||
let codeSpans = self.confirmedCodeSpans(in: markdown)
|
||||
var codeSpanIndex = 0
|
||||
|
||||
while cursor < markdown.endIndex {
|
||||
if codeSpanIndex < codeSpans.count,
|
||||
cursor == codeSpans[codeSpanIndex].lowerBound
|
||||
{
|
||||
cursor = codeSpans[codeSpanIndex].upperBound
|
||||
codeSpanIndex += 1
|
||||
continue
|
||||
}
|
||||
|
||||
guard markdown[cursor...].hasPrefix(#"\("#),
|
||||
!self.isEscaped(at: cursor, in: markdown)
|
||||
else {
|
||||
cursor = markdown.index(after: cursor)
|
||||
continue
|
||||
}
|
||||
|
||||
if textStart < cursor {
|
||||
pieces.append(.markdown(String(markdown[textStart..<cursor])))
|
||||
}
|
||||
let opener = cursor
|
||||
let contentStart = markdown.index(cursor, offsetBy: 2)
|
||||
guard let candidate = self.candidate(
|
||||
startingAt: contentStart,
|
||||
in: markdown,
|
||||
codeSpans: codeSpans)
|
||||
else {
|
||||
pieces.append(.literal(String(markdown[opener...])))
|
||||
return pieces
|
||||
}
|
||||
|
||||
spanCount += 1
|
||||
let source = String(markdown[opener..<candidate.end])
|
||||
let latex = String(markdown[contentStart..<candidate.closeStart])
|
||||
if spanCount <= self.maxSpanCount,
|
||||
!candidate.containsNewline,
|
||||
source.utf8.count <= self.maxSourceBytes
|
||||
{
|
||||
pieces.append(.math(latex: latex, source: source))
|
||||
} else {
|
||||
pieces.append(.literal(source))
|
||||
}
|
||||
cursor = candidate.end
|
||||
textStart = cursor
|
||||
}
|
||||
|
||||
if textStart < markdown.endIndex {
|
||||
pieces.append(.markdown(String(markdown[textStart...])))
|
||||
}
|
||||
return pieces
|
||||
}
|
||||
|
||||
private struct Candidate {
|
||||
let closeStart: String.Index
|
||||
let end: String.Index
|
||||
let containsNewline: Bool
|
||||
}
|
||||
|
||||
private static func candidate(
|
||||
startingAt start: String.Index,
|
||||
in markdown: String,
|
||||
codeSpans: [Range<String.Index>]) -> Candidate?
|
||||
{
|
||||
var cursor = start
|
||||
var codeSpanIndex = self.firstCodeSpan(endingAfter: start, in: codeSpans)
|
||||
var containsNewline = false
|
||||
while cursor < markdown.endIndex {
|
||||
if codeSpanIndex < codeSpans.count,
|
||||
cursor == codeSpans[codeSpanIndex].lowerBound
|
||||
{
|
||||
cursor = codeSpans[codeSpanIndex].upperBound
|
||||
codeSpanIndex += 1
|
||||
continue
|
||||
}
|
||||
let character = markdown[cursor]
|
||||
if character == "\n" || character == "\r" {
|
||||
containsNewline = true
|
||||
}
|
||||
if markdown[cursor...].hasPrefix(#"\)"#),
|
||||
!self.isEscaped(at: cursor, in: markdown)
|
||||
{
|
||||
return Candidate(
|
||||
closeStart: cursor,
|
||||
end: markdown.index(cursor, offsetBy: 2),
|
||||
containsNewline: containsNewline)
|
||||
}
|
||||
cursor = markdown.index(after: cursor)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private struct BacktickRun {
|
||||
let start: String.Index
|
||||
let end: String.Index
|
||||
let length: Int
|
||||
let canOpen: Bool
|
||||
}
|
||||
|
||||
private static func confirmedCodeSpans(in markdown: String) -> [Range<String.Index>] {
|
||||
var runs: [BacktickRun] = []
|
||||
var cursor = markdown.startIndex
|
||||
while cursor < markdown.endIndex {
|
||||
guard markdown[cursor] == "`" else {
|
||||
cursor = markdown.index(after: cursor)
|
||||
continue
|
||||
}
|
||||
let end = self.endOfBacktickRun(at: cursor, in: markdown)
|
||||
runs.append(BacktickRun(
|
||||
start: cursor,
|
||||
end: end,
|
||||
length: markdown.distance(from: cursor, to: end),
|
||||
canOpen: !self.isEscaped(at: cursor, in: markdown)))
|
||||
cursor = end
|
||||
}
|
||||
|
||||
var nextMatchingRun = [Int?](repeating: nil, count: runs.count)
|
||||
var nextIndexByLength: [Int: Int] = [:]
|
||||
for index in runs.indices.reversed() {
|
||||
nextMatchingRun[index] = nextIndexByLength[runs[index].length]
|
||||
nextIndexByLength[runs[index].length] = index
|
||||
}
|
||||
|
||||
var spans: [Range<String.Index>] = []
|
||||
var index = 0
|
||||
while index < runs.count {
|
||||
guard runs[index].canOpen,
|
||||
let closeIndex = nextMatchingRun[index]
|
||||
else {
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
spans.append(runs[index].start..<runs[closeIndex].end)
|
||||
index = closeIndex + 1
|
||||
}
|
||||
return spans
|
||||
}
|
||||
|
||||
private static func firstCodeSpan(
|
||||
endingAfter index: String.Index,
|
||||
in spans: [Range<String.Index>]) -> Int
|
||||
{
|
||||
var lower = 0
|
||||
var upper = spans.count
|
||||
while lower < upper {
|
||||
let middle = lower + (upper - lower) / 2
|
||||
if spans[middle].upperBound <= index {
|
||||
lower = middle + 1
|
||||
} else {
|
||||
upper = middle
|
||||
}
|
||||
}
|
||||
return lower
|
||||
}
|
||||
|
||||
private static func endOfBacktickRun(at start: String.Index, in markdown: String) -> String.Index {
|
||||
var end = start
|
||||
while end < markdown.endIndex, markdown[end] == "`" {
|
||||
end = markdown.index(after: end)
|
||||
}
|
||||
return end
|
||||
}
|
||||
|
||||
private static func isEscaped(at index: String.Index, in markdown: String) -> Bool {
|
||||
var cursor = index
|
||||
var slashCount = 0
|
||||
while cursor > markdown.startIndex {
|
||||
let previous = markdown.index(before: cursor)
|
||||
guard markdown[previous] == "\\" else { break }
|
||||
slashCount += 1
|
||||
cursor = previous
|
||||
}
|
||||
return slashCount.isMultiple(of: 2) == false
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsed math is stable after its delimiter closes. A bounded cache avoids
|
||||
/// repeating SwiftMath parsing as later streaming deltas rerender old blocks.
|
||||
@MainActor
|
||||
enum ChatMathParseCache {
|
||||
private enum Result {
|
||||
case parsed(MTMathList)
|
||||
case invalid
|
||||
}
|
||||
|
||||
private static var cache: [String: Result] = [:]
|
||||
private static let capacity = 80
|
||||
private static let maxNestingDepth = 64
|
||||
private static let maxCommandCount = 128
|
||||
private static let unsafeCommands = [#"\color"#, #"\colorbox"#, #"\textcolor"#]
|
||||
|
||||
static func mathList(latex: String) -> MTMathList? {
|
||||
guard !latex.isEmpty else { return nil }
|
||||
// SwiftMath silently drops unsupported Unicode instead of reporting a
|
||||
// parse error. Preserve the source through the raw-text fallback.
|
||||
guard latex.unicodeScalars.allSatisfy(\.isASCII) else { return nil }
|
||||
// SwiftMath recursively parses groups. Bound hostile nesting before
|
||||
// entering the dependency so a short expression cannot exhaust stack.
|
||||
guard self.isWithinParserLimits(latex) else { return nil }
|
||||
// SwiftMath 1.7.3 traps while typesetting empty color-command bodies.
|
||||
// Chat owns the surrounding color, so preserve these as raw source.
|
||||
guard !self.unsafeCommands.contains(where: latex.contains) else { return nil }
|
||||
if let hit = self.cache[latex] {
|
||||
if case let .parsed(mathList) = hit {
|
||||
return mathList
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
let result = MTMathListBuilder.build(fromString: latex)
|
||||
.map(Result.parsed) ?? .invalid
|
||||
if self.cache.count >= self.capacity {
|
||||
self.cache.removeAll(keepingCapacity: true)
|
||||
}
|
||||
self.cache[latex] = result
|
||||
if case let .parsed(mathList) = result {
|
||||
return mathList
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private static func isWithinParserLimits(_ latex: String) -> Bool {
|
||||
var depth = 0
|
||||
var commandCount = 0
|
||||
var escaped = false
|
||||
for character in latex {
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if character == "\\" {
|
||||
commandCount += 1
|
||||
if commandCount > self.maxCommandCount {
|
||||
return false
|
||||
}
|
||||
escaped = true
|
||||
} else if character == "{" {
|
||||
depth += 1
|
||||
if depth > self.maxNestingDepth {
|
||||
return false
|
||||
}
|
||||
} else if character == "}" {
|
||||
depth = max(0, depth - 1)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
enum ChatInlineMathImageCache {
|
||||
struct RenderedImage {
|
||||
let image: ChatInlineMathPlatformImage
|
||||
let baselineOffset: CGFloat
|
||||
}
|
||||
|
||||
private struct Key: Hashable {
|
||||
let latex: String
|
||||
let fontSize: CGFloat
|
||||
let colorHash: Int
|
||||
}
|
||||
|
||||
private static var cache: [Key: RenderedImage] = [:]
|
||||
private static let capacity = 80
|
||||
private static let maxRenderedPixelArea: CGFloat = 262_144
|
||||
private static let maxRenderedPixelDimension: CGFloat = 4096
|
||||
|
||||
static func image(
|
||||
latex: String,
|
||||
fontSize: CGFloat,
|
||||
textColor: Color,
|
||||
colorScheme: ColorScheme) -> RenderedImage?
|
||||
{
|
||||
guard fontSize.isFinite, fontSize > 0,
|
||||
let mathList = ChatMathParseCache.mathList(latex: latex)
|
||||
else { return nil }
|
||||
let platformColor = self.platformColor(textColor, colorScheme: colorScheme)
|
||||
let key = Key(
|
||||
latex: latex,
|
||||
fontSize: fontSize,
|
||||
colorHash: self.colorHash(platformColor))
|
||||
if let cached = self.cache[key] {
|
||||
return cached
|
||||
}
|
||||
|
||||
let metricsLabel = MTMathUILabel()
|
||||
metricsLabel.mathList = mathList
|
||||
metricsLabel.labelMode = .text
|
||||
metricsLabel.fontSize = fontSize
|
||||
metricsLabel.textColor = platformColor
|
||||
let size: CGSize
|
||||
#if os(macOS)
|
||||
size = metricsLabel.fittingSize
|
||||
metricsLabel.frame = CGRect(origin: .zero, size: size)
|
||||
metricsLabel.layoutSubtreeIfNeeded()
|
||||
#else
|
||||
size = metricsLabel.intrinsicContentSize
|
||||
metricsLabel.frame = CGRect(origin: .zero, size: size)
|
||||
metricsLabel.layoutIfNeeded()
|
||||
#endif
|
||||
guard self.isSafeImageSize(size, scale: self.imageScale) else { return nil }
|
||||
|
||||
// MTMathImage allocates immediately in asImage(). Measure with the same
|
||||
// text-mode typesetter first so hostile input cannot request an empty or
|
||||
// oversized backing bitmap.
|
||||
let imageRenderer = MTMathImage(
|
||||
latex: latex,
|
||||
fontSize: fontSize,
|
||||
textColor: platformColor,
|
||||
labelMode: .text,
|
||||
textAlignment: .left)
|
||||
guard let image = imageRenderer.asImage().1 else { return nil }
|
||||
|
||||
// Text(Image) places the image bottom on the surrounding baseline.
|
||||
// SwiftMath exposes its baseline descent through displayList, so lower
|
||||
// the image by that descent to align the two typographic baselines.
|
||||
let rendered = RenderedImage(
|
||||
image: image,
|
||||
baselineOffset: -(metricsLabel.displayList?.descent ?? 0))
|
||||
if self.cache.count >= self.capacity {
|
||||
self.cache.removeAll(keepingCapacity: true)
|
||||
}
|
||||
self.cache[key] = rendered
|
||||
return rendered
|
||||
}
|
||||
|
||||
static func isSafeImageSize(_ size: CGSize, scale: CGFloat) -> Bool {
|
||||
guard size.width.isFinite, size.height.isFinite, scale.isFinite,
|
||||
size.width > 0, size.height > 0, scale > 0
|
||||
else { return false }
|
||||
let pixelWidth = size.width * scale
|
||||
let pixelHeight = size.height * scale
|
||||
return pixelWidth.isFinite && pixelHeight.isFinite &&
|
||||
pixelWidth <= self.maxRenderedPixelDimension &&
|
||||
pixelHeight <= self.maxRenderedPixelDimension &&
|
||||
pixelWidth * pixelHeight <= self.maxRenderedPixelArea
|
||||
}
|
||||
|
||||
private static var imageScale: CGFloat {
|
||||
#if os(macOS)
|
||||
NSScreen.main?.backingScaleFactor ?? 2
|
||||
#else
|
||||
UIScreen.main.scale
|
||||
#endif
|
||||
}
|
||||
|
||||
private static func platformColor(_ color: Color, colorScheme: ColorScheme) -> MTColor {
|
||||
#if os(macOS)
|
||||
guard let appearance = NSAppearance(
|
||||
named: colorScheme == .dark ? .darkAqua : .aqua)
|
||||
else { return NSColor(color) }
|
||||
var platformColor = NSColor.clear
|
||||
appearance.performAsCurrentDrawingAppearance {
|
||||
let resolved = NSColor(color)
|
||||
platformColor = resolved.usingColorSpace(.deviceRGB) ?? resolved
|
||||
}
|
||||
return platformColor
|
||||
#else
|
||||
let style: UIUserInterfaceStyle = colorScheme == .dark ? .dark : .light
|
||||
return UIColor(color).resolvedColor(with: UITraitCollection(userInterfaceStyle: style))
|
||||
#endif
|
||||
}
|
||||
|
||||
private static func colorHash(_ color: MTColor) -> Int {
|
||||
var red: CGFloat = 0
|
||||
var green: CGFloat = 0
|
||||
var blue: CGFloat = 0
|
||||
var alpha: CGFloat = 0
|
||||
color.getRed(&red, green: &green, blue: &blue, alpha: &alpha)
|
||||
var hasher = Hasher()
|
||||
hasher.combine(red)
|
||||
hasher.combine(green)
|
||||
hasher.combine(blue)
|
||||
hasher.combine(alpha)
|
||||
return hasher.finalize()
|
||||
}
|
||||
}
|
||||
|
|
@ -77,71 +77,6 @@ struct ChatMathBlockView: View {
|
|||
}
|
||||
}
|
||||
|
||||
/// Parsed math is stable after its delimiter closes. A bounded cache avoids
|
||||
/// repeating SwiftMath parsing as later streaming deltas rerender old blocks.
|
||||
@MainActor
|
||||
private enum ChatMathParseCache {
|
||||
private enum Result {
|
||||
case parsed(MTMathList)
|
||||
case invalid
|
||||
}
|
||||
|
||||
private static var cache: [String: Result] = [:]
|
||||
private static let capacity = 80
|
||||
private static let maxNestingDepth = 64
|
||||
private static let maxCommandCount = 128
|
||||
private static let unsafeCommands = [#"\color"#, #"\colorbox"#, #"\textcolor"#]
|
||||
|
||||
static func mathList(latex: String) -> MTMathList? {
|
||||
guard !latex.isEmpty else { return nil }
|
||||
// SwiftMath silently drops unsupported Unicode instead of reporting a
|
||||
// parse error. Preserve the source through the raw-text fallback.
|
||||
guard latex.unicodeScalars.allSatisfy(\.isASCII) else { return nil }
|
||||
// SwiftMath recursively parses groups. Bound hostile nesting before
|
||||
// entering the dependency so a short expression cannot exhaust stack.
|
||||
guard self.isWithinParserLimits(latex) else { return nil }
|
||||
// SwiftMath 1.7.3 traps while typesetting empty color-command bodies.
|
||||
// Chat owns the surrounding color, so preserve these as raw source.
|
||||
guard !self.unsafeCommands.contains(where: latex.contains) else { return nil }
|
||||
if let hit = self.cache[latex] {
|
||||
if case let .parsed(mathList) = hit { return mathList }
|
||||
return nil
|
||||
}
|
||||
|
||||
let result = MTMathListBuilder.build(fromString: latex)
|
||||
.map(Result.parsed) ?? .invalid
|
||||
if self.cache.count >= self.capacity {
|
||||
self.cache.removeAll(keepingCapacity: true)
|
||||
}
|
||||
self.cache[latex] = result
|
||||
if case let .parsed(mathList) = result { return mathList }
|
||||
return nil
|
||||
}
|
||||
|
||||
private static func isWithinParserLimits(_ latex: String) -> Bool {
|
||||
var depth = 0
|
||||
var commandCount = 0
|
||||
var escaped = false
|
||||
for character in latex {
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if character == "\\" {
|
||||
commandCount += 1
|
||||
if commandCount > self.maxCommandCount { return false }
|
||||
escaped = true
|
||||
} else if character == "{" {
|
||||
depth += 1
|
||||
if depth > self.maxNestingDepth { return false }
|
||||
} else if character == "}" {
|
||||
depth = max(0, depth - 1)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
#if os(macOS)
|
||||
@MainActor
|
||||
private struct ChatMathPlatformView: NSViewRepresentable {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,14 @@ struct ChatMarkdownRenderer: View {
|
|||
case assistant
|
||||
}
|
||||
|
||||
struct InlineMathTypography {
|
||||
static let body = Self(size: OpenClawChatTypography.bodySize, relativeTo: .body)
|
||||
static let callout = Self(size: 16, relativeTo: .callout)
|
||||
|
||||
let size: CGFloat
|
||||
let relativeTo: Font.TextStyle
|
||||
}
|
||||
|
||||
let snapshot: ChatMarkdownRenderSnapshot
|
||||
let context: Context
|
||||
let variant: ChatMarkdownVariant
|
||||
|
|
@ -21,12 +29,16 @@ struct ChatMarkdownRenderer: View {
|
|||
let textColor: Color
|
||||
var reveal: ChatMarkdownProseReveal?
|
||||
|
||||
@ScaledMetric private var inlineMathFontSize: CGFloat
|
||||
@Environment(\.colorScheme) private var colorScheme
|
||||
|
||||
init(
|
||||
text: String,
|
||||
context: Context,
|
||||
variant: ChatMarkdownVariant,
|
||||
font: Font,
|
||||
textColor: Color,
|
||||
inlineMathTypography: InlineMathTypography = .body,
|
||||
isComplete: Bool = true)
|
||||
{
|
||||
self.init(
|
||||
|
|
@ -34,7 +46,8 @@ struct ChatMarkdownRenderer: View {
|
|||
context: context,
|
||||
variant: variant,
|
||||
font: font,
|
||||
textColor: textColor)
|
||||
textColor: textColor,
|
||||
inlineMathTypography: inlineMathTypography)
|
||||
}
|
||||
|
||||
init(
|
||||
|
|
@ -43,6 +56,7 @@ struct ChatMarkdownRenderer: View {
|
|||
variant: ChatMarkdownVariant,
|
||||
font: Font,
|
||||
textColor: Color,
|
||||
inlineMathTypography: InlineMathTypography = .body,
|
||||
reveal: ChatMarkdownProseReveal? = nil)
|
||||
{
|
||||
self.snapshot = snapshot
|
||||
|
|
@ -51,6 +65,9 @@ struct ChatMarkdownRenderer: View {
|
|||
self.font = font
|
||||
self.textColor = textColor
|
||||
self.reveal = reveal
|
||||
self._inlineMathFontSize = ScaledMetric(
|
||||
wrappedValue: inlineMathTypography.size,
|
||||
relativeTo: inlineMathTypography.relativeTo)
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
|
|
@ -75,6 +92,7 @@ struct ChatMarkdownRenderer: View {
|
|||
.tint(self.linkColor)
|
||||
.textSelection(.enabled)
|
||||
.lineSpacing(self.variant == .compact ? 2 : 4)
|
||||
.modifier(ChatInlineMathAccessibilityModifier(label: prose.inlineAccessibilityText))
|
||||
case let .code(code):
|
||||
ChatCodeBlockView(block: code)
|
||||
case let .math(math):
|
||||
|
|
@ -86,7 +104,10 @@ struct ChatMarkdownRenderer: View {
|
|||
|
||||
private func proseText(_ prose: ChatMarkdownProse, index: Int) -> SwiftUI.Text {
|
||||
guard let reveal = self.reveal, reveal.blockIndex == index else {
|
||||
return SwiftUI.Text(prose.attributed)
|
||||
return prose.renderedText(
|
||||
fontSize: self.inlineMathFontSize,
|
||||
textColor: self.textColor,
|
||||
colorScheme: self.colorScheme)
|
||||
}
|
||||
return prose.revealedText(
|
||||
frame: revealedOpacities(state: reveal.state, now: reveal.now),
|
||||
|
|
@ -104,6 +125,7 @@ struct ChatMarkdownProseReveal {
|
|||
let now: TimeInterval
|
||||
}
|
||||
|
||||
@MainActor
|
||||
struct ChatMarkdownRenderSnapshot {
|
||||
let blocks: [ChatMarkdownRenderedBlock]
|
||||
let images: [ChatMarkdownPreprocessor.InlineImage]
|
||||
|
|
@ -115,7 +137,10 @@ struct ChatMarkdownRenderSnapshot {
|
|||
isComplete: isComplete).map { block in
|
||||
switch block {
|
||||
case let .prose(markdown):
|
||||
.prose(ChatMarkdownProse(markdown: markdown, preparesReveal: preparesReveal))
|
||||
.prose(ChatMarkdownProse(
|
||||
markdown: markdown,
|
||||
isComplete: isComplete,
|
||||
preparesReveal: preparesReveal))
|
||||
case let .code(code):
|
||||
.code(code)
|
||||
case let .math(math):
|
||||
|
|
@ -129,7 +154,9 @@ struct ChatMarkdownRenderSnapshot {
|
|||
|
||||
var lastProseIndex: Int? {
|
||||
self.blocks.lastIndex {
|
||||
if case .prose = $0 { return true }
|
||||
if case .prose = $0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -142,6 +169,7 @@ enum ChatMarkdownRenderedBlock {
|
|||
case table(ChatMarkdownTable)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
struct ChatMarkdownProse {
|
||||
struct TailPiece {
|
||||
let attributed: AttributedString
|
||||
|
|
@ -152,14 +180,23 @@ struct ChatMarkdownProse {
|
|||
let plainText: String
|
||||
let prefix: AttributedString
|
||||
let tail: [TailPiece]
|
||||
let inlineContent: [InlineContent]?
|
||||
|
||||
init(markdown: String, preparesReveal: Bool) {
|
||||
let displayMarkdown = ChatMarkdownDisplayPreprocessor.preserveChatSoftBreaks(in: markdown)
|
||||
let options = AttributedString.MarkdownParsingOptions(
|
||||
interpretedSyntax: .full,
|
||||
failurePolicy: .returnPartiallyParsedIfPossible)
|
||||
let attributed = (try? AttributedString(markdown: displayMarkdown, options: options))
|
||||
?? AttributedString(displayMarkdown)
|
||||
enum InlineContent {
|
||||
case text(AttributedString)
|
||||
case math(ChatInlineMathSpan)
|
||||
}
|
||||
|
||||
init(markdown: String, isComplete: Bool, preparesReveal: Bool) {
|
||||
// preparesReveal belongs only to the streaming snapshot. Keep that path
|
||||
// on its single AttributedString build; completed snapshots alone split
|
||||
// prose for inline image interpolation.
|
||||
let inlineContent = isComplete && !preparesReveal
|
||||
? Self.makeInlineContent(markdown: markdown)
|
||||
: nil
|
||||
let attributed = inlineContent == nil
|
||||
? Self.parseMarkdown(markdown)
|
||||
: AttributedString()
|
||||
let plainText = preparesReveal ? String(attributed.characters) : ""
|
||||
let wordRanges = preparesReveal
|
||||
? Array(chatStreamingWordRanges(in: plainText).suffix(24))
|
||||
|
|
@ -168,6 +205,7 @@ struct ChatMarkdownProse {
|
|||
|
||||
self.attributed = attributed
|
||||
self.plainText = plainText
|
||||
self.inlineContent = inlineContent
|
||||
if preparesReveal {
|
||||
self.prefix = Self.slice(attributed, characterRange: 0..<tailStart)
|
||||
self.tail = Self.tailPieces(
|
||||
|
|
@ -181,6 +219,54 @@ struct ChatMarkdownProse {
|
|||
}
|
||||
}
|
||||
|
||||
var inlineMathLatex: [String] {
|
||||
self.inlineContent?.compactMap { content in
|
||||
if case let .math(span) = content {
|
||||
return span.latex
|
||||
}
|
||||
return nil
|
||||
} ?? []
|
||||
}
|
||||
|
||||
var inlineAccessibilityText: String? {
|
||||
guard let inlineContent else { return nil }
|
||||
return inlineContent.reduce(into: "") { text, content in
|
||||
switch content {
|
||||
case let .text(attributed):
|
||||
text += String(attributed.characters)
|
||||
case let .math(span):
|
||||
text += span.latex
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func renderedText(
|
||||
fontSize: CGFloat,
|
||||
textColor: Color,
|
||||
colorScheme: ColorScheme) -> SwiftUI.Text
|
||||
{
|
||||
guard let inlineContent else { return SwiftUI.Text(self.attributed) }
|
||||
return inlineContent.reduce(SwiftUI.Text("")) { text, content in
|
||||
switch content {
|
||||
case let .text(attributed):
|
||||
return text + SwiftUI.Text(attributed)
|
||||
case let .math(span):
|
||||
guard let rendered = ChatInlineMathImageCache.image(
|
||||
latex: span.latex,
|
||||
fontSize: fontSize,
|
||||
textColor: textColor,
|
||||
colorScheme: colorScheme)
|
||||
else { return text + SwiftUI.Text(span.source) }
|
||||
#if os(macOS)
|
||||
let image = Image(nsImage: rendered.image)
|
||||
#else
|
||||
let image = Image(uiImage: rendered.image)
|
||||
#endif
|
||||
return text + SwiftUI.Text(image).baselineOffset(rendered.baselineOffset)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func revealedText(frame: ChatStreamingRevealFrame, textColor: Color) -> SwiftUI.Text {
|
||||
self.tail.reduce(SwiftUI.Text(self.prefix)) { text, piece in
|
||||
var attributed = piece.attributed
|
||||
|
|
@ -193,6 +279,118 @@ struct ChatMarkdownProse {
|
|||
}
|
||||
}
|
||||
|
||||
private static func makeInlineContent(markdown: String) -> [InlineContent]? {
|
||||
let pieces = ChatInlineMathScanner.pieces(in: markdown)
|
||||
guard pieces.contains(where: { piece in
|
||||
if case .markdown = piece {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}) else { return nil }
|
||||
|
||||
var substitutedMarkdown = ""
|
||||
var replacements: [InlineReplacement] = []
|
||||
var markerValue: UInt32 = 0xE000
|
||||
var occupiedMarkerValues = Set(markdown.unicodeScalars.map(\.value))
|
||||
for piece in pieces {
|
||||
switch piece {
|
||||
case let .markdown(source):
|
||||
substitutedMarkdown += source
|
||||
case let .literal(source):
|
||||
substitutedMarkdown += Self.markdownEscapedLiteral(source)
|
||||
case let .math(latex, source):
|
||||
guard ChatMathParseCache.mathList(latex: latex) != nil else {
|
||||
substitutedMarkdown += Self.markdownEscapedLiteral(source)
|
||||
continue
|
||||
}
|
||||
let marker = Self.nextMarker(
|
||||
startingAt: &markerValue,
|
||||
occupiedValues: &occupiedMarkerValues)
|
||||
substitutedMarkdown.append(marker)
|
||||
replacements.append(InlineReplacement(
|
||||
marker: marker,
|
||||
span: ChatInlineMathSpan(latex: latex, source: source)))
|
||||
}
|
||||
}
|
||||
|
||||
let attributed = self.parseMarkdown(substitutedMarkdown)
|
||||
var content: [InlineContent] = []
|
||||
var cursor = attributed.startIndex
|
||||
for replacement in replacements {
|
||||
guard let markerIndex = attributed.characters[cursor...]
|
||||
.firstIndex(of: replacement.marker)
|
||||
else { return nil }
|
||||
if cursor < markerIndex {
|
||||
content.append(.text(AttributedString(attributed[cursor..<markerIndex])))
|
||||
}
|
||||
let markerEnd = attributed.characters.index(after: markerIndex)
|
||||
let attributes = attributed[markerIndex..<markerEnd].runs.first?.attributes
|
||||
if attributes?.link != nil {
|
||||
// Text(Image) cannot carry an AttributedString link. Keep math
|
||||
// used as a link label literal so the destination remains
|
||||
// visible and tappable instead of silently dropping the link.
|
||||
var literal = AttributedString(replacement.span.source)
|
||||
if let attributes {
|
||||
literal.mergeAttributes(attributes)
|
||||
}
|
||||
content.append(.text(literal))
|
||||
} else {
|
||||
content.append(.math(replacement.span))
|
||||
}
|
||||
cursor = markerEnd
|
||||
}
|
||||
if cursor < attributed.endIndex {
|
||||
content.append(.text(AttributedString(attributed[cursor...])))
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
private struct InlineReplacement {
|
||||
let marker: Character
|
||||
let span: ChatInlineMathSpan
|
||||
}
|
||||
|
||||
private static func nextMarker(
|
||||
startingAt value: inout UInt32,
|
||||
occupiedValues: inout Set<UInt32>) -> Character
|
||||
{
|
||||
while occupiedValues.contains(value) {
|
||||
value += 1
|
||||
}
|
||||
guard let scalar = UnicodeScalar(value) else {
|
||||
preconditionFailure("inline math marker range exhausted")
|
||||
}
|
||||
occupiedValues.insert(value)
|
||||
let marker = Character(String(scalar))
|
||||
value += 1
|
||||
return marker
|
||||
}
|
||||
|
||||
private static func markdownEscapedLiteral(_ source: String) -> String {
|
||||
source.reduce(into: "") { escaped, character in
|
||||
if character.unicodeScalars.count == 1,
|
||||
let scalar = character.unicodeScalars.first,
|
||||
scalar.isASCII,
|
||||
(33...47).contains(scalar.value) ||
|
||||
(58...64).contains(scalar.value) ||
|
||||
(91...96).contains(scalar.value) ||
|
||||
(123...126).contains(scalar.value)
|
||||
{
|
||||
escaped.append("\\")
|
||||
}
|
||||
escaped.append(character)
|
||||
}
|
||||
}
|
||||
|
||||
private static func parseMarkdown(_ markdown: String) -> AttributedString {
|
||||
let displayMarkdown = ChatMarkdownDisplayPreprocessor.preserveChatSoftBreaks(in: markdown)
|
||||
let options = AttributedString.MarkdownParsingOptions(
|
||||
interpretedSyntax: .full,
|
||||
failurePolicy: .returnPartiallyParsedIfPossible)
|
||||
return (try? AttributedString(markdown: displayMarkdown, options: options))
|
||||
?? AttributedString(displayMarkdown)
|
||||
}
|
||||
|
||||
private static func tailPieces(
|
||||
attributed: AttributedString,
|
||||
textLength: Int,
|
||||
|
|
@ -235,6 +433,18 @@ struct ChatMarkdownProse {
|
|||
}
|
||||
}
|
||||
|
||||
private struct ChatInlineMathAccessibilityModifier: ViewModifier {
|
||||
let label: String?
|
||||
|
||||
func body(content: Content) -> some View {
|
||||
if let label {
|
||||
content.accessibilityLabel(Text(label))
|
||||
} else {
|
||||
content
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fenced code, display math, and GFM tables are split out by `ChatMarkdownBlockSegmenter`
|
||||
/// before this runs, so prose only needs chat-style soft-break preservation.
|
||||
enum ChatMarkdownDisplayPreprocessor {
|
||||
|
|
|
|||
|
|
@ -920,12 +920,16 @@ private struct ChatAssistantTextBody: View {
|
|||
let font = segment.kind == .thinking
|
||||
? OpenClawChatTypography.callout.italic()
|
||||
: OpenClawChatTypography.body
|
||||
let inlineMathTypography: ChatMarkdownRenderer.InlineMathTypography = segment.kind == .thinking
|
||||
? .callout
|
||||
: .body
|
||||
ChatMarkdownRenderer(
|
||||
text: segment.text,
|
||||
context: .assistant,
|
||||
variant: self.markdownVariant,
|
||||
font: font,
|
||||
textColor: OpenClawChatTheme.assistantText,
|
||||
inlineMathTypography: inlineMathTypography,
|
||||
isComplete: self.isComplete)
|
||||
}
|
||||
}
|
||||
|
|
@ -1003,6 +1007,9 @@ private struct ChatStreamingAssistantTextBody: View {
|
|||
let font = segment.kind == .thinking
|
||||
? OpenClawChatTypography.callout.italic()
|
||||
: OpenClawChatTypography.body
|
||||
let inlineMathTypography: ChatMarkdownRenderer.InlineMathTypography = segment.kind == .thinking
|
||||
? .callout
|
||||
: .body
|
||||
let reveal = self.reveal(
|
||||
segmentIndex: entry.offset,
|
||||
now: now)
|
||||
|
|
@ -1012,6 +1019,7 @@ private struct ChatStreamingAssistantTextBody: View {
|
|||
variant: self.markdownVariant,
|
||||
font: font,
|
||||
textColor: OpenClawChatTheme.assistantText,
|
||||
inlineMathTypography: inlineMathTypography,
|
||||
reveal: reveal)
|
||||
}
|
||||
}
|
||||
|
|
@ -1060,6 +1068,7 @@ private struct ChatStreamingAssistantTextBody: View {
|
|||
return deadline
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private struct Snapshot {
|
||||
struct Segment {
|
||||
let kind: AssistantTextSegment.Kind
|
||||
|
|
|
|||
|
|
@ -0,0 +1,161 @@
|
|||
import CoreGraphics
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import OpenClawChatUI
|
||||
|
||||
struct ChatInlineMathTests {
|
||||
@Test func `basic span splits from prose`() {
|
||||
#expect(ChatInlineMathScanner.pieces(in: #"before \(x + 1\) after"#) == [
|
||||
.markdown("before "),
|
||||
.math(latex: "x + 1", source: #"\(x + 1\)"#),
|
||||
.markdown(" after"),
|
||||
])
|
||||
}
|
||||
|
||||
@Test func `multiple spans split independently`() {
|
||||
let pieces = ChatInlineMathScanner.pieces(in: #"\(x\), then \(y\)"#)
|
||||
#expect(pieces.compactMap(\.mathLatex) == ["x", "y"])
|
||||
}
|
||||
|
||||
@Test func `span inside backticks stays markdown`() {
|
||||
let source = #"use `\(not math\)` here"#
|
||||
#expect(ChatInlineMathScanner.pieces(in: source) == [.markdown(source)])
|
||||
}
|
||||
|
||||
@Test func `unmatched backtick does not hide later math`() {
|
||||
#expect(ChatInlineMathScanner.pieces(in: #"stray ` before \(x\)"#) == [
|
||||
.markdown("stray ` before "),
|
||||
.math(latex: "x", source: #"\(x\)"#),
|
||||
])
|
||||
}
|
||||
|
||||
@Test func `escaped delimiters stay markdown`() {
|
||||
let source = #"escaped \\(x\\)"#
|
||||
#expect(ChatInlineMathScanner.pieces(in: source) == [.markdown(source)])
|
||||
}
|
||||
|
||||
@Test func `unclosed opener stays literal`() {
|
||||
#expect(ChatInlineMathScanner.pieces(in: #"before \(x"#) == [
|
||||
.markdown("before "),
|
||||
.literal(#"\(x"#),
|
||||
])
|
||||
}
|
||||
|
||||
@Test func `newline span stays literal`() {
|
||||
let source = "before \\(x\ny\\) after"
|
||||
#expect(ChatInlineMathScanner.pieces(in: source) == [
|
||||
.markdown("before "),
|
||||
.literal("\\(x\ny\\)"),
|
||||
.markdown(" after"),
|
||||
])
|
||||
}
|
||||
|
||||
@Test func `only first sixteen spans are candidates`() {
|
||||
let source = (0..<17).map { #"\(x\#($0)\)"# }.joined(separator: " ")
|
||||
let pieces = ChatInlineMathScanner.pieces(in: source)
|
||||
#expect(pieces.compactMap(\.mathLatex).count == ChatInlineMathScanner.maxSpanCount)
|
||||
#expect(pieces.contains(.literal(#"\(x16\)"#)))
|
||||
}
|
||||
|
||||
@Test @MainActor func `markdown after excess spans remains parsed`() throws {
|
||||
let spans = (0..<17).map { #"\(x\#($0)\)"# }.joined(separator: " ")
|
||||
let snapshot = ChatMarkdownRenderSnapshot(
|
||||
text: spans + " [docs](https://example.com)",
|
||||
isComplete: true)
|
||||
let prose = try self.prose(in: snapshot)
|
||||
let hasLink = prose.inlineContent?.contains { content in
|
||||
guard case let .text(attributed) = content else { return false }
|
||||
return attributed.runs.contains { $0.link != nil }
|
||||
}
|
||||
|
||||
#expect(hasLink == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func `math link label stays literal and linked`() throws {
|
||||
let snapshot = ChatMarkdownRenderSnapshot(
|
||||
text: #"[\(x\)](https://example.com)"#,
|
||||
isComplete: true)
|
||||
let prose = try self.prose(in: snapshot)
|
||||
let linkedText = prose.inlineContent?.compactMap { content -> AttributedString? in
|
||||
guard case let .text(attributed) = content,
|
||||
attributed.runs.contains(where: { $0.link != nil })
|
||||
else { return nil }
|
||||
return attributed
|
||||
}.first
|
||||
|
||||
#expect(prose.inlineMathLatex.isEmpty)
|
||||
#expect(try String(#require(linkedText).characters) == #"\(x\)"#)
|
||||
}
|
||||
|
||||
@Test func `two hundred byte cap is inclusive`() {
|
||||
let acceptedLatex = String(repeating: "x", count: ChatInlineMathScanner.maxSourceBytes - 4)
|
||||
let rejectedLatex = acceptedLatex + "x"
|
||||
#expect(ChatInlineMathScanner.pieces(in: "\\(\(acceptedLatex)\\)").compactMap(\.mathLatex) == [
|
||||
acceptedLatex,
|
||||
])
|
||||
#expect(ChatInlineMathScanner.pieces(in: "\\(\(rejectedLatex)\\)") == [
|
||||
.literal("\\(\(rejectedLatex)\\)"),
|
||||
])
|
||||
}
|
||||
|
||||
@Test @MainActor func `only completed prose prepares inline math`() throws {
|
||||
let complete = ChatMarkdownRenderSnapshot(text: #"value \(x^2\)"#, isComplete: true)
|
||||
let streaming = ChatMarkdownRenderSnapshot(
|
||||
text: #"value \(x^2\)"#,
|
||||
isComplete: false,
|
||||
preparesReveal: true)
|
||||
|
||||
#expect(try self.prose(in: complete).inlineMathLatex == ["x^2"])
|
||||
#expect(try self.prose(in: streaming).inlineMathLatex.isEmpty)
|
||||
}
|
||||
|
||||
@Test @MainActor func `display math guards also reject inline math`() throws {
|
||||
for source in [#"value \(α + β\)"#, #"value \(x\color{red}\)"#] {
|
||||
let snapshot = ChatMarkdownRenderSnapshot(text: source, isComplete: true)
|
||||
#expect(try self.prose(in: snapshot).inlineMathLatex.isEmpty)
|
||||
}
|
||||
}
|
||||
|
||||
@Test @MainActor func `markdown attributes can cross inline math`() throws {
|
||||
let snapshot = ChatMarkdownRenderSnapshot(text: #"**value \(x\)**"#, isComplete: true)
|
||||
let prose = try self.prose(in: snapshot)
|
||||
|
||||
#expect(prose.inlineMathLatex == ["x"])
|
||||
#expect(prose.inlineAccessibilityText == "value x")
|
||||
}
|
||||
|
||||
@Test @MainActor func `image sizing rejects unsafe bitmap bounds`() {
|
||||
#expect(!ChatInlineMathImageCache.isSafeImageSize(.zero, scale: 2))
|
||||
#expect(!ChatInlineMathImageCache.isSafeImageSize(
|
||||
CGSize(width: 4097, height: 1),
|
||||
scale: 1))
|
||||
#expect(!ChatInlineMathImageCache.isSafeImageSize(
|
||||
CGSize(width: 1024, height: 1024),
|
||||
scale: 1))
|
||||
#expect(ChatInlineMathImageCache.isSafeImageSize(
|
||||
CGSize(width: 256, height: 256),
|
||||
scale: 2))
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private func prose(in snapshot: ChatMarkdownRenderSnapshot) throws -> ChatMarkdownProse {
|
||||
guard case let .prose(prose) = try #require(snapshot.blocks.first) else {
|
||||
Issue.record("expected prose block")
|
||||
throw InlineMathTestError.expectedProse
|
||||
}
|
||||
return prose
|
||||
}
|
||||
}
|
||||
|
||||
private enum InlineMathTestError: Error {
|
||||
case expectedProse
|
||||
}
|
||||
|
||||
extension ChatInlineMathScanner.Piece {
|
||||
fileprivate var mathLatex: String? {
|
||||
if case let .math(latex, _) = self {
|
||||
return latex
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
@ -16,6 +16,7 @@ const iosSourceRoots = [
|
|||
const sharedSwiftFiles = [
|
||||
"../shared/OpenClawKit/Sources/OpenClawChatUI/ChatComposer.swift",
|
||||
"../shared/OpenClawKit/Sources/OpenClawChatUI/ChatCodeHighlighter.swift",
|
||||
"../shared/OpenClawKit/Sources/OpenClawChatUI/ChatInlineMath.swift",
|
||||
"../shared/OpenClawKit/Sources/OpenClawChatUI/ChatLinkPreview.swift",
|
||||
"../shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockSegmenter.swift",
|
||||
"../shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownBlockViews.swift",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue