feat(voice): universal talk waveform driven by real audio levels (#102901)

* feat(voice): universal talk waveform driven by real audio levels

One shared Siri-style talk animation across iOS, watchOS, macOS, and
Android (TalkWaveformView + exact Compose port), replacing per-platform
fakes: synthetic speaking pulses, constant speech-detected power, static
Android bar rows, flat realtime listening. Listening/recording follow
live mic levels on every route; agent speech follows the real playback
envelope (AVAudioPlayer metering + playback-aligned PCM envelope +
WebRTC stats); voice-note recording shows a live capture wave.

* fix(android): order waveform imports and sync native i18n inventory

* chore(i18n): resync native inventory after rebase
This commit is contained in:
Peter Steinberger 2026-07-09 16:33:25 +01:00 committed by GitHub
parent 3c048ef052
commit 20816f676f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 1872 additions and 641 deletions

File diff suppressed because it is too large Load diff

View file

@ -262,6 +262,9 @@ class MainViewModel(
val talkModeEnabled: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeEnabled }
val talkModeListening: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeListening }
val talkModeSpeaking: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeSpeaking }
val talkInputLevel: StateFlow<Float> = runtimeState(initial = 0f) { it.talkInputLevel }
val talkOutputLevel: StateFlow<Float?> = runtimeState(initial = null) { it.talkOutputLevel }
val talkSpeechActive: StateFlow<Boolean> = runtimeState(initial = false) { it.talkSpeechActive }
val talkModeStatusText: StateFlow<String> = runtimeState(initial = "Off") { it.talkModeStatusText }
val talkModeConversation: StateFlow<List<VoiceConversationEntry>> =
runtimeState(initial = emptyList()) { it.talkModeConversation }

View file

@ -1259,6 +1259,15 @@ class NodeRuntime private constructor(
val talkModeSpeaking: StateFlow<Boolean>
get() = talkMode.isSpeaking
val talkInputLevel: StateFlow<Float>
get() = talkMode.inputLevel
val talkOutputLevel: StateFlow<Float?>
get() = talkMode.outputLevel
val talkSpeechActive: StateFlow<Boolean>
get() = talkMode.speechActive
val talkModeStatusText: StateFlow<String>
get() = talkMode.statusText

View file

@ -1,5 +1,6 @@
package ai.openclaw.app.chat
import ai.openclaw.app.voice.smoothedAudioLevel
import android.content.Context
import android.media.MediaRecorder
import android.os.SystemClock
@ -44,6 +45,9 @@ internal interface VoiceNoteRecordingEngine {
fun stop(): Long
fun cancel()
/** Peak abs PCM amplitude (0..32767) since the last poll; 0 when not recording. */
fun pollAmplitude(): Int
}
/** Owns voice-note recording state and temporary-file cleanup. */
@ -64,6 +68,9 @@ internal class VoiceNoteRecorderController(
private val _elapsedMs = MutableStateFlow(0L)
val elapsedMs: StateFlow<Long> = _elapsedMs.asStateFlow()
private val _inputLevel = MutableStateFlow(0f)
val inputLevel: StateFlow<Float> = _inputLevel.asStateFlow()
private var outputFile: File? = null
private var elapsedJob: Job? = null
private var ownsMic = false
@ -143,6 +150,7 @@ internal class VoiceNoteRecorderController(
// coroutine is composition-scoped and may be cancelled before it runs,
// so cancel() must still be able to delete the handed-off recording.
_elapsedMs.value = 0L
_inputLevel.value = 0f
_state.value = VoiceNoteRecorderState.Preparing
releaseMicLocked()
VoiceNoteRecording(file = file, durationMs = durationMs)
@ -171,6 +179,7 @@ internal class VoiceNoteRecorderController(
outputFile?.delete()
outputFile = null
_elapsedMs.value = 0L
_inputLevel.value = 0f
_state.value = VoiceNoteRecorderState.Idle
}
}
@ -190,13 +199,18 @@ internal class VoiceNoteRecorderController(
while (isActive && state.value is VoiceNoteRecorderState.Recording) {
val elapsed = (elapsedRealtimeMillis() - startedAt).coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS)
_elapsedMs.value = elapsed
// MediaRecorder reports the peak since the last poll; at 10Hz that
// reads close to the mean-abs mic levels used by the other waveform
// surfaces, so peak/32767 needs no extra curve.
val rawLevel = (engine.pollAmplitude().coerceIn(0, 32_767)) / 32_767f
_inputLevel.value = smoothedAudioLevel(_inputLevel.value, rawLevel)
// MediaRecorder's duration callback races its asynchronous auto-stop.
// Own the cap here so every successful finish calls stop() exactly once.
if (elapsed >= VOICE_NOTE_MAX_DURATION_MS) {
finish()
return@launch
}
delay(250L)
delay(100L)
}
}
}
@ -213,6 +227,7 @@ internal class VoiceNoteRecorderController(
releaseMicLocked()
outputFile = null
_elapsedMs.value = 0L
_inputLevel.value = 0f
_state.value = VoiceNoteRecorderState.Failure(message)
}
@ -282,4 +297,8 @@ internal class AndroidVoiceNoteRecordingEngine(
runCatching { active.stop() }
active.release()
}
// maxAmplitude throws until sampling starts on some OEMs; a dead meter beats
// killing the recording.
override fun pollAmplitude(): Int = recorder?.let { active -> runCatching { active.maxAmplitude }.getOrDefault(0) } ?: 0
}

View file

@ -49,6 +49,8 @@ import ai.openclaw.app.ui.design.ClawTextBadge
import ai.openclaw.app.ui.design.ClawTextField
import ai.openclaw.app.ui.design.ClawTheme
import ai.openclaw.app.ui.design.OpenClawMascot
import ai.openclaw.app.ui.design.TalkWaveform
import ai.openclaw.app.ui.design.TalkWaveformPhase
import android.Manifest
import android.content.ClipData
import android.content.ClipboardManager
@ -628,16 +630,12 @@ private fun SettingsWaveformPanel(
horizontalArrangement = Arrangement.spacedBy(5.dp),
) {
Icon(imageVector = Icons.Default.PlayArrow, contentDescription = null, modifier = Modifier.size(24.dp), tint = ClawTheme.colors.text)
Row(modifier = Modifier.weight(1f), horizontalArrangement = Arrangement.SpaceEvenly, verticalAlignment = Alignment.CenterVertically) {
listOf(6, 12, 18, 11, 28, 34, 18, 10, 8, 24, 38, 31, 12, 8, 18, 30, 40, 22, 12, 8, 20, 29, 16, 8).forEachIndexed { index, height ->
Box(
modifier =
Modifier
.size(width = 2.dp, height = (if (active) height else 7 + index % 4 * 4).dp)
.background(if (active) ClawTheme.colors.text else ClawTheme.colors.textSubtle, RoundedCornerShape(999.dp)),
)
}
}
// Thinking is the preview phase: no capture runs on this screen, so the
// synthetic swell demonstrates the animation without touching the mic.
TalkWaveform(
phase = if (active) TalkWaveformPhase.Thinking else TalkWaveformPhase.Idle,
modifier = Modifier.weight(1f).height(48.dp),
)
}
}
}

View file

@ -14,6 +14,9 @@ import ai.openclaw.app.ui.design.ClawStatus
import ai.openclaw.app.ui.design.ClawStatusPill
import ai.openclaw.app.ui.design.ClawTheme
import ai.openclaw.app.ui.design.OpenClawMascot
import ai.openclaw.app.ui.design.TalkWaveform
import ai.openclaw.app.ui.design.TalkWaveformPalette
import ai.openclaw.app.ui.design.TalkWaveformPhase
import ai.openclaw.app.voice.VoiceConversationEntry
import ai.openclaw.app.voice.VoiceConversationRole
import android.Manifest
@ -106,6 +109,10 @@ fun VoiceScreen(
val talkModeStatusText by viewModel.talkModeStatusText.collectAsState()
val talkModeConversation by viewModel.talkModeConversation.collectAsState()
val talkSetupReadiness by viewModel.talkSetupReadiness.collectAsState()
val micInputLevel by viewModel.micInputLevel.collectAsState()
val talkInputLevel by viewModel.talkInputLevel.collectAsState()
val talkOutputLevel by viewModel.talkOutputLevel.collectAsState()
val talkSpeechActive by viewModel.talkSpeechActive.collectAsState()
var pendingAction by remember { mutableStateOf<VoiceAction?>(null) }
var hasMicPermission by remember { mutableStateOf(context.hasRecordAudioPermission()) }
@ -167,6 +174,10 @@ fun VoiceScreen(
entries = talkModeConversation,
listening = talkModeListening,
speaking = talkModeSpeaking,
statusText = talkModeStatusText,
inputLevel = talkInputLevel,
outputLevel = talkOutputLevel,
speechActive = talkSpeechActive,
speakerEnabled = speakerEnabled,
onToggleSpeaker = { viewModel.setSpeakerEnabled(!speakerEnabled) },
onEndTalk = { viewModel.setTalkModeEnabled(false) },
@ -183,6 +194,7 @@ fun VoiceScreen(
conversation = micConversation,
listening = micEnabled,
sending = micIsSending,
inputLevel = micInputLevel,
statusText = activeStatus,
gatewayStatus = gatewayStatus,
onCancel = { viewModel.cancelMicCapture() },
@ -209,11 +221,21 @@ fun VoiceScreen(
VoiceHero(
gatewayStatus = gatewayStatus,
voiceCaptureMode = voiceCaptureMode,
micEnabled = micEnabled,
talkModeEnabled = talkModeEnabled,
talkModeListening = talkModeListening,
talkModeSpeaking = talkModeSpeaking,
orbPhase =
voiceHeroWaveformPhase(
micEnabled = micEnabled,
micInputLevel = micInputLevel,
talkModeEnabled = talkModeEnabled,
talkModeListening = talkModeListening,
talkModeSpeaking = talkModeSpeaking,
talkInputLevel = talkInputLevel,
talkOutputLevel = talkOutputLevel,
talkSpeechActive = talkSpeechActive,
),
micLiveTranscript = micLiveTranscript,
gatewayReady = gatewayReady,
voiceAttentionStatus = voiceAttentionStatus,
@ -271,6 +293,7 @@ private fun DictationScreen(
conversation: List<VoiceConversationEntry>,
listening: Boolean,
sending: Boolean,
inputLevel: Float,
statusText: String,
gatewayStatus: String,
onCancel: () -> Unit,
@ -314,7 +337,10 @@ private fun DictationScreen(
overflow = TextOverflow.Ellipsis,
)
Column(horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.spacedBy(10.dp)) {
DictationWaveform(active = listening || sending)
TalkWaveform(
phase = TalkWaveformPhase.Listening(level = inputLevel, speechActive = false),
modifier = Modifier.fillMaxWidth().height(56.dp),
)
Row(horizontalArrangement = Arrangement.spacedBy(7.dp), verticalAlignment = Alignment.CenterVertically) {
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(15.dp), tint = if (listening) ClawTheme.colors.success else ClawTheme.colors.textMuted)
Text(text = displayStatusText, style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
@ -404,27 +430,15 @@ private fun DictationScreen(
}
}
@Composable
private fun DictationWaveform(active: Boolean) {
Row(horizontalArrangement = Arrangement.spacedBy(4.dp), verticalAlignment = Alignment.CenterVertically) {
List(48) { index ->
val height = if (active) 3 + ((index * 7) % 16) else 3 + (index % 3) * 2
Box(
modifier =
Modifier
.size(width = 2.dp, height = height.dp)
.clip(RoundedCornerShape(999.dp))
.background(if (active) ClawTheme.colors.text else ClawTheme.colors.textSubtle),
)
}
}
}
@Composable
private fun TalkSessionScreen(
entries: List<VoiceConversationEntry>,
listening: Boolean,
speaking: Boolean,
statusText: String,
inputLevel: Float,
outputLevel: Float?,
speechActive: Boolean,
speakerEnabled: Boolean,
onToggleSpeaker: () -> Unit,
onEndTalk: () -> Unit,
@ -467,9 +481,18 @@ private fun TalkSessionScreen(
color = ClawTheme.colors.canvas,
border = BorderStroke(1.dp, ClawTheme.colors.borderStrong),
) {
Box(contentAlignment = Alignment.Center) {
TalkWaveform(active = listening || speaking)
}
TalkWaveform(
phase =
talkSessionWaveformPhase(
speaking = speaking,
listening = listening,
statusText = statusText,
inputLevel = inputLevel,
speechActive = speechActive,
outputLevel = outputLevel,
),
modifier = Modifier.fillMaxSize(),
)
}
Column(modifier = Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(8.dp)) {
@ -583,21 +606,6 @@ private fun TalkControl(
}
}
@Composable
private fun TalkWaveform(active: Boolean) {
Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) {
listOf(4, 12, 24, 34, 46, 28, 12, 38, 44, 24, 12, 30, 42, 18, 6).forEachIndexed { index, height ->
Box(
modifier =
Modifier
.size(width = 3.dp, height = (if (active) height else 6 + index % 4 * 5).dp)
.clip(RoundedCornerShape(999.dp))
.background(if (active) ClawTheme.colors.text else ClawTheme.colors.textSubtle),
)
}
}
}
@Composable
private fun VoiceHeader(
statusText: String,
@ -647,11 +655,11 @@ private fun VoiceHeader(
@Composable
private fun VoiceHero(
gatewayStatus: String,
voiceCaptureMode: VoiceCaptureMode,
micEnabled: Boolean,
talkModeEnabled: Boolean,
talkModeListening: Boolean,
talkModeSpeaking: Boolean,
orbPhase: TalkWaveformPhase,
micLiveTranscript: String?,
gatewayReady: Boolean,
voiceAttentionStatus: String?,
@ -664,11 +672,7 @@ private fun VoiceHero(
val talkNeedsSetup = gatewayReady && talkSetupReadiness.realtimeTalk.requiresSetup
val dictationNeedsSetup = gatewayReady && talkSetupReadiness.dictation.requiresSetup
Column(horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.spacedBy(9.dp)) {
VoiceOrb(
active = micEnabled || talkModeEnabled,
listening = talkModeListening || voiceCaptureMode == VoiceCaptureMode.ManualMic,
speaking = talkModeSpeaking,
)
VoiceOrb(phase = orbPhase)
Row(
modifier = Modifier.fillMaxWidth(),
@ -958,51 +962,30 @@ private fun VoicePrimaryAction(
}
}
// White wave stack for the tinted orb background (standard palette reds would
// vanish against the blue), mirroring how the macOS orb passes its own colors.
private val voiceOrbPalette =
TalkWaveformPalette(
active = listOf(Color.White, Color.White.copy(alpha = 0.75f), Color.White.copy(alpha = 0.5f)),
inactive = listOf(Color.White.copy(alpha = 0.62f), Color.White.copy(alpha = 0.5f), Color.White.copy(alpha = 0.38f)),
)
@Composable
private fun VoiceOrb(
active: Boolean,
listening: Boolean,
speaking: Boolean,
) {
private fun VoiceOrb(phase: TalkWaveformPhase) {
Surface(
modifier = Modifier.size(112.dp),
shape = CircleShape,
color = if (active || listening || speaking) Color(0xFF1976D2) else Color(0xFF123B63),
color = if (phase != TalkWaveformPhase.Idle) Color(0xFF1976D2) else Color(0xFF123B63),
contentColor = Color.White,
tonalElevation = 3.dp,
shadowElevation = 7.dp,
) {
Box(contentAlignment = Alignment.Center) {
Column(horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.spacedBy(8.dp)) {
Icon(
imageVector =
when {
speaking -> Icons.Default.RecordVoiceOver
listening -> Icons.Default.GraphicEq
else -> Icons.Default.Mic
},
contentDescription = null,
modifier = Modifier.size(32.dp),
tint = Color.White,
)
Waveform(active = active)
}
}
}
}
@Composable
private fun Waveform(active: Boolean) {
Row(horizontalArrangement = Arrangement.spacedBy(3.dp), verticalAlignment = Alignment.CenterVertically) {
listOf(6, 11, 17, 23, 14, 9, 20, 14, 7).forEachIndexed { index, height ->
Box(
modifier =
Modifier
.size(width = 2.dp, height = (if (active) height else 6 + index % 3 * 3).dp)
.clip(RoundedCornerShape(999.dp))
.background(if (active) Color.White else Color.White.copy(alpha = 0.52f)),
)
}
// The circular surface clips the wave, matching the macOS orb treatment.
TalkWaveform(
phase = phase,
modifier = Modifier.fillMaxSize().padding(horizontal = 8.dp),
palette = voiceOrbPalette,
)
}
}
@ -1116,6 +1099,43 @@ private fun runVoiceAction(
}
}
// Status literals TalkModeManager publishes while a turn runs but no audio
// flows yet; they are the only signal that the agent is being awaited.
private val talkAwaitingAgentStatuses = setOf("Thinking…", "Connecting…", "Generating voice…")
internal fun talkSessionWaveformPhase(
speaking: Boolean,
listening: Boolean,
statusText: String,
inputLevel: Float,
speechActive: Boolean,
outputLevel: Float?,
): TalkWaveformPhase =
when {
speaking -> TalkWaveformPhase.Speaking(outputLevel)
statusText.trim() in talkAwaitingAgentStatuses -> TalkWaveformPhase.Thinking
listening -> TalkWaveformPhase.Listening(level = inputLevel, speechActive = speechActive)
else -> TalkWaveformPhase.Idle
}
internal fun voiceHeroWaveformPhase(
micEnabled: Boolean,
micInputLevel: Float,
talkModeEnabled: Boolean,
talkModeListening: Boolean,
talkModeSpeaking: Boolean,
talkInputLevel: Float,
talkOutputLevel: Float?,
talkSpeechActive: Boolean,
): TalkWaveformPhase =
when {
talkModeSpeaking -> TalkWaveformPhase.Speaking(talkOutputLevel)
talkModeListening -> TalkWaveformPhase.Listening(level = talkInputLevel, speechActive = talkSpeechActive)
micEnabled -> TalkWaveformPhase.Listening(level = micInputLevel, speechActive = false)
talkModeEnabled -> TalkWaveformPhase.Thinking
else -> TalkWaveformPhase.Idle
}
internal fun voiceStatusLabel(
gatewayStatus: String,
voiceCaptureMode: VoiceCaptureMode,

View file

@ -146,6 +146,7 @@ internal fun ChatComposer(
onRemoveAttachment: (id: String) -> Unit,
voiceNoteState: VoiceNoteRecorderState,
voiceNoteElapsedMs: Long,
voiceNoteLevel: Float,
recordVoiceNoteEnabled: Boolean,
onStartVoiceNote: () -> Unit,
onCancelVoiceNote: () -> Unit,
@ -227,6 +228,7 @@ internal fun ChatComposer(
if (recordingVoiceNote) {
VoiceNoteRecordingControls(
elapsedMs = voiceNoteElapsedMs,
level = voiceNoteLevel,
onCancel = onCancelVoiceNote,
onDone = onFinishVoiceNote,
)

View file

@ -208,6 +208,7 @@ fun ChatScreen(
)
val voiceNoteState by voiceNoteRecorder.state.collectAsState()
val voiceNoteElapsedMs by voiceNoteRecorder.elapsedMs.collectAsState()
val voiceNoteLevel by voiceNoteRecorder.inputLevel.collectAsState()
val pickImages =
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult
@ -368,6 +369,7 @@ fun ChatScreen(
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
voiceNoteState = voiceNoteState,
voiceNoteElapsedMs = voiceNoteElapsedMs,
voiceNoteLevel = voiceNoteLevel,
recordVoiceNoteEnabled = pendingRunCount == 0 && !micCaptureActive,
onStartVoiceNote = { scope.launch { voiceNoteRecorder.start() } },
onCancelVoiceNote = voiceNoteRecorder::cancel,
@ -1109,6 +1111,7 @@ private fun ChatComposer(
onRemoveAttachment: (String) -> Unit,
voiceNoteState: VoiceNoteRecorderState,
voiceNoteElapsedMs: Long,
voiceNoteLevel: Float,
recordVoiceNoteEnabled: Boolean,
onStartVoiceNote: () -> Unit,
onCancelVoiceNote: () -> Unit,
@ -1168,6 +1171,7 @@ private fun ChatComposer(
if (voiceNoteState is VoiceNoteRecorderState.Recording) {
VoiceNoteRecordingControls(
elapsedMs = voiceNoteElapsedMs,
level = voiceNoteLevel,
onCancel = onCancelVoiceNote,
onDone = onFinishVoiceNote,
modifier = Modifier.weight(1f),

View file

@ -6,12 +6,15 @@ import ai.openclaw.app.chat.ChatMessageContent
import ai.openclaw.app.chat.VoiceNoteRecorderController
import ai.openclaw.app.chat.VoiceNoteRecorderState
import ai.openclaw.app.ui.design.ClawTheme
import ai.openclaw.app.ui.design.TalkWaveform
import ai.openclaw.app.ui.design.TalkWaveformPhase
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
@ -134,6 +137,7 @@ internal fun VoiceNoteRecordButton(
@Composable
internal fun VoiceNoteRecordingControls(
elapsedMs: Long,
level: Float,
onCancel: () -> Unit,
onDone: () -> Unit,
modifier: Modifier = Modifier,
@ -154,7 +158,10 @@ internal fun VoiceNoteRecordingControls(
Text(
text = formatVoiceNoteDuration(elapsedMs),
style = ClawTheme.type.label.copy(fontWeight = FontWeight.SemiBold),
modifier = Modifier.weight(1f),
)
TalkWaveform(
phase = TalkWaveformPhase.Listening(level = level, speechActive = false),
modifier = Modifier.weight(1f).height(30.dp),
)
Surface(
onClick = onCancel,

View file

@ -0,0 +1,266 @@
package ai.openclaw.app.ui.design
import android.provider.Settings
import androidx.compose.animation.core.withInfiniteAnimationFrameNanos
import androidx.compose.foundation.Canvas
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableDoubleStateOf
import androidx.compose.runtime.mutableLongStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.BlendMode
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.CompositingStrategy
import androidx.compose.ui.graphics.Path
import androidx.compose.ui.graphics.graphicsLayer
import androidx.compose.ui.graphics.luminance
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import kotlin.math.PI
import kotlin.math.abs
import kotlin.math.max
import kotlin.math.pow
import kotlin.math.sin
/**
* Universal OpenClaw talk animation: Compose port of the shared Siri-style
* waveform in `apps/shared/OpenClawKit/Sources/OpenClawChatUI/TalkWaveformView.swift`.
* Every constant mirrors the Swift original; change them there first.
*/
internal sealed interface TalkWaveformPhase {
/** Voice surface is off or unavailable: flat, static, dimmed. */
data object Idle : TalkWaveformPhase
/**
* Connecting or waiting on the agent. No audio exists in this state, so the
* wave breathes on a slow synthetic swell by design.
*/
data object Thinking : TalkWaveformPhase
/**
* Capturing the user's voice. [level] is the live microphone level in 0..1;
* [speechActive] raises the floor once endpointing detects actual speech.
*/
data class Listening(
val level: Float,
val speechActive: Boolean,
) : TalkWaveformPhase
/**
* Agent speech playback. [level] is the live playback envelope in 0..1.
* `null` means the active voice path exposes no envelope (system TTS and
* talk.speak compressed playback have no metering); the wave then falls
* back to a synthetic pulse rather than freezing.
*/
data class Speaking(
val level: Float?,
) : TalkWaveformPhase
}
/**
* Wave colors, front to back. Surfaces embedding the wave on tinted backgrounds
* (for example the voice orb) pass their own colors.
*/
internal data class TalkWaveformPalette(
val active: List<Color>,
val inactive: List<Color>,
) {
companion object {
val standard =
TalkWaveformPalette(
active =
listOf(
Color(red = 198 / 255f, green = 62 / 255f, blue = 56 / 255f),
Color(red = 0.95f, green = 0.45f, blue = 0.30f),
Color(red = 0.45f, green = 0.08f, blue = 0.12f),
),
inactive =
listOf(
Color(red = 0.62f, green = 0.62f, blue = 0.62f),
Color(red = 0.72f, green = 0.72f, blue = 0.72f),
Color(red = 0.82f, green = 0.82f, blue = 0.82f),
),
)
}
}
@Composable
internal fun TalkWaveform(
phase: TalkWaveformPhase,
modifier: Modifier = Modifier,
palette: TalkWaveformPalette = TalkWaveformPalette.standard,
) {
val context = LocalContext.current
// Compose frame clocks ignore the system animator scale; honor the OS
// "remove animations" setting explicitly (same pattern as OpenClawMascot).
val animationsEnabled =
remember(context) {
Settings.Global.getFloat(context.contentResolver, Settings.Global.ANIMATOR_DURATION_SCALE, 1f) > 0f
}
val idle = phase == TalkWaveformPhase.Idle
val frozen = !animationsEnabled || idle
var timeSeconds by remember { mutableDoubleStateOf(0.0) }
var bornNanos by remember { mutableLongStateOf(Long.MIN_VALUE) }
LaunchedEffect(frozen) {
if (frozen) {
timeSeconds = 0.0
return@LaunchedEffect
}
var lastFrameNanos = Long.MIN_VALUE
while (true) {
withInfiniteAnimationFrameNanos { frameNanos ->
if (bornNanos == Long.MIN_VALUE) bornNanos = frameNanos
// ~30fps like the iOS TimelineView(minimumInterval: 1/30): skip
// intermediate vsync callbacks instead of redrawing at display rate.
if (lastFrameNanos == Long.MIN_VALUE || frameNanos - lastFrameNanos >= 33_000_000L) {
lastFrameNanos = frameNanos
timeSeconds = (frameNanos - bornNanos) / 1_000_000_000.0
}
}
}
}
val power = TalkWaveformMath.power(phase, timeSeconds)
val colors = if (idle) palette.inactive else palette.active
// The applied Claw theme (not the system setting) decides dark rendering.
val dark = ClawTheme.colors.canvas.luminance() < 0.5f
val midlineColor = ClawTheme.colors.textMuted.copy(alpha = 0.30f)
Canvas(
modifier =
modifier.graphicsLayer {
alpha = if (idle) 0.6f else 1f
// Waves must blend with each other (screen on dark), not with whatever
// happens to be drawn behind the widget.
compositingStrategy = CompositingStrategy.Offscreen
},
) {
val midY = size.height / 2f
drawLine(
color = midlineColor,
start = Offset(0f, midY),
end = Offset(size.width, midY),
strokeWidth = 1.dp.toPx(),
)
// Screen blend pops on dark; opacity overlap reads better on light.
val blendMode = if (dark) BlendMode.Screen else BlendMode.SrcOver
val opacity = if (dark) 0.9f else 0.55f
colors.forEachIndexed { index, color ->
drawPath(
path = wavePath(size = size, time = timeSeconds, seed = index * 7.31, power = power),
color = color.copy(alpha = color.alpha * opacity),
blendMode = blendMode,
)
}
}
}
/** One wave = max envelope of three drifting lobes, mirrored around the midline. */
private fun wavePath(
size: Size,
time: Double,
seed: Double,
power: Double,
): Path {
val midX = size.width / 2.0
val midY = size.height / 2.0
val lobes = TalkWaveformMath.lobes(time, seed)
val upper = ArrayList<Offset>()
var x = -midX
while (x <= midX) {
val graphX = x / (midX / 9.0)
var y = 0.0
for (lobe in lobes) {
val amplitude = lobe.amplitude * midY * power
y = max(y, TalkWaveformMath.attenuatedSine(x = graphX, amplitude = amplitude, k = lobe.k, t = lobe.t))
}
upper.add(Offset((midX + x).toFloat(), (midY - y).toFloat()))
x += 2.0
}
val path = Path()
path.moveTo(0f, midY.toFloat())
for (point in upper) {
path.lineTo(point.x, point.y)
}
for (index in upper.indices.reversed()) {
val point = upper[index]
path.lineTo(point.x, (2.0 * midY).toFloat() - point.y)
}
path.close()
return path
}
/**
* Pure waveform math, split from the composable for unit testing. Canonical
* reference for every constant is TalkWaveformMath in TalkWaveformView.swift.
*/
internal object TalkWaveformMath {
data class Lobe(
val amplitude: Double,
val k: Double,
val t: Double,
)
/** Per-phase drive for the wave amplitude in 0..1. */
fun power(
phase: TalkWaveformPhase,
time: Double,
): Double =
when (phase) {
TalkWaveformPhase.Idle -> 0.05
TalkWaveformPhase.Thinking -> 0.16 + 0.10 * (0.5 + 0.5 * sin(time * 1.6))
is TalkWaveformPhase.Listening -> {
val clamped = phase.level.toDouble().coerceIn(0.0, 1.0)
// Detected speech lifts the floor so the wave visibly commits to the
// user even when the mic level dips between words.
if (phase.speechActive) 0.55 + 0.45 * clamped else 0.30 + 0.65 * clamped
}
is TalkWaveformPhase.Speaking -> {
val level = phase.level
if (level == null) {
// Synthetic pulse for voice paths with no playback metering.
0.70 * (0.55 + 0.45 * abs(sin(time * 5.0)))
} else {
0.25 + 0.75 * level.toDouble().coerceIn(0.0, 1.0)
}
}
}
/**
* Lobe parameters oscillate smoothly so peaks sweep back and forth across
* the line instead of scrolling off-screen.
*/
fun lobes(
time: Double,
seed: Double,
): List<Lobe> =
(0 until 3).map { index ->
val f = index.toDouble()
val ampFrequency = 0.9 + 0.23 * f
val ampPhase = time * ampFrequency + seed * 2.4 + f * 2.1
val amplitude = 0.30 + 0.70 * (0.5 + 0.5 * sin(ampPhase))
val k = 0.62 + 0.11 * f
val driftFrequency = 0.45 + 0.17 * f
val driftPhase = time * driftFrequency + seed + f * 1.9
Lobe(amplitude = amplitude, k = k, t = 2.8 * sin(driftPhase))
}
/** |A·sin(kx t)| shaped by the bell envelope g = (K/(K+(kxt)²))^K, K = 4. */
fun attenuatedSine(
x: Double,
amplitude: Double,
k: Double,
t: Double,
): Double {
val sine = amplitude * sin(k * x - t)
val tPrime = t - PI / 2
val envelope = (4.0 / (4.0 + (k * x - tPrime).pow(2))).pow(4.0)
return abs(sine * envelope)
}
}

View file

@ -203,6 +203,37 @@ private class BluetoothCommunicationRoute {
private val bluetoothCommunicationRoute = BluetoothCommunicationRoute()
/**
* Mean-abs level of little-endian PCM16 audio in 0..1. Single implementation
* shared by mic capture (dictation, realtime talk) and playback metering so
* every waveform surface reads the same scale.
*/
internal fun pcm16MeanAbsLevel(
frame: ByteArray,
length: Int,
): Float {
var total = 0L
var count = 0
var index = 0
val limit = length - (length % 2)
while (index < limit) {
val sample =
(frame[index].toInt() and 0xff) or
(frame[index + 1].toInt() shl 8)
total += kotlin.math.abs(sample.toShort().toInt())
count += 1
index += 2
}
if (count == 0) return 0f
return ((total / count).toFloat() / Short.MAX_VALUE).coerceIn(0f, 1f)
}
/** iOS-parity level smoothing (new = old*0.8 + raw*0.2) for waveform meters. */
internal fun smoothedAudioLevel(
previous: Float,
raw: Float,
): Float = previous * 0.8f + raw * 0.2f
/** Converts AudioRecord's negative return codes into capture-session failures. */
internal fun checkAudioRecordReadResult(result: Int): Int {
if (result >= 0) return result

View file

@ -702,7 +702,7 @@ internal class MicCaptureManager(
while (coroutineContext.isActive && _micEnabled.value && transcriptionSession == session) {
val read = audioInput.read(buffer, 0, buffer.size)
if (read <= 0) continue
_inputLevel.value = pcm16Level(buffer, read)
_inputLevel.value = pcm16MeanAbsLevel(buffer, read)
audioFrames.trySend(buffer.copyOf(read))
}
} catch (err: Throwable) {
@ -785,26 +785,6 @@ internal class MicCaptureManager(
else -> "Listening"
}
private fun pcm16Level(
frame: ByteArray,
length: Int,
): Float {
var total = 0L
var count = 0
var index = 0
val limit = length - (length % 2)
while (index < limit) {
val sample =
(frame[index].toInt() and 0xff) or
(frame[index + 1].toInt() shl 8)
total += kotlin.math.abs(sample.toShort().toInt())
count += 1
index += 2
}
if (count == 0) return 0f
return ((total / count).toFloat() / Short.MAX_VALUE).coerceIn(0f, 1f)
}
private fun pcm16ToPcmu(pcm16: ByteArray): ByteArray {
val output = ByteArray(pcm16.size / 2)
var inputIndex = 0

View file

@ -181,6 +181,20 @@ class TalkModeManager internal constructor(
private val _isSpeaking = MutableStateFlow(false)
val isSpeaking: StateFlow<Boolean> = _isSpeaking
private val _inputLevel = MutableStateFlow(0f)
val inputLevel: StateFlow<Float> = _inputLevel
// Null while no metered PCM playback is active. System TTS and talk.speak
// compressed playback expose no envelope; the waveform then shows the
// synthetic Speaking(null) pulse instead of a frozen line.
private val _outputLevel = MutableStateFlow<Float?>(null)
val outputLevel: StateFlow<Float?> = _outputLevel
// True while the realtime provider streams a non-final user transcript, the
// closest Android has to iOS endpointing's "speech detected" signal.
private val _speechActive = MutableStateFlow(false)
val speechActive: StateFlow<Boolean> = _speechActive
private val _statusText = MutableStateFlow("Off")
val statusText: StateFlow<String> = _statusText
@ -1058,6 +1072,7 @@ class TalkModeManager internal constructor(
while (coroutineContext.isActive && _isEnabled.value && realtimeSessionId == sessionId) {
val read = audioInput.read(buffer, 0, buffer.size)
if (read <= 0) continue
_inputLevel.value = smoothedAudioLevel(_inputLevel.value, pcm16MeanAbsLevel(buffer, read))
if (!shouldAppendRealtimeCapturedFrame(read)) continue
audioFrames.trySend(buffer.copyOf(read))
}
@ -1068,6 +1083,7 @@ class TalkModeManager internal constructor(
} finally {
audioFrames.close()
audioInput?.close()
_inputLevel.value = 0f
}
}
}
@ -1124,6 +1140,11 @@ class TalkModeManager internal constructor(
"transcript" -> {
val role = obj["role"].asStringOrNull()
val isFinal = obj["final"].asBooleanOrNull() == true
// A streaming (non-final) user transcript is the provider's speech
// signal; it raises the waveform floor like iOS endpointing does.
if (role == "user") {
_speechActive.value = !isFinal
}
val text = realtimeTranscriptText(obj["text"].asStringOrNull(), isFinal)
var assistantText: String? = null
if (text != null) {
@ -1272,6 +1293,10 @@ class TalkModeManager internal constructor(
if (track.playState != AudioTrack.PLAYSTATE_PLAYING) {
track.play()
}
// Blocking MODE_STREAM writes are playback-paced once the track buffer
// fills, so per-write metering tracks what the speaker actually plays.
_outputLevel.value =
smoothedAudioLevel(_outputLevel.value ?: 0f, pcm16MeanAbsLevel(bytes, writtenBytes))
_isSpeaking.value = true
_statusText.value = "Speaking…"
val durationMs = ((writtenBytes / 2.0) / realtimeSampleRateHz * 1000.0).toLong()
@ -1290,7 +1315,10 @@ class TalkModeManager internal constructor(
val idle =
synchronized(realtimePlaybackLock) {
val playbackIdle = SystemClock.elapsedRealtime() >= realtimePlaybackEndsAtMs
if (playbackIdle) _isSpeaking.value = false
if (playbackIdle) {
_isSpeaking.value = false
_outputLevel.value = null
}
playbackIdle
}
if (idle && _isEnabled.value && realtimeSessionId != null) {
@ -1322,6 +1350,7 @@ class TalkModeManager internal constructor(
realtimeAudioTrack = null
}
_isSpeaking.value = false
_outputLevel.value = null
if (_isEnabled.value) {
_statusText.value = "Listening"
}
@ -1362,6 +1391,8 @@ class TalkModeManager internal constructor(
realtimeUserEntryAwaitingFinal = false
realtimeUserEntryAwaitingFinalStartedAtMs = null
realtimeAssistantEntryId = null
_speechActive.value = false
_inputLevel.value = 0f
stopRealtimePlayback()
if (preserveStatus) {
_statusText.value = status

View file

@ -21,6 +21,7 @@ class VoiceNoteRecorderControllerTest {
var stopCount = 0
var cancelCount = 0
var outputFile: File? = null
var amplitude = 0
override fun start(outputFile: File) {
startCount += 1
@ -36,6 +37,8 @@ class VoiceNoteRecorderControllerTest {
override fun cancel() {
cancelCount += 1
}
override fun pollAmplitude(): Int = amplitude
}
@Test
@ -103,6 +106,27 @@ class VoiceNoteRecorderControllerTest {
directory.deleteRecursively()
}
@Test
fun recordingPublishesSmoothedLevelAndCancelClearsIt() =
runTest {
val directory = Files.createTempDirectory("voice-note-test").toFile()
val engine = FakeEngine()
val controller = controller(directory, engine)
controller.start()
engine.amplitude = 32_767
advanceTimeBy(50L)
runCurrent()
assertEquals(0.2f, controller.inputLevel.value, 1e-4f)
advanceTimeBy(100L)
runCurrent()
assertEquals(0.36f, controller.inputLevel.value, 1e-4f)
controller.cancel()
assertEquals(0f, controller.inputLevel.value, 0f)
directory.deleteRecursively()
}
@Test
fun cancelDeletesTemporaryFileAndReturnsIdle() =
runTest {

View file

@ -1,6 +1,7 @@
package ai.openclaw.app.ui
import ai.openclaw.app.VoiceCaptureMode
import ai.openclaw.app.ui.design.TalkWaveformPhase
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
@ -72,4 +73,68 @@ class VoiceScreenLogicTest {
voiceRuntimeAttentionStatus("Transcription unavailable: UNAVAILABLE: Error: No realtime transcription provider registered"),
)
}
@Test
fun talkSessionWaveformPhaseFollowsTalkState() {
assertEquals(
TalkWaveformPhase.Speaking(0.4f),
talkSessionWaveformPhase(speaking = true, listening = true, statusText = "Speaking…", inputLevel = 0.2f, speechActive = true, outputLevel = 0.4f),
)
// Thinking statuses win over the still-running capture loop.
assertEquals(
TalkWaveformPhase.Thinking,
talkSessionWaveformPhase(speaking = false, listening = true, statusText = "Thinking…", inputLevel = 0.2f, speechActive = false, outputLevel = null),
)
assertEquals(
TalkWaveformPhase.Listening(level = 0.2f, speechActive = true),
talkSessionWaveformPhase(speaking = false, listening = true, statusText = "Listening", inputLevel = 0.2f, speechActive = true, outputLevel = null),
)
assertEquals(
TalkWaveformPhase.Idle,
talkSessionWaveformPhase(speaking = false, listening = false, statusText = "Off", inputLevel = 0f, speechActive = false, outputLevel = null),
)
}
@Test
fun voiceHeroWaveformPhasePrefersTalkOverDictation() {
assertEquals(
TalkWaveformPhase.Speaking(null),
voiceHeroWaveformPhase(
micEnabled = true,
micInputLevel = 0.5f,
talkModeEnabled = true,
talkModeListening = true,
talkModeSpeaking = true,
talkInputLevel = 0.1f,
talkOutputLevel = null,
talkSpeechActive = false,
),
)
assertEquals(
TalkWaveformPhase.Listening(level = 0.5f, speechActive = false),
voiceHeroWaveformPhase(
micEnabled = true,
micInputLevel = 0.5f,
talkModeEnabled = false,
talkModeListening = false,
talkModeSpeaking = false,
talkInputLevel = 0f,
talkOutputLevel = null,
talkSpeechActive = false,
),
)
assertEquals(
TalkWaveformPhase.Thinking,
voiceHeroWaveformPhase(
micEnabled = false,
micInputLevel = 0f,
talkModeEnabled = true,
talkModeListening = false,
talkModeSpeaking = false,
talkInputLevel = 0f,
talkOutputLevel = null,
talkSpeechActive = false,
),
)
}
}

View file

@ -0,0 +1,81 @@
package ai.openclaw.app.ui.design
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import kotlin.math.PI
/** Guards the exact constants ported from TalkWaveformView.swift. */
class TalkWaveformMathTest {
@Test
fun idlePowerIsStaticRegardlessOfTime() {
assertEquals(0.05, TalkWaveformMath.power(TalkWaveformPhase.Idle, 0.0), 1e-9)
assertEquals(0.05, TalkWaveformMath.power(TalkWaveformPhase.Idle, 42.5), 1e-9)
}
@Test
fun thinkingBreathesInsideItsBand() {
assertEquals(0.21, TalkWaveformMath.power(TalkWaveformPhase.Thinking, 0.0), 1e-9)
var time = 0.0
while (time < 10.0) {
val power = TalkWaveformMath.power(TalkWaveformPhase.Thinking, time)
assertTrue(power in 0.16..0.26)
time += 0.1
}
}
@Test
fun listeningClampsLevelAndSpeechRaisesFloor() {
assertEquals(0.30, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = -0.5f, speechActive = false), 0.0), 1e-9)
assertEquals(0.95, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = 1.5f, speechActive = false), 0.0), 1e-9)
assertEquals(0.56, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = 0.4f, speechActive = false), 0.0), 1e-6)
assertEquals(0.73, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = 0.4f, speechActive = true), 0.0), 1e-6)
assertEquals(1.0, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = 2f, speechActive = true), 0.0), 1e-9)
}
@Test
fun speakingClampsMeteredLevel() {
assertEquals(0.25, TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = -1f), 0.0), 1e-9)
assertEquals(1.0, TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = 2f), 0.0), 1e-9)
assertEquals(0.70, TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = 0.6f), 0.0), 1e-6)
}
@Test
fun speakingWithoutEnvelopePulsesSynthetically() {
val trough = TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = null), 0.0)
val peak = TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = null), PI / 10.0)
assertEquals(0.70 * 0.55, trough, 1e-9)
assertEquals(0.70, peak, 1e-9)
var time = 0.0
while (time < 5.0) {
val power = TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = null), time)
assertTrue(power in trough..peak)
time += 0.05
}
}
@Test
fun lobesAreDeterministicWithFixedShapeConstants() {
val lobes = TalkWaveformMath.lobes(time = 1.25, seed = 7.31)
assertEquals(lobes, TalkWaveformMath.lobes(time = 1.25, seed = 7.31))
assertEquals(3, lobes.size)
assertEquals(0.62, lobes[0].k, 1e-9)
assertEquals(0.73, lobes[1].k, 1e-9)
assertEquals(0.84, lobes[2].k, 1e-9)
for (lobe in lobes) {
assertTrue(lobe.amplitude in 0.30..1.0)
assertTrue(lobe.t in -2.8..2.8)
}
}
@Test
fun attenuatedSinePeaksAtFullAmplitudeAndStaysNonNegative() {
// At kx t = −π/2 the bell envelope is exactly 1 and |sin| is 1.
assertEquals(3.0, TalkWaveformMath.attenuatedSine(x = -PI / 2, amplitude = 3.0, k = 1.0, t = 0.0), 1e-9)
var x = -9.0
while (x <= 9.0) {
assertTrue(TalkWaveformMath.attenuatedSine(x = x, amplitude = 1.0, k = 0.73, t = 1.4) >= 0.0)
x += 0.25
}
}
}

View file

@ -0,0 +1,58 @@
package ai.openclaw.app.voice
import org.junit.Assert.assertEquals
import org.junit.Test
class AudioLevelsTest {
@Test
fun silenceMetersToZero() {
val silence = ByteArray(640)
assertEquals(0f, pcm16MeanAbsLevel(silence, silence.size), 0f)
}
@Test
fun fullScaleMetersToOne() {
val frame = pcm16Frame(samples = 160, sample = Short.MAX_VALUE)
assertEquals(1f, pcm16MeanAbsLevel(frame, frame.size), 1e-6f)
}
@Test
fun negativeFullScaleClampsToOne() {
// abs(-32768) exceeds Short.MAX_VALUE by one; the level must stay in 0..1.
val frame = pcm16Frame(samples = 160, sample = Short.MIN_VALUE)
assertEquals(1f, pcm16MeanAbsLevel(frame, frame.size), 0f)
}
@Test
fun midScaleMetersToHalf() {
val frame = pcm16Frame(samples = 160, sample = (Short.MAX_VALUE / 2).toShort())
assertEquals(0.5f, pcm16MeanAbsLevel(frame, frame.size), 1e-3f)
}
@Test
fun trailingOddByteAndEmptyLengthAreIgnored() {
val frame = pcm16Frame(samples = 2, sample = Short.MAX_VALUE) + byteArrayOf(0x7F)
assertEquals(1f, pcm16MeanAbsLevel(frame, frame.size), 1e-6f)
assertEquals(0f, pcm16MeanAbsLevel(frame, 0), 0f)
assertEquals(0f, pcm16MeanAbsLevel(frame, 1), 0f)
}
@Test
fun smoothingMatchesIosWeighting() {
assertEquals(0.2f, smoothedAudioLevel(previous = 0f, raw = 1f), 1e-6f)
assertEquals(0.36f, smoothedAudioLevel(previous = 0.2f, raw = 1f), 1e-6f)
assertEquals(0.8f, smoothedAudioLevel(previous = 1f, raw = 0f), 1e-6f)
}
private fun pcm16Frame(
samples: Int,
sample: Short,
): ByteArray {
val frame = ByteArray(samples * 2)
for (index in 0 until samples) {
frame[index * 2] = (sample.toInt() and 0xff).toByte()
frame[index * 2 + 1] = ((sample.toInt() shr 8) and 0xff).toByte()
}
return frame
}
}

View file

@ -359,6 +359,19 @@ class TalkModeManagerTest {
)
}
@Test
fun realtimeUserTranscriptsDriveSpeechActive() {
val manager = createManager()
setPrivateField(manager, "realtimeSessionId", "relay-1")
assertFalse(manager.speechActive.value)
manager.handleGatewayEvent("talk.event", realtimeTranscriptPayload(role = "user", text = "hello"))
assertTrue(manager.speechActive.value)
manager.handleGatewayEvent("talk.event", realtimeTranscriptPayload(role = "user", text = "hello world", final = true))
assertFalse(manager.speechActive.value)
}
@Test
fun realtimeTranscriptDeltasAccumulateVoiceConversation() {
val manager = createManager()

View file

@ -1,3 +1,4 @@
import OpenClawChatUI
import SwiftUI
struct TalkProTab: View {
@ -116,7 +117,11 @@ struct TalkProTab: View {
private var heroSection: some View {
Section {
VStack(spacing: 16) {
TalkSiriWaveView(mode: self.state.waveformMode(micLevel: self.appModel.talkMode.micLevel))
TalkWaveformView(
phase: self.state.waveformPhase(
micLevel: self.appModel.talkMode.micLevel,
playbackLevel: self.appModel.talkMode.playbackLevel),
palette: .openClawBrand)
.frame(height: 130)
.accessibilityHidden(true)
@ -325,12 +330,20 @@ enum TalkProPrimaryAction: Equatable {
case waiting
}
enum TalkProWaveformMode: Equatable {
case level(Double)
case inputSpeech
case speaking
case indeterminate
case still
extension TalkWaveformPalette {
/// iOS app branding for the shared wave: adaptive accent front lobe plus
/// system grays so the idle wave tracks light/dark appearance.
static let openClawBrand = TalkWaveformPalette(
active: [
OpenClawBrand.accent,
Color(red: 0.95, green: 0.45, blue: 0.30),
Color(red: 0.45, green: 0.08, blue: 0.12),
],
inactive: [
Color(uiColor: .systemGray2),
Color(uiColor: .systemGray3),
Color(uiColor: .systemGray4),
])
}
struct TalkProState: Equatable {
@ -432,24 +445,23 @@ struct TalkProState: Equatable {
}
}
func waveformMode(micLevel: Double) -> TalkProWaveformMode {
if self.isDemoMode { return .still }
if !self.gatewayConnected { return .still }
func waveformPhase(micLevel: Double, playbackLevel: Double?) -> TalkWaveformPhase {
if self.isDemoMode { return .idle }
if !self.gatewayConnected { return .idle }
switch self.permissionState {
case .requestingUpgrade, .upgradeRequested:
return .indeterminate
return .thinking
case .missingScope, .requestFailed, .apiKeyMissing, .loadFailed:
return .still
return .idle
default:
break
}
if !self.isConfigLoaded { return .still }
if self.isSpeaking { return .speaking }
if self.isListening, self.isUserSpeechDetected { return .inputSpeech }
if self.isListening { return .level(micLevel) }
if !self.isConfigLoaded { return .idle }
if self.isSpeaking { return .speaking(level: playbackLevel) }
if self.isListening { return .listening(level: micLevel, speechActive: self.isUserSpeechDetected) }
if self.normalizedStatus.contains("connecting") || self.normalizedStatus.contains("thinking") {
return .indeterminate
return .thinking
}
return self.isEnabled ? .indeterminate : .still
return self.isEnabled ? .thinking : .idle
}
}

View file

@ -1,128 +0,0 @@
import SwiftUI
/// iOS 9-style Siri waveform for the Talk screen, driven by the runtime mode.
/// Math adapted from noahchalifour/swiftui-siri-waveform-view (MIT), as packaged
/// by alfianlosari/SiriWaveView; redrawn with Canvas + TimelineView so lobes flow
/// continuously instead of re-randomizing per power change.
///
/// State rendering: off = flat static grey; connecting/thinking = slow low
/// breathing; listening = amplitude follows mic level; speech detected = full
/// power; agent speaking = strong TTS-style pulse.
struct TalkSiriWaveView: View {
var mode: TalkProWaveformMode
@Environment(\.colorScheme) private var colorScheme
@Environment(\.accessibilityReduceMotion) private var reduceMotion
private static let born = Date()
private static let monoColors: [Color] = [
OpenClawBrand.accent,
Color(red: 0.95, green: 0.45, blue: 0.30),
Color(red: 0.45, green: 0.08, blue: 0.12),
]
private static let inactiveColors: [Color] = [
Color(uiColor: .systemGray2),
Color(uiColor: .systemGray3),
Color(uiColor: .systemGray4),
]
var body: some View {
let frozen = self.reduceMotion || self.mode == .still
TimelineView(.animation(minimumInterval: 1.0 / 30.0, paused: frozen)) { timeline in
let time = frozen ? 0 : timeline.date.timeIntervalSince(Self.born)
let power = Self.power(for: self.mode, time: time)
Canvas { context, size in
let midY = size.height / 2
var line = Path()
line.move(to: CGPoint(x: 0, y: midY))
line.addLine(to: CGPoint(x: size.width, y: midY))
context.stroke(line, with: .color(.secondary.opacity(0.30)), lineWidth: 1)
// Screen blend pops on dark; opacity overlap reads better on light.
context.blendMode = self.colorScheme == .dark ? .screen : .normal
let opacity = self.colorScheme == .dark ? 0.9 : 0.55
for (index, color) in self.colors.enumerated() {
let path = Self.wavePath(
in: size,
time: time,
seed: Double(index) * 7.31,
power: power)
context.fill(path, with: .color(color.opacity(opacity)))
}
}
}
.opacity(self.mode == .still ? 0.6 : 1.0)
}
private var colors: [Color] {
self.mode == .still ? Self.inactiveColors : Self.monoColors
}
/// Per-state drive: thinking breathes low and slow, speaking pulses like
/// TTS playback, listening follows the live mic level.
private static func power(for mode: TalkProWaveformMode, time: Double) -> Double {
switch mode {
case .still:
0.05
case .indeterminate:
0.16 + 0.10 * (0.5 + 0.5 * sin(time * 1.6))
case let .level(level):
0.30 + 0.65 * min(max(level, 0), 1)
case .inputSpeech:
0.95
case .speaking:
0.70 * (0.55 + 0.45 * abs(sin(time * 5.0)))
}
}
/// One wave = max envelope of three drifting lobes, mirrored around the midline.
private static func wavePath(in size: CGSize, time: Double, seed: Double, power: Double) -> Path {
let midX = Double(size.width) / 2
let midY = Double(size.height) / 2
// Lobe parameters oscillate smoothly so peaks sweep back and forth
// across the line instead of scrolling off-screen.
let lobes: [(A: Double, k: Double, t: Double)] = (0..<3).map { index in
let f = Double(index)
let ampFrequency = 0.9 + 0.23 * f
let ampPhase = time * ampFrequency + seed * 2.4 + f * 2.1
let amp = 0.30 + 0.70 * (0.5 + 0.5 * sin(ampPhase))
let k = 0.62 + 0.11 * f
let driftFrequency = 0.45 + 0.17 * f
let driftPhase = time * driftFrequency + seed + f * 1.9
let t = 2.8 * sin(driftPhase)
return (A: amp, k: k, t: t)
}
var upper: [CGPoint] = []
var x = -midX
while x <= midX {
let graphX = x / (midX / 9.0)
var y: Double = 0
for lobe in lobes {
let amplitude = lobe.A * midY * power
y = max(y, Self.attenuatedSine(x: graphX, A: amplitude, k: lobe.k, t: lobe.t))
}
upper.append(CGPoint(x: midX + x, y: midY - y))
x += 2
}
var path = Path()
path.move(to: CGPoint(x: 0, y: midY))
path.addLines(upper)
for point in upper.reversed() {
path.addLine(to: CGPoint(x: point.x, y: 2 * midY - point.y))
}
path.closeSubpath()
return path
}
/// |A·sin(kx t)| shaped by the bell envelope g = (K/(K+(kxt)²))^K, K = 4.
private static func attenuatedSine(x: Double, A: Double, k: Double, t: Double) -> Double {
let sine = A * sin(k * x - t)
let tPrime = t - .pi / 2
let envelope = pow(4.0 / (4.0 + pow(k * x - tPrime, 2)), 4.0)
return abs(sine * envelope)
}
}

View file

@ -125,6 +125,11 @@ final class RealtimeTalkRelaySession {
private let onStatus: (String) -> Void
private let onIssue: (TalkRuntimeIssue) -> Void
private let onSpeakingChanged: (Bool) -> Void
private let onInputLevel: (Double) -> Void
private let onOutputLevel: (Double?) -> Void
/// Playback-time-aligned envelope of the assistant PCM the relay schedules;
/// drives the speaking waveform with real audio instead of a synthetic pulse.
private var outputEnvelope: PCMPlaybackEnvelope?
private let audioEngine = AVAudioEngine()
private var relaySessionId: String?
@ -165,7 +170,9 @@ final class RealtimeTalkRelaySession {
pcmPlayer: PCMStreamingAudioPlaying,
onStatus: @escaping (String) -> Void,
onIssue: @escaping (TalkRuntimeIssue) -> Void = { _ in },
onSpeakingChanged: @escaping (Bool) -> Void)
onSpeakingChanged: @escaping (Bool) -> Void,
onInputLevel: @escaping (Double) -> Void = { _ in },
onOutputLevel: @escaping (Double?) -> Void = { _ in })
{
self.gateway = gateway
self.options = options
@ -173,6 +180,8 @@ final class RealtimeTalkRelaySession {
self.onStatus = onStatus
self.onIssue = onIssue
self.onSpeakingChanged = onSpeakingChanged
self.onInputLevel = onInputLevel
self.onOutputLevel = onOutputLevel
}
func start() async throws {
@ -360,6 +369,7 @@ final class RealtimeTalkRelaySession {
return
}
self.ensureOutputPlaybackStarted()
self.outputEnvelope?.append(data)
self.outputContinuation?.yield(data)
case "audioDone":
self.finishOutputPlaybackStream()
@ -729,6 +739,7 @@ final class RealtimeTalkRelaySession {
private func recordMicrophoneFrame(byteCount: Int, rms: Float, timestampMs: Double) {
guard !self.isClosed else { return }
self.onInputLevel(TalkAudioLevel.normalized(rms: Double(rms)))
self.micLogFrameCount += 1
self.micLogByteCount += byteCount
self.micLogMaxRms = max(self.micLogMaxRms, rms)
@ -767,6 +778,11 @@ final class RealtimeTalkRelaySession {
guard self.outputContinuation == nil, self.outputTask == nil else { return }
self.outputSessionId += 1
let sessionId = self.outputSessionId
let envelope = self.outputEnvelope ?? PCMPlaybackEnvelope { [weak self] level in
self?.onOutputLevel(level)
}
envelope.begin(sampleRate: self.outputSampleRateHz)
self.outputEnvelope = envelope
let stream = AsyncThrowingStream<Data, Error> { continuation in
self.outputContinuation = continuation
}
@ -810,6 +826,7 @@ final class RealtimeTalkRelaySession {
for chunk in chunks {
self.markOutputAudioStarted(byteCount: chunk.count, nowMs: ProcessInfo.processInfo.systemUptime * 1000)
self.onSpeakingChanged(true)
self.outputEnvelope?.append(chunk)
self.outputContinuation?.yield(chunk)
}
if shouldFinish {
@ -846,6 +863,7 @@ final class RealtimeTalkRelaySession {
self.isOutputPlaying = false
self.outputStartedAtMs = nil
self.outputPlaybackExpectedEndMs = 0
self.outputEnvelope?.cancel()
self.onSpeakingChanged(false)
}
@ -863,6 +881,7 @@ final class RealtimeTalkRelaySession {
self.isOutputPlaying = false
self.outputStartedAtMs = nil
self.outputPlaybackExpectedEndMs = 0
self.outputEnvelope?.cancel()
self.onSpeakingChanged(false)
}

View file

@ -119,6 +119,12 @@ final class TalkGatewaySpeechClient: TalkGatewaySpeechSynthesizing {
protocol TalkBufferedAudioPlaying {
func play(data: Data) async -> StreamingPlaybackResult
func stop() -> Double?
func setLevelHandler(_ handler: (@MainActor (Double?) -> Void)?)
}
extension TalkBufferedAudioPlaying {
/// Level metering is a UI nicety; test doubles and future players may skip it.
func setLevelHandler(_: (@MainActor (Double?) -> Void)?) {}
}
@MainActor
@ -163,6 +169,12 @@ final class TalkBufferedAudioPlayer: NSObject, TalkBufferedAudioPlaying, @precon
private let logger = Logger(subsystem: "ai.openclaw", category: "talk.tts")
private var player: AVAudioPlayer?
private var playback: Playback?
private var levelHandler: (@MainActor (Double?) -> Void)?
private var levelMeter: AudioPlayerLevelMeter?
func setLevelHandler(_ handler: (@MainActor (Double?) -> Void)?) {
self.levelHandler = handler
}
func play(data: Data) async -> StreamingPlaybackResult {
self.stopInternal()
@ -176,6 +188,11 @@ final class TalkBufferedAudioPlayer: NSObject, TalkBufferedAudioPlaying, @precon
self.player = player
player.delegate = self
player.prepareToPlay()
if let levelHandler {
let meter = AudioPlayerLevelMeter(onLevel: levelHandler)
meter.attach(player)
self.levelMeter = meter
}
self.armWatchdog(playback: playback)
if !player.play() {
self.logger.error("talk buffered audio player refused to play")
@ -236,6 +253,8 @@ final class TalkBufferedAudioPlayer: NSObject, TalkBufferedAudioPlaying, @precon
guard self.playback === playback else { return }
self.playback = nil
self.levelMeter?.detach()
self.levelMeter = nil
self.player?.stop()
self.player = nil
}

View file

@ -46,6 +46,10 @@ final class TalkModeManager: NSObject {
var statusText: String = "Off"
/// 0..1-ish (not calibrated). Intended for UI feedback only.
var micLevel: Double = 0
/// Live agent playback envelope in 0...1 while speaking. nil means the active
/// voice path exposes no real level (system voice, compressed streaming); the
/// waveform then falls back to a synthetic pulse.
var playbackLevel: Double?
var gatewayTalkConfigLoaded: Bool = false
var gatewayTalkApiKeyConfigured: Bool = false
var gatewayTalkDefaultModelId: String?
@ -161,6 +165,12 @@ final class TalkModeManager: NSObject {
var mp3Player: StreamingAudioPlaying = StreamingAudioPlayer.shared
var bufferedPlayer: TalkBufferedAudioPlaying = TalkBufferedAudioPlayer.shared
/// Meters PCM speech bytes on their way into the streaming player so the
/// speaking waveform tracks the audible envelope, not network arrival.
@ObservationIgnored private lazy var pcmPlaybackEnvelope = PCMPlaybackEnvelope { [weak self] level in
self?.playbackLevel = level
}
private var gateway: GatewayNodeSession?
private var gatewayConnected = false
private var talkConfigLoadedAt: Date?
@ -1347,6 +1357,14 @@ final class TalkModeManager: NSObject {
if speaking {
self.isListening = false
}
},
onInputLevel: { [weak self] level in
guard let self, self.isListening else { return }
// Same smoothing as the SFSpeech tap so route switches keep the wave feel.
self.micLevel = (self.micLevel * 0.80) + (level * 0.20)
},
onOutputLevel: { [weak self] level in
self?.playbackLevel = level
})
self.realtimeRelaySession = relaySession
do {
@ -1687,6 +1705,7 @@ final class TalkModeManager: NSObject {
if self.speechGeneration == speechGeneration {
self.stopRecognition()
self.isSpeaking = false
self.playbackLevel = nil
self.restoreConfiguredVoiceModeDescriptor()
}
}
@ -1773,7 +1792,10 @@ final class TalkModeManager: NSObject {
let streamFailure = StreamFailureBox()
let stream = Self.monitorStreamFailures(rawStream, failureBox: streamFailure)
self.lastPlaybackWasPCM = true
var playback = await pcmPlayer.play(stream: stream, sampleRate: sampleRate)
var playback = await pcmPlayer.play(
stream: self.meteredPCMStream(stream, sampleRate: sampleRate),
sampleRate: sampleRate)
self.pcmPlaybackEnvelope.cancel()
if !playback.finished, playback.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100_128")
self.logger.warning("pcm playback failed; retrying mp3")
@ -1869,9 +1891,15 @@ final class TalkModeManager: NSObject {
case let .pcm(sampleRate):
self.lastPlaybackWasPCM = true
let stream = Self.makeBufferedAudioStream(chunks: [audio.data])
result = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
result = await self.pcmPlayer.play(
stream: self.meteredPCMStream(stream, sampleRate: sampleRate),
sampleRate: sampleRate)
self.pcmPlaybackEnvelope.cancel()
case .buffered:
self.lastPlaybackWasPCM = false
self.bufferedPlayer.setLevelHandler { [weak self] level in
self?.playbackLevel = level
}
result = await self.bufferedPlayer.play(data: audio.data)
case let .unsupportedRaw(codec):
throw NSError(domain: "TalkGatewaySpeech", code: 2, userInfo: [
@ -1970,7 +1998,9 @@ final class TalkModeManager: NSObject {
self.stopRecognition()
TalkSystemSpeechSynthesizer.shared.stop()
self.cancelIncrementalSpeech()
self.pcmPlaybackEnvelope.cancel()
self.isSpeaking = false
self.playbackLevel = nil
restoreConfiguredVoiceModeDescriptor()
}
@ -2422,6 +2452,32 @@ final class TalkModeManager: NSObject {
}
}
/// Passes PCM chunks through to the player while feeding the playback
/// envelope so `playbackLevel` follows the audible speech.
private func meteredPCMStream(
_ stream: AsyncThrowingStream<Data, Error>,
sampleRate: Double) -> AsyncThrowingStream<Data, Error>
{
let envelope = self.pcmPlaybackEnvelope
envelope.begin(sampleRate: sampleRate)
return AsyncThrowingStream { continuation in
let task = Task { @MainActor in
do {
for try await chunk in stream {
envelope.append(chunk)
continuation.yield(chunk)
}
continuation.finish()
} catch {
continuation.finish(throwing: error)
}
}
continuation.onTermination = { _ in
task.cancel()
}
}
}
private func speakIncrementalSegment(
_ text: String,
context preferredContext: IncrementalSpeechContext? = nil,
@ -2465,7 +2521,10 @@ final class TalkModeManager: NSObject {
let streamFailure = StreamFailureBox()
let stream = Self.monitorStreamFailures(rawStream, failureBox: streamFailure)
self.lastPlaybackWasPCM = true
var playback = await pcmPlayer.play(stream: stream, sampleRate: sampleRate)
var playback = await pcmPlayer.play(
stream: self.meteredPCMStream(stream, sampleRate: sampleRate),
sampleRate: sampleRate)
self.pcmPlaybackEnvelope.cancel()
if !playback.finished, playback.interruptedAt == nil {
self.logger.warning("pcm playback failed; retrying mp3")
if Self.isPCMFormatRejectedByAPI(streamFailure.value) {
@ -3275,6 +3334,9 @@ extension TalkModeManager: TalkRealtimeWebRTCSessionDelegate {
self.isSpeaking = false
self.isUserSpeechDetected = false
}
if !self.isSpeaking {
self.playbackLevel = nil
}
}
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didDetectInputSpeech active: Bool) {
@ -3285,6 +3347,17 @@ extension TalkModeManager: TalkRealtimeWebRTCSessionDelegate {
}
}
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didUpdateAudioLevels input: Double?, output: Double?) {
guard session === self.realtimeSession else { return }
if self.isListening, let input {
// Same smoothing as the SFSpeech tap so route switches keep the wave feel.
self.micLevel = (self.micLevel * 0.80) + (input * 0.20)
}
if self.isSpeaking, let output {
self.playbackLevel = output
}
}
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didReceiveUserTranscript text: String) {
guard session === self.realtimeSession else { return }
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)

View file

@ -10,6 +10,9 @@ import OSLog
protocol TalkRealtimeWebRTCSessionDelegate: AnyObject {
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didChangeStatus status: String)
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didDetectInputSpeech active: Bool)
/// Live mic/playback levels (normalized 0...1) polled from WebRTC stats; nil
/// when the report carries no audio level for that direction.
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didUpdateAudioLevels input: Double?, output: Double?)
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didReceiveUserTranscript text: String)
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didReceiveAssistantTranscript text: String)
func realtimeSessionDidFinish(_ session: TalkRealtimeWebRTCSession)
@ -51,6 +54,7 @@ final class TalkRealtimeWebRTCSession: NSObject {
private var assistantAudioActive = false
private var assistantAudioFinishTask: Task<Void, Never>?
private var ownsAudioSessionActivation = false
private var audioLevelPollTask: Task<Void, Never>?
private struct ToolBuffer {
var name: String
@ -161,11 +165,53 @@ final class TalkRealtimeWebRTCSession: NSObject {
self.trace("remote description set")
try self.checkNotStopped()
self.delegate?.realtimeSession(self, didChangeStatus: "Listening")
self.startAudioLevelPolling()
}
/// WebRTC owns capture and playback in this transport, so the app never sees
/// PCM; peer-connection stats are the only real level source for the waveform.
private func startAudioLevelPolling() {
self.audioLevelPollTask?.cancel()
self.audioLevelPollTask = Task { [weak self] in
while !Task.isCancelled {
guard let self, !self.stopped, let peer = self.peerConnection else { return }
let levels = await Self.audioLevels(peer: peer)
guard !Task.isCancelled, !self.stopped else { return }
self.delegate?.realtimeSession(
self,
didUpdateAudioLevels: levels.input.map { TalkAudioLevel.normalized(rms: $0) },
output: levels.output.map { TalkAudioLevel.normalized(rms: $0) })
try? await Task.sleep(nanoseconds: 100_000_000)
}
}
}
private nonisolated static func audioLevels(
peer: RTCPeerConnection) async -> (input: Double?, output: Double?)
{
await withCheckedContinuation { continuation in
peer.statistics { report in
var input: Double?
var output: Double?
for stat in report.statistics.values {
guard (stat.values["kind"] as? String) == "audio",
let level = stat.values["audioLevel"] as? NSNumber
else { continue }
// Per the WebRTC stats spec audioLevel is linear 0...1:
// media-source is the local mic, inbound-rtp the remote voice.
if stat.type == "media-source" { input = level.doubleValue }
if stat.type == "inbound-rtp" { output = level.doubleValue }
}
continuation.resume(returning: (input, output))
}
}
}
func stop() {
let shouldNotify = !self.stopped
self.stopped = true
self.audioLevelPollTask?.cancel()
self.audioLevelPollTask = nil
self.cancelActiveToolCalls()
self.toolBuffers.removeAll()
self.dataChannel?.close()

View file

@ -1,3 +1,4 @@
import OpenClawChatUI
import Testing
@testable import OpenClaw
@ -17,7 +18,7 @@ struct TalkProStateTests {
#expect(state.title == "Voice config unavailable")
#expect(state.primaryAction == .start)
#expect(state.primaryButtonTitle == "Start Talk")
#expect(state.waveformMode(micLevel: 0.8) == .still)
#expect(state.waveformPhase(micLevel: 0.8, playbackLevel: nil) == .idle)
}
@Test func `enabled talk without loaded config can be stopped`() {
@ -35,7 +36,7 @@ struct TalkProStateTests {
#expect(state.title == "Voice config unavailable")
#expect(state.primaryAction == .stop)
#expect(state.primaryButtonTitle == "Stop Talk")
#expect(state.waveformMode(micLevel: 0.8) == .still)
#expect(state.waveformPhase(micLevel: 0.8, playbackLevel: nil) == .idle)
}
@Test func `enabled talk with loaded config can be stopped`() {
@ -87,6 +88,41 @@ struct TalkProStateTests {
#expect(state.primaryAction == .waiting)
#expect(state.primaryButtonTitle == "Demo Mode Only")
#expect(state.primaryButtonIcon == "lock.fill")
#expect(state.waveformMode(micLevel: 0.8) == .still)
#expect(state.waveformPhase(micLevel: 0.8, playbackLevel: nil) == .idle)
}
@Test func `listening drives the wave with the real mic level`() {
let state = Self.readyState(isListening: true)
#expect(state.waveformPhase(micLevel: 0.4, playbackLevel: nil)
== .listening(level: 0.4, speechActive: false))
}
@Test func `detected speech keeps the real mic level and marks speech active`() {
let state = Self.readyState(isListening: true, isUserSpeechDetected: true)
#expect(state.waveformPhase(micLevel: 0.7, playbackLevel: nil)
== .listening(level: 0.7, speechActive: true))
}
@Test func `speaking forwards the playback envelope when available`() {
let state = Self.readyState(isSpeaking: true)
#expect(state.waveformPhase(micLevel: 0, playbackLevel: 0.55) == .speaking(level: 0.55))
#expect(state.waveformPhase(micLevel: 0, playbackLevel: nil) == .speaking(level: nil))
}
private static func readyState(
isListening: Bool = false,
isSpeaking: Bool = false,
isUserSpeechDetected: Bool = false) -> TalkProState
{
TalkProState(
gatewayConnected: true,
isDemoMode: false,
isEnabled: true,
statusText: "Ready",
isConfigLoaded: true,
isListening: isListening,
isSpeaking: isSpeaking,
isUserSpeechDetected: isUserSpeechDetected,
permissionState: .ready)
}
}

View file

@ -1090,7 +1090,15 @@ private struct WatchChatTimelineView: View {
}
if let voiceStatusText = self.voiceStatusText {
WatchTinyStatus(text: voiceStatusText)
VStack(alignment: .leading, spacing: 3) {
// Watch TTS runs through AVSpeechSynthesizer, which has no
// metering API, so speaking uses the wave's synthetic pulse.
TalkWaveformView(
phase: self.speechPlayback.isSpeaking ? .speaking(level: nil) : .thinking)
.frame(height: 24)
.accessibilityHidden(true)
WatchTinyStatus(text: voiceStatusText)
}
}
WatchSecondaryButton(title: "Refresh") {

View file

@ -313,6 +313,9 @@ targets:
- Info.plist
- path: Sources/Services/WatchSessionActivationGate.swift
group: Sources/Services
# Universal talk waveform, compiled directly since the watch links no packages.
- path: ../shared/OpenClawKit/Sources/OpenClawChatUI/TalkWaveformView.swift
group: Shared
- path: Sources/Fonts
buildPhase: resources
- path: Resources/Localizable.xcstrings

View file

@ -1,5 +1,6 @@
import AVFoundation
import Foundation
import OpenClawKit
import OSLog
@MainActor
@ -9,6 +10,12 @@ final class TalkAudioPlayer: NSObject, @preconcurrency AVAudioPlayerDelegate {
private let logger = Logger(subsystem: "ai.openclaw", category: "talk.tts")
private var player: AVAudioPlayer?
private var playback: Playback?
private var levelHandler: (@MainActor (Double?) -> Void)?
private var levelMeter: AudioPlayerLevelMeter?
func setLevelHandler(_ handler: (@MainActor (Double?) -> Void)?) {
self.levelHandler = handler
}
private final class Playback: @unchecked Sendable {
private let lock = NSLock()
@ -63,6 +70,11 @@ final class TalkAudioPlayer: NSObject, @preconcurrency AVAudioPlayerDelegate {
player.delegate = self
player.prepareToPlay()
if let levelHandler {
let meter = AudioPlayerLevelMeter(onLevel: levelHandler)
meter.attach(player)
self.levelMeter = meter
}
self.armWatchdog(playback: playback)
@ -101,6 +113,8 @@ final class TalkAudioPlayer: NSObject, @preconcurrency AVAudioPlayerDelegate {
guard self.playback === playback else { return }
self.playback = nil
self.levelMeter?.detach()
self.levelMeter = nil
self.player?.stop()
self.player = nil
}

View file

@ -1,5 +1,6 @@
import AppKit
import Observation
import OpenClawKit
@MainActor
@Observable
@ -11,6 +12,12 @@ final class TalkModeController {
private(set) var phase: TalkModePhase = .idle
private(set) var isPaused: Bool = false
/// Meters streamed PCM speech so the orb waveform follows the audible
/// envelope instead of a synthetic pulse.
@ObservationIgnored private lazy var playbackEnvelope = PCMPlaybackEnvelope { [weak self] level in
self?.updateSpeakingLevel(level)
}
func setEnabled(_ enabled: Bool) async {
self.logger.info("talk enabled=\(enabled)")
if enabled {
@ -72,6 +79,43 @@ final class TalkModeController {
TalkOverlayController.shared.updateLevel(level)
}
/// Playback level published while agent speech plays; nil (path without
/// metering, or playback ended) settles the wave back to its floor.
func updateSpeakingLevel(_ level: Double?) {
guard self.phase == .speaking else { return }
TalkOverlayController.shared.updateLevel(level ?? 0)
}
/// Passes streamed PCM speech through to the player while feeding the
/// playback envelope; call `endSpeechMetering` once playback returns.
func meteredSpeechStream(
_ stream: AsyncThrowingStream<Data, Error>,
sampleRate: Double) -> AsyncThrowingStream<Data, Error>
{
let envelope = self.playbackEnvelope
envelope.begin(sampleRate: sampleRate)
return AsyncThrowingStream { continuation in
let task = Task { @MainActor in
do {
for try await chunk in stream {
envelope.append(chunk)
continuation.yield(chunk)
}
continuation.finish()
} catch {
continuation.finish(throwing: error)
}
}
continuation.onTermination = { _ in
task.cancel()
}
}
}
func endSpeechMetering() {
self.playbackEnvelope.cancel()
}
func setPaused(_ paused: Bool) {
guard self.isPaused != paused else { return }
self.logger.info("talk paused=\(paused)")

View file

@ -1079,9 +1079,13 @@ extension TalkModeRuntime {
stream: AsyncThrowingStream<Data, Error>,
sampleRate: Double) async -> StreamingPlaybackResult
{
await PCMStreamingAudioPlayer.shared.play(stream: stream, sampleRate: sampleRate)
let metered = TalkModeController.shared.meteredSpeechStream(stream, sampleRate: sampleRate)
let result = await PCMStreamingAudioPlayer.shared.play(stream: metered, sampleRate: sampleRate)
TalkModeController.shared.endSpeechMetering()
return result
}
/// MP3 streaming has no metering hook; the wave falls back to its floor.
@MainActor
private func playMP3(stream: AsyncThrowingStream<Data, Error>) async -> StreamingPlaybackResult {
await StreamingAudioPlayer.shared.play(stream: stream)
@ -1099,7 +1103,10 @@ extension TalkModeRuntime {
@MainActor
private func playTalkAudio(data: Data) async -> TalkPlaybackResult {
await TalkAudioPlayer.shared.play(data: data)
TalkAudioPlayer.shared.setLevelHandler { level in
TalkModeController.shared.updateSpeakingLevel(level)
}
return await TalkAudioPlayer.shared.play(data: data)
}
@MainActor

View file

@ -1,4 +1,5 @@
import AppKit
import OpenClawChatUI
import SwiftUI
struct TalkOverlayView: View {
@ -134,32 +135,39 @@ private struct TalkOrbView: View {
let isPaused: Bool
var body: some View {
if self.isPaused {
ZStack {
Circle()
.fill(self.orbGradient)
.overlay(Circle().stroke(Color.white.opacity(0.35), lineWidth: 1))
.shadow(color: Color.black.opacity(0.18), radius: 10, x: 0, y: 5)
} else {
TimelineView(.animation) { context in
let t = context.date.timeIntervalSinceReferenceDate
let listenScale = self.phase == .listening ? (1 + CGFloat(self.level) * 0.12) : 1
let pulse = self.phase == .speaking ? (1 + 0.06 * sin(t * 6)) : 1
.overlay(Circle().stroke(Color.white.opacity(self.isPaused ? 0.35 : 0.45), lineWidth: 1))
.shadow(color: Color.black.opacity(self.isPaused ? 0.18 : 0.22), radius: 10, x: 0, y: 5)
.scaleEffect(self.orbScale)
ZStack {
Circle()
.fill(self.orbGradient)
.overlay(Circle().stroke(Color.white.opacity(0.45), lineWidth: 1))
.shadow(color: Color.black.opacity(0.22), radius: 10, x: 0, y: 5)
.scaleEffect(pulse * listenScale)
TalkWaveRings(phase: self.phase, level: self.level, time: t, accent: self.accent)
if self.phase == .thinking {
TalkOrbitArcs(time: t)
}
}
if !self.isPaused {
// The universal talk waveform (shared with iOS/watchOS/Android)
// rendered inside the orb; the level is real mic/playback audio.
// 0.82 x 0.56 keeps the wave rect's corners inside the orb circle.
TalkWaveformView(phase: self.wavePhase, palette: .talkOrb)
.frame(
width: TalkOverlayController.orbSize * 0.82,
height: TalkOverlayController.orbSize * 0.56)
.accessibilityHidden(true)
}
}
.animation(.easeOut(duration: 0.12), value: self.orbScale)
}
private var wavePhase: TalkWaveformPhase {
switch self.phase {
case .idle: .idle
case .thinking: .thinking
case .listening: .listening(level: self.level, speechActive: false)
case .speaking: .speaking(level: self.level)
}
}
private var orbScale: CGFloat {
guard !self.isPaused, self.phase == .listening || self.phase == .speaking else { return 1 }
return 1 + CGFloat(self.level) * 0.08
}
private var orbGradient: RadialGradient {
@ -171,44 +179,17 @@ private struct TalkOrbView: View {
}
}
private struct TalkWaveRings: View {
let phase: TalkModePhase
let level: Double
let time: TimeInterval
let accent: Color
var body: some View {
ZStack {
ForEach(0..<3, id: \.self) { idx in
let speed = self.phase == .speaking ? 1.4 : self.phase == .listening ? 0.9 : 0.6
let progress = (time * speed + Double(idx) * 0.28).truncatingRemainder(dividingBy: 1)
let amplitude = self.phase == .speaking ? 0.95 : self.phase == .listening ? 0.5 + self
.level * 0.7 : 0.35
let scale = 0.75 + progress * amplitude + (self.phase == .listening ? self.level * 0.15 : 0)
let alpha = self.phase == .speaking ? 0.72 : self.phase == .listening ? 0.58 + self.level * 0.28 : 0.4
Circle()
.stroke(self.accent.opacity(alpha - progress * 0.3), lineWidth: 1.6)
.scaleEffect(scale)
.opacity(alpha - progress * 0.6)
}
}
}
}
private struct TalkOrbitArcs: View {
let time: TimeInterval
var body: some View {
ZStack {
Circle()
.trim(from: 0.08, to: 0.26)
.stroke(Color.white.opacity(0.88), style: StrokeStyle(lineWidth: 1.6, lineCap: .round))
.rotationEffect(.degrees(self.time * 42))
Circle()
.trim(from: 0.62, to: 0.86)
.stroke(Color.white.opacity(0.7), style: StrokeStyle(lineWidth: 1.4, lineCap: .round))
.rotationEffect(.degrees(-self.time * 35))
}
.scaleEffect(1.08)
}
extension TalkWaveformPalette {
/// High-contrast wave tones for the tinted orb background.
fileprivate static let talkOrb = TalkWaveformPalette(
active: [
Color.white,
Color(white: 0.9),
Color(white: 0.72),
],
inactive: [
Color.white.opacity(0.7),
Color.white.opacity(0.55),
Color.white.opacity(0.4),
])
}

View file

@ -0,0 +1,172 @@
import SwiftUI
/// Universal OpenClaw talk animation: an iOS 9-style Siri waveform shared by the
/// iOS, watchOS, and macOS apps; the Android app ports the same math in Compose
/// (`TalkWaveform.kt`). Math adapted from noahchalifour/swiftui-siri-waveform-view
/// (MIT), as packaged by alfianlosari/SiriWaveView; redrawn with Canvas +
/// TimelineView so lobes flow continuously instead of re-randomizing per power
/// change.
///
/// This file is also compiled directly into the watch target, which links no
/// packages (see `apps/ios/project.yml`). Keep it dependency-free SwiftUI.
public enum TalkWaveformPhase: Equatable, Sendable {
/// Voice surface is off or unavailable: flat, static, dimmed.
case idle
/// Connecting or waiting on the agent. No audio exists in this state, so the
/// wave breathes on a slow synthetic swell by design.
case thinking
/// Capturing the user's voice. `level` is the live microphone level in 0...1;
/// `speechActive` raises the floor once endpointing detects actual speech.
case listening(level: Double, speechActive: Bool)
/// Agent speech playback. `level` is the live playback envelope in 0...1.
/// `nil` means the active voice path exposes no envelope (AVSpeechSynthesizer
/// and compressed streaming playback have no metering API); the wave then
/// falls back to a synthetic pulse rather than freezing.
case speaking(level: Double?)
}
/// Wave colors, front to back. Surfaces embedding the wave on tinted backgrounds
/// (for example the macOS orb) pass their own colors.
public struct TalkWaveformPalette: Equatable, Sendable {
public var active: [Color]
public var inactive: [Color]
public init(active: [Color], inactive: [Color]) {
self.active = active
self.inactive = inactive
}
public static let standard = TalkWaveformPalette(
active: [
Color(red: 198 / 255.0, green: 62 / 255.0, blue: 56 / 255.0),
Color(red: 0.95, green: 0.45, blue: 0.30),
Color(red: 0.45, green: 0.08, blue: 0.12),
],
inactive: [
Color(white: 0.62),
Color(white: 0.72),
Color(white: 0.82),
])
}
public struct TalkWaveformView: View {
public var phase: TalkWaveformPhase
public var palette: TalkWaveformPalette
@Environment(\.colorScheme) private var colorScheme
@Environment(\.accessibilityReduceMotion) private var reduceMotion
private static let born = Date()
public init(phase: TalkWaveformPhase, palette: TalkWaveformPalette = .standard) {
self.phase = phase
self.palette = palette
}
public var body: some View {
let frozen = self.reduceMotion || self.phase == .idle
TimelineView(.animation(minimumInterval: 1.0 / 30.0, paused: frozen)) { timeline in
let time = frozen ? 0 : timeline.date.timeIntervalSince(Self.born)
let power = TalkWaveformMath.power(for: self.phase, time: time)
Canvas { context, size in
let midY = size.height / 2
var line = Path()
line.move(to: CGPoint(x: 0, y: midY))
line.addLine(to: CGPoint(x: size.width, y: midY))
context.stroke(line, with: .color(.secondary.opacity(0.30)), lineWidth: 1)
// Screen blend pops on dark; opacity overlap reads better on light.
context.blendMode = self.colorScheme == .dark ? .screen : .normal
let opacity = self.colorScheme == .dark ? 0.9 : 0.55
for (index, color) in self.colors.enumerated() {
let path = TalkWaveformMath.wavePath(
in: size,
time: time,
seed: Double(index) * 7.31,
power: power)
context.fill(path, with: .color(color.opacity(opacity)))
}
}
}
.opacity(self.phase == .idle ? 0.6 : 1.0)
}
private var colors: [Color] {
self.phase == .idle ? self.palette.inactive : self.palette.active
}
}
/// Pure waveform math, split from the view for unit testing and so the Android
/// port has one canonical reference for every constant.
public enum TalkWaveformMath {
/// Per-phase drive for the wave amplitude in 0...1.
public static func power(for phase: TalkWaveformPhase, time: Double) -> Double {
switch phase {
case .idle:
return 0.05
case .thinking:
return 0.16 + 0.10 * (0.5 + 0.5 * sin(time * 1.6))
case let .listening(level, speechActive):
let clamped = min(max(level, 0), 1)
// Detected speech lifts the floor so the wave visibly commits to the
// user even when the mic level dips between words.
return speechActive ? 0.55 + 0.45 * clamped : 0.30 + 0.65 * clamped
case let .speaking(level):
guard let level else {
// Synthetic pulse for voice paths with no playback metering.
return 0.70 * (0.55 + 0.45 * abs(sin(time * 5.0)))
}
return 0.25 + 0.75 * min(max(level, 0), 1)
}
}
/// One wave = max envelope of three drifting lobes, mirrored around the midline.
public static func wavePath(in size: CGSize, time: Double, seed: Double, power: Double) -> Path {
let midX = Double(size.width) / 2
let midY = Double(size.height) / 2
// Lobe parameters oscillate smoothly so peaks sweep back and forth
// across the line instead of scrolling off-screen.
let lobes: [(A: Double, k: Double, t: Double)] = (0..<3).map { index in
let f = Double(index)
let ampFrequency = 0.9 + 0.23 * f
let ampPhase = time * ampFrequency + seed * 2.4 + f * 2.1
let amp = 0.30 + 0.70 * (0.5 + 0.5 * sin(ampPhase))
let k = 0.62 + 0.11 * f
let driftFrequency = 0.45 + 0.17 * f
let driftPhase = time * driftFrequency + seed + f * 1.9
let t = 2.8 * sin(driftPhase)
return (A: amp, k: k, t: t)
}
var upper: [CGPoint] = []
var x = -midX
while x <= midX {
let graphX = x / (midX / 9.0)
var y: Double = 0
for lobe in lobes {
let amplitude = lobe.A * midY * power
y = max(y, Self.attenuatedSine(x: graphX, A: amplitude, k: lobe.k, t: lobe.t))
}
upper.append(CGPoint(x: midX + x, y: midY - y))
x += 2
}
var path = Path()
path.move(to: CGPoint(x: 0, y: midY))
path.addLines(upper)
for point in upper.reversed() {
path.addLine(to: CGPoint(x: point.x, y: 2 * midY - point.y))
}
path.closeSubpath()
return path
}
/// |A·sin(kx t)| shaped by the bell envelope g = (K/(K+(kxt)²))^K, K = 4.
private static func attenuatedSine(x: Double, A: Double, k: Double, t: Double) -> Double {
let sine = A * sin(k * x - t)
let tPrime = t - .pi / 2
let envelope = pow(4.0 / (4.0 + pow(k * x - tPrime, 2)), 4.0)
return abs(sine * envelope)
}
}

View file

@ -56,8 +56,11 @@ struct OpenClawVoiceNoteRecordingRow: View {
.fill(OpenClawChatTheme.danger)
.frame(width: 9, height: 9)
Text("Recording")
.font(OpenClawChatTypography.footnoteSemiBold)
// Live capture wave replaces a static "Recording" label; the level is
// real recorder metering, so silence reads flat and speech moves.
TalkWaveformView(phase: .listening(level: self.recorder.level, speechActive: false))
.frame(maxWidth: .infinity, minHeight: 26, maxHeight: 26)
.accessibilityLabel("Recording")
Text(openClawVoiceNoteDurationLabel(self.recorder.elapsedSeconds))
.font(OpenClawChatTypography.mono(size: 13, relativeTo: .footnote))

View file

@ -1,6 +1,7 @@
import AVFAudio
import Foundation
import Observation
import OpenClawKit
private let voiceNoteMaximumDurationSeconds: TimeInterval = 180
@ -21,6 +22,14 @@ public protocol VoiceNoteAudioCapture: AnyObject {
/// Reports capture loss after a recording has started.
func setFailureHandler(_ handler: @escaping @MainActor () -> Void)
/// Latest capture level in 0...1 for the live waveform, when supported.
func currentLevel() -> Double?
}
extension VoiceNoteAudioCapture {
/// Metering is a UI nicety; test captures may skip it.
public func currentLevel() -> Double? { nil }
}
/// A completed voice-note recording ready to stage as a chat attachment.
@ -51,6 +60,8 @@ public final class OpenClawVoiceNoteRecorder {
public private(set) var state: State = .idle
public private(set) var elapsedSeconds: TimeInterval = 0
/// Live capture level in 0...1 while recording; drives the recording waveform.
public private(set) var level: Double = 0
@ObservationIgnored public var onRecordingActiveChanged: (@MainActor (Bool) -> Void)?
@ -179,6 +190,7 @@ public final class OpenClawVoiceNoteRecorder {
let duration = max(0, self.capture.stop())
let recording = OpenClawVoiceNoteRecording(fileURL: fileURL, durationSeconds: duration)
self.elapsedSeconds = duration
self.level = 0
self.state = .finished(recording: recording)
self.onRecordingActiveChanged?(false)
return recording
@ -205,6 +217,7 @@ public final class OpenClawVoiceNoteRecorder {
}
let wasRecording = self.isRecording
self.elapsedSeconds = 0
self.level = 0
self.state = .idle
if wasRecording {
self.onRecordingActiveChanged?(false)
@ -220,6 +233,10 @@ public final class OpenClawVoiceNoteRecorder {
guard !Task.isCancelled else { return }
guard case let .recording(startedAt, _) = self.state else { return }
self.elapsedSeconds = max(0, self.now().timeIntervalSince(startedAt))
if let captureLevel = self.capture.currentLevel() {
// Light smoothing keeps the 10 Hz meter poll from stepping visibly.
self.level = (self.level * 0.55) + (captureLevel * 0.45)
}
if self.elapsedSeconds >= self.durationLimit {
self.finish()
return
@ -233,6 +250,7 @@ public final class OpenClawVoiceNoteRecorder {
self.timerTask = nil
let wasRecording = self.isRecording
self.elapsedSeconds = 0
self.level = 0
self.state = .failed(message: message)
if wasRecording {
self.onRecordingActiveChanged?(false)
@ -317,6 +335,7 @@ public final class OpenClawVoiceNoteAudioCapture: NSObject, VoiceNoteAudioCaptur
]
let recorder = try AVAudioRecorder(url: url, settings: settings)
recorder.delegate = self
recorder.isMeteringEnabled = true
guard recorder.record() else {
throw NSError(
domain: "OpenClawVoiceNoteAudioCapture",
@ -345,6 +364,12 @@ public final class OpenClawVoiceNoteAudioCapture: NSObject, VoiceNoteAudioCaptur
self.deactivateAudioSession()
}
public func currentLevel() -> Double? {
guard let recorder, recorder.isRecording else { return nil }
recorder.updateMeters()
return TalkAudioLevel.normalized(decibels: Double(recorder.averagePower(forChannel: 0)))
}
public nonisolated func audioRecorderEncodeErrorDidOccur(_ recorder: AVAudioRecorder, error: (any Error)?) {
Task { @MainActor [weak self] in
self?.captureDidFail()

View file

@ -0,0 +1,169 @@
import AVFoundation
import Foundation
/// Normalizes an RMS value to the shared 0...1 UI level scale used by every
/// talk animation (mic, playback, recording) so the wave reads identically
/// across surfaces: dB full scale mapped over a 50 dB window.
public enum TalkAudioLevel {
public static func normalized(rms: Double) -> Double {
self.normalized(decibels: 20 * log10(max(rms, 1e-7)))
}
public static func normalized(decibels: Double) -> Double {
max(0, min(1, (decibels + 50) / 50))
}
/// RMS of little-endian PCM16 mono bytes; 0 for empty or odd-length data.
public static func pcm16RMS(_ data: Data) -> Double {
let sampleCount = data.count / 2
guard sampleCount > 0 else { return 0 }
var sum: Double = 0
data.withUnsafeBytes { raw in
let samples = raw.bindMemory(to: Int16.self)
for index in 0..<sampleCount {
let sample = Double(Int16(littleEndian: samples[index])) / Double(Int16.max)
sum += sample * sample
}
}
return (sum / Double(sampleCount)).squareRoot()
}
}
/// Builds a playback-time-aligned level envelope from PCM16 chunks that stream
/// through the app faster than real time (gateway TTS, ElevenLabs PCM, realtime
/// relay output). Chunks are RMS-metered on arrival but scheduled at their
/// expected playback offset, so the published level tracks what is audible
/// instead of network arrival bursts.
@MainActor
public final class PCMPlaybackEnvelope {
private struct Segment {
let start: TimeInterval
let end: TimeInterval
let level: Double
}
private let onLevel: @MainActor (Double?) -> Void
private var segments: [Segment] = []
private var bytesPerSecond: Double = 0
private var startedAt: ContinuousClock.Instant?
private var scheduleEnd: TimeInterval = 0
private var publishTask: Task<Void, Never>?
public init(onLevel: @escaping @MainActor (Double?) -> Void) {
self.onLevel = onLevel
}
/// Starts a new envelope; the playback clock is anchored to the first chunk.
public func begin(sampleRate: Double) {
self.cancel()
self.bytesPerSecond = max(1, sampleRate * Double(MemoryLayout<Int16>.size))
}
public func append(_ chunk: Data) {
guard self.bytesPerSecond > 1, !chunk.isEmpty else { return }
let now = ContinuousClock.now
if self.startedAt == nil {
self.startedAt = now
self.startPublishing()
}
guard let startedAt = self.startedAt else { return }
let elapsed = Self.seconds(startedAt.duration(to: now))
// Chunks queue behind whatever is already scheduled; a stalled stream
// resumes at "now" instead of leaving a phantom backlog gap.
var start = max(elapsed, self.scheduleEnd)
// Meter in ~50 ms windows: a whole clip can arrive as one chunk, and a
// single RMS for it would render a flat line instead of an envelope.
let windowBytes = max(2, Int(self.bytesPerSecond * 0.05) & ~1)
var offset = chunk.startIndex
while offset < chunk.endIndex {
let end = min(offset + windowBytes, chunk.endIndex)
let window = chunk[offset..<end]
let duration = Double(window.count) / self.bytesPerSecond
self.segments.append(Segment(
start: start,
end: start + duration,
level: TalkAudioLevel.normalized(rms: TalkAudioLevel.pcm16RMS(Data(window)))))
start += duration
offset = end
}
self.scheduleEnd = start
}
/// Stops immediately (interruption/teardown) and clears the published level.
public func cancel() {
self.publishTask?.cancel()
self.publishTask = nil
self.segments = []
self.startedAt = nil
self.scheduleEnd = 0
self.onLevel(nil)
}
private func startPublishing() {
self.publishTask?.cancel()
self.publishTask = Task { [weak self] in
while !Task.isCancelled {
guard let self else { return }
guard self.publish() else {
self.cancel()
return
}
try? await Task.sleep(nanoseconds: 33_000_000)
}
}
}
/// Publishes the level at the current playback position; false once the
/// scheduled tail (plus a drain grace) has fully played out.
private func publish() -> Bool {
guard let startedAt = self.startedAt else { return false }
let elapsed = Self.seconds(startedAt.duration(to: .now))
if elapsed > self.scheduleEnd + 0.5 {
return false
}
self.segments.removeAll { $0.end < elapsed }
let level = self.segments.first { elapsed >= $0.start && elapsed < $0.end }?.level ?? 0
self.onLevel(level)
return true
}
private static func seconds(_ duration: Duration) -> TimeInterval {
let parts = duration.components
return Double(parts.seconds) + Double(parts.attoseconds) * 1e-18
}
}
/// Publishes the live output level of an `AVAudioPlayer` (buffered TTS clips)
/// via its built-in metering at ~30 Hz. Detach clears the level to nil so the
/// consumer can distinguish "silent" from "not playing".
@MainActor
public final class AudioPlayerLevelMeter {
private let onLevel: @MainActor (Double?) -> Void
private var pollTask: Task<Void, Never>?
private weak var player: AVAudioPlayer?
public init(onLevel: @escaping @MainActor (Double?) -> Void) {
self.onLevel = onLevel
}
public func attach(_ player: AVAudioPlayer) {
self.detach()
player.isMeteringEnabled = true
self.player = player
self.pollTask = Task { [weak self] in
while !Task.isCancelled {
guard let self, let player = self.player else { return }
player.updateMeters()
self.onLevel(TalkAudioLevel.normalized(decibels: Double(player.averagePower(forChannel: 0))))
try? await Task.sleep(nanoseconds: 33_000_000)
}
}
}
public func detach() {
self.pollTask?.cancel()
self.pollTask = nil
self.player = nil
self.onLevel(nil)
}
}

View file

@ -0,0 +1,90 @@
import Foundation
import OpenClawChatUI
import OpenClawKit
import Testing
struct TalkWaveformMathTests {
@Test
func `idle is A flat floor`() {
#expect(TalkWaveformMath.power(for: .idle, time: 0) == 0.05)
#expect(TalkWaveformMath.power(for: .idle, time: 12.7) == 0.05)
}
@Test
func `thinking breathes inside its band`() {
for time in stride(from: 0.0, through: 10.0, by: 0.25) {
let power = TalkWaveformMath.power(for: .thinking, time: time)
#expect(power >= 0.16)
#expect(power <= 0.26 + 1e-9)
}
}
@Test
func `listening follows the mic level and clamps`() {
#expect(TalkWaveformMath.power(for: .listening(level: 0, speechActive: false), time: 1) == 0.30)
#expect(TalkWaveformMath.power(for: .listening(level: 1, speechActive: false), time: 1) == 0.95)
#expect(TalkWaveformMath.power(for: .listening(level: 2.5, speechActive: false), time: 1) == 0.95)
#expect(TalkWaveformMath.power(for: .listening(level: -1, speechActive: false), time: 1) == 0.30)
}
@Test
func `detected speech raises the floor but stays level driven`() {
let quiet = TalkWaveformMath.power(for: .listening(level: 0, speechActive: true), time: 1)
let loud = TalkWaveformMath.power(for: .listening(level: 1, speechActive: true), time: 1)
#expect(quiet == 0.55)
#expect(loud == 1.0)
}
@Test
func `speaking follows the playback envelope`() {
#expect(TalkWaveformMath.power(for: .speaking(level: 0), time: 1) == 0.25)
#expect(TalkWaveformMath.power(for: .speaking(level: 1), time: 1) == 1.0)
#expect(TalkWaveformMath.power(for: .speaking(level: 3), time: 1) == 1.0)
}
@Test
func `speaking without an envelope pulses synthetically`() {
var seen: Set<Int> = []
for time in stride(from: 0.0, through: 3.0, by: 0.05) {
let power = TalkWaveformMath.power(for: .speaking(level: nil), time: time)
#expect(power >= 0.70 * 0.55 - 1e-9)
#expect(power <= 0.70 + 1e-9)
seen.insert(Int(power * 1000))
}
// The fallback must move over time, not freeze.
#expect(seen.count > 5)
}
}
struct TalkAudioLevelTests {
@Test
func `full scale RMS normalizes to one`() {
#expect(TalkAudioLevel.normalized(rms: 1.0) == 1.0)
}
@Test
func `silence normalizes to zero`() {
#expect(TalkAudioLevel.normalized(rms: 0) == 0)
#expect(TalkAudioLevel.normalized(rms: 1e-9) == 0)
}
@Test
func `pcm 16 RMS measures real samples`() {
let silence = Data(repeating: 0, count: 512)
#expect(TalkAudioLevel.pcm16RMS(silence) == 0)
var fullScale = Data()
for _ in 0..<256 {
withUnsafeBytes(of: Int16.max.littleEndian) { fullScale.append(contentsOf: $0) }
}
let rms = TalkAudioLevel.pcm16RMS(fullScale)
#expect(abs(rms - 1.0) < 0.001)
#expect(TalkAudioLevel.normalized(rms: rms) > 0.99)
}
@Test
func `pcm 16 RMS ignores empty and odd data`() {
#expect(TalkAudioLevel.pcm16RMS(Data()) == 0)
#expect(TalkAudioLevel.pcm16RMS(Data([0x7F])) == 0)
}
}

View file

@ -37,6 +37,12 @@ private final class FakeVoiceNoteAudioCapture: VoiceNoteAudioCapture {
self.failureHandler = handler
}
var meterLevel: Double?
func currentLevel() -> Double? {
self.meterLevel
}
func failCapture() {
self.failureHandler?()
}
@ -217,6 +223,27 @@ final class VoiceNoteRecorderTests: XCTestCase {
XCTAssertEqual(openClawVoiceNoteDurationLabel(-1), "0:00")
}
@MainActor
func testRecordingPublishesCaptureLevelsAndResetsOnFinish() async throws {
let capture = FakeVoiceNoteAudioCapture()
capture.meterLevel = 0.8
let recorder = OpenClawVoiceNoteRecorder(
capture: capture,
timerIntervalNanoseconds: 2_000_000)
let started = await recorder.start()
XCTAssertTrue(started)
XCTAssertEqual(recorder.level, 0)
for _ in 0..<200 where recorder.level == 0 {
try await Task.sleep(nanoseconds: 3_000_000)
}
XCTAssertGreaterThan(recorder.level, 0)
_ = try XCTUnwrap(recorder.finish())
XCTAssertEqual(recorder.level, 0)
}
@MainActor
func testRecordButtonRequiresAttachmentInput() {
let recorder = OpenClawVoiceNoteRecorder(capture: FakeVoiceNoteAudioCapture())

View file

@ -110,7 +110,7 @@ Supported keys: `voice` / `voice_id` / `voiceId`, `model` / `model_id` / `modelI
- Menu bar toggle: **Talk**
- Config tab: **Talk Mode** group (voice id + interrupt toggle)
- Overlay: Listening (cloud pulses with mic level) &rarr; Thinking (sinking animation) &rarr; Speaking (radiating rings). Click the cloud to stop speaking, click X to exit Talk mode.
- Overlay: the orb renders the universal talk waveform (shared with iOS, watchOS, and Android). Listening follows the live mic level, Speaking follows the actual TTS playback envelope, Thinking breathes softly. Click the orb to pause/resume, double-click to stop speaking, click X to exit Talk mode.
## Android UI