mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
feat(voice): universal talk waveform driven by real audio levels (#102901)
* feat(voice): universal talk waveform driven by real audio levels One shared Siri-style talk animation across iOS, watchOS, macOS, and Android (TalkWaveformView + exact Compose port), replacing per-platform fakes: synthetic speaking pulses, constant speech-detected power, static Android bar rows, flat realtime listening. Listening/recording follow live mic levels on every route; agent speech follows the real playback envelope (AVAudioPlayer metering + playback-aligned PCM envelope + WebRTC stats); voice-note recording shows a live capture wave. * fix(android): order waveform imports and sync native i18n inventory * chore(i18n): resync native inventory after rebase
This commit is contained in:
parent
3c048ef052
commit
20816f676f
38 changed files with 1872 additions and 641 deletions
File diff suppressed because it is too large
Load diff
|
|
@ -262,6 +262,9 @@ class MainViewModel(
|
|||
val talkModeEnabled: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeEnabled }
|
||||
val talkModeListening: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeListening }
|
||||
val talkModeSpeaking: StateFlow<Boolean> = runtimeState(initial = false) { it.talkModeSpeaking }
|
||||
val talkInputLevel: StateFlow<Float> = runtimeState(initial = 0f) { it.talkInputLevel }
|
||||
val talkOutputLevel: StateFlow<Float?> = runtimeState(initial = null) { it.talkOutputLevel }
|
||||
val talkSpeechActive: StateFlow<Boolean> = runtimeState(initial = false) { it.talkSpeechActive }
|
||||
val talkModeStatusText: StateFlow<String> = runtimeState(initial = "Off") { it.talkModeStatusText }
|
||||
val talkModeConversation: StateFlow<List<VoiceConversationEntry>> =
|
||||
runtimeState(initial = emptyList()) { it.talkModeConversation }
|
||||
|
|
|
|||
|
|
@ -1259,6 +1259,15 @@ class NodeRuntime private constructor(
|
|||
val talkModeSpeaking: StateFlow<Boolean>
|
||||
get() = talkMode.isSpeaking
|
||||
|
||||
val talkInputLevel: StateFlow<Float>
|
||||
get() = talkMode.inputLevel
|
||||
|
||||
val talkOutputLevel: StateFlow<Float?>
|
||||
get() = talkMode.outputLevel
|
||||
|
||||
val talkSpeechActive: StateFlow<Boolean>
|
||||
get() = talkMode.speechActive
|
||||
|
||||
val talkModeStatusText: StateFlow<String>
|
||||
get() = talkMode.statusText
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
package ai.openclaw.app.chat
|
||||
|
||||
import ai.openclaw.app.voice.smoothedAudioLevel
|
||||
import android.content.Context
|
||||
import android.media.MediaRecorder
|
||||
import android.os.SystemClock
|
||||
|
|
@ -44,6 +45,9 @@ internal interface VoiceNoteRecordingEngine {
|
|||
fun stop(): Long
|
||||
|
||||
fun cancel()
|
||||
|
||||
/** Peak abs PCM amplitude (0..32767) since the last poll; 0 when not recording. */
|
||||
fun pollAmplitude(): Int
|
||||
}
|
||||
|
||||
/** Owns voice-note recording state and temporary-file cleanup. */
|
||||
|
|
@ -64,6 +68,9 @@ internal class VoiceNoteRecorderController(
|
|||
private val _elapsedMs = MutableStateFlow(0L)
|
||||
val elapsedMs: StateFlow<Long> = _elapsedMs.asStateFlow()
|
||||
|
||||
private val _inputLevel = MutableStateFlow(0f)
|
||||
val inputLevel: StateFlow<Float> = _inputLevel.asStateFlow()
|
||||
|
||||
private var outputFile: File? = null
|
||||
private var elapsedJob: Job? = null
|
||||
private var ownsMic = false
|
||||
|
|
@ -143,6 +150,7 @@ internal class VoiceNoteRecorderController(
|
|||
// coroutine is composition-scoped and may be cancelled before it runs,
|
||||
// so cancel() must still be able to delete the handed-off recording.
|
||||
_elapsedMs.value = 0L
|
||||
_inputLevel.value = 0f
|
||||
_state.value = VoiceNoteRecorderState.Preparing
|
||||
releaseMicLocked()
|
||||
VoiceNoteRecording(file = file, durationMs = durationMs)
|
||||
|
|
@ -171,6 +179,7 @@ internal class VoiceNoteRecorderController(
|
|||
outputFile?.delete()
|
||||
outputFile = null
|
||||
_elapsedMs.value = 0L
|
||||
_inputLevel.value = 0f
|
||||
_state.value = VoiceNoteRecorderState.Idle
|
||||
}
|
||||
}
|
||||
|
|
@ -190,13 +199,18 @@ internal class VoiceNoteRecorderController(
|
|||
while (isActive && state.value is VoiceNoteRecorderState.Recording) {
|
||||
val elapsed = (elapsedRealtimeMillis() - startedAt).coerceIn(0L, VOICE_NOTE_MAX_DURATION_MS)
|
||||
_elapsedMs.value = elapsed
|
||||
// MediaRecorder reports the peak since the last poll; at 10Hz that
|
||||
// reads close to the mean-abs mic levels used by the other waveform
|
||||
// surfaces, so peak/32767 needs no extra curve.
|
||||
val rawLevel = (engine.pollAmplitude().coerceIn(0, 32_767)) / 32_767f
|
||||
_inputLevel.value = smoothedAudioLevel(_inputLevel.value, rawLevel)
|
||||
// MediaRecorder's duration callback races its asynchronous auto-stop.
|
||||
// Own the cap here so every successful finish calls stop() exactly once.
|
||||
if (elapsed >= VOICE_NOTE_MAX_DURATION_MS) {
|
||||
finish()
|
||||
return@launch
|
||||
}
|
||||
delay(250L)
|
||||
delay(100L)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -213,6 +227,7 @@ internal class VoiceNoteRecorderController(
|
|||
releaseMicLocked()
|
||||
outputFile = null
|
||||
_elapsedMs.value = 0L
|
||||
_inputLevel.value = 0f
|
||||
_state.value = VoiceNoteRecorderState.Failure(message)
|
||||
}
|
||||
|
||||
|
|
@ -282,4 +297,8 @@ internal class AndroidVoiceNoteRecordingEngine(
|
|||
runCatching { active.stop() }
|
||||
active.release()
|
||||
}
|
||||
|
||||
// maxAmplitude throws until sampling starts on some OEMs; a dead meter beats
|
||||
// killing the recording.
|
||||
override fun pollAmplitude(): Int = recorder?.let { active -> runCatching { active.maxAmplitude }.getOrDefault(0) } ?: 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ import ai.openclaw.app.ui.design.ClawTextBadge
|
|||
import ai.openclaw.app.ui.design.ClawTextField
|
||||
import ai.openclaw.app.ui.design.ClawTheme
|
||||
import ai.openclaw.app.ui.design.OpenClawMascot
|
||||
import ai.openclaw.app.ui.design.TalkWaveform
|
||||
import ai.openclaw.app.ui.design.TalkWaveformPhase
|
||||
import android.Manifest
|
||||
import android.content.ClipData
|
||||
import android.content.ClipboardManager
|
||||
|
|
@ -628,18 +630,14 @@ private fun SettingsWaveformPanel(
|
|||
horizontalArrangement = Arrangement.spacedBy(5.dp),
|
||||
) {
|
||||
Icon(imageVector = Icons.Default.PlayArrow, contentDescription = null, modifier = Modifier.size(24.dp), tint = ClawTheme.colors.text)
|
||||
Row(modifier = Modifier.weight(1f), horizontalArrangement = Arrangement.SpaceEvenly, verticalAlignment = Alignment.CenterVertically) {
|
||||
listOf(6, 12, 18, 11, 28, 34, 18, 10, 8, 24, 38, 31, 12, 8, 18, 30, 40, 22, 12, 8, 20, 29, 16, 8).forEachIndexed { index, height ->
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.size(width = 2.dp, height = (if (active) height else 7 + index % 4 * 4).dp)
|
||||
.background(if (active) ClawTheme.colors.text else ClawTheme.colors.textSubtle, RoundedCornerShape(999.dp)),
|
||||
// Thinking is the preview phase: no capture runs on this screen, so the
|
||||
// synthetic swell demonstrates the animation without touching the mic.
|
||||
TalkWaveform(
|
||||
phase = if (active) TalkWaveformPhase.Thinking else TalkWaveformPhase.Idle,
|
||||
modifier = Modifier.weight(1f).height(48.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun playVoiceSetupTone() {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ import ai.openclaw.app.ui.design.ClawStatus
|
|||
import ai.openclaw.app.ui.design.ClawStatusPill
|
||||
import ai.openclaw.app.ui.design.ClawTheme
|
||||
import ai.openclaw.app.ui.design.OpenClawMascot
|
||||
import ai.openclaw.app.ui.design.TalkWaveform
|
||||
import ai.openclaw.app.ui.design.TalkWaveformPalette
|
||||
import ai.openclaw.app.ui.design.TalkWaveformPhase
|
||||
import ai.openclaw.app.voice.VoiceConversationEntry
|
||||
import ai.openclaw.app.voice.VoiceConversationRole
|
||||
import android.Manifest
|
||||
|
|
@ -106,6 +109,10 @@ fun VoiceScreen(
|
|||
val talkModeStatusText by viewModel.talkModeStatusText.collectAsState()
|
||||
val talkModeConversation by viewModel.talkModeConversation.collectAsState()
|
||||
val talkSetupReadiness by viewModel.talkSetupReadiness.collectAsState()
|
||||
val micInputLevel by viewModel.micInputLevel.collectAsState()
|
||||
val talkInputLevel by viewModel.talkInputLevel.collectAsState()
|
||||
val talkOutputLevel by viewModel.talkOutputLevel.collectAsState()
|
||||
val talkSpeechActive by viewModel.talkSpeechActive.collectAsState()
|
||||
|
||||
var pendingAction by remember { mutableStateOf<VoiceAction?>(null) }
|
||||
var hasMicPermission by remember { mutableStateOf(context.hasRecordAudioPermission()) }
|
||||
|
|
@ -167,6 +174,10 @@ fun VoiceScreen(
|
|||
entries = talkModeConversation,
|
||||
listening = talkModeListening,
|
||||
speaking = talkModeSpeaking,
|
||||
statusText = talkModeStatusText,
|
||||
inputLevel = talkInputLevel,
|
||||
outputLevel = talkOutputLevel,
|
||||
speechActive = talkSpeechActive,
|
||||
speakerEnabled = speakerEnabled,
|
||||
onToggleSpeaker = { viewModel.setSpeakerEnabled(!speakerEnabled) },
|
||||
onEndTalk = { viewModel.setTalkModeEnabled(false) },
|
||||
|
|
@ -183,6 +194,7 @@ fun VoiceScreen(
|
|||
conversation = micConversation,
|
||||
listening = micEnabled,
|
||||
sending = micIsSending,
|
||||
inputLevel = micInputLevel,
|
||||
statusText = activeStatus,
|
||||
gatewayStatus = gatewayStatus,
|
||||
onCancel = { viewModel.cancelMicCapture() },
|
||||
|
|
@ -209,11 +221,21 @@ fun VoiceScreen(
|
|||
|
||||
VoiceHero(
|
||||
gatewayStatus = gatewayStatus,
|
||||
voiceCaptureMode = voiceCaptureMode,
|
||||
micEnabled = micEnabled,
|
||||
talkModeEnabled = talkModeEnabled,
|
||||
talkModeListening = talkModeListening,
|
||||
talkModeSpeaking = talkModeSpeaking,
|
||||
orbPhase =
|
||||
voiceHeroWaveformPhase(
|
||||
micEnabled = micEnabled,
|
||||
micInputLevel = micInputLevel,
|
||||
talkModeEnabled = talkModeEnabled,
|
||||
talkModeListening = talkModeListening,
|
||||
talkModeSpeaking = talkModeSpeaking,
|
||||
talkInputLevel = talkInputLevel,
|
||||
talkOutputLevel = talkOutputLevel,
|
||||
talkSpeechActive = talkSpeechActive,
|
||||
),
|
||||
micLiveTranscript = micLiveTranscript,
|
||||
gatewayReady = gatewayReady,
|
||||
voiceAttentionStatus = voiceAttentionStatus,
|
||||
|
|
@ -271,6 +293,7 @@ private fun DictationScreen(
|
|||
conversation: List<VoiceConversationEntry>,
|
||||
listening: Boolean,
|
||||
sending: Boolean,
|
||||
inputLevel: Float,
|
||||
statusText: String,
|
||||
gatewayStatus: String,
|
||||
onCancel: () -> Unit,
|
||||
|
|
@ -314,7 +337,10 @@ private fun DictationScreen(
|
|||
overflow = TextOverflow.Ellipsis,
|
||||
)
|
||||
Column(horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.spacedBy(10.dp)) {
|
||||
DictationWaveform(active = listening || sending)
|
||||
TalkWaveform(
|
||||
phase = TalkWaveformPhase.Listening(level = inputLevel, speechActive = false),
|
||||
modifier = Modifier.fillMaxWidth().height(56.dp),
|
||||
)
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(7.dp), verticalAlignment = Alignment.CenterVertically) {
|
||||
Icon(imageVector = Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(15.dp), tint = if (listening) ClawTheme.colors.success else ClawTheme.colors.textMuted)
|
||||
Text(text = displayStatusText, style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
|
||||
|
|
@ -404,27 +430,15 @@ private fun DictationScreen(
|
|||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun DictationWaveform(active: Boolean) {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(4.dp), verticalAlignment = Alignment.CenterVertically) {
|
||||
List(48) { index ->
|
||||
val height = if (active) 3 + ((index * 7) % 16) else 3 + (index % 3) * 2
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.size(width = 2.dp, height = height.dp)
|
||||
.clip(RoundedCornerShape(999.dp))
|
||||
.background(if (active) ClawTheme.colors.text else ClawTheme.colors.textSubtle),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun TalkSessionScreen(
|
||||
entries: List<VoiceConversationEntry>,
|
||||
listening: Boolean,
|
||||
speaking: Boolean,
|
||||
statusText: String,
|
||||
inputLevel: Float,
|
||||
outputLevel: Float?,
|
||||
speechActive: Boolean,
|
||||
speakerEnabled: Boolean,
|
||||
onToggleSpeaker: () -> Unit,
|
||||
onEndTalk: () -> Unit,
|
||||
|
|
@ -467,9 +481,18 @@ private fun TalkSessionScreen(
|
|||
color = ClawTheme.colors.canvas,
|
||||
border = BorderStroke(1.dp, ClawTheme.colors.borderStrong),
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
TalkWaveform(active = listening || speaking)
|
||||
}
|
||||
TalkWaveform(
|
||||
phase =
|
||||
talkSessionWaveformPhase(
|
||||
speaking = speaking,
|
||||
listening = listening,
|
||||
statusText = statusText,
|
||||
inputLevel = inputLevel,
|
||||
speechActive = speechActive,
|
||||
outputLevel = outputLevel,
|
||||
),
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
)
|
||||
}
|
||||
|
||||
Column(modifier = Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
|
|
@ -583,21 +606,6 @@ private fun TalkControl(
|
|||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun TalkWaveform(active: Boolean) {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) {
|
||||
listOf(4, 12, 24, 34, 46, 28, 12, 38, 44, 24, 12, 30, 42, 18, 6).forEachIndexed { index, height ->
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.size(width = 3.dp, height = (if (active) height else 6 + index % 4 * 5).dp)
|
||||
.clip(RoundedCornerShape(999.dp))
|
||||
.background(if (active) ClawTheme.colors.text else ClawTheme.colors.textSubtle),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun VoiceHeader(
|
||||
statusText: String,
|
||||
|
|
@ -647,11 +655,11 @@ private fun VoiceHeader(
|
|||
@Composable
|
||||
private fun VoiceHero(
|
||||
gatewayStatus: String,
|
||||
voiceCaptureMode: VoiceCaptureMode,
|
||||
micEnabled: Boolean,
|
||||
talkModeEnabled: Boolean,
|
||||
talkModeListening: Boolean,
|
||||
talkModeSpeaking: Boolean,
|
||||
orbPhase: TalkWaveformPhase,
|
||||
micLiveTranscript: String?,
|
||||
gatewayReady: Boolean,
|
||||
voiceAttentionStatus: String?,
|
||||
|
|
@ -664,11 +672,7 @@ private fun VoiceHero(
|
|||
val talkNeedsSetup = gatewayReady && talkSetupReadiness.realtimeTalk.requiresSetup
|
||||
val dictationNeedsSetup = gatewayReady && talkSetupReadiness.dictation.requiresSetup
|
||||
Column(horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.spacedBy(9.dp)) {
|
||||
VoiceOrb(
|
||||
active = micEnabled || talkModeEnabled,
|
||||
listening = talkModeListening || voiceCaptureMode == VoiceCaptureMode.ManualMic,
|
||||
speaking = talkModeSpeaking,
|
||||
)
|
||||
VoiceOrb(phase = orbPhase)
|
||||
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
|
|
@ -958,51 +962,30 @@ private fun VoicePrimaryAction(
|
|||
}
|
||||
}
|
||||
|
||||
// White wave stack for the tinted orb background (standard palette reds would
|
||||
// vanish against the blue), mirroring how the macOS orb passes its own colors.
|
||||
private val voiceOrbPalette =
|
||||
TalkWaveformPalette(
|
||||
active = listOf(Color.White, Color.White.copy(alpha = 0.75f), Color.White.copy(alpha = 0.5f)),
|
||||
inactive = listOf(Color.White.copy(alpha = 0.62f), Color.White.copy(alpha = 0.5f), Color.White.copy(alpha = 0.38f)),
|
||||
)
|
||||
|
||||
@Composable
|
||||
private fun VoiceOrb(
|
||||
active: Boolean,
|
||||
listening: Boolean,
|
||||
speaking: Boolean,
|
||||
) {
|
||||
private fun VoiceOrb(phase: TalkWaveformPhase) {
|
||||
Surface(
|
||||
modifier = Modifier.size(112.dp),
|
||||
shape = CircleShape,
|
||||
color = if (active || listening || speaking) Color(0xFF1976D2) else Color(0xFF123B63),
|
||||
color = if (phase != TalkWaveformPhase.Idle) Color(0xFF1976D2) else Color(0xFF123B63),
|
||||
contentColor = Color.White,
|
||||
tonalElevation = 3.dp,
|
||||
shadowElevation = 7.dp,
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Column(horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
Icon(
|
||||
imageVector =
|
||||
when {
|
||||
speaking -> Icons.Default.RecordVoiceOver
|
||||
listening -> Icons.Default.GraphicEq
|
||||
else -> Icons.Default.Mic
|
||||
},
|
||||
contentDescription = null,
|
||||
modifier = Modifier.size(32.dp),
|
||||
tint = Color.White,
|
||||
// The circular surface clips the wave, matching the macOS orb treatment.
|
||||
TalkWaveform(
|
||||
phase = phase,
|
||||
modifier = Modifier.fillMaxSize().padding(horizontal = 8.dp),
|
||||
palette = voiceOrbPalette,
|
||||
)
|
||||
Waveform(active = active)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun Waveform(active: Boolean) {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(3.dp), verticalAlignment = Alignment.CenterVertically) {
|
||||
listOf(6, 11, 17, 23, 14, 9, 20, 14, 7).forEachIndexed { index, height ->
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.size(width = 2.dp, height = (if (active) height else 6 + index % 3 * 3).dp)
|
||||
.clip(RoundedCornerShape(999.dp))
|
||||
.background(if (active) Color.White else Color.White.copy(alpha = 0.52f)),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1116,6 +1099,43 @@ private fun runVoiceAction(
|
|||
}
|
||||
}
|
||||
|
||||
// Status literals TalkModeManager publishes while a turn runs but no audio
|
||||
// flows yet; they are the only signal that the agent is being awaited.
|
||||
private val talkAwaitingAgentStatuses = setOf("Thinking…", "Connecting…", "Generating voice…")
|
||||
|
||||
internal fun talkSessionWaveformPhase(
|
||||
speaking: Boolean,
|
||||
listening: Boolean,
|
||||
statusText: String,
|
||||
inputLevel: Float,
|
||||
speechActive: Boolean,
|
||||
outputLevel: Float?,
|
||||
): TalkWaveformPhase =
|
||||
when {
|
||||
speaking -> TalkWaveformPhase.Speaking(outputLevel)
|
||||
statusText.trim() in talkAwaitingAgentStatuses -> TalkWaveformPhase.Thinking
|
||||
listening -> TalkWaveformPhase.Listening(level = inputLevel, speechActive = speechActive)
|
||||
else -> TalkWaveformPhase.Idle
|
||||
}
|
||||
|
||||
internal fun voiceHeroWaveformPhase(
|
||||
micEnabled: Boolean,
|
||||
micInputLevel: Float,
|
||||
talkModeEnabled: Boolean,
|
||||
talkModeListening: Boolean,
|
||||
talkModeSpeaking: Boolean,
|
||||
talkInputLevel: Float,
|
||||
talkOutputLevel: Float?,
|
||||
talkSpeechActive: Boolean,
|
||||
): TalkWaveformPhase =
|
||||
when {
|
||||
talkModeSpeaking -> TalkWaveformPhase.Speaking(talkOutputLevel)
|
||||
talkModeListening -> TalkWaveformPhase.Listening(level = talkInputLevel, speechActive = talkSpeechActive)
|
||||
micEnabled -> TalkWaveformPhase.Listening(level = micInputLevel, speechActive = false)
|
||||
talkModeEnabled -> TalkWaveformPhase.Thinking
|
||||
else -> TalkWaveformPhase.Idle
|
||||
}
|
||||
|
||||
internal fun voiceStatusLabel(
|
||||
gatewayStatus: String,
|
||||
voiceCaptureMode: VoiceCaptureMode,
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ internal fun ChatComposer(
|
|||
onRemoveAttachment: (id: String) -> Unit,
|
||||
voiceNoteState: VoiceNoteRecorderState,
|
||||
voiceNoteElapsedMs: Long,
|
||||
voiceNoteLevel: Float,
|
||||
recordVoiceNoteEnabled: Boolean,
|
||||
onStartVoiceNote: () -> Unit,
|
||||
onCancelVoiceNote: () -> Unit,
|
||||
|
|
@ -227,6 +228,7 @@ internal fun ChatComposer(
|
|||
if (recordingVoiceNote) {
|
||||
VoiceNoteRecordingControls(
|
||||
elapsedMs = voiceNoteElapsedMs,
|
||||
level = voiceNoteLevel,
|
||||
onCancel = onCancelVoiceNote,
|
||||
onDone = onFinishVoiceNote,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -208,6 +208,7 @@ fun ChatScreen(
|
|||
)
|
||||
val voiceNoteState by voiceNoteRecorder.state.collectAsState()
|
||||
val voiceNoteElapsedMs by voiceNoteRecorder.elapsedMs.collectAsState()
|
||||
val voiceNoteLevel by voiceNoteRecorder.inputLevel.collectAsState()
|
||||
val pickImages =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
|
||||
if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult
|
||||
|
|
@ -368,6 +369,7 @@ fun ChatScreen(
|
|||
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
|
||||
voiceNoteState = voiceNoteState,
|
||||
voiceNoteElapsedMs = voiceNoteElapsedMs,
|
||||
voiceNoteLevel = voiceNoteLevel,
|
||||
recordVoiceNoteEnabled = pendingRunCount == 0 && !micCaptureActive,
|
||||
onStartVoiceNote = { scope.launch { voiceNoteRecorder.start() } },
|
||||
onCancelVoiceNote = voiceNoteRecorder::cancel,
|
||||
|
|
@ -1109,6 +1111,7 @@ private fun ChatComposer(
|
|||
onRemoveAttachment: (String) -> Unit,
|
||||
voiceNoteState: VoiceNoteRecorderState,
|
||||
voiceNoteElapsedMs: Long,
|
||||
voiceNoteLevel: Float,
|
||||
recordVoiceNoteEnabled: Boolean,
|
||||
onStartVoiceNote: () -> Unit,
|
||||
onCancelVoiceNote: () -> Unit,
|
||||
|
|
@ -1168,6 +1171,7 @@ private fun ChatComposer(
|
|||
if (voiceNoteState is VoiceNoteRecorderState.Recording) {
|
||||
VoiceNoteRecordingControls(
|
||||
elapsedMs = voiceNoteElapsedMs,
|
||||
level = voiceNoteLevel,
|
||||
onCancel = onCancelVoiceNote,
|
||||
onDone = onFinishVoiceNote,
|
||||
modifier = Modifier.weight(1f),
|
||||
|
|
|
|||
|
|
@ -6,12 +6,15 @@ import ai.openclaw.app.chat.ChatMessageContent
|
|||
import ai.openclaw.app.chat.VoiceNoteRecorderController
|
||||
import ai.openclaw.app.chat.VoiceNoteRecorderState
|
||||
import ai.openclaw.app.ui.design.ClawTheme
|
||||
import ai.openclaw.app.ui.design.TalkWaveform
|
||||
import ai.openclaw.app.ui.design.TalkWaveformPhase
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.heightIn
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
|
|
@ -134,6 +137,7 @@ internal fun VoiceNoteRecordButton(
|
|||
@Composable
|
||||
internal fun VoiceNoteRecordingControls(
|
||||
elapsedMs: Long,
|
||||
level: Float,
|
||||
onCancel: () -> Unit,
|
||||
onDone: () -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
|
|
@ -154,7 +158,10 @@ internal fun VoiceNoteRecordingControls(
|
|||
Text(
|
||||
text = formatVoiceNoteDuration(elapsedMs),
|
||||
style = ClawTheme.type.label.copy(fontWeight = FontWeight.SemiBold),
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
TalkWaveform(
|
||||
phase = TalkWaveformPhase.Listening(level = level, speechActive = false),
|
||||
modifier = Modifier.weight(1f).height(30.dp),
|
||||
)
|
||||
Surface(
|
||||
onClick = onCancel,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,266 @@
|
|||
package ai.openclaw.app.ui.design
|
||||
|
||||
import android.provider.Settings
|
||||
import androidx.compose.animation.core.withInfiniteAnimationFrameNanos
|
||||
import androidx.compose.foundation.Canvas
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableDoubleStateOf
|
||||
import androidx.compose.runtime.mutableLongStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.geometry.Offset
|
||||
import androidx.compose.ui.geometry.Size
|
||||
import androidx.compose.ui.graphics.BlendMode
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.CompositingStrategy
|
||||
import androidx.compose.ui.graphics.Path
|
||||
import androidx.compose.ui.graphics.graphicsLayer
|
||||
import androidx.compose.ui.graphics.luminance
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.unit.dp
|
||||
import kotlin.math.PI
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.max
|
||||
import kotlin.math.pow
|
||||
import kotlin.math.sin
|
||||
|
||||
/**
|
||||
* Universal OpenClaw talk animation: Compose port of the shared Siri-style
|
||||
* waveform in `apps/shared/OpenClawKit/Sources/OpenClawChatUI/TalkWaveformView.swift`.
|
||||
* Every constant mirrors the Swift original; change them there first.
|
||||
*/
|
||||
internal sealed interface TalkWaveformPhase {
|
||||
/** Voice surface is off or unavailable: flat, static, dimmed. */
|
||||
data object Idle : TalkWaveformPhase
|
||||
|
||||
/**
|
||||
* Connecting or waiting on the agent. No audio exists in this state, so the
|
||||
* wave breathes on a slow synthetic swell by design.
|
||||
*/
|
||||
data object Thinking : TalkWaveformPhase
|
||||
|
||||
/**
|
||||
* Capturing the user's voice. [level] is the live microphone level in 0..1;
|
||||
* [speechActive] raises the floor once endpointing detects actual speech.
|
||||
*/
|
||||
data class Listening(
|
||||
val level: Float,
|
||||
val speechActive: Boolean,
|
||||
) : TalkWaveformPhase
|
||||
|
||||
/**
|
||||
* Agent speech playback. [level] is the live playback envelope in 0..1.
|
||||
* `null` means the active voice path exposes no envelope (system TTS and
|
||||
* talk.speak compressed playback have no metering); the wave then falls
|
||||
* back to a synthetic pulse rather than freezing.
|
||||
*/
|
||||
data class Speaking(
|
||||
val level: Float?,
|
||||
) : TalkWaveformPhase
|
||||
}
|
||||
|
||||
/**
|
||||
* Wave colors, front to back. Surfaces embedding the wave on tinted backgrounds
|
||||
* (for example the voice orb) pass their own colors.
|
||||
*/
|
||||
internal data class TalkWaveformPalette(
|
||||
val active: List<Color>,
|
||||
val inactive: List<Color>,
|
||||
) {
|
||||
companion object {
|
||||
val standard =
|
||||
TalkWaveformPalette(
|
||||
active =
|
||||
listOf(
|
||||
Color(red = 198 / 255f, green = 62 / 255f, blue = 56 / 255f),
|
||||
Color(red = 0.95f, green = 0.45f, blue = 0.30f),
|
||||
Color(red = 0.45f, green = 0.08f, blue = 0.12f),
|
||||
),
|
||||
inactive =
|
||||
listOf(
|
||||
Color(red = 0.62f, green = 0.62f, blue = 0.62f),
|
||||
Color(red = 0.72f, green = 0.72f, blue = 0.72f),
|
||||
Color(red = 0.82f, green = 0.82f, blue = 0.82f),
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun TalkWaveform(
|
||||
phase: TalkWaveformPhase,
|
||||
modifier: Modifier = Modifier,
|
||||
palette: TalkWaveformPalette = TalkWaveformPalette.standard,
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
// Compose frame clocks ignore the system animator scale; honor the OS
|
||||
// "remove animations" setting explicitly (same pattern as OpenClawMascot).
|
||||
val animationsEnabled =
|
||||
remember(context) {
|
||||
Settings.Global.getFloat(context.contentResolver, Settings.Global.ANIMATOR_DURATION_SCALE, 1f) > 0f
|
||||
}
|
||||
val idle = phase == TalkWaveformPhase.Idle
|
||||
val frozen = !animationsEnabled || idle
|
||||
var timeSeconds by remember { mutableDoubleStateOf(0.0) }
|
||||
var bornNanos by remember { mutableLongStateOf(Long.MIN_VALUE) }
|
||||
LaunchedEffect(frozen) {
|
||||
if (frozen) {
|
||||
timeSeconds = 0.0
|
||||
return@LaunchedEffect
|
||||
}
|
||||
var lastFrameNanos = Long.MIN_VALUE
|
||||
while (true) {
|
||||
withInfiniteAnimationFrameNanos { frameNanos ->
|
||||
if (bornNanos == Long.MIN_VALUE) bornNanos = frameNanos
|
||||
// ~30fps like the iOS TimelineView(minimumInterval: 1/30): skip
|
||||
// intermediate vsync callbacks instead of redrawing at display rate.
|
||||
if (lastFrameNanos == Long.MIN_VALUE || frameNanos - lastFrameNanos >= 33_000_000L) {
|
||||
lastFrameNanos = frameNanos
|
||||
timeSeconds = (frameNanos - bornNanos) / 1_000_000_000.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val power = TalkWaveformMath.power(phase, timeSeconds)
|
||||
val colors = if (idle) palette.inactive else palette.active
|
||||
// The applied Claw theme (not the system setting) decides dark rendering.
|
||||
val dark = ClawTheme.colors.canvas.luminance() < 0.5f
|
||||
val midlineColor = ClawTheme.colors.textMuted.copy(alpha = 0.30f)
|
||||
Canvas(
|
||||
modifier =
|
||||
modifier.graphicsLayer {
|
||||
alpha = if (idle) 0.6f else 1f
|
||||
// Waves must blend with each other (screen on dark), not with whatever
|
||||
// happens to be drawn behind the widget.
|
||||
compositingStrategy = CompositingStrategy.Offscreen
|
||||
},
|
||||
) {
|
||||
val midY = size.height / 2f
|
||||
drawLine(
|
||||
color = midlineColor,
|
||||
start = Offset(0f, midY),
|
||||
end = Offset(size.width, midY),
|
||||
strokeWidth = 1.dp.toPx(),
|
||||
)
|
||||
// Screen blend pops on dark; opacity overlap reads better on light.
|
||||
val blendMode = if (dark) BlendMode.Screen else BlendMode.SrcOver
|
||||
val opacity = if (dark) 0.9f else 0.55f
|
||||
colors.forEachIndexed { index, color ->
|
||||
drawPath(
|
||||
path = wavePath(size = size, time = timeSeconds, seed = index * 7.31, power = power),
|
||||
color = color.copy(alpha = color.alpha * opacity),
|
||||
blendMode = blendMode,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** One wave = max envelope of three drifting lobes, mirrored around the midline. */
|
||||
private fun wavePath(
|
||||
size: Size,
|
||||
time: Double,
|
||||
seed: Double,
|
||||
power: Double,
|
||||
): Path {
|
||||
val midX = size.width / 2.0
|
||||
val midY = size.height / 2.0
|
||||
val lobes = TalkWaveformMath.lobes(time, seed)
|
||||
|
||||
val upper = ArrayList<Offset>()
|
||||
var x = -midX
|
||||
while (x <= midX) {
|
||||
val graphX = x / (midX / 9.0)
|
||||
var y = 0.0
|
||||
for (lobe in lobes) {
|
||||
val amplitude = lobe.amplitude * midY * power
|
||||
y = max(y, TalkWaveformMath.attenuatedSine(x = graphX, amplitude = amplitude, k = lobe.k, t = lobe.t))
|
||||
}
|
||||
upper.add(Offset((midX + x).toFloat(), (midY - y).toFloat()))
|
||||
x += 2.0
|
||||
}
|
||||
|
||||
val path = Path()
|
||||
path.moveTo(0f, midY.toFloat())
|
||||
for (point in upper) {
|
||||
path.lineTo(point.x, point.y)
|
||||
}
|
||||
for (index in upper.indices.reversed()) {
|
||||
val point = upper[index]
|
||||
path.lineTo(point.x, (2.0 * midY).toFloat() - point.y)
|
||||
}
|
||||
path.close()
|
||||
return path
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure waveform math, split from the composable for unit testing. Canonical
|
||||
* reference for every constant is TalkWaveformMath in TalkWaveformView.swift.
|
||||
*/
|
||||
internal object TalkWaveformMath {
|
||||
data class Lobe(
|
||||
val amplitude: Double,
|
||||
val k: Double,
|
||||
val t: Double,
|
||||
)
|
||||
|
||||
/** Per-phase drive for the wave amplitude in 0..1. */
|
||||
fun power(
|
||||
phase: TalkWaveformPhase,
|
||||
time: Double,
|
||||
): Double =
|
||||
when (phase) {
|
||||
TalkWaveformPhase.Idle -> 0.05
|
||||
TalkWaveformPhase.Thinking -> 0.16 + 0.10 * (0.5 + 0.5 * sin(time * 1.6))
|
||||
is TalkWaveformPhase.Listening -> {
|
||||
val clamped = phase.level.toDouble().coerceIn(0.0, 1.0)
|
||||
// Detected speech lifts the floor so the wave visibly commits to the
|
||||
// user even when the mic level dips between words.
|
||||
if (phase.speechActive) 0.55 + 0.45 * clamped else 0.30 + 0.65 * clamped
|
||||
}
|
||||
is TalkWaveformPhase.Speaking -> {
|
||||
val level = phase.level
|
||||
if (level == null) {
|
||||
// Synthetic pulse for voice paths with no playback metering.
|
||||
0.70 * (0.55 + 0.45 * abs(sin(time * 5.0)))
|
||||
} else {
|
||||
0.25 + 0.75 * level.toDouble().coerceIn(0.0, 1.0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lobe parameters oscillate smoothly so peaks sweep back and forth across
|
||||
* the line instead of scrolling off-screen.
|
||||
*/
|
||||
fun lobes(
|
||||
time: Double,
|
||||
seed: Double,
|
||||
): List<Lobe> =
|
||||
(0 until 3).map { index ->
|
||||
val f = index.toDouble()
|
||||
val ampFrequency = 0.9 + 0.23 * f
|
||||
val ampPhase = time * ampFrequency + seed * 2.4 + f * 2.1
|
||||
val amplitude = 0.30 + 0.70 * (0.5 + 0.5 * sin(ampPhase))
|
||||
val k = 0.62 + 0.11 * f
|
||||
val driftFrequency = 0.45 + 0.17 * f
|
||||
val driftPhase = time * driftFrequency + seed + f * 1.9
|
||||
Lobe(amplitude = amplitude, k = k, t = 2.8 * sin(driftPhase))
|
||||
}
|
||||
|
||||
/** |A·sin(kx − t)| shaped by the bell envelope g = (K/(K+(kx−t′)²))^K, K = 4. */
|
||||
fun attenuatedSine(
|
||||
x: Double,
|
||||
amplitude: Double,
|
||||
k: Double,
|
||||
t: Double,
|
||||
): Double {
|
||||
val sine = amplitude * sin(k * x - t)
|
||||
val tPrime = t - PI / 2
|
||||
val envelope = (4.0 / (4.0 + (k * x - tPrime).pow(2))).pow(4.0)
|
||||
return abs(sine * envelope)
|
||||
}
|
||||
}
|
||||
|
|
@ -203,6 +203,37 @@ private class BluetoothCommunicationRoute {
|
|||
|
||||
private val bluetoothCommunicationRoute = BluetoothCommunicationRoute()
|
||||
|
||||
/**
|
||||
* Mean-abs level of little-endian PCM16 audio in 0..1. Single implementation
|
||||
* shared by mic capture (dictation, realtime talk) and playback metering so
|
||||
* every waveform surface reads the same scale.
|
||||
*/
|
||||
internal fun pcm16MeanAbsLevel(
|
||||
frame: ByteArray,
|
||||
length: Int,
|
||||
): Float {
|
||||
var total = 0L
|
||||
var count = 0
|
||||
var index = 0
|
||||
val limit = length - (length % 2)
|
||||
while (index < limit) {
|
||||
val sample =
|
||||
(frame[index].toInt() and 0xff) or
|
||||
(frame[index + 1].toInt() shl 8)
|
||||
total += kotlin.math.abs(sample.toShort().toInt())
|
||||
count += 1
|
||||
index += 2
|
||||
}
|
||||
if (count == 0) return 0f
|
||||
return ((total / count).toFloat() / Short.MAX_VALUE).coerceIn(0f, 1f)
|
||||
}
|
||||
|
||||
/** iOS-parity level smoothing (new = old*0.8 + raw*0.2) for waveform meters. */
|
||||
internal fun smoothedAudioLevel(
|
||||
previous: Float,
|
||||
raw: Float,
|
||||
): Float = previous * 0.8f + raw * 0.2f
|
||||
|
||||
/** Converts AudioRecord's negative return codes into capture-session failures. */
|
||||
internal fun checkAudioRecordReadResult(result: Int): Int {
|
||||
if (result >= 0) return result
|
||||
|
|
|
|||
|
|
@ -702,7 +702,7 @@ internal class MicCaptureManager(
|
|||
while (coroutineContext.isActive && _micEnabled.value && transcriptionSession == session) {
|
||||
val read = audioInput.read(buffer, 0, buffer.size)
|
||||
if (read <= 0) continue
|
||||
_inputLevel.value = pcm16Level(buffer, read)
|
||||
_inputLevel.value = pcm16MeanAbsLevel(buffer, read)
|
||||
audioFrames.trySend(buffer.copyOf(read))
|
||||
}
|
||||
} catch (err: Throwable) {
|
||||
|
|
@ -785,26 +785,6 @@ internal class MicCaptureManager(
|
|||
else -> "Listening"
|
||||
}
|
||||
|
||||
private fun pcm16Level(
|
||||
frame: ByteArray,
|
||||
length: Int,
|
||||
): Float {
|
||||
var total = 0L
|
||||
var count = 0
|
||||
var index = 0
|
||||
val limit = length - (length % 2)
|
||||
while (index < limit) {
|
||||
val sample =
|
||||
(frame[index].toInt() and 0xff) or
|
||||
(frame[index + 1].toInt() shl 8)
|
||||
total += kotlin.math.abs(sample.toShort().toInt())
|
||||
count += 1
|
||||
index += 2
|
||||
}
|
||||
if (count == 0) return 0f
|
||||
return ((total / count).toFloat() / Short.MAX_VALUE).coerceIn(0f, 1f)
|
||||
}
|
||||
|
||||
private fun pcm16ToPcmu(pcm16: ByteArray): ByteArray {
|
||||
val output = ByteArray(pcm16.size / 2)
|
||||
var inputIndex = 0
|
||||
|
|
|
|||
|
|
@ -181,6 +181,20 @@ class TalkModeManager internal constructor(
|
|||
private val _isSpeaking = MutableStateFlow(false)
|
||||
val isSpeaking: StateFlow<Boolean> = _isSpeaking
|
||||
|
||||
private val _inputLevel = MutableStateFlow(0f)
|
||||
val inputLevel: StateFlow<Float> = _inputLevel
|
||||
|
||||
// Null while no metered PCM playback is active. System TTS and talk.speak
|
||||
// compressed playback expose no envelope; the waveform then shows the
|
||||
// synthetic Speaking(null) pulse instead of a frozen line.
|
||||
private val _outputLevel = MutableStateFlow<Float?>(null)
|
||||
val outputLevel: StateFlow<Float?> = _outputLevel
|
||||
|
||||
// True while the realtime provider streams a non-final user transcript, the
|
||||
// closest Android has to iOS endpointing's "speech detected" signal.
|
||||
private val _speechActive = MutableStateFlow(false)
|
||||
val speechActive: StateFlow<Boolean> = _speechActive
|
||||
|
||||
private val _statusText = MutableStateFlow("Off")
|
||||
val statusText: StateFlow<String> = _statusText
|
||||
|
||||
|
|
@ -1058,6 +1072,7 @@ class TalkModeManager internal constructor(
|
|||
while (coroutineContext.isActive && _isEnabled.value && realtimeSessionId == sessionId) {
|
||||
val read = audioInput.read(buffer, 0, buffer.size)
|
||||
if (read <= 0) continue
|
||||
_inputLevel.value = smoothedAudioLevel(_inputLevel.value, pcm16MeanAbsLevel(buffer, read))
|
||||
if (!shouldAppendRealtimeCapturedFrame(read)) continue
|
||||
audioFrames.trySend(buffer.copyOf(read))
|
||||
}
|
||||
|
|
@ -1068,6 +1083,7 @@ class TalkModeManager internal constructor(
|
|||
} finally {
|
||||
audioFrames.close()
|
||||
audioInput?.close()
|
||||
_inputLevel.value = 0f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1124,6 +1140,11 @@ class TalkModeManager internal constructor(
|
|||
"transcript" -> {
|
||||
val role = obj["role"].asStringOrNull()
|
||||
val isFinal = obj["final"].asBooleanOrNull() == true
|
||||
// A streaming (non-final) user transcript is the provider's speech
|
||||
// signal; it raises the waveform floor like iOS endpointing does.
|
||||
if (role == "user") {
|
||||
_speechActive.value = !isFinal
|
||||
}
|
||||
val text = realtimeTranscriptText(obj["text"].asStringOrNull(), isFinal)
|
||||
var assistantText: String? = null
|
||||
if (text != null) {
|
||||
|
|
@ -1272,6 +1293,10 @@ class TalkModeManager internal constructor(
|
|||
if (track.playState != AudioTrack.PLAYSTATE_PLAYING) {
|
||||
track.play()
|
||||
}
|
||||
// Blocking MODE_STREAM writes are playback-paced once the track buffer
|
||||
// fills, so per-write metering tracks what the speaker actually plays.
|
||||
_outputLevel.value =
|
||||
smoothedAudioLevel(_outputLevel.value ?: 0f, pcm16MeanAbsLevel(bytes, writtenBytes))
|
||||
_isSpeaking.value = true
|
||||
_statusText.value = "Speaking…"
|
||||
val durationMs = ((writtenBytes / 2.0) / realtimeSampleRateHz * 1000.0).toLong()
|
||||
|
|
@ -1290,7 +1315,10 @@ class TalkModeManager internal constructor(
|
|||
val idle =
|
||||
synchronized(realtimePlaybackLock) {
|
||||
val playbackIdle = SystemClock.elapsedRealtime() >= realtimePlaybackEndsAtMs
|
||||
if (playbackIdle) _isSpeaking.value = false
|
||||
if (playbackIdle) {
|
||||
_isSpeaking.value = false
|
||||
_outputLevel.value = null
|
||||
}
|
||||
playbackIdle
|
||||
}
|
||||
if (idle && _isEnabled.value && realtimeSessionId != null) {
|
||||
|
|
@ -1322,6 +1350,7 @@ class TalkModeManager internal constructor(
|
|||
realtimeAudioTrack = null
|
||||
}
|
||||
_isSpeaking.value = false
|
||||
_outputLevel.value = null
|
||||
if (_isEnabled.value) {
|
||||
_statusText.value = "Listening"
|
||||
}
|
||||
|
|
@ -1362,6 +1391,8 @@ class TalkModeManager internal constructor(
|
|||
realtimeUserEntryAwaitingFinal = false
|
||||
realtimeUserEntryAwaitingFinalStartedAtMs = null
|
||||
realtimeAssistantEntryId = null
|
||||
_speechActive.value = false
|
||||
_inputLevel.value = 0f
|
||||
stopRealtimePlayback()
|
||||
if (preserveStatus) {
|
||||
_statusText.value = status
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ class VoiceNoteRecorderControllerTest {
|
|||
var stopCount = 0
|
||||
var cancelCount = 0
|
||||
var outputFile: File? = null
|
||||
var amplitude = 0
|
||||
|
||||
override fun start(outputFile: File) {
|
||||
startCount += 1
|
||||
|
|
@ -36,6 +37,8 @@ class VoiceNoteRecorderControllerTest {
|
|||
override fun cancel() {
|
||||
cancelCount += 1
|
||||
}
|
||||
|
||||
override fun pollAmplitude(): Int = amplitude
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -103,6 +106,27 @@ class VoiceNoteRecorderControllerTest {
|
|||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun recordingPublishesSmoothedLevelAndCancelClearsIt() =
|
||||
runTest {
|
||||
val directory = Files.createTempDirectory("voice-note-test").toFile()
|
||||
val engine = FakeEngine()
|
||||
val controller = controller(directory, engine)
|
||||
|
||||
controller.start()
|
||||
engine.amplitude = 32_767
|
||||
advanceTimeBy(50L)
|
||||
runCurrent()
|
||||
assertEquals(0.2f, controller.inputLevel.value, 1e-4f)
|
||||
advanceTimeBy(100L)
|
||||
runCurrent()
|
||||
assertEquals(0.36f, controller.inputLevel.value, 1e-4f)
|
||||
|
||||
controller.cancel()
|
||||
assertEquals(0f, controller.inputLevel.value, 0f)
|
||||
directory.deleteRecursively()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun cancelDeletesTemporaryFileAndReturnsIdle() =
|
||||
runTest {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package ai.openclaw.app.ui
|
||||
|
||||
import ai.openclaw.app.VoiceCaptureMode
|
||||
import ai.openclaw.app.ui.design.TalkWaveformPhase
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Test
|
||||
|
|
@ -72,4 +73,68 @@ class VoiceScreenLogicTest {
|
|||
voiceRuntimeAttentionStatus("Transcription unavailable: UNAVAILABLE: Error: No realtime transcription provider registered"),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun talkSessionWaveformPhaseFollowsTalkState() {
|
||||
assertEquals(
|
||||
TalkWaveformPhase.Speaking(0.4f),
|
||||
talkSessionWaveformPhase(speaking = true, listening = true, statusText = "Speaking…", inputLevel = 0.2f, speechActive = true, outputLevel = 0.4f),
|
||||
)
|
||||
// Thinking statuses win over the still-running capture loop.
|
||||
assertEquals(
|
||||
TalkWaveformPhase.Thinking,
|
||||
talkSessionWaveformPhase(speaking = false, listening = true, statusText = "Thinking…", inputLevel = 0.2f, speechActive = false, outputLevel = null),
|
||||
)
|
||||
assertEquals(
|
||||
TalkWaveformPhase.Listening(level = 0.2f, speechActive = true),
|
||||
talkSessionWaveformPhase(speaking = false, listening = true, statusText = "Listening", inputLevel = 0.2f, speechActive = true, outputLevel = null),
|
||||
)
|
||||
assertEquals(
|
||||
TalkWaveformPhase.Idle,
|
||||
talkSessionWaveformPhase(speaking = false, listening = false, statusText = "Off", inputLevel = 0f, speechActive = false, outputLevel = null),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun voiceHeroWaveformPhasePrefersTalkOverDictation() {
|
||||
assertEquals(
|
||||
TalkWaveformPhase.Speaking(null),
|
||||
voiceHeroWaveformPhase(
|
||||
micEnabled = true,
|
||||
micInputLevel = 0.5f,
|
||||
talkModeEnabled = true,
|
||||
talkModeListening = true,
|
||||
talkModeSpeaking = true,
|
||||
talkInputLevel = 0.1f,
|
||||
talkOutputLevel = null,
|
||||
talkSpeechActive = false,
|
||||
),
|
||||
)
|
||||
assertEquals(
|
||||
TalkWaveformPhase.Listening(level = 0.5f, speechActive = false),
|
||||
voiceHeroWaveformPhase(
|
||||
micEnabled = true,
|
||||
micInputLevel = 0.5f,
|
||||
talkModeEnabled = false,
|
||||
talkModeListening = false,
|
||||
talkModeSpeaking = false,
|
||||
talkInputLevel = 0f,
|
||||
talkOutputLevel = null,
|
||||
talkSpeechActive = false,
|
||||
),
|
||||
)
|
||||
assertEquals(
|
||||
TalkWaveformPhase.Thinking,
|
||||
voiceHeroWaveformPhase(
|
||||
micEnabled = false,
|
||||
micInputLevel = 0f,
|
||||
talkModeEnabled = true,
|
||||
talkModeListening = false,
|
||||
talkModeSpeaking = false,
|
||||
talkInputLevel = 0f,
|
||||
talkOutputLevel = null,
|
||||
talkSpeechActive = false,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,81 @@
|
|||
package ai.openclaw.app.ui.design
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import kotlin.math.PI
|
||||
|
||||
/** Guards the exact constants ported from TalkWaveformView.swift. */
|
||||
class TalkWaveformMathTest {
|
||||
@Test
|
||||
fun idlePowerIsStaticRegardlessOfTime() {
|
||||
assertEquals(0.05, TalkWaveformMath.power(TalkWaveformPhase.Idle, 0.0), 1e-9)
|
||||
assertEquals(0.05, TalkWaveformMath.power(TalkWaveformPhase.Idle, 42.5), 1e-9)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun thinkingBreathesInsideItsBand() {
|
||||
assertEquals(0.21, TalkWaveformMath.power(TalkWaveformPhase.Thinking, 0.0), 1e-9)
|
||||
var time = 0.0
|
||||
while (time < 10.0) {
|
||||
val power = TalkWaveformMath.power(TalkWaveformPhase.Thinking, time)
|
||||
assertTrue(power in 0.16..0.26)
|
||||
time += 0.1
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun listeningClampsLevelAndSpeechRaisesFloor() {
|
||||
assertEquals(0.30, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = -0.5f, speechActive = false), 0.0), 1e-9)
|
||||
assertEquals(0.95, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = 1.5f, speechActive = false), 0.0), 1e-9)
|
||||
assertEquals(0.56, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = 0.4f, speechActive = false), 0.0), 1e-6)
|
||||
assertEquals(0.73, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = 0.4f, speechActive = true), 0.0), 1e-6)
|
||||
assertEquals(1.0, TalkWaveformMath.power(TalkWaveformPhase.Listening(level = 2f, speechActive = true), 0.0), 1e-9)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun speakingClampsMeteredLevel() {
|
||||
assertEquals(0.25, TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = -1f), 0.0), 1e-9)
|
||||
assertEquals(1.0, TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = 2f), 0.0), 1e-9)
|
||||
assertEquals(0.70, TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = 0.6f), 0.0), 1e-6)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun speakingWithoutEnvelopePulsesSynthetically() {
|
||||
val trough = TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = null), 0.0)
|
||||
val peak = TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = null), PI / 10.0)
|
||||
assertEquals(0.70 * 0.55, trough, 1e-9)
|
||||
assertEquals(0.70, peak, 1e-9)
|
||||
var time = 0.0
|
||||
while (time < 5.0) {
|
||||
val power = TalkWaveformMath.power(TalkWaveformPhase.Speaking(level = null), time)
|
||||
assertTrue(power in trough..peak)
|
||||
time += 0.05
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun lobesAreDeterministicWithFixedShapeConstants() {
|
||||
val lobes = TalkWaveformMath.lobes(time = 1.25, seed = 7.31)
|
||||
assertEquals(lobes, TalkWaveformMath.lobes(time = 1.25, seed = 7.31))
|
||||
assertEquals(3, lobes.size)
|
||||
assertEquals(0.62, lobes[0].k, 1e-9)
|
||||
assertEquals(0.73, lobes[1].k, 1e-9)
|
||||
assertEquals(0.84, lobes[2].k, 1e-9)
|
||||
for (lobe in lobes) {
|
||||
assertTrue(lobe.amplitude in 0.30..1.0)
|
||||
assertTrue(lobe.t in -2.8..2.8)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun attenuatedSinePeaksAtFullAmplitudeAndStaysNonNegative() {
|
||||
// At kx − t = −π/2 the bell envelope is exactly 1 and |sin| is 1.
|
||||
assertEquals(3.0, TalkWaveformMath.attenuatedSine(x = -PI / 2, amplitude = 3.0, k = 1.0, t = 0.0), 1e-9)
|
||||
var x = -9.0
|
||||
while (x <= 9.0) {
|
||||
assertTrue(TalkWaveformMath.attenuatedSine(x = x, amplitude = 1.0, k = 0.73, t = 1.4) >= 0.0)
|
||||
x += 0.25
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
package ai.openclaw.app.voice
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Test
|
||||
|
||||
class AudioLevelsTest {
|
||||
@Test
|
||||
fun silenceMetersToZero() {
|
||||
val silence = ByteArray(640)
|
||||
assertEquals(0f, pcm16MeanAbsLevel(silence, silence.size), 0f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun fullScaleMetersToOne() {
|
||||
val frame = pcm16Frame(samples = 160, sample = Short.MAX_VALUE)
|
||||
assertEquals(1f, pcm16MeanAbsLevel(frame, frame.size), 1e-6f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun negativeFullScaleClampsToOne() {
|
||||
// abs(-32768) exceeds Short.MAX_VALUE by one; the level must stay in 0..1.
|
||||
val frame = pcm16Frame(samples = 160, sample = Short.MIN_VALUE)
|
||||
assertEquals(1f, pcm16MeanAbsLevel(frame, frame.size), 0f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun midScaleMetersToHalf() {
|
||||
val frame = pcm16Frame(samples = 160, sample = (Short.MAX_VALUE / 2).toShort())
|
||||
assertEquals(0.5f, pcm16MeanAbsLevel(frame, frame.size), 1e-3f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun trailingOddByteAndEmptyLengthAreIgnored() {
|
||||
val frame = pcm16Frame(samples = 2, sample = Short.MAX_VALUE) + byteArrayOf(0x7F)
|
||||
assertEquals(1f, pcm16MeanAbsLevel(frame, frame.size), 1e-6f)
|
||||
assertEquals(0f, pcm16MeanAbsLevel(frame, 0), 0f)
|
||||
assertEquals(0f, pcm16MeanAbsLevel(frame, 1), 0f)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun smoothingMatchesIosWeighting() {
|
||||
assertEquals(0.2f, smoothedAudioLevel(previous = 0f, raw = 1f), 1e-6f)
|
||||
assertEquals(0.36f, smoothedAudioLevel(previous = 0.2f, raw = 1f), 1e-6f)
|
||||
assertEquals(0.8f, smoothedAudioLevel(previous = 1f, raw = 0f), 1e-6f)
|
||||
}
|
||||
|
||||
private fun pcm16Frame(
|
||||
samples: Int,
|
||||
sample: Short,
|
||||
): ByteArray {
|
||||
val frame = ByteArray(samples * 2)
|
||||
for (index in 0 until samples) {
|
||||
frame[index * 2] = (sample.toInt() and 0xff).toByte()
|
||||
frame[index * 2 + 1] = ((sample.toInt() shr 8) and 0xff).toByte()
|
||||
}
|
||||
return frame
|
||||
}
|
||||
}
|
||||
|
|
@ -359,6 +359,19 @@ class TalkModeManagerTest {
|
|||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun realtimeUserTranscriptsDriveSpeechActive() {
|
||||
val manager = createManager()
|
||||
|
||||
setPrivateField(manager, "realtimeSessionId", "relay-1")
|
||||
|
||||
assertFalse(manager.speechActive.value)
|
||||
manager.handleGatewayEvent("talk.event", realtimeTranscriptPayload(role = "user", text = "hello"))
|
||||
assertTrue(manager.speechActive.value)
|
||||
manager.handleGatewayEvent("talk.event", realtimeTranscriptPayload(role = "user", text = "hello world", final = true))
|
||||
assertFalse(manager.speechActive.value)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun realtimeTranscriptDeltasAccumulateVoiceConversation() {
|
||||
val manager = createManager()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import OpenClawChatUI
|
||||
import SwiftUI
|
||||
|
||||
struct TalkProTab: View {
|
||||
|
|
@ -116,7 +117,11 @@ struct TalkProTab: View {
|
|||
private var heroSection: some View {
|
||||
Section {
|
||||
VStack(spacing: 16) {
|
||||
TalkSiriWaveView(mode: self.state.waveformMode(micLevel: self.appModel.talkMode.micLevel))
|
||||
TalkWaveformView(
|
||||
phase: self.state.waveformPhase(
|
||||
micLevel: self.appModel.talkMode.micLevel,
|
||||
playbackLevel: self.appModel.talkMode.playbackLevel),
|
||||
palette: .openClawBrand)
|
||||
.frame(height: 130)
|
||||
.accessibilityHidden(true)
|
||||
|
||||
|
|
@ -325,12 +330,20 @@ enum TalkProPrimaryAction: Equatable {
|
|||
case waiting
|
||||
}
|
||||
|
||||
enum TalkProWaveformMode: Equatable {
|
||||
case level(Double)
|
||||
case inputSpeech
|
||||
case speaking
|
||||
case indeterminate
|
||||
case still
|
||||
extension TalkWaveformPalette {
|
||||
/// iOS app branding for the shared wave: adaptive accent front lobe plus
|
||||
/// system grays so the idle wave tracks light/dark appearance.
|
||||
static let openClawBrand = TalkWaveformPalette(
|
||||
active: [
|
||||
OpenClawBrand.accent,
|
||||
Color(red: 0.95, green: 0.45, blue: 0.30),
|
||||
Color(red: 0.45, green: 0.08, blue: 0.12),
|
||||
],
|
||||
inactive: [
|
||||
Color(uiColor: .systemGray2),
|
||||
Color(uiColor: .systemGray3),
|
||||
Color(uiColor: .systemGray4),
|
||||
])
|
||||
}
|
||||
|
||||
struct TalkProState: Equatable {
|
||||
|
|
@ -432,24 +445,23 @@ struct TalkProState: Equatable {
|
|||
}
|
||||
}
|
||||
|
||||
func waveformMode(micLevel: Double) -> TalkProWaveformMode {
|
||||
if self.isDemoMode { return .still }
|
||||
if !self.gatewayConnected { return .still }
|
||||
func waveformPhase(micLevel: Double, playbackLevel: Double?) -> TalkWaveformPhase {
|
||||
if self.isDemoMode { return .idle }
|
||||
if !self.gatewayConnected { return .idle }
|
||||
switch self.permissionState {
|
||||
case .requestingUpgrade, .upgradeRequested:
|
||||
return .indeterminate
|
||||
return .thinking
|
||||
case .missingScope, .requestFailed, .apiKeyMissing, .loadFailed:
|
||||
return .still
|
||||
return .idle
|
||||
default:
|
||||
break
|
||||
}
|
||||
if !self.isConfigLoaded { return .still }
|
||||
if self.isSpeaking { return .speaking }
|
||||
if self.isListening, self.isUserSpeechDetected { return .inputSpeech }
|
||||
if self.isListening { return .level(micLevel) }
|
||||
if !self.isConfigLoaded { return .idle }
|
||||
if self.isSpeaking { return .speaking(level: playbackLevel) }
|
||||
if self.isListening { return .listening(level: micLevel, speechActive: self.isUserSpeechDetected) }
|
||||
if self.normalizedStatus.contains("connecting") || self.normalizedStatus.contains("thinking") {
|
||||
return .indeterminate
|
||||
return .thinking
|
||||
}
|
||||
return self.isEnabled ? .indeterminate : .still
|
||||
return self.isEnabled ? .thinking : .idle
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,128 +0,0 @@
|
|||
import SwiftUI
|
||||
|
||||
/// iOS 9-style Siri waveform for the Talk screen, driven by the runtime mode.
|
||||
/// Math adapted from noahchalifour/swiftui-siri-waveform-view (MIT), as packaged
|
||||
/// by alfianlosari/SiriWaveView; redrawn with Canvas + TimelineView so lobes flow
|
||||
/// continuously instead of re-randomizing per power change.
|
||||
///
|
||||
/// State rendering: off = flat static grey; connecting/thinking = slow low
|
||||
/// breathing; listening = amplitude follows mic level; speech detected = full
|
||||
/// power; agent speaking = strong TTS-style pulse.
|
||||
struct TalkSiriWaveView: View {
|
||||
var mode: TalkProWaveformMode
|
||||
|
||||
@Environment(\.colorScheme) private var colorScheme
|
||||
@Environment(\.accessibilityReduceMotion) private var reduceMotion
|
||||
|
||||
private static let born = Date()
|
||||
|
||||
private static let monoColors: [Color] = [
|
||||
OpenClawBrand.accent,
|
||||
Color(red: 0.95, green: 0.45, blue: 0.30),
|
||||
Color(red: 0.45, green: 0.08, blue: 0.12),
|
||||
]
|
||||
private static let inactiveColors: [Color] = [
|
||||
Color(uiColor: .systemGray2),
|
||||
Color(uiColor: .systemGray3),
|
||||
Color(uiColor: .systemGray4),
|
||||
]
|
||||
|
||||
var body: some View {
|
||||
let frozen = self.reduceMotion || self.mode == .still
|
||||
TimelineView(.animation(minimumInterval: 1.0 / 30.0, paused: frozen)) { timeline in
|
||||
let time = frozen ? 0 : timeline.date.timeIntervalSince(Self.born)
|
||||
let power = Self.power(for: self.mode, time: time)
|
||||
Canvas { context, size in
|
||||
let midY = size.height / 2
|
||||
var line = Path()
|
||||
line.move(to: CGPoint(x: 0, y: midY))
|
||||
line.addLine(to: CGPoint(x: size.width, y: midY))
|
||||
context.stroke(line, with: .color(.secondary.opacity(0.30)), lineWidth: 1)
|
||||
|
||||
// Screen blend pops on dark; opacity overlap reads better on light.
|
||||
context.blendMode = self.colorScheme == .dark ? .screen : .normal
|
||||
let opacity = self.colorScheme == .dark ? 0.9 : 0.55
|
||||
for (index, color) in self.colors.enumerated() {
|
||||
let path = Self.wavePath(
|
||||
in: size,
|
||||
time: time,
|
||||
seed: Double(index) * 7.31,
|
||||
power: power)
|
||||
context.fill(path, with: .color(color.opacity(opacity)))
|
||||
}
|
||||
}
|
||||
}
|
||||
.opacity(self.mode == .still ? 0.6 : 1.0)
|
||||
}
|
||||
|
||||
private var colors: [Color] {
|
||||
self.mode == .still ? Self.inactiveColors : Self.monoColors
|
||||
}
|
||||
|
||||
/// Per-state drive: thinking breathes low and slow, speaking pulses like
|
||||
/// TTS playback, listening follows the live mic level.
|
||||
private static func power(for mode: TalkProWaveformMode, time: Double) -> Double {
|
||||
switch mode {
|
||||
case .still:
|
||||
0.05
|
||||
case .indeterminate:
|
||||
0.16 + 0.10 * (0.5 + 0.5 * sin(time * 1.6))
|
||||
case let .level(level):
|
||||
0.30 + 0.65 * min(max(level, 0), 1)
|
||||
case .inputSpeech:
|
||||
0.95
|
||||
case .speaking:
|
||||
0.70 * (0.55 + 0.45 * abs(sin(time * 5.0)))
|
||||
}
|
||||
}
|
||||
|
||||
/// One wave = max envelope of three drifting lobes, mirrored around the midline.
|
||||
private static func wavePath(in size: CGSize, time: Double, seed: Double, power: Double) -> Path {
|
||||
let midX = Double(size.width) / 2
|
||||
let midY = Double(size.height) / 2
|
||||
|
||||
// Lobe parameters oscillate smoothly so peaks sweep back and forth
|
||||
// across the line instead of scrolling off-screen.
|
||||
let lobes: [(A: Double, k: Double, t: Double)] = (0..<3).map { index in
|
||||
let f = Double(index)
|
||||
let ampFrequency = 0.9 + 0.23 * f
|
||||
let ampPhase = time * ampFrequency + seed * 2.4 + f * 2.1
|
||||
let amp = 0.30 + 0.70 * (0.5 + 0.5 * sin(ampPhase))
|
||||
let k = 0.62 + 0.11 * f
|
||||
let driftFrequency = 0.45 + 0.17 * f
|
||||
let driftPhase = time * driftFrequency + seed + f * 1.9
|
||||
let t = 2.8 * sin(driftPhase)
|
||||
return (A: amp, k: k, t: t)
|
||||
}
|
||||
|
||||
var upper: [CGPoint] = []
|
||||
var x = -midX
|
||||
while x <= midX {
|
||||
let graphX = x / (midX / 9.0)
|
||||
var y: Double = 0
|
||||
for lobe in lobes {
|
||||
let amplitude = lobe.A * midY * power
|
||||
y = max(y, Self.attenuatedSine(x: graphX, A: amplitude, k: lobe.k, t: lobe.t))
|
||||
}
|
||||
upper.append(CGPoint(x: midX + x, y: midY - y))
|
||||
x += 2
|
||||
}
|
||||
|
||||
var path = Path()
|
||||
path.move(to: CGPoint(x: 0, y: midY))
|
||||
path.addLines(upper)
|
||||
for point in upper.reversed() {
|
||||
path.addLine(to: CGPoint(x: point.x, y: 2 * midY - point.y))
|
||||
}
|
||||
path.closeSubpath()
|
||||
return path
|
||||
}
|
||||
|
||||
/// |A·sin(kx − t)| shaped by the bell envelope g = (K/(K+(kx−t′)²))^K, K = 4.
|
||||
private static func attenuatedSine(x: Double, A: Double, k: Double, t: Double) -> Double {
|
||||
let sine = A * sin(k * x - t)
|
||||
let tPrime = t - .pi / 2
|
||||
let envelope = pow(4.0 / (4.0 + pow(k * x - tPrime, 2)), 4.0)
|
||||
return abs(sine * envelope)
|
||||
}
|
||||
}
|
||||
|
|
@ -125,6 +125,11 @@ final class RealtimeTalkRelaySession {
|
|||
private let onStatus: (String) -> Void
|
||||
private let onIssue: (TalkRuntimeIssue) -> Void
|
||||
private let onSpeakingChanged: (Bool) -> Void
|
||||
private let onInputLevel: (Double) -> Void
|
||||
private let onOutputLevel: (Double?) -> Void
|
||||
/// Playback-time-aligned envelope of the assistant PCM the relay schedules;
|
||||
/// drives the speaking waveform with real audio instead of a synthetic pulse.
|
||||
private var outputEnvelope: PCMPlaybackEnvelope?
|
||||
|
||||
private let audioEngine = AVAudioEngine()
|
||||
private var relaySessionId: String?
|
||||
|
|
@ -165,7 +170,9 @@ final class RealtimeTalkRelaySession {
|
|||
pcmPlayer: PCMStreamingAudioPlaying,
|
||||
onStatus: @escaping (String) -> Void,
|
||||
onIssue: @escaping (TalkRuntimeIssue) -> Void = { _ in },
|
||||
onSpeakingChanged: @escaping (Bool) -> Void)
|
||||
onSpeakingChanged: @escaping (Bool) -> Void,
|
||||
onInputLevel: @escaping (Double) -> Void = { _ in },
|
||||
onOutputLevel: @escaping (Double?) -> Void = { _ in })
|
||||
{
|
||||
self.gateway = gateway
|
||||
self.options = options
|
||||
|
|
@ -173,6 +180,8 @@ final class RealtimeTalkRelaySession {
|
|||
self.onStatus = onStatus
|
||||
self.onIssue = onIssue
|
||||
self.onSpeakingChanged = onSpeakingChanged
|
||||
self.onInputLevel = onInputLevel
|
||||
self.onOutputLevel = onOutputLevel
|
||||
}
|
||||
|
||||
func start() async throws {
|
||||
|
|
@ -360,6 +369,7 @@ final class RealtimeTalkRelaySession {
|
|||
return
|
||||
}
|
||||
self.ensureOutputPlaybackStarted()
|
||||
self.outputEnvelope?.append(data)
|
||||
self.outputContinuation?.yield(data)
|
||||
case "audioDone":
|
||||
self.finishOutputPlaybackStream()
|
||||
|
|
@ -729,6 +739,7 @@ final class RealtimeTalkRelaySession {
|
|||
|
||||
private func recordMicrophoneFrame(byteCount: Int, rms: Float, timestampMs: Double) {
|
||||
guard !self.isClosed else { return }
|
||||
self.onInputLevel(TalkAudioLevel.normalized(rms: Double(rms)))
|
||||
self.micLogFrameCount += 1
|
||||
self.micLogByteCount += byteCount
|
||||
self.micLogMaxRms = max(self.micLogMaxRms, rms)
|
||||
|
|
@ -767,6 +778,11 @@ final class RealtimeTalkRelaySession {
|
|||
guard self.outputContinuation == nil, self.outputTask == nil else { return }
|
||||
self.outputSessionId += 1
|
||||
let sessionId = self.outputSessionId
|
||||
let envelope = self.outputEnvelope ?? PCMPlaybackEnvelope { [weak self] level in
|
||||
self?.onOutputLevel(level)
|
||||
}
|
||||
envelope.begin(sampleRate: self.outputSampleRateHz)
|
||||
self.outputEnvelope = envelope
|
||||
let stream = AsyncThrowingStream<Data, Error> { continuation in
|
||||
self.outputContinuation = continuation
|
||||
}
|
||||
|
|
@ -810,6 +826,7 @@ final class RealtimeTalkRelaySession {
|
|||
for chunk in chunks {
|
||||
self.markOutputAudioStarted(byteCount: chunk.count, nowMs: ProcessInfo.processInfo.systemUptime * 1000)
|
||||
self.onSpeakingChanged(true)
|
||||
self.outputEnvelope?.append(chunk)
|
||||
self.outputContinuation?.yield(chunk)
|
||||
}
|
||||
if shouldFinish {
|
||||
|
|
@ -846,6 +863,7 @@ final class RealtimeTalkRelaySession {
|
|||
self.isOutputPlaying = false
|
||||
self.outputStartedAtMs = nil
|
||||
self.outputPlaybackExpectedEndMs = 0
|
||||
self.outputEnvelope?.cancel()
|
||||
self.onSpeakingChanged(false)
|
||||
}
|
||||
|
||||
|
|
@ -863,6 +881,7 @@ final class RealtimeTalkRelaySession {
|
|||
self.isOutputPlaying = false
|
||||
self.outputStartedAtMs = nil
|
||||
self.outputPlaybackExpectedEndMs = 0
|
||||
self.outputEnvelope?.cancel()
|
||||
self.onSpeakingChanged(false)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -119,6 +119,12 @@ final class TalkGatewaySpeechClient: TalkGatewaySpeechSynthesizing {
|
|||
protocol TalkBufferedAudioPlaying {
|
||||
func play(data: Data) async -> StreamingPlaybackResult
|
||||
func stop() -> Double?
|
||||
func setLevelHandler(_ handler: (@MainActor (Double?) -> Void)?)
|
||||
}
|
||||
|
||||
extension TalkBufferedAudioPlaying {
|
||||
/// Level metering is a UI nicety; test doubles and future players may skip it.
|
||||
func setLevelHandler(_: (@MainActor (Double?) -> Void)?) {}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
|
|
@ -163,6 +169,12 @@ final class TalkBufferedAudioPlayer: NSObject, TalkBufferedAudioPlaying, @precon
|
|||
private let logger = Logger(subsystem: "ai.openclaw", category: "talk.tts")
|
||||
private var player: AVAudioPlayer?
|
||||
private var playback: Playback?
|
||||
private var levelHandler: (@MainActor (Double?) -> Void)?
|
||||
private var levelMeter: AudioPlayerLevelMeter?
|
||||
|
||||
func setLevelHandler(_ handler: (@MainActor (Double?) -> Void)?) {
|
||||
self.levelHandler = handler
|
||||
}
|
||||
|
||||
func play(data: Data) async -> StreamingPlaybackResult {
|
||||
self.stopInternal()
|
||||
|
|
@ -176,6 +188,11 @@ final class TalkBufferedAudioPlayer: NSObject, TalkBufferedAudioPlaying, @precon
|
|||
self.player = player
|
||||
player.delegate = self
|
||||
player.prepareToPlay()
|
||||
if let levelHandler {
|
||||
let meter = AudioPlayerLevelMeter(onLevel: levelHandler)
|
||||
meter.attach(player)
|
||||
self.levelMeter = meter
|
||||
}
|
||||
self.armWatchdog(playback: playback)
|
||||
if !player.play() {
|
||||
self.logger.error("talk buffered audio player refused to play")
|
||||
|
|
@ -236,6 +253,8 @@ final class TalkBufferedAudioPlayer: NSObject, TalkBufferedAudioPlaying, @precon
|
|||
|
||||
guard self.playback === playback else { return }
|
||||
self.playback = nil
|
||||
self.levelMeter?.detach()
|
||||
self.levelMeter = nil
|
||||
self.player?.stop()
|
||||
self.player = nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,6 +46,10 @@ final class TalkModeManager: NSObject {
|
|||
var statusText: String = "Off"
|
||||
/// 0..1-ish (not calibrated). Intended for UI feedback only.
|
||||
var micLevel: Double = 0
|
||||
/// Live agent playback envelope in 0...1 while speaking. nil means the active
|
||||
/// voice path exposes no real level (system voice, compressed streaming); the
|
||||
/// waveform then falls back to a synthetic pulse.
|
||||
var playbackLevel: Double?
|
||||
var gatewayTalkConfigLoaded: Bool = false
|
||||
var gatewayTalkApiKeyConfigured: Bool = false
|
||||
var gatewayTalkDefaultModelId: String?
|
||||
|
|
@ -161,6 +165,12 @@ final class TalkModeManager: NSObject {
|
|||
var mp3Player: StreamingAudioPlaying = StreamingAudioPlayer.shared
|
||||
var bufferedPlayer: TalkBufferedAudioPlaying = TalkBufferedAudioPlayer.shared
|
||||
|
||||
/// Meters PCM speech bytes on their way into the streaming player so the
|
||||
/// speaking waveform tracks the audible envelope, not network arrival.
|
||||
@ObservationIgnored private lazy var pcmPlaybackEnvelope = PCMPlaybackEnvelope { [weak self] level in
|
||||
self?.playbackLevel = level
|
||||
}
|
||||
|
||||
private var gateway: GatewayNodeSession?
|
||||
private var gatewayConnected = false
|
||||
private var talkConfigLoadedAt: Date?
|
||||
|
|
@ -1347,6 +1357,14 @@ final class TalkModeManager: NSObject {
|
|||
if speaking {
|
||||
self.isListening = false
|
||||
}
|
||||
},
|
||||
onInputLevel: { [weak self] level in
|
||||
guard let self, self.isListening else { return }
|
||||
// Same smoothing as the SFSpeech tap so route switches keep the wave feel.
|
||||
self.micLevel = (self.micLevel * 0.80) + (level * 0.20)
|
||||
},
|
||||
onOutputLevel: { [weak self] level in
|
||||
self?.playbackLevel = level
|
||||
})
|
||||
self.realtimeRelaySession = relaySession
|
||||
do {
|
||||
|
|
@ -1687,6 +1705,7 @@ final class TalkModeManager: NSObject {
|
|||
if self.speechGeneration == speechGeneration {
|
||||
self.stopRecognition()
|
||||
self.isSpeaking = false
|
||||
self.playbackLevel = nil
|
||||
self.restoreConfiguredVoiceModeDescriptor()
|
||||
}
|
||||
}
|
||||
|
|
@ -1773,7 +1792,10 @@ final class TalkModeManager: NSObject {
|
|||
let streamFailure = StreamFailureBox()
|
||||
let stream = Self.monitorStreamFailures(rawStream, failureBox: streamFailure)
|
||||
self.lastPlaybackWasPCM = true
|
||||
var playback = await pcmPlayer.play(stream: stream, sampleRate: sampleRate)
|
||||
var playback = await pcmPlayer.play(
|
||||
stream: self.meteredPCMStream(stream, sampleRate: sampleRate),
|
||||
sampleRate: sampleRate)
|
||||
self.pcmPlaybackEnvelope.cancel()
|
||||
if !playback.finished, playback.interruptedAt == nil {
|
||||
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100_128")
|
||||
self.logger.warning("pcm playback failed; retrying mp3")
|
||||
|
|
@ -1869,9 +1891,15 @@ final class TalkModeManager: NSObject {
|
|||
case let .pcm(sampleRate):
|
||||
self.lastPlaybackWasPCM = true
|
||||
let stream = Self.makeBufferedAudioStream(chunks: [audio.data])
|
||||
result = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
|
||||
result = await self.pcmPlayer.play(
|
||||
stream: self.meteredPCMStream(stream, sampleRate: sampleRate),
|
||||
sampleRate: sampleRate)
|
||||
self.pcmPlaybackEnvelope.cancel()
|
||||
case .buffered:
|
||||
self.lastPlaybackWasPCM = false
|
||||
self.bufferedPlayer.setLevelHandler { [weak self] level in
|
||||
self?.playbackLevel = level
|
||||
}
|
||||
result = await self.bufferedPlayer.play(data: audio.data)
|
||||
case let .unsupportedRaw(codec):
|
||||
throw NSError(domain: "TalkGatewaySpeech", code: 2, userInfo: [
|
||||
|
|
@ -1970,7 +1998,9 @@ final class TalkModeManager: NSObject {
|
|||
self.stopRecognition()
|
||||
TalkSystemSpeechSynthesizer.shared.stop()
|
||||
self.cancelIncrementalSpeech()
|
||||
self.pcmPlaybackEnvelope.cancel()
|
||||
self.isSpeaking = false
|
||||
self.playbackLevel = nil
|
||||
restoreConfiguredVoiceModeDescriptor()
|
||||
}
|
||||
|
||||
|
|
@ -2422,6 +2452,32 @@ final class TalkModeManager: NSObject {
|
|||
}
|
||||
}
|
||||
|
||||
/// Passes PCM chunks through to the player while feeding the playback
|
||||
/// envelope so `playbackLevel` follows the audible speech.
|
||||
private func meteredPCMStream(
|
||||
_ stream: AsyncThrowingStream<Data, Error>,
|
||||
sampleRate: Double) -> AsyncThrowingStream<Data, Error>
|
||||
{
|
||||
let envelope = self.pcmPlaybackEnvelope
|
||||
envelope.begin(sampleRate: sampleRate)
|
||||
return AsyncThrowingStream { continuation in
|
||||
let task = Task { @MainActor in
|
||||
do {
|
||||
for try await chunk in stream {
|
||||
envelope.append(chunk)
|
||||
continuation.yield(chunk)
|
||||
}
|
||||
continuation.finish()
|
||||
} catch {
|
||||
continuation.finish(throwing: error)
|
||||
}
|
||||
}
|
||||
continuation.onTermination = { _ in
|
||||
task.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func speakIncrementalSegment(
|
||||
_ text: String,
|
||||
context preferredContext: IncrementalSpeechContext? = nil,
|
||||
|
|
@ -2465,7 +2521,10 @@ final class TalkModeManager: NSObject {
|
|||
let streamFailure = StreamFailureBox()
|
||||
let stream = Self.monitorStreamFailures(rawStream, failureBox: streamFailure)
|
||||
self.lastPlaybackWasPCM = true
|
||||
var playback = await pcmPlayer.play(stream: stream, sampleRate: sampleRate)
|
||||
var playback = await pcmPlayer.play(
|
||||
stream: self.meteredPCMStream(stream, sampleRate: sampleRate),
|
||||
sampleRate: sampleRate)
|
||||
self.pcmPlaybackEnvelope.cancel()
|
||||
if !playback.finished, playback.interruptedAt == nil {
|
||||
self.logger.warning("pcm playback failed; retrying mp3")
|
||||
if Self.isPCMFormatRejectedByAPI(streamFailure.value) {
|
||||
|
|
@ -3275,6 +3334,9 @@ extension TalkModeManager: TalkRealtimeWebRTCSessionDelegate {
|
|||
self.isSpeaking = false
|
||||
self.isUserSpeechDetected = false
|
||||
}
|
||||
if !self.isSpeaking {
|
||||
self.playbackLevel = nil
|
||||
}
|
||||
}
|
||||
|
||||
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didDetectInputSpeech active: Bool) {
|
||||
|
|
@ -3285,6 +3347,17 @@ extension TalkModeManager: TalkRealtimeWebRTCSessionDelegate {
|
|||
}
|
||||
}
|
||||
|
||||
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didUpdateAudioLevels input: Double?, output: Double?) {
|
||||
guard session === self.realtimeSession else { return }
|
||||
if self.isListening, let input {
|
||||
// Same smoothing as the SFSpeech tap so route switches keep the wave feel.
|
||||
self.micLevel = (self.micLevel * 0.80) + (input * 0.20)
|
||||
}
|
||||
if self.isSpeaking, let output {
|
||||
self.playbackLevel = output
|
||||
}
|
||||
}
|
||||
|
||||
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didReceiveUserTranscript text: String) {
|
||||
guard session === self.realtimeSession else { return }
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ import OSLog
|
|||
protocol TalkRealtimeWebRTCSessionDelegate: AnyObject {
|
||||
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didChangeStatus status: String)
|
||||
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didDetectInputSpeech active: Bool)
|
||||
/// Live mic/playback levels (normalized 0...1) polled from WebRTC stats; nil
|
||||
/// when the report carries no audio level for that direction.
|
||||
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didUpdateAudioLevels input: Double?, output: Double?)
|
||||
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didReceiveUserTranscript text: String)
|
||||
func realtimeSession(_ session: TalkRealtimeWebRTCSession, didReceiveAssistantTranscript text: String)
|
||||
func realtimeSessionDidFinish(_ session: TalkRealtimeWebRTCSession)
|
||||
|
|
@ -51,6 +54,7 @@ final class TalkRealtimeWebRTCSession: NSObject {
|
|||
private var assistantAudioActive = false
|
||||
private var assistantAudioFinishTask: Task<Void, Never>?
|
||||
private var ownsAudioSessionActivation = false
|
||||
private var audioLevelPollTask: Task<Void, Never>?
|
||||
|
||||
private struct ToolBuffer {
|
||||
var name: String
|
||||
|
|
@ -161,11 +165,53 @@ final class TalkRealtimeWebRTCSession: NSObject {
|
|||
self.trace("remote description set")
|
||||
try self.checkNotStopped()
|
||||
self.delegate?.realtimeSession(self, didChangeStatus: "Listening")
|
||||
self.startAudioLevelPolling()
|
||||
}
|
||||
|
||||
/// WebRTC owns capture and playback in this transport, so the app never sees
|
||||
/// PCM; peer-connection stats are the only real level source for the waveform.
|
||||
private func startAudioLevelPolling() {
|
||||
self.audioLevelPollTask?.cancel()
|
||||
self.audioLevelPollTask = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
guard let self, !self.stopped, let peer = self.peerConnection else { return }
|
||||
let levels = await Self.audioLevels(peer: peer)
|
||||
guard !Task.isCancelled, !self.stopped else { return }
|
||||
self.delegate?.realtimeSession(
|
||||
self,
|
||||
didUpdateAudioLevels: levels.input.map { TalkAudioLevel.normalized(rms: $0) },
|
||||
output: levels.output.map { TalkAudioLevel.normalized(rms: $0) })
|
||||
try? await Task.sleep(nanoseconds: 100_000_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func audioLevels(
|
||||
peer: RTCPeerConnection) async -> (input: Double?, output: Double?)
|
||||
{
|
||||
await withCheckedContinuation { continuation in
|
||||
peer.statistics { report in
|
||||
var input: Double?
|
||||
var output: Double?
|
||||
for stat in report.statistics.values {
|
||||
guard (stat.values["kind"] as? String) == "audio",
|
||||
let level = stat.values["audioLevel"] as? NSNumber
|
||||
else { continue }
|
||||
// Per the WebRTC stats spec audioLevel is linear 0...1:
|
||||
// media-source is the local mic, inbound-rtp the remote voice.
|
||||
if stat.type == "media-source" { input = level.doubleValue }
|
||||
if stat.type == "inbound-rtp" { output = level.doubleValue }
|
||||
}
|
||||
continuation.resume(returning: (input, output))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
let shouldNotify = !self.stopped
|
||||
self.stopped = true
|
||||
self.audioLevelPollTask?.cancel()
|
||||
self.audioLevelPollTask = nil
|
||||
self.cancelActiveToolCalls()
|
||||
self.toolBuffers.removeAll()
|
||||
self.dataChannel?.close()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import OpenClawChatUI
|
||||
import Testing
|
||||
@testable import OpenClaw
|
||||
|
||||
|
|
@ -17,7 +18,7 @@ struct TalkProStateTests {
|
|||
#expect(state.title == "Voice config unavailable")
|
||||
#expect(state.primaryAction == .start)
|
||||
#expect(state.primaryButtonTitle == "Start Talk")
|
||||
#expect(state.waveformMode(micLevel: 0.8) == .still)
|
||||
#expect(state.waveformPhase(micLevel: 0.8, playbackLevel: nil) == .idle)
|
||||
}
|
||||
|
||||
@Test func `enabled talk without loaded config can be stopped`() {
|
||||
|
|
@ -35,7 +36,7 @@ struct TalkProStateTests {
|
|||
#expect(state.title == "Voice config unavailable")
|
||||
#expect(state.primaryAction == .stop)
|
||||
#expect(state.primaryButtonTitle == "Stop Talk")
|
||||
#expect(state.waveformMode(micLevel: 0.8) == .still)
|
||||
#expect(state.waveformPhase(micLevel: 0.8, playbackLevel: nil) == .idle)
|
||||
}
|
||||
|
||||
@Test func `enabled talk with loaded config can be stopped`() {
|
||||
|
|
@ -87,6 +88,41 @@ struct TalkProStateTests {
|
|||
#expect(state.primaryAction == .waiting)
|
||||
#expect(state.primaryButtonTitle == "Demo Mode Only")
|
||||
#expect(state.primaryButtonIcon == "lock.fill")
|
||||
#expect(state.waveformMode(micLevel: 0.8) == .still)
|
||||
#expect(state.waveformPhase(micLevel: 0.8, playbackLevel: nil) == .idle)
|
||||
}
|
||||
|
||||
@Test func `listening drives the wave with the real mic level`() {
|
||||
let state = Self.readyState(isListening: true)
|
||||
#expect(state.waveformPhase(micLevel: 0.4, playbackLevel: nil)
|
||||
== .listening(level: 0.4, speechActive: false))
|
||||
}
|
||||
|
||||
@Test func `detected speech keeps the real mic level and marks speech active`() {
|
||||
let state = Self.readyState(isListening: true, isUserSpeechDetected: true)
|
||||
#expect(state.waveformPhase(micLevel: 0.7, playbackLevel: nil)
|
||||
== .listening(level: 0.7, speechActive: true))
|
||||
}
|
||||
|
||||
@Test func `speaking forwards the playback envelope when available`() {
|
||||
let state = Self.readyState(isSpeaking: true)
|
||||
#expect(state.waveformPhase(micLevel: 0, playbackLevel: 0.55) == .speaking(level: 0.55))
|
||||
#expect(state.waveformPhase(micLevel: 0, playbackLevel: nil) == .speaking(level: nil))
|
||||
}
|
||||
|
||||
private static func readyState(
|
||||
isListening: Bool = false,
|
||||
isSpeaking: Bool = false,
|
||||
isUserSpeechDetected: Bool = false) -> TalkProState
|
||||
{
|
||||
TalkProState(
|
||||
gatewayConnected: true,
|
||||
isDemoMode: false,
|
||||
isEnabled: true,
|
||||
statusText: "Ready",
|
||||
isConfigLoaded: true,
|
||||
isListening: isListening,
|
||||
isSpeaking: isSpeaking,
|
||||
isUserSpeechDetected: isUserSpeechDetected,
|
||||
permissionState: .ready)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1090,8 +1090,16 @@ private struct WatchChatTimelineView: View {
|
|||
}
|
||||
|
||||
if let voiceStatusText = self.voiceStatusText {
|
||||
VStack(alignment: .leading, spacing: 3) {
|
||||
// Watch TTS runs through AVSpeechSynthesizer, which has no
|
||||
// metering API, so speaking uses the wave's synthetic pulse.
|
||||
TalkWaveformView(
|
||||
phase: self.speechPlayback.isSpeaking ? .speaking(level: nil) : .thinking)
|
||||
.frame(height: 24)
|
||||
.accessibilityHidden(true)
|
||||
WatchTinyStatus(text: voiceStatusText)
|
||||
}
|
||||
}
|
||||
|
||||
WatchSecondaryButton(title: "Refresh") {
|
||||
self.onRefresh?()
|
||||
|
|
|
|||
|
|
@ -313,6 +313,9 @@ targets:
|
|||
- Info.plist
|
||||
- path: Sources/Services/WatchSessionActivationGate.swift
|
||||
group: Sources/Services
|
||||
# Universal talk waveform, compiled directly since the watch links no packages.
|
||||
- path: ../shared/OpenClawKit/Sources/OpenClawChatUI/TalkWaveformView.swift
|
||||
group: Shared
|
||||
- path: Sources/Fonts
|
||||
buildPhase: resources
|
||||
- path: Resources/Localizable.xcstrings
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import AVFoundation
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
import OSLog
|
||||
|
||||
@MainActor
|
||||
|
|
@ -9,6 +10,12 @@ final class TalkAudioPlayer: NSObject, @preconcurrency AVAudioPlayerDelegate {
|
|||
private let logger = Logger(subsystem: "ai.openclaw", category: "talk.tts")
|
||||
private var player: AVAudioPlayer?
|
||||
private var playback: Playback?
|
||||
private var levelHandler: (@MainActor (Double?) -> Void)?
|
||||
private var levelMeter: AudioPlayerLevelMeter?
|
||||
|
||||
func setLevelHandler(_ handler: (@MainActor (Double?) -> Void)?) {
|
||||
self.levelHandler = handler
|
||||
}
|
||||
|
||||
private final class Playback: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
|
|
@ -63,6 +70,11 @@ final class TalkAudioPlayer: NSObject, @preconcurrency AVAudioPlayerDelegate {
|
|||
|
||||
player.delegate = self
|
||||
player.prepareToPlay()
|
||||
if let levelHandler {
|
||||
let meter = AudioPlayerLevelMeter(onLevel: levelHandler)
|
||||
meter.attach(player)
|
||||
self.levelMeter = meter
|
||||
}
|
||||
|
||||
self.armWatchdog(playback: playback)
|
||||
|
||||
|
|
@ -101,6 +113,8 @@ final class TalkAudioPlayer: NSObject, @preconcurrency AVAudioPlayerDelegate {
|
|||
|
||||
guard self.playback === playback else { return }
|
||||
self.playback = nil
|
||||
self.levelMeter?.detach()
|
||||
self.levelMeter = nil
|
||||
self.player?.stop()
|
||||
self.player = nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import AppKit
|
||||
import Observation
|
||||
import OpenClawKit
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
|
|
@ -11,6 +12,12 @@ final class TalkModeController {
|
|||
private(set) var phase: TalkModePhase = .idle
|
||||
private(set) var isPaused: Bool = false
|
||||
|
||||
/// Meters streamed PCM speech so the orb waveform follows the audible
|
||||
/// envelope instead of a synthetic pulse.
|
||||
@ObservationIgnored private lazy var playbackEnvelope = PCMPlaybackEnvelope { [weak self] level in
|
||||
self?.updateSpeakingLevel(level)
|
||||
}
|
||||
|
||||
func setEnabled(_ enabled: Bool) async {
|
||||
self.logger.info("talk enabled=\(enabled)")
|
||||
if enabled {
|
||||
|
|
@ -72,6 +79,43 @@ final class TalkModeController {
|
|||
TalkOverlayController.shared.updateLevel(level)
|
||||
}
|
||||
|
||||
/// Playback level published while agent speech plays; nil (path without
|
||||
/// metering, or playback ended) settles the wave back to its floor.
|
||||
func updateSpeakingLevel(_ level: Double?) {
|
||||
guard self.phase == .speaking else { return }
|
||||
TalkOverlayController.shared.updateLevel(level ?? 0)
|
||||
}
|
||||
|
||||
/// Passes streamed PCM speech through to the player while feeding the
|
||||
/// playback envelope; call `endSpeechMetering` once playback returns.
|
||||
func meteredSpeechStream(
|
||||
_ stream: AsyncThrowingStream<Data, Error>,
|
||||
sampleRate: Double) -> AsyncThrowingStream<Data, Error>
|
||||
{
|
||||
let envelope = self.playbackEnvelope
|
||||
envelope.begin(sampleRate: sampleRate)
|
||||
return AsyncThrowingStream { continuation in
|
||||
let task = Task { @MainActor in
|
||||
do {
|
||||
for try await chunk in stream {
|
||||
envelope.append(chunk)
|
||||
continuation.yield(chunk)
|
||||
}
|
||||
continuation.finish()
|
||||
} catch {
|
||||
continuation.finish(throwing: error)
|
||||
}
|
||||
}
|
||||
continuation.onTermination = { _ in
|
||||
task.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func endSpeechMetering() {
|
||||
self.playbackEnvelope.cancel()
|
||||
}
|
||||
|
||||
func setPaused(_ paused: Bool) {
|
||||
guard self.isPaused != paused else { return }
|
||||
self.logger.info("talk paused=\(paused)")
|
||||
|
|
|
|||
|
|
@ -1079,9 +1079,13 @@ extension TalkModeRuntime {
|
|||
stream: AsyncThrowingStream<Data, Error>,
|
||||
sampleRate: Double) async -> StreamingPlaybackResult
|
||||
{
|
||||
await PCMStreamingAudioPlayer.shared.play(stream: stream, sampleRate: sampleRate)
|
||||
let metered = TalkModeController.shared.meteredSpeechStream(stream, sampleRate: sampleRate)
|
||||
let result = await PCMStreamingAudioPlayer.shared.play(stream: metered, sampleRate: sampleRate)
|
||||
TalkModeController.shared.endSpeechMetering()
|
||||
return result
|
||||
}
|
||||
|
||||
/// MP3 streaming has no metering hook; the wave falls back to its floor.
|
||||
@MainActor
|
||||
private func playMP3(stream: AsyncThrowingStream<Data, Error>) async -> StreamingPlaybackResult {
|
||||
await StreamingAudioPlayer.shared.play(stream: stream)
|
||||
|
|
@ -1099,7 +1103,10 @@ extension TalkModeRuntime {
|
|||
|
||||
@MainActor
|
||||
private func playTalkAudio(data: Data) async -> TalkPlaybackResult {
|
||||
await TalkAudioPlayer.shared.play(data: data)
|
||||
TalkAudioPlayer.shared.setLevelHandler { level in
|
||||
TalkModeController.shared.updateSpeakingLevel(level)
|
||||
}
|
||||
return await TalkAudioPlayer.shared.play(data: data)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import AppKit
|
||||
import OpenClawChatUI
|
||||
import SwiftUI
|
||||
|
||||
struct TalkOverlayView: View {
|
||||
|
|
@ -134,32 +135,39 @@ private struct TalkOrbView: View {
|
|||
let isPaused: Bool
|
||||
|
||||
var body: some View {
|
||||
if self.isPaused {
|
||||
Circle()
|
||||
.fill(self.orbGradient)
|
||||
.overlay(Circle().stroke(Color.white.opacity(0.35), lineWidth: 1))
|
||||
.shadow(color: Color.black.opacity(0.18), radius: 10, x: 0, y: 5)
|
||||
} else {
|
||||
TimelineView(.animation) { context in
|
||||
let t = context.date.timeIntervalSinceReferenceDate
|
||||
let listenScale = self.phase == .listening ? (1 + CGFloat(self.level) * 0.12) : 1
|
||||
let pulse = self.phase == .speaking ? (1 + 0.06 * sin(t * 6)) : 1
|
||||
|
||||
ZStack {
|
||||
Circle()
|
||||
.fill(self.orbGradient)
|
||||
.overlay(Circle().stroke(Color.white.opacity(0.45), lineWidth: 1))
|
||||
.shadow(color: Color.black.opacity(0.22), radius: 10, x: 0, y: 5)
|
||||
.scaleEffect(pulse * listenScale)
|
||||
.overlay(Circle().stroke(Color.white.opacity(self.isPaused ? 0.35 : 0.45), lineWidth: 1))
|
||||
.shadow(color: Color.black.opacity(self.isPaused ? 0.18 : 0.22), radius: 10, x: 0, y: 5)
|
||||
.scaleEffect(self.orbScale)
|
||||
|
||||
TalkWaveRings(phase: self.phase, level: self.level, time: t, accent: self.accent)
|
||||
if !self.isPaused {
|
||||
// The universal talk waveform (shared with iOS/watchOS/Android)
|
||||
// rendered inside the orb; the level is real mic/playback audio.
|
||||
// 0.82 x 0.56 keeps the wave rect's corners inside the orb circle.
|
||||
TalkWaveformView(phase: self.wavePhase, palette: .talkOrb)
|
||||
.frame(
|
||||
width: TalkOverlayController.orbSize * 0.82,
|
||||
height: TalkOverlayController.orbSize * 0.56)
|
||||
.accessibilityHidden(true)
|
||||
}
|
||||
}
|
||||
.animation(.easeOut(duration: 0.12), value: self.orbScale)
|
||||
}
|
||||
|
||||
if self.phase == .thinking {
|
||||
TalkOrbitArcs(time: t)
|
||||
}
|
||||
}
|
||||
private var wavePhase: TalkWaveformPhase {
|
||||
switch self.phase {
|
||||
case .idle: .idle
|
||||
case .thinking: .thinking
|
||||
case .listening: .listening(level: self.level, speechActive: false)
|
||||
case .speaking: .speaking(level: self.level)
|
||||
}
|
||||
}
|
||||
|
||||
private var orbScale: CGFloat {
|
||||
guard !self.isPaused, self.phase == .listening || self.phase == .speaking else { return 1 }
|
||||
return 1 + CGFloat(self.level) * 0.08
|
||||
}
|
||||
|
||||
private var orbGradient: RadialGradient {
|
||||
|
|
@ -171,44 +179,17 @@ private struct TalkOrbView: View {
|
|||
}
|
||||
}
|
||||
|
||||
private struct TalkWaveRings: View {
|
||||
let phase: TalkModePhase
|
||||
let level: Double
|
||||
let time: TimeInterval
|
||||
let accent: Color
|
||||
|
||||
var body: some View {
|
||||
ZStack {
|
||||
ForEach(0..<3, id: \.self) { idx in
|
||||
let speed = self.phase == .speaking ? 1.4 : self.phase == .listening ? 0.9 : 0.6
|
||||
let progress = (time * speed + Double(idx) * 0.28).truncatingRemainder(dividingBy: 1)
|
||||
let amplitude = self.phase == .speaking ? 0.95 : self.phase == .listening ? 0.5 + self
|
||||
.level * 0.7 : 0.35
|
||||
let scale = 0.75 + progress * amplitude + (self.phase == .listening ? self.level * 0.15 : 0)
|
||||
let alpha = self.phase == .speaking ? 0.72 : self.phase == .listening ? 0.58 + self.level * 0.28 : 0.4
|
||||
Circle()
|
||||
.stroke(self.accent.opacity(alpha - progress * 0.3), lineWidth: 1.6)
|
||||
.scaleEffect(scale)
|
||||
.opacity(alpha - progress * 0.6)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private struct TalkOrbitArcs: View {
|
||||
let time: TimeInterval
|
||||
|
||||
var body: some View {
|
||||
ZStack {
|
||||
Circle()
|
||||
.trim(from: 0.08, to: 0.26)
|
||||
.stroke(Color.white.opacity(0.88), style: StrokeStyle(lineWidth: 1.6, lineCap: .round))
|
||||
.rotationEffect(.degrees(self.time * 42))
|
||||
Circle()
|
||||
.trim(from: 0.62, to: 0.86)
|
||||
.stroke(Color.white.opacity(0.7), style: StrokeStyle(lineWidth: 1.4, lineCap: .round))
|
||||
.rotationEffect(.degrees(-self.time * 35))
|
||||
}
|
||||
.scaleEffect(1.08)
|
||||
}
|
||||
extension TalkWaveformPalette {
|
||||
/// High-contrast wave tones for the tinted orb background.
|
||||
fileprivate static let talkOrb = TalkWaveformPalette(
|
||||
active: [
|
||||
Color.white,
|
||||
Color(white: 0.9),
|
||||
Color(white: 0.72),
|
||||
],
|
||||
inactive: [
|
||||
Color.white.opacity(0.7),
|
||||
Color.white.opacity(0.55),
|
||||
Color.white.opacity(0.4),
|
||||
])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,172 @@
|
|||
import SwiftUI
|
||||
|
||||
/// Universal OpenClaw talk animation: an iOS 9-style Siri waveform shared by the
|
||||
/// iOS, watchOS, and macOS apps; the Android app ports the same math in Compose
|
||||
/// (`TalkWaveform.kt`). Math adapted from noahchalifour/swiftui-siri-waveform-view
|
||||
/// (MIT), as packaged by alfianlosari/SiriWaveView; redrawn with Canvas +
|
||||
/// TimelineView so lobes flow continuously instead of re-randomizing per power
|
||||
/// change.
|
||||
///
|
||||
/// This file is also compiled directly into the watch target, which links no
|
||||
/// packages (see `apps/ios/project.yml`). Keep it dependency-free SwiftUI.
|
||||
public enum TalkWaveformPhase: Equatable, Sendable {
|
||||
/// Voice surface is off or unavailable: flat, static, dimmed.
|
||||
case idle
|
||||
/// Connecting or waiting on the agent. No audio exists in this state, so the
|
||||
/// wave breathes on a slow synthetic swell by design.
|
||||
case thinking
|
||||
/// Capturing the user's voice. `level` is the live microphone level in 0...1;
|
||||
/// `speechActive` raises the floor once endpointing detects actual speech.
|
||||
case listening(level: Double, speechActive: Bool)
|
||||
/// Agent speech playback. `level` is the live playback envelope in 0...1.
|
||||
/// `nil` means the active voice path exposes no envelope (AVSpeechSynthesizer
|
||||
/// and compressed streaming playback have no metering API); the wave then
|
||||
/// falls back to a synthetic pulse rather than freezing.
|
||||
case speaking(level: Double?)
|
||||
}
|
||||
|
||||
/// Wave colors, front to back. Surfaces embedding the wave on tinted backgrounds
|
||||
/// (for example the macOS orb) pass their own colors.
|
||||
public struct TalkWaveformPalette: Equatable, Sendable {
|
||||
public var active: [Color]
|
||||
public var inactive: [Color]
|
||||
|
||||
public init(active: [Color], inactive: [Color]) {
|
||||
self.active = active
|
||||
self.inactive = inactive
|
||||
}
|
||||
|
||||
public static let standard = TalkWaveformPalette(
|
||||
active: [
|
||||
Color(red: 198 / 255.0, green: 62 / 255.0, blue: 56 / 255.0),
|
||||
Color(red: 0.95, green: 0.45, blue: 0.30),
|
||||
Color(red: 0.45, green: 0.08, blue: 0.12),
|
||||
],
|
||||
inactive: [
|
||||
Color(white: 0.62),
|
||||
Color(white: 0.72),
|
||||
Color(white: 0.82),
|
||||
])
|
||||
}
|
||||
|
||||
public struct TalkWaveformView: View {
|
||||
public var phase: TalkWaveformPhase
|
||||
public var palette: TalkWaveformPalette
|
||||
|
||||
@Environment(\.colorScheme) private var colorScheme
|
||||
@Environment(\.accessibilityReduceMotion) private var reduceMotion
|
||||
|
||||
private static let born = Date()
|
||||
|
||||
public init(phase: TalkWaveformPhase, palette: TalkWaveformPalette = .standard) {
|
||||
self.phase = phase
|
||||
self.palette = palette
|
||||
}
|
||||
|
||||
public var body: some View {
|
||||
let frozen = self.reduceMotion || self.phase == .idle
|
||||
TimelineView(.animation(minimumInterval: 1.0 / 30.0, paused: frozen)) { timeline in
|
||||
let time = frozen ? 0 : timeline.date.timeIntervalSince(Self.born)
|
||||
let power = TalkWaveformMath.power(for: self.phase, time: time)
|
||||
Canvas { context, size in
|
||||
let midY = size.height / 2
|
||||
var line = Path()
|
||||
line.move(to: CGPoint(x: 0, y: midY))
|
||||
line.addLine(to: CGPoint(x: size.width, y: midY))
|
||||
context.stroke(line, with: .color(.secondary.opacity(0.30)), lineWidth: 1)
|
||||
|
||||
// Screen blend pops on dark; opacity overlap reads better on light.
|
||||
context.blendMode = self.colorScheme == .dark ? .screen : .normal
|
||||
let opacity = self.colorScheme == .dark ? 0.9 : 0.55
|
||||
for (index, color) in self.colors.enumerated() {
|
||||
let path = TalkWaveformMath.wavePath(
|
||||
in: size,
|
||||
time: time,
|
||||
seed: Double(index) * 7.31,
|
||||
power: power)
|
||||
context.fill(path, with: .color(color.opacity(opacity)))
|
||||
}
|
||||
}
|
||||
}
|
||||
.opacity(self.phase == .idle ? 0.6 : 1.0)
|
||||
}
|
||||
|
||||
private var colors: [Color] {
|
||||
self.phase == .idle ? self.palette.inactive : self.palette.active
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure waveform math, split from the view for unit testing and so the Android
|
||||
/// port has one canonical reference for every constant.
|
||||
public enum TalkWaveformMath {
|
||||
/// Per-phase drive for the wave amplitude in 0...1.
|
||||
public static func power(for phase: TalkWaveformPhase, time: Double) -> Double {
|
||||
switch phase {
|
||||
case .idle:
|
||||
return 0.05
|
||||
case .thinking:
|
||||
return 0.16 + 0.10 * (0.5 + 0.5 * sin(time * 1.6))
|
||||
case let .listening(level, speechActive):
|
||||
let clamped = min(max(level, 0), 1)
|
||||
// Detected speech lifts the floor so the wave visibly commits to the
|
||||
// user even when the mic level dips between words.
|
||||
return speechActive ? 0.55 + 0.45 * clamped : 0.30 + 0.65 * clamped
|
||||
case let .speaking(level):
|
||||
guard let level else {
|
||||
// Synthetic pulse for voice paths with no playback metering.
|
||||
return 0.70 * (0.55 + 0.45 * abs(sin(time * 5.0)))
|
||||
}
|
||||
return 0.25 + 0.75 * min(max(level, 0), 1)
|
||||
}
|
||||
}
|
||||
|
||||
/// One wave = max envelope of three drifting lobes, mirrored around the midline.
|
||||
public static func wavePath(in size: CGSize, time: Double, seed: Double, power: Double) -> Path {
|
||||
let midX = Double(size.width) / 2
|
||||
let midY = Double(size.height) / 2
|
||||
|
||||
// Lobe parameters oscillate smoothly so peaks sweep back and forth
|
||||
// across the line instead of scrolling off-screen.
|
||||
let lobes: [(A: Double, k: Double, t: Double)] = (0..<3).map { index in
|
||||
let f = Double(index)
|
||||
let ampFrequency = 0.9 + 0.23 * f
|
||||
let ampPhase = time * ampFrequency + seed * 2.4 + f * 2.1
|
||||
let amp = 0.30 + 0.70 * (0.5 + 0.5 * sin(ampPhase))
|
||||
let k = 0.62 + 0.11 * f
|
||||
let driftFrequency = 0.45 + 0.17 * f
|
||||
let driftPhase = time * driftFrequency + seed + f * 1.9
|
||||
let t = 2.8 * sin(driftPhase)
|
||||
return (A: amp, k: k, t: t)
|
||||
}
|
||||
|
||||
var upper: [CGPoint] = []
|
||||
var x = -midX
|
||||
while x <= midX {
|
||||
let graphX = x / (midX / 9.0)
|
||||
var y: Double = 0
|
||||
for lobe in lobes {
|
||||
let amplitude = lobe.A * midY * power
|
||||
y = max(y, Self.attenuatedSine(x: graphX, A: amplitude, k: lobe.k, t: lobe.t))
|
||||
}
|
||||
upper.append(CGPoint(x: midX + x, y: midY - y))
|
||||
x += 2
|
||||
}
|
||||
|
||||
var path = Path()
|
||||
path.move(to: CGPoint(x: 0, y: midY))
|
||||
path.addLines(upper)
|
||||
for point in upper.reversed() {
|
||||
path.addLine(to: CGPoint(x: point.x, y: 2 * midY - point.y))
|
||||
}
|
||||
path.closeSubpath()
|
||||
return path
|
||||
}
|
||||
|
||||
/// |A·sin(kx − t)| shaped by the bell envelope g = (K/(K+(kx−t′)²))^K, K = 4.
|
||||
private static func attenuatedSine(x: Double, A: Double, k: Double, t: Double) -> Double {
|
||||
let sine = A * sin(k * x - t)
|
||||
let tPrime = t - .pi / 2
|
||||
let envelope = pow(4.0 / (4.0 + pow(k * x - tPrime, 2)), 4.0)
|
||||
return abs(sine * envelope)
|
||||
}
|
||||
}
|
||||
|
|
@ -56,8 +56,11 @@ struct OpenClawVoiceNoteRecordingRow: View {
|
|||
.fill(OpenClawChatTheme.danger)
|
||||
.frame(width: 9, height: 9)
|
||||
|
||||
Text("Recording")
|
||||
.font(OpenClawChatTypography.footnoteSemiBold)
|
||||
// Live capture wave replaces a static "Recording" label; the level is
|
||||
// real recorder metering, so silence reads flat and speech moves.
|
||||
TalkWaveformView(phase: .listening(level: self.recorder.level, speechActive: false))
|
||||
.frame(maxWidth: .infinity, minHeight: 26, maxHeight: 26)
|
||||
.accessibilityLabel("Recording")
|
||||
|
||||
Text(openClawVoiceNoteDurationLabel(self.recorder.elapsedSeconds))
|
||||
.font(OpenClawChatTypography.mono(size: 13, relativeTo: .footnote))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import AVFAudio
|
||||
import Foundation
|
||||
import Observation
|
||||
import OpenClawKit
|
||||
|
||||
private let voiceNoteMaximumDurationSeconds: TimeInterval = 180
|
||||
|
||||
|
|
@ -21,6 +22,14 @@ public protocol VoiceNoteAudioCapture: AnyObject {
|
|||
|
||||
/// Reports capture loss after a recording has started.
|
||||
func setFailureHandler(_ handler: @escaping @MainActor () -> Void)
|
||||
|
||||
/// Latest capture level in 0...1 for the live waveform, when supported.
|
||||
func currentLevel() -> Double?
|
||||
}
|
||||
|
||||
extension VoiceNoteAudioCapture {
|
||||
/// Metering is a UI nicety; test captures may skip it.
|
||||
public func currentLevel() -> Double? { nil }
|
||||
}
|
||||
|
||||
/// A completed voice-note recording ready to stage as a chat attachment.
|
||||
|
|
@ -51,6 +60,8 @@ public final class OpenClawVoiceNoteRecorder {
|
|||
|
||||
public private(set) var state: State = .idle
|
||||
public private(set) var elapsedSeconds: TimeInterval = 0
|
||||
/// Live capture level in 0...1 while recording; drives the recording waveform.
|
||||
public private(set) var level: Double = 0
|
||||
|
||||
@ObservationIgnored public var onRecordingActiveChanged: (@MainActor (Bool) -> Void)?
|
||||
|
||||
|
|
@ -179,6 +190,7 @@ public final class OpenClawVoiceNoteRecorder {
|
|||
let duration = max(0, self.capture.stop())
|
||||
let recording = OpenClawVoiceNoteRecording(fileURL: fileURL, durationSeconds: duration)
|
||||
self.elapsedSeconds = duration
|
||||
self.level = 0
|
||||
self.state = .finished(recording: recording)
|
||||
self.onRecordingActiveChanged?(false)
|
||||
return recording
|
||||
|
|
@ -205,6 +217,7 @@ public final class OpenClawVoiceNoteRecorder {
|
|||
}
|
||||
let wasRecording = self.isRecording
|
||||
self.elapsedSeconds = 0
|
||||
self.level = 0
|
||||
self.state = .idle
|
||||
if wasRecording {
|
||||
self.onRecordingActiveChanged?(false)
|
||||
|
|
@ -220,6 +233,10 @@ public final class OpenClawVoiceNoteRecorder {
|
|||
guard !Task.isCancelled else { return }
|
||||
guard case let .recording(startedAt, _) = self.state else { return }
|
||||
self.elapsedSeconds = max(0, self.now().timeIntervalSince(startedAt))
|
||||
if let captureLevel = self.capture.currentLevel() {
|
||||
// Light smoothing keeps the 10 Hz meter poll from stepping visibly.
|
||||
self.level = (self.level * 0.55) + (captureLevel * 0.45)
|
||||
}
|
||||
if self.elapsedSeconds >= self.durationLimit {
|
||||
self.finish()
|
||||
return
|
||||
|
|
@ -233,6 +250,7 @@ public final class OpenClawVoiceNoteRecorder {
|
|||
self.timerTask = nil
|
||||
let wasRecording = self.isRecording
|
||||
self.elapsedSeconds = 0
|
||||
self.level = 0
|
||||
self.state = .failed(message: message)
|
||||
if wasRecording {
|
||||
self.onRecordingActiveChanged?(false)
|
||||
|
|
@ -317,6 +335,7 @@ public final class OpenClawVoiceNoteAudioCapture: NSObject, VoiceNoteAudioCaptur
|
|||
]
|
||||
let recorder = try AVAudioRecorder(url: url, settings: settings)
|
||||
recorder.delegate = self
|
||||
recorder.isMeteringEnabled = true
|
||||
guard recorder.record() else {
|
||||
throw NSError(
|
||||
domain: "OpenClawVoiceNoteAudioCapture",
|
||||
|
|
@ -345,6 +364,12 @@ public final class OpenClawVoiceNoteAudioCapture: NSObject, VoiceNoteAudioCaptur
|
|||
self.deactivateAudioSession()
|
||||
}
|
||||
|
||||
public func currentLevel() -> Double? {
|
||||
guard let recorder, recorder.isRecording else { return nil }
|
||||
recorder.updateMeters()
|
||||
return TalkAudioLevel.normalized(decibels: Double(recorder.averagePower(forChannel: 0)))
|
||||
}
|
||||
|
||||
public nonisolated func audioRecorderEncodeErrorDidOccur(_ recorder: AVAudioRecorder, error: (any Error)?) {
|
||||
Task { @MainActor [weak self] in
|
||||
self?.captureDidFail()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,169 @@
|
|||
import AVFoundation
|
||||
import Foundation
|
||||
|
||||
/// Normalizes an RMS value to the shared 0...1 UI level scale used by every
|
||||
/// talk animation (mic, playback, recording) so the wave reads identically
|
||||
/// across surfaces: dB full scale mapped over a 50 dB window.
|
||||
public enum TalkAudioLevel {
|
||||
public static func normalized(rms: Double) -> Double {
|
||||
self.normalized(decibels: 20 * log10(max(rms, 1e-7)))
|
||||
}
|
||||
|
||||
public static func normalized(decibels: Double) -> Double {
|
||||
max(0, min(1, (decibels + 50) / 50))
|
||||
}
|
||||
|
||||
/// RMS of little-endian PCM16 mono bytes; 0 for empty or odd-length data.
|
||||
public static func pcm16RMS(_ data: Data) -> Double {
|
||||
let sampleCount = data.count / 2
|
||||
guard sampleCount > 0 else { return 0 }
|
||||
var sum: Double = 0
|
||||
data.withUnsafeBytes { raw in
|
||||
let samples = raw.bindMemory(to: Int16.self)
|
||||
for index in 0..<sampleCount {
|
||||
let sample = Double(Int16(littleEndian: samples[index])) / Double(Int16.max)
|
||||
sum += sample * sample
|
||||
}
|
||||
}
|
||||
return (sum / Double(sampleCount)).squareRoot()
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds a playback-time-aligned level envelope from PCM16 chunks that stream
|
||||
/// through the app faster than real time (gateway TTS, ElevenLabs PCM, realtime
|
||||
/// relay output). Chunks are RMS-metered on arrival but scheduled at their
|
||||
/// expected playback offset, so the published level tracks what is audible
|
||||
/// instead of network arrival bursts.
|
||||
@MainActor
|
||||
public final class PCMPlaybackEnvelope {
|
||||
private struct Segment {
|
||||
let start: TimeInterval
|
||||
let end: TimeInterval
|
||||
let level: Double
|
||||
}
|
||||
|
||||
private let onLevel: @MainActor (Double?) -> Void
|
||||
private var segments: [Segment] = []
|
||||
private var bytesPerSecond: Double = 0
|
||||
private var startedAt: ContinuousClock.Instant?
|
||||
private var scheduleEnd: TimeInterval = 0
|
||||
private var publishTask: Task<Void, Never>?
|
||||
|
||||
public init(onLevel: @escaping @MainActor (Double?) -> Void) {
|
||||
self.onLevel = onLevel
|
||||
}
|
||||
|
||||
/// Starts a new envelope; the playback clock is anchored to the first chunk.
|
||||
public func begin(sampleRate: Double) {
|
||||
self.cancel()
|
||||
self.bytesPerSecond = max(1, sampleRate * Double(MemoryLayout<Int16>.size))
|
||||
}
|
||||
|
||||
public func append(_ chunk: Data) {
|
||||
guard self.bytesPerSecond > 1, !chunk.isEmpty else { return }
|
||||
let now = ContinuousClock.now
|
||||
if self.startedAt == nil {
|
||||
self.startedAt = now
|
||||
self.startPublishing()
|
||||
}
|
||||
guard let startedAt = self.startedAt else { return }
|
||||
let elapsed = Self.seconds(startedAt.duration(to: now))
|
||||
// Chunks queue behind whatever is already scheduled; a stalled stream
|
||||
// resumes at "now" instead of leaving a phantom backlog gap.
|
||||
var start = max(elapsed, self.scheduleEnd)
|
||||
// Meter in ~50 ms windows: a whole clip can arrive as one chunk, and a
|
||||
// single RMS for it would render a flat line instead of an envelope.
|
||||
let windowBytes = max(2, Int(self.bytesPerSecond * 0.05) & ~1)
|
||||
var offset = chunk.startIndex
|
||||
while offset < chunk.endIndex {
|
||||
let end = min(offset + windowBytes, chunk.endIndex)
|
||||
let window = chunk[offset..<end]
|
||||
let duration = Double(window.count) / self.bytesPerSecond
|
||||
self.segments.append(Segment(
|
||||
start: start,
|
||||
end: start + duration,
|
||||
level: TalkAudioLevel.normalized(rms: TalkAudioLevel.pcm16RMS(Data(window)))))
|
||||
start += duration
|
||||
offset = end
|
||||
}
|
||||
self.scheduleEnd = start
|
||||
}
|
||||
|
||||
/// Stops immediately (interruption/teardown) and clears the published level.
|
||||
public func cancel() {
|
||||
self.publishTask?.cancel()
|
||||
self.publishTask = nil
|
||||
self.segments = []
|
||||
self.startedAt = nil
|
||||
self.scheduleEnd = 0
|
||||
self.onLevel(nil)
|
||||
}
|
||||
|
||||
private func startPublishing() {
|
||||
self.publishTask?.cancel()
|
||||
self.publishTask = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
guard let self else { return }
|
||||
guard self.publish() else {
|
||||
self.cancel()
|
||||
return
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 33_000_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Publishes the level at the current playback position; false once the
|
||||
/// scheduled tail (plus a drain grace) has fully played out.
|
||||
private func publish() -> Bool {
|
||||
guard let startedAt = self.startedAt else { return false }
|
||||
let elapsed = Self.seconds(startedAt.duration(to: .now))
|
||||
if elapsed > self.scheduleEnd + 0.5 {
|
||||
return false
|
||||
}
|
||||
self.segments.removeAll { $0.end < elapsed }
|
||||
let level = self.segments.first { elapsed >= $0.start && elapsed < $0.end }?.level ?? 0
|
||||
self.onLevel(level)
|
||||
return true
|
||||
}
|
||||
|
||||
private static func seconds(_ duration: Duration) -> TimeInterval {
|
||||
let parts = duration.components
|
||||
return Double(parts.seconds) + Double(parts.attoseconds) * 1e-18
|
||||
}
|
||||
}
|
||||
|
||||
/// Publishes the live output level of an `AVAudioPlayer` (buffered TTS clips)
|
||||
/// via its built-in metering at ~30 Hz. Detach clears the level to nil so the
|
||||
/// consumer can distinguish "silent" from "not playing".
|
||||
@MainActor
|
||||
public final class AudioPlayerLevelMeter {
|
||||
private let onLevel: @MainActor (Double?) -> Void
|
||||
private var pollTask: Task<Void, Never>?
|
||||
private weak var player: AVAudioPlayer?
|
||||
|
||||
public init(onLevel: @escaping @MainActor (Double?) -> Void) {
|
||||
self.onLevel = onLevel
|
||||
}
|
||||
|
||||
public func attach(_ player: AVAudioPlayer) {
|
||||
self.detach()
|
||||
player.isMeteringEnabled = true
|
||||
self.player = player
|
||||
self.pollTask = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
guard let self, let player = self.player else { return }
|
||||
player.updateMeters()
|
||||
self.onLevel(TalkAudioLevel.normalized(decibels: Double(player.averagePower(forChannel: 0))))
|
||||
try? await Task.sleep(nanoseconds: 33_000_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public func detach() {
|
||||
self.pollTask?.cancel()
|
||||
self.pollTask = nil
|
||||
self.player = nil
|
||||
self.onLevel(nil)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
import Foundation
|
||||
import OpenClawChatUI
|
||||
import OpenClawKit
|
||||
import Testing
|
||||
|
||||
struct TalkWaveformMathTests {
|
||||
@Test
|
||||
func `idle is A flat floor`() {
|
||||
#expect(TalkWaveformMath.power(for: .idle, time: 0) == 0.05)
|
||||
#expect(TalkWaveformMath.power(for: .idle, time: 12.7) == 0.05)
|
||||
}
|
||||
|
||||
@Test
|
||||
func `thinking breathes inside its band`() {
|
||||
for time in stride(from: 0.0, through: 10.0, by: 0.25) {
|
||||
let power = TalkWaveformMath.power(for: .thinking, time: time)
|
||||
#expect(power >= 0.16)
|
||||
#expect(power <= 0.26 + 1e-9)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
func `listening follows the mic level and clamps`() {
|
||||
#expect(TalkWaveformMath.power(for: .listening(level: 0, speechActive: false), time: 1) == 0.30)
|
||||
#expect(TalkWaveformMath.power(for: .listening(level: 1, speechActive: false), time: 1) == 0.95)
|
||||
#expect(TalkWaveformMath.power(for: .listening(level: 2.5, speechActive: false), time: 1) == 0.95)
|
||||
#expect(TalkWaveformMath.power(for: .listening(level: -1, speechActive: false), time: 1) == 0.30)
|
||||
}
|
||||
|
||||
@Test
|
||||
func `detected speech raises the floor but stays level driven`() {
|
||||
let quiet = TalkWaveformMath.power(for: .listening(level: 0, speechActive: true), time: 1)
|
||||
let loud = TalkWaveformMath.power(for: .listening(level: 1, speechActive: true), time: 1)
|
||||
#expect(quiet == 0.55)
|
||||
#expect(loud == 1.0)
|
||||
}
|
||||
|
||||
@Test
|
||||
func `speaking follows the playback envelope`() {
|
||||
#expect(TalkWaveformMath.power(for: .speaking(level: 0), time: 1) == 0.25)
|
||||
#expect(TalkWaveformMath.power(for: .speaking(level: 1), time: 1) == 1.0)
|
||||
#expect(TalkWaveformMath.power(for: .speaking(level: 3), time: 1) == 1.0)
|
||||
}
|
||||
|
||||
@Test
|
||||
func `speaking without an envelope pulses synthetically`() {
|
||||
var seen: Set<Int> = []
|
||||
for time in stride(from: 0.0, through: 3.0, by: 0.05) {
|
||||
let power = TalkWaveformMath.power(for: .speaking(level: nil), time: time)
|
||||
#expect(power >= 0.70 * 0.55 - 1e-9)
|
||||
#expect(power <= 0.70 + 1e-9)
|
||||
seen.insert(Int(power * 1000))
|
||||
}
|
||||
// The fallback must move over time, not freeze.
|
||||
#expect(seen.count > 5)
|
||||
}
|
||||
}
|
||||
|
||||
struct TalkAudioLevelTests {
|
||||
@Test
|
||||
func `full scale RMS normalizes to one`() {
|
||||
#expect(TalkAudioLevel.normalized(rms: 1.0) == 1.0)
|
||||
}
|
||||
|
||||
@Test
|
||||
func `silence normalizes to zero`() {
|
||||
#expect(TalkAudioLevel.normalized(rms: 0) == 0)
|
||||
#expect(TalkAudioLevel.normalized(rms: 1e-9) == 0)
|
||||
}
|
||||
|
||||
@Test
|
||||
func `pcm 16 RMS measures real samples`() {
|
||||
let silence = Data(repeating: 0, count: 512)
|
||||
#expect(TalkAudioLevel.pcm16RMS(silence) == 0)
|
||||
|
||||
var fullScale = Data()
|
||||
for _ in 0..<256 {
|
||||
withUnsafeBytes(of: Int16.max.littleEndian) { fullScale.append(contentsOf: $0) }
|
||||
}
|
||||
let rms = TalkAudioLevel.pcm16RMS(fullScale)
|
||||
#expect(abs(rms - 1.0) < 0.001)
|
||||
#expect(TalkAudioLevel.normalized(rms: rms) > 0.99)
|
||||
}
|
||||
|
||||
@Test
|
||||
func `pcm 16 RMS ignores empty and odd data`() {
|
||||
#expect(TalkAudioLevel.pcm16RMS(Data()) == 0)
|
||||
#expect(TalkAudioLevel.pcm16RMS(Data([0x7F])) == 0)
|
||||
}
|
||||
}
|
||||
|
|
@ -37,6 +37,12 @@ private final class FakeVoiceNoteAudioCapture: VoiceNoteAudioCapture {
|
|||
self.failureHandler = handler
|
||||
}
|
||||
|
||||
var meterLevel: Double?
|
||||
|
||||
func currentLevel() -> Double? {
|
||||
self.meterLevel
|
||||
}
|
||||
|
||||
func failCapture() {
|
||||
self.failureHandler?()
|
||||
}
|
||||
|
|
@ -217,6 +223,27 @@ final class VoiceNoteRecorderTests: XCTestCase {
|
|||
XCTAssertEqual(openClawVoiceNoteDurationLabel(-1), "0:00")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testRecordingPublishesCaptureLevelsAndResetsOnFinish() async throws {
|
||||
let capture = FakeVoiceNoteAudioCapture()
|
||||
capture.meterLevel = 0.8
|
||||
let recorder = OpenClawVoiceNoteRecorder(
|
||||
capture: capture,
|
||||
timerIntervalNanoseconds: 2_000_000)
|
||||
|
||||
let started = await recorder.start()
|
||||
XCTAssertTrue(started)
|
||||
XCTAssertEqual(recorder.level, 0)
|
||||
|
||||
for _ in 0..<200 where recorder.level == 0 {
|
||||
try await Task.sleep(nanoseconds: 3_000_000)
|
||||
}
|
||||
XCTAssertGreaterThan(recorder.level, 0)
|
||||
|
||||
_ = try XCTUnwrap(recorder.finish())
|
||||
XCTAssertEqual(recorder.level, 0)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testRecordButtonRequiresAttachmentInput() {
|
||||
let recorder = OpenClawVoiceNoteRecorder(capture: FakeVoiceNoteAudioCapture())
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ Supported keys: `voice` / `voice_id` / `voiceId`, `model` / `model_id` / `modelI
|
|||
|
||||
- Menu bar toggle: **Talk**
|
||||
- Config tab: **Talk Mode** group (voice id + interrupt toggle)
|
||||
- Overlay: Listening (cloud pulses with mic level) → Thinking (sinking animation) → Speaking (radiating rings). Click the cloud to stop speaking, click X to exit Talk mode.
|
||||
- Overlay: the orb renders the universal talk waveform (shared with iOS, watchOS, and Android). Listening follows the live mic level, Speaking follows the actual TTS playback envelope, Thinking breathes softly. Click the orb to pause/resume, double-click to stop speaking, click X to exit Talk mode.
|
||||
|
||||
## Android UI
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue