diff --git a/preload.py b/preload.py index 0c8118e49..2494a7091 100644 --- a/preload.py +++ b/preload.py @@ -1,6 +1,7 @@ import asyncio from python.helpers import runtime, whisper, settings from python.helpers.print_style import PrintStyle +from python.helpers import kokoro_tts import models PrintStyle().print("Running preload...") @@ -24,17 +25,22 @@ async def preload(): try: # Use the new LiteLLM-based model system emb_mod = models.get_embedding_model( - models.ModelProvider.HUGGINGFACE, - set["embed_model_name"] + models.ModelProvider.HUGGINGFACE, set["embed_model_name"] ) emb_txt = await emb_mod.aembed_query("test") return emb_txt except Exception as e: PrintStyle().error(f"Error in preload_embedding: {e}") + # preload kokoro tts model if enabled + async def preload_kokoro(): + try: + return await kokoro_tts.preload() + except Exception as e: + PrintStyle().error(f"Error in preload_kokoro: {e}") # async tasks to preload - tasks = [preload_whisper(), preload_embedding()] + tasks = [preload_whisper(), preload_embedding(), preload_kokoro()] await asyncio.gather(*tasks, return_exceptions=True) PrintStyle().print("Preload completed") diff --git a/python/api/synthesize.py b/python/api/synthesize.py new file mode 100644 index 000000000..4d33254f8 --- /dev/null +++ b/python/api/synthesize.py @@ -0,0 +1,92 @@ +# api/synthesize.py + +import re +from python.helpers.api import ApiHandler +from flask import Request, Response + +from python.helpers import runtime, settings, kokoro_tts + +class Synthesize(ApiHandler): + async def process(self, input: dict, request: Request) -> dict | Response: + text = input.get("text", "") + ctxid = input.get("ctxid", "") + + context = self.get_context(ctxid) + if await kokoro_tts.is_downloading(): + context.log.log(type="info", content="Kokoro TTS model is currently being downloaded, please wait...") + + try: + # Clean and chunk text for long responses + cleaned_text = self._clean_text(text) + chunks = self._chunk_text(cleaned_text) + + if len(chunks) == 1: + # Single chunk - return as before + audio = await kokoro_tts.synthesize_sentences(chunks) + return {"audio": audio, "success": True} + else: + # Multiple chunks - return as sequence + audio_parts = [] + for chunk in chunks: + chunk_audio = await kokoro_tts.synthesize_sentences([chunk]) + audio_parts.append(chunk_audio) + return {"audio_parts": audio_parts, "success": True} + except Exception as e: + return {"error": str(e), "success": False} + + def _clean_text(self, text: str) -> str: + """Clean text by removing markdown, tables, code blocks, and other formatting""" + # Remove code blocks + text = re.sub(r'```[\s\S]*?```', '', text) + text = re.sub(r'`[^`]*`', '', text) + + # Remove markdown links + text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text) + + # Remove markdown formatting + text = re.sub(r'[*_#]+', '', text) + + # Remove tables (basic cleanup) + text = re.sub(r'\|[^\n]*\|', '', text) + + # Remove extra whitespace and newlines + text = re.sub(r'\n+', ' ', text) + text = re.sub(r'\s+', ' ', text) + + # Remove URLs + text = re.sub(r'https?://[^\s]+', '', text) + + # Remove email addresses + text = re.sub(r'\S+@\S+', '', text) + + return text.strip() + + def _chunk_text(self, text: str) -> list[str]: + """Split text into manageable chunks for TTS""" + # If text is short enough, return as single chunk + if len(text) <= 300: + return [text] + + # Split into sentences first + sentences = re.split(r'(?<=[.!?])\s+', text) + + chunks = [] + current_chunk = "" + + for sentence in sentences: + sentence = sentence.strip() + if not sentence: + continue + + # If adding this sentence would make chunk too long, start new chunk + if current_chunk and len(current_chunk + " " + sentence) > 300: + chunks.append(current_chunk.strip()) + current_chunk = sentence + else: + current_chunk += (" " if current_chunk else "") + sentence + + # Add the last chunk if it has content + if current_chunk.strip(): + chunks.append(current_chunk.strip()) + + return chunks if chunks else [text] \ No newline at end of file diff --git a/python/helpers/kokoro_tts.py b/python/helpers/kokoro_tts.py new file mode 100644 index 000000000..20f6036f0 --- /dev/null +++ b/python/helpers/kokoro_tts.py @@ -0,0 +1,91 @@ +# kokoro_tts.py + +import base64 +import io +import warnings +import asyncio +import soundfile as sf +from python.helpers import runtime +from python.helpers.print_style import PrintStyle + +warnings.filterwarnings("ignore", category=FutureWarning) + +_pipeline = None +_voice = "am_puck,am_onyx" +_speed = 1.1 +is_updating_model = False + +async def preload(): + try: + return await runtime.call_development_function(_preload) + except Exception as e: + if not runtime.is_development(): + raise e + # Fallback to direct execution if RFC fails in development + PrintStyle.standard("RFC failed, falling back to direct execution...") + return await _preload() + +async def _preload(): + global _pipeline, is_updating_model + + while is_updating_model: + await asyncio.sleep(0.1) + + try: + is_updating_model = True + if not _pipeline: + PrintStyle.standard("Loading Kokoro TTS model...") + from kokoro import KPipeline + _pipeline = KPipeline(lang_code='a') + finally: + is_updating_model = False + +async def is_downloading(): + try: + return await runtime.call_development_function(_is_downloading) + except Exception as e: + if not runtime.is_development(): + raise e + # Fallback to direct execution if RFC fails in development + return _is_downloading() + +def _is_downloading(): + return is_updating_model + +async def synthesize_sentences(sentences: list[str]): + """Generate audio for multiple sentences and return concatenated base64 audio""" + try: + return await runtime.call_development_function(_synthesize_sentences, sentences) + except Exception as e: + if not runtime.is_development(): + raise e + # Fallback to direct execution if RFC fails in development + return await _synthesize_sentences(sentences) + +async def _synthesize_sentences(sentences: list[str]): + await _preload() + + combined_audio = [] + + try: + for sentence in sentences: + if sentence.strip(): + segments = _pipeline(sentence.strip(), voice=_voice, speed=_speed) + segment_list = list(segments) + + for segment in segment_list: + audio_tensor = segment.audio + audio_numpy = audio_tensor.detach().cpu().numpy() + combined_audio.extend(audio_numpy) + + # Convert combined audio to bytes + buffer = io.BytesIO() + sf.write(buffer, combined_audio, 24000, format='WAV') + audio_bytes = buffer.getvalue() + + # Return base64 encoded audio + return base64.b64encode(audio_bytes).decode('utf-8') + + except Exception as e: + PrintStyle.error(f"Error in Kokoro TTS synthesis: {e}") + raise \ No newline at end of file diff --git a/python/helpers/settings.py b/python/helpers/settings.py index 1b806de76..2b1ba258f 100644 --- a/python/helpers/settings.py +++ b/python/helpers/settings.py @@ -692,10 +692,12 @@ def convert_out(settings: Settings) -> SettingsOutput: ) stt_section: SettingsSection = { - "id": "stt", - "title": "Speech to Text", - "description": "Voice transcription preferences and server turn detection settings.", - "fields": stt_fields, + # TTS fields + tts_fields: list[SettingsField] = [] + + tts_fields.append( + { + "id": "tts_enabled", "tab": "agent", } @@ -825,6 +827,7 @@ def convert_out(settings: Settings) -> SettingsOutput: browser_model_section, embed_model_section, stt_section, + speech_section, api_keys_section, auth_section, mcp_client_section, @@ -1016,6 +1019,7 @@ def get_default_settings() -> Settings: stt_silence_threshold=0.3, stt_silence_duration=1000, stt_waiting_timeout=2000, + tts_enabled=False, mcp_servers='{\n "mcpServers": {}\n}', mcp_client_init_timeout=10, mcp_client_tool_timeout=120, diff --git a/requirements.txt b/requirements.txt index 2a8f994b9..38e741d80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,3 +38,5 @@ pdf2image==1.17.0 crontab==1.0.1 pathspec>=0.12.1 psutil>=7.0.0 +kokoro>=0.9.2 +soundfile \ No newline at end of file diff --git a/webui/index.html b/webui/index.html index b782da944..d07691fbd 100644 --- a/webui/index.html +++ b/webui/index.html @@ -100,7 +100,6 @@ - @@ -361,15 +360,12 @@ -
+

|>

-

- Stop Speech +

+ Stop Speech

+ aria-label="Start/Stop recording" + @click="$store.speech.handleMicrophoneClick()" + x-effect="$store.speech.updateMicrophoneButtonUI()"> diff --git a/webui/index.js b/webui/index.js index 8354dc100..db01fc256 100644 --- a/webui/index.js +++ b/webui/index.js @@ -1,9 +1,9 @@ import * as msgs from "./js/messages.js"; -import { speech } from "./js/speech.js"; import * as api from "./js/api.js"; import * as css from "./js/css.js"; import { sleep } from "./js/sleep.js"; import { store as attachmentsStore } from "./components/chat/attachments/attachmentsStore.js"; +import { store as speechStore } from "./js/speech-store.js"; window.fetchApi = api.fetchApi; // TODO - backward compatibility for non-modular scripts, remove once refactored to alpine @@ -509,7 +509,7 @@ function speakMessages(logs) { if (log.type == "response" && log.kvps && log.kvps.finished) { if (log.no > lastSpokenNo) { lastSpokenNo = log.no; - speech.speak(log.content); + speechStore.speak(log.content); return; } // finished LLM headline, not response @@ -522,7 +522,7 @@ function speakMessages(logs) { ) { if (log.no > lastSpokenNo) { lastSpokenNo = log.no; - speech.speak(log.kvps.headline); + speechStore.speak(log.kvps.headline); return; } } @@ -715,6 +715,9 @@ export const setContext = function (id) { lastLogGuid = ""; lastLogVersion = 0; lastSpokenNo = 0; + + // Stop speech when switching chats + speechStore.stop(); // Clear the chat history immediately to avoid showing stale content chatHistory.innerHTML = ""; @@ -775,7 +778,7 @@ window.toggleDarkMode = function (isDark) { window.toggleSpeech = function (isOn) { console.log("Speech:", isOn); localStorage.setItem("speech", isOn); - if (!isOn) speech.stop(); + if (!isOn) speechStore.stop(); }; window.nudge = async function () { diff --git a/webui/js/speech-store.js b/webui/js/speech-store.js new file mode 100644 index 000000000..1cbab090e --- /dev/null +++ b/webui/js/speech-store.js @@ -0,0 +1,580 @@ +import { createStore } from "./AlpineStore.js"; +import { updateChatInput, sendMessage } from "../index.js"; + +const Status = { + INACTIVE: "inactive", + ACTIVATING: "activating", + LISTENING: "listening", + RECORDING: "recording", + WAITING: "waiting", + PROCESSING: "processing", +}; + +// Create the speech store +const model = { + // STT Settings + stt_model_size: "tiny", + stt_language: "en", + stt_silence_threshold: 0.05, + stt_silence_duration: 1000, + stt_waiting_timeout: 2000, + + // TTS Settings + tts_enabled: false, + + // TTS State + isSpeaking: false, + currentAudio: null, + audioContext: null, + userHasInteracted: false, + + // STT State + microphoneInput: null, + isProcessingClick: false, + + // Getter for micStatus - delegates to microphoneInput + get micStatus() { + return this.microphoneInput?.status || Status.INACTIVE; + }, + + updateMicrophoneButtonUI() { + const microphoneButton = document.getElementById("microphone-button"); + if (!microphoneButton) return; + const status = this.micStatus; + microphoneButton.classList.remove( + 'mic-inactive', 'mic-activating', 'mic-listening', 'mic-recording', 'mic-waiting', 'mic-processing' + ); + microphoneButton.classList.add(`mic-${status.toLowerCase()}`); + microphoneButton.setAttribute("data-status", status); + }, + + async handleMicrophoneClick() { + if (this.isProcessingClick) return; + this.isProcessingClick = true; + try { + if (!this.microphoneInput) { + await this.initMicrophone(); + } + + if (this.microphoneInput) { + await this.microphoneInput.toggle(); + } + } finally { + setTimeout(() => { + this.isProcessingClick = false; + }, 300); + } + }, + + + // Initialize speech functionality + async init() { + await this.loadSettings(); + this.setupBrowserTTS(); + this.setupUserInteractionHandling(); + }, + + // Load settings from server + async loadSettings() { + try { + const response = await fetchApi("/settings_get", { method: "POST" }); + const data = await response.json(); + const speechSection = data.settings.sections.find(s => s.title === "Speech"); + + if (speechSection) { + speechSection.fields.forEach(field => { + if (this.hasOwnProperty(field.id)) { + this[field.id] = field.value; + } + }); + } + } catch (error) { + window.toastFetchError("Failed to load speech settings", error); + console.error("Failed to load speech settings:", error); + } + }, + + // Setup browser TTS + setupBrowserTTS() { + this.synth = window.speechSynthesis; + this.browserUtterance = null; + }, + + // Setup user interaction handling for autoplay policy + setupUserInteractionHandling() { + const enableAudio = () => { + if (!this.userHasInteracted) { + this.userHasInteracted = true; + console.log("User interaction detected - audio playback enabled"); + + // Create a dummy audio context to "unlock" audio + try { + this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); + this.audioContext.resume(); + } catch (e) { + console.log("AudioContext not available"); + } + } + }; + + // Listen for any user interaction + const events = ['click', 'touchstart', 'keydown', 'mousedown']; + events.forEach(event => { + document.addEventListener(event, enableAudio, { once: true, passive: true }); + }); + }, + + // Main speak function + async speak(text) { + if (!this.tts_enabled) return; + if (this.isSpeaking) return; + + text = this.cleanText(text); + if (!text.trim()) return; + + if (!this.userHasInteracted) { + this.showAudioPermissionPrompt(); + return; + } + + try { + await this.speakWithKokoro(text); + } catch (error) { + console.error("TTS error:", error); + this.speakWithBrowser(text); + } + }, + + // Show a prompt to user to enable audio + showAudioPermissionPrompt() { + if (window.toast) { + window.toast("Click anywhere to enable audio playback", "info", 5000); + } else { + console.log("Please click anywhere on the page to enable audio playback"); + } + }, + + // Browser TTS + speakWithBrowser(text) { + this.browserUtterance = new SpeechSynthesisUtterance(text); + this.browserUtterance.onstart = () => { this.isSpeaking = true; }; + this.browserUtterance.onend = () => { this.isSpeaking = false; }; + this.synth.speak(this.browserUtterance); + }, + + // Kokoro TTS + async speakWithKokoro(text) { + try { + const response = await sendJsonData("/synthesize", { text }); + + if (response.success) { + if (response.audio_parts) { + // Multiple chunks - play sequentially + for (const audioPart of response.audio_parts) { + await this.playAudio(audioPart); + await new Promise(resolve => setTimeout(resolve, 100)); // Brief pause + } + } else if (response.audio) { + // Single audio + await this.playAudio(response.audio); + } + } else { + console.error("Kokoro TTS error:", response.error); + this.speakWithBrowser(text); + } + } catch (error) { + console.error("Kokoro TTS error:", error); + this.speakWithBrowser(text); + } + }, + + + // Play base64 audio + async playAudio(base64Audio) { + return new Promise((resolve, reject) => { + const audio = new Audio(); + + audio.onplay = () => { + this.isSpeaking = true; + }; + audio.onended = () => { + this.isSpeaking = false; + this.currentAudio = null; + resolve(); + }; + audio.onerror = (error) => { + this.isSpeaking = false; + this.currentAudio = null; + reject(error); + }; + + audio.src = `data:audio/wav;base64,${base64Audio}`; + this.currentAudio = audio; + + audio.play().catch(error => { + this.isSpeaking = false; + this.currentAudio = null; + + if (error.name === 'NotAllowedError') { + this.showAudioPermissionPrompt(); + this.userHasInteracted = false; + } + reject(error); + }); + }); + }, + + // Stop all speech + stop() { + if (this.synth?.speaking) { + this.synth.cancel(); + } + + if (this.currentAudio) { + this.currentAudio.pause(); + this.currentAudio.currentTime = 0; + this.currentAudio = null; + } + + this.isSpeaking = false; + }, + + // Clean text for TTS + cleanText(text) { + return text + .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, "") + .replace(/https?:\/\/[^\s]+/g, "") + .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "") + .replace(/\s+/g, " ") + .trim(); + }, + + // Initialize microphone input + async initMicrophone() { + if (this.microphoneInput) return this.microphoneInput; + + this.microphoneInput = new MicrophoneInput( + async (text, isFinal) => { + if (isFinal) { + updateChatInput(text); + if (!this.microphoneInput.messageSent) { + this.microphoneInput.messageSent = true; + await sendMessage(); + } + } + } + ); + + const initialized = await this.microphoneInput.initialize(); + return initialized ? this.microphoneInput : null; + }, + + // Request microphone permission - delegate to MicrophoneInput + async requestMicrophonePermission() { + return this.microphoneInput ? + this.microphoneInput.requestPermission() : + MicrophoneInput.prototype.requestPermission.call(null); + } +}; + +// Microphone Input Class (simplified for store integration) +class MicrophoneInput { + constructor(updateCallback) { + this.mediaRecorder = null; + this.audioChunks = []; + this.lastChunk = []; + this.updateCallback = updateCallback; + this.messageSent = false; + this.audioContext = null; + this.mediaStreamSource = null; + this.analyserNode = null; + this._status = Status.INACTIVE; + this.lastAudioTime = null; + this.waitingTimer = null; + this.silenceStartTime = null; + this.hasStartedRecording = false; + this.analysisFrame = null; + } + + get status() { + return this._status; + } + + set status(newStatus) { + if (this._status === newStatus) return; + + const oldStatus = this._status; + this._status = newStatus; + console.log(`Mic status changed from ${oldStatus} to ${newStatus}`); + + this.handleStatusChange(oldStatus, newStatus); + } + + async initialize() { + // Set status to activating at the start of initialization + this.status = Status.ACTIVATING; + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 } + }); + + // Log which device is being used + const audioTrack = stream.getAudioTracks()[0]; + const deviceId = audioTrack.getSettings().deviceId; + if (deviceId) { + const devices = await navigator.mediaDevices.enumerateDevices(); + const device = devices.find(d => d.deviceId === deviceId); + if (device) { + console.log(`Microphone input device: ${device.label} (ID: ${device.deviceId})`); + } else { + console.log(`Microphone input device ID: ${deviceId} (label not found)`); + } + } else { + console.log('Microphone input deviceId not available'); + } + + this.mediaRecorder = new MediaRecorder(stream); + this.mediaRecorder.ondataavailable = (event) => { + if (event.data.size > 0 && (this.status === Status.RECORDING || this.status === Status.WAITING)) { + if (this.lastChunk) { + this.audioChunks.push(this.lastChunk); + this.lastChunk = null; + } + this.audioChunks.push(event.data); + } else if (this.status === Status.LISTENING) { + this.lastChunk = event.data; + } + }; + + this.setupAudioAnalysis(stream); + return true; + } catch (error) { + console.error("Microphone initialization error:", error); + toast("Failed to access microphone. Please check permissions.", "error"); + return false; + } + } + + handleStatusChange(oldStatus, newStatus) { + if (newStatus != Status.RECORDING) { + this.lastChunk = null; + } + + switch (newStatus) { + case Status.INACTIVE: + this.handleInactiveState(); + break; + case Status.LISTENING: + this.handleListeningState(); + break; + case Status.RECORDING: + this.handleRecordingState(); + break; + case Status.WAITING: + this.handleWaitingState(); + break; + case Status.PROCESSING: + this.handleProcessingState(); + break; + } + } + + handleInactiveState() { + this.stopRecording(); + this.stopAudioAnalysis(); + if (this.waitingTimer) { + clearTimeout(this.waitingTimer); + this.waitingTimer = null; + } + } + + handleListeningState() { + this.stopRecording(); + this.audioChunks = []; + this.hasStartedRecording = false; + this.silenceStartTime = null; + this.lastAudioTime = null; + this.messageSent = false; + this.startAudioAnalysis(); + } + + handleRecordingState() { + if (!this.hasStartedRecording && this.mediaRecorder.state !== "recording") { + this.hasStartedRecording = true; + this.mediaRecorder.start(1000); + console.log("Speech started"); + } + if (this.waitingTimer) { + clearTimeout(this.waitingTimer); + this.waitingTimer = null; + } + } + + handleWaitingState() { + this.waitingTimer = setTimeout(() => { + if (this.status === Status.WAITING) { + this.status = Status.PROCESSING; + } + }, store.stt_waiting_timeout); + } + + handleProcessingState() { + this.stopRecording(); + this.process(); + } + + setupAudioAnalysis(stream) { + this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); + this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream); + this.analyserNode = this.audioContext.createAnalyser(); + this.analyserNode.fftSize = 2048; + this.analyserNode.minDecibels = -90; + this.analyserNode.maxDecibels = -10; + this.analyserNode.smoothingTimeConstant = 0.85; + this.mediaStreamSource.connect(this.analyserNode); + } + + startAudioAnalysis() { + const analyzeFrame = () => { + if (this.status === Status.INACTIVE) return; + + const dataArray = new Uint8Array(this.analyserNode.fftSize); + this.analyserNode.getByteTimeDomainData(dataArray); + + let sum = 0; + for (let i = 0; i < dataArray.length; i++) { + const amplitude = (dataArray[i] - 128) / 128; + sum += amplitude * amplitude; + } + const rms = Math.sqrt(sum / dataArray.length); + const now = Date.now(); + + // Update status based on audio level (ignore if TTS is speaking) + if (rms > this.densify(store.stt_silence_threshold)) { + this.lastAudioTime = now; + this.silenceStartTime = null; + + if ((this.status === Status.LISTENING || this.status === Status.WAITING) && !store.isSpeaking && !store.isGenerating) { + this.status = Status.RECORDING; + } + } else if (this.status === Status.RECORDING) { + if (!this.silenceStartTime) { + this.silenceStartTime = now; + } + + const silenceDuration = now - this.silenceStartTime; + if (silenceDuration >= store.stt_silence_duration) { + this.status = Status.WAITING; + } + } + + this.analysisFrame = requestAnimationFrame(analyzeFrame); + }; + + this.analysisFrame = requestAnimationFrame(analyzeFrame); + } + + stopAudioAnalysis() { + if (this.analysisFrame) { + cancelAnimationFrame(this.analysisFrame); + this.analysisFrame = null; + } + } + + stopRecording() { + if (this.mediaRecorder?.state === "recording") { + this.mediaRecorder.stop(); + this.hasStartedRecording = false; + } + } + + densify(x) { + return Math.exp(-5 * (1 - x)); + } + + async process() { + if (this.audioChunks.length === 0) { + this.status = Status.LISTENING; + return; + } + + const audioBlob = new Blob(this.audioChunks, { type: "audio/wav" }); + const base64 = await this.convertBlobToBase64Wav(audioBlob); + + try { + const result = await sendJsonData("/transcribe", { audio: base64 }); + const text = this.filterResult(result.text || ""); + + if (text) { + console.log("Transcription:", result.text); + await this.updateCallback(result.text, true); + } + } catch (error) { + window.toastFetchError("Transcription error", error); + console.error("Transcription error:", error); + } finally { + this.audioChunks = []; + this.status = Status.LISTENING; + } + } + + convertBlobToBase64Wav(audioBlob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => { + const base64Data = reader.result.split(",")[1]; + resolve(base64Data); + }; + reader.onerror = (error) => reject(error); + reader.readAsDataURL(audioBlob); + }); + } + + filterResult(text) { + text = text.trim(); + let ok = false; + while (!ok) { + if (!text) break; + if (text[0] === "{" && text[text.length - 1] === "}") break; + if (text[0] === "(" && text[text.length - 1] === ")") break; + if (text[0] === "[" && text[text.length - 1] === "]") break; + ok = true; + } + if (ok) return text; + else console.log(`Discarding transcription: ${text}`); + } + + // Toggle microphone between active and inactive states + async toggle() { + const hasPermission = await this.requestPermission(); + if (!hasPermission) return; + + // Toggle between listening and inactive + if (this.status === Status.INACTIVE || this.status === Status.ACTIVATING) { + this.status = Status.LISTENING; + } else { + this.status = Status.INACTIVE; + } + } + + // Request microphone permission + async requestPermission() { + try { + await navigator.mediaDevices.getUserMedia({ audio: true }); + return true; + } catch (err) { + console.error("Error accessing microphone:", err); + toast("Microphone access denied. Please enable microphone access in your browser settings.", "error"); + return false; + } + } +} + +export const store = createStore('speech', model); + +// Initialize speech store +// window.speechStore = speechStore; + +// Event listeners +document.addEventListener("settings-updated", () => store.loadSettings()); +// document.addEventListener("DOMContentLoaded", () => speechStore.init()); \ No newline at end of file diff --git a/webui/js/speech.js b/webui/js/speech.js deleted file mode 100644 index 040d0b783..000000000 --- a/webui/js/speech.js +++ /dev/null @@ -1,492 +0,0 @@ -// import { pipeline, read_audio } from '../transformers@3.0.2.js'; -import { updateChatInput, sendMessage } from "../index.js"; -import { fetchApi } from "./api.js"; - -const microphoneButton = document.getElementById("microphone-button"); -let microphoneInput = null; -let isProcessingClick = false; - -const Status = { - INACTIVE: "inactive", - ACTIVATING: "activating", - LISTENING: "listening", - RECORDING: "recording", - WAITING: "waiting", - PROCESSING: "processing", -}; - -const micSettings = { - stt_model_size: "tiny", - stt_language: "en", - stt_silence_threshold: 0.05, - stt_silence_duration: 1000, - stt_waiting_timeout: 2000, -}; -window.micSettings = micSettings; -loadMicSettings(); - -function densify(x) { - return Math.exp(-5 * (1 - x)); -} - -async function loadMicSettings() { - try { - const response = await fetchApi("/settings_get", { - method: "POST", - }); - const data = await response.json(); - const sttSettings = data.settings.sections.find( - (s) => s.title === "Speech to Text" - ); - - if (sttSettings) { - // Update options from server settings - sttSettings.fields.forEach((field) => { - const key = field.id; //.split('.')[1]; // speech_to_text.model_size -> model_size - micSettings[key] = field.value; - }); - } - } catch (error) { - window.toastFetchError("Failed to load speech settings", error); - console.error("Failed to load speech settings:", error); - } -} - -class MicrophoneInput { - constructor(updateCallback, options = {}) { - this.mediaRecorder = null; - this.audioChunks = []; - this.lastChunk = []; - this.updateCallback = updateCallback; - this.messageSent = false; - - // Audio analysis properties - this.audioContext = null; - this.mediaStreamSource = null; - this.analyserNode = null; - this._status = Status.INACTIVE; - - // Timing properties - this.lastAudioTime = null; - this.waitingTimer = null; - this.silenceStartTime = null; - this.hasStartedRecording = false; - this.analysisFrame = null; - } - - get status() { - return this._status; - } - - set status(newStatus) { - if (this._status === newStatus) return; - - const oldStatus = this._status; - this._status = newStatus; - console.log(`Mic status changed from ${oldStatus} to ${newStatus}`); - - // Update UI - microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`); - microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`); - microphoneButton.setAttribute("data-status", newStatus); - - // Handle state-specific behaviors - this.handleStatusChange(oldStatus, newStatus); - } - - handleStatusChange(oldStatus, newStatus) { - //last chunk kept only for transition to recording status - if (newStatus != Status.RECORDING) { - this.lastChunk = null; - } - - switch (newStatus) { - case Status.INACTIVE: - this.handleInactiveState(); - break; - case Status.LISTENING: - this.handleListeningState(); - break; - case Status.RECORDING: - this.handleRecordingState(); - break; - case Status.WAITING: - this.handleWaitingState(); - break; - case Status.PROCESSING: - this.handleProcessingState(); - break; - } - } - - handleInactiveState() { - this.stopRecording(); - this.stopAudioAnalysis(); - if (this.waitingTimer) { - clearTimeout(this.waitingTimer); - this.waitingTimer = null; - } - } - - handleListeningState() { - this.stopRecording(); - this.audioChunks = []; - this.hasStartedRecording = false; - this.silenceStartTime = null; - this.lastAudioTime = null; - this.messageSent = false; - this.startAudioAnalysis(); - } - - handleRecordingState() { - if (!this.hasStartedRecording && this.mediaRecorder.state !== "recording") { - this.hasStartedRecording = true; - this.mediaRecorder.start(1000); - console.log("Speech started"); - } - if (this.waitingTimer) { - clearTimeout(this.waitingTimer); - this.waitingTimer = null; - } - } - - handleWaitingState() { - // Don't stop recording during waiting state - this.waitingTimer = setTimeout(() => { - if (this.status === Status.WAITING) { - this.status = Status.PROCESSING; - } - }, micSettings.stt_waiting_timeout); - } - - handleProcessingState() { - this.stopRecording(); - this.process(); - } - - stopRecording() { - if (this.mediaRecorder?.state === "recording") { - this.mediaRecorder.stop(); - this.hasStartedRecording = false; - } - } - - async initialize() { - try { - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - echoCancellation: true, - noiseSuppression: true, - channelCount: 1, - }, - }); - - this.mediaRecorder = new MediaRecorder(stream); - this.mediaRecorder.ondataavailable = (event) => { - if ( - event.data.size > 0 && - (this.status === Status.RECORDING || this.status === Status.WAITING) - ) { - if (this.lastChunk) { - this.audioChunks.push(this.lastChunk); - this.lastChunk = null; - } - this.audioChunks.push(event.data); - console.log( - "Audio chunk received, total chunks:", - this.audioChunks.length - ); - } else if (this.status === Status.LISTENING) { - this.lastChunk = event.data; - } - }; - - this.setupAudioAnalysis(stream); - return true; - } catch (error) { - console.error("Microphone initialization error:", error); - toast("Failed to access microphone. Please check permissions.", "error"); - return false; - } - } - - setupAudioAnalysis(stream) { - this.audioContext = new (window.AudioContext || - window.webkitAudioContext)(); - this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream); - this.analyserNode = this.audioContext.createAnalyser(); - this.analyserNode.fftSize = 2048; - this.analyserNode.minDecibels = -90; - this.analyserNode.maxDecibels = -10; - this.analyserNode.smoothingTimeConstant = 0.85; - this.mediaStreamSource.connect(this.analyserNode); - } - - startAudioAnalysis() { - const analyzeFrame = () => { - if (this.status === Status.INACTIVE) return; - - const dataArray = new Uint8Array(this.analyserNode.fftSize); - this.analyserNode.getByteTimeDomainData(dataArray); - - // Calculate RMS volume - let sum = 0; - for (let i = 0; i < dataArray.length; i++) { - const amplitude = (dataArray[i] - 128) / 128; - sum += amplitude * amplitude; - } - const rms = Math.sqrt(sum / dataArray.length); - - const now = Date.now(); - - // Update status based on audio level - if (rms > densify(micSettings.stt_silence_threshold)) { - this.lastAudioTime = now; - this.silenceStartTime = null; - - if ( - this.status === Status.LISTENING || - this.status === Status.WAITING - ) { - if (!speech.isSpeaking()) - // TODO? a better way to ignore agent's voice? - this.status = Status.RECORDING; - } - } else if (this.status === Status.RECORDING) { - if (!this.silenceStartTime) { - this.silenceStartTime = now; - } - - const silenceDuration = now - this.silenceStartTime; - if (silenceDuration >= micSettings.stt_silence_duration) { - this.status = Status.WAITING; - } - } - - this.analysisFrame = requestAnimationFrame(analyzeFrame); - }; - - this.analysisFrame = requestAnimationFrame(analyzeFrame); - } - - stopAudioAnalysis() { - if (this.analysisFrame) { - cancelAnimationFrame(this.analysisFrame); - this.analysisFrame = null; - } - } - - async process() { - if (this.audioChunks.length === 0) { - this.status = Status.LISTENING; - return; - } - - const audioBlob = new Blob(this.audioChunks, { type: "audio/wav" }); - const base64 = await this.convertBlobToBase64Wav(audioBlob); - - try { - const result = await sendJsonData("/transcribe", { audio: base64 }); - - const text = this.filterResult(result.text || ""); - - if (text) { - console.log("Transcription:", result.text); - await this.updateCallback(result.text, true); - } - } catch (error) { - window.toastFetchError("Transcription error", error); - console.error("Transcription error:", error); - } finally { - this.audioChunks = []; - this.status = Status.LISTENING; - } - } - - convertBlobToBase64Wav(audioBlob) { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - - // Read the Blob as a Data URL - reader.onloadend = () => { - const base64Data = reader.result.split(",")[1]; // Extract Base64 data - resolve(base64Data); - }; - - reader.onerror = (error) => { - reject(error); - }; - - reader.readAsDataURL(audioBlob); // Start reading the Blob - }); - } - - filterResult(text) { - text = text.trim(); - let ok = false; - while (!ok) { - if (!text) break; - if (text[0] === "{" && text[text.length - 1] === "}") break; - if (text[0] === "(" && text[text.length - 1] === ")") break; - if (text[0] === "[" && text[text.length - 1] === "]") break; - ok = true; - } - if (ok) return text; - else console.log(`Discarding transcription: ${text}`); - } -} - -// Initialize and handle click events -async function initializeMicrophoneInput() { - window.microphoneInput = microphoneInput = new MicrophoneInput( - async (text, isFinal) => { - if (isFinal) { - updateChatInput(text); - if (!microphoneInput.messageSent) { - microphoneInput.messageSent = true; - await sendMessage(); - } - } - } - ); - microphoneInput.status = Status.ACTIVATING; - - return await microphoneInput.initialize(); -} - -microphoneButton.addEventListener("click", async () => { - if (isProcessingClick) return; - isProcessingClick = true; - - const hasPermission = await requestMicrophonePermission(); - if (!hasPermission) return; - - try { - if (!microphoneInput && !(await initializeMicrophoneInput())) { - return; - } - - // Simply toggle between INACTIVE and LISTENING states - microphoneInput.status = - microphoneInput.status === Status.INACTIVE || - microphoneInput.status === Status.ACTIVATING - ? Status.LISTENING - : Status.INACTIVE; - } finally { - setTimeout(() => { - isProcessingClick = false; - }, 300); - } -}); - -// Some error handling for microphone input -async function requestMicrophonePermission() { - try { - await navigator.mediaDevices.getUserMedia({ audio: true }); - return true; - } catch (err) { - console.error("Error accessing microphone:", err); - toast( - "Microphone access denied. Please enable microphone access in your browser settings.", - "error" - ); - return false; - } -} - -class Speech { - constructor() { - this.synth = window.speechSynthesis; - this.utterance = null; - } - - stripEmojis(str) { - // Remove emojis - return str.replace( - /([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, - "" - ); - } - - normalizeWhitespace(str) { - // Replace multiple spaces with a single space while preserving newlines - str = str.split('\n').map(line => { - return line.replace(/\s+/g, ' ').trim(); - }).join('\n'); - - return str.trim(); - } - - speak(text) { - console.log("Starting speak:", text); - // Stop any current utterance - this.stop(); - - // Process text for speech synthesis - text = this.stripEmojis(text); // remove emojis - text = this.normalizeWhitespace(text); // normalize whitespace while preserving newlines - text = this.replaceURLs(text); - text = this.replaceGuids(text); - - console.log("Speaking text:", text); - this.utterance = new SpeechSynthesisUtterance(text); - - // Speak the new utterance - this.synth.speak(this.utterance); - } - - replaceURLs(text) { - const urlRegex = - /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b(www\.)[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b[-A-Z0-9+&@#\/%?=~_|!:,.;]*\.(?:[A-Z]{2,})[-A-Z0-9+&@#\/%?=~_|])/gi; - return text.replace(urlRegex, (url) => { - let text = url; - // if contains ://, split by it - if (text.includes("://")) text = text.split("://")[1]; - // if contains /, split by it - if (text.includes("/")) text = text.split("/")[0]; - - // if contains ., split by it - if (text.includes(".")) { - const doms = text.split("."); - //up to last two - return doms[doms.length - 2] + "." + doms[doms.length - 1]; - } else { - return text; - } - }); - } - - replaceGuids(text) { - const guidRegex = - /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g; - return text.replace(guidRegex, ""); - } - - replaceNonText(text) { - const nonTextRegex = /\w[^\w\s]*\w(?=\s|$)|[^\w\s]+/g; - text = text.replace(nonTextRegex, (match) => { - return ``; - }); - const longStringRegex = /\S{25,}/g; - text = text.replace(longStringRegex, (match) => { - return ``; - }); - return text; - } - - stop() { - if (this.isSpeaking()) { - this.synth.cancel(); - } - } - - isSpeaking() { - return this.synth?.speaking || false; - } -} - -export const speech = new Speech(); -window.speech = speech; - -// Add event listener for settings changes -document.addEventListener("settings-updated", loadMicSettings); diff --git a/webui/public/speech.svg b/webui/public/speech.svg new file mode 100644 index 000000000..b5a2de211 --- /dev/null +++ b/webui/public/speech.svg @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file