diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23537df940..00f78b071f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,10 +177,10 @@ jobs: run: | gnome-keyring-daemon --components=secrets --daemonize --unlock <<< 'foobar' export CARGO_INCREMENTAL=0 - cargo test -p goose --no-default-features --features ${{ matrix.tls-feature }},code-mode - cargo test -p goose-providers --no-default-features --features ${{ matrix.tls-feature }} - cargo test -p goose-cli --no-default-features --features ${{ matrix.tls-feature }},code-mode -- --skip scenario_tests::scenarios::tests - cargo test -p goose-cli --no-default-features --features ${{ matrix.tls-feature }},code-mode --jobs 1 scenario_tests::scenarios::tests + cargo test -p goose --no-default-features --features ${{ matrix.tls-feature }},code-mode,live-voice + cargo test -p goose-providers --no-default-features --features ${{ matrix.tls-feature }},live-websocket + cargo test -p goose-cli --no-default-features --features ${{ matrix.tls-feature }},code-mode,live-voice -- --skip scenario_tests::scenarios::tests + cargo test -p goose-cli --no-default-features --features ${{ matrix.tls-feature }},code-mode,live-voice --jobs 1 scenario_tests::scenarios::tests env: RUST_MIN_STACK: 8388608 diff --git a/.gitignore b/.gitignore index 98a576419d..0fd03cd519 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,5 @@ gooseselftest/ /crates/goose-sdk/examples/uniffi/kotlin/.gradle/ /crates/goose-sdk/examples/uniffi/kotlin/.kotlin/ /crates/goose-sdk/examples/uniffi/kotlin/build/ + +live-in-desktop-design diff --git a/crates/goose-acp-macros/src/lib.rs b/crates/goose-acp-macros/src/lib.rs index beaa3e62bb..96c4616d18 100644 --- a/crates/goose-acp-macros/src/lib.rs +++ b/crates/goose-acp-macros/src/lib.rs @@ -25,7 +25,8 @@ use syn::{ /// /// # Handler signatures /// -/// Handlers may take zero or one parameter (beyond `&self`): +/// Handlers may take zero or one request parameter (beyond `&self`). A handler +/// that needs the active ACP connection may take it before the request: /// /// ```ignore /// // No params — called for requests with no/empty params @@ -35,6 +36,14 @@ use syn::{ /// // Typed params — JSON params auto-deserialized /// #[custom_method(GetSessionRequest)] /// async fn on_get_session(&self, req: GetSessionRequest) -> Result { .. } +/// +/// // Connection and typed request +/// #[custom_method(StartCallRequest)] +/// async fn on_start_call( +/// &self, +/// cx: &agent_client_protocol::ConnectionTo, +/// req: StartCallRequest, +/// ) -> Result { .. } /// ``` /// /// The return type must be `Result` where `T: Serialize`. @@ -64,7 +73,13 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream { if let Some(req_type) = request_type { let fn_ident = method.sig.ident.clone(); - let param_type = extract_param_type(&method.sig); + let parameter_types = extract_parameter_types(&method.sig); + let (uses_connection, param_type) = match parameter_types.as_slice() { + [] => (false, None), + [request] => (false, Some(request.clone())), + [_connection, request] => (true, Some(request.clone())), + _ => panic!("custom method handlers accept at most a connection and request"), + }; let return_type = extract_return_type(&method.sig); let ok_type = extract_result_ok_type(&method.sig); @@ -72,6 +87,7 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream { request_type: req_type, fn_ident, param_type, + uses_connection, return_type, ok_type, }); @@ -85,6 +101,11 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream { .map(|route| { let req_type = &route.request_type; let fn_ident = &route.fn_ident; + let call = if route.uses_connection { + quote! { self.#fn_ident(cx, req).await? } + } else { + quote! { self.#fn_ident(req).await? } + }; match &route.param_type { Some(_) => { @@ -92,7 +113,7 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream { if <#req_type as agent_client_protocol::JsonRpcMessage>::matches_method(method) { let req = serde_json::from_value(params) .map_err(|e| agent_client_protocol::Error::invalid_params().data(e.to_string()))?; - let result = self.#fn_ident(req).await?; + let result = #call; return serde_json::to_value(&result) .map_err(|e| agent_client_protocol::Error::internal_error().data(e.to_string())); } @@ -183,7 +204,8 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream { // Generate the handle_custom_request method. let dispatcher = quote! { async fn handle_custom_request( - &self, + self: &std::sync::Arc, + cx: &agent_client_protocol::ConnectionTo, method: &str, params: serde_json::Value, ) -> Result { @@ -216,24 +238,26 @@ struct Route { request_type: Type, fn_ident: syn::Ident, param_type: Option, + uses_connection: bool, #[allow(dead_code)] return_type: Option, ok_type: Option, } -/// Extract the type of the first non-self parameter, if any. -fn extract_param_type(sig: &syn::Signature) -> Option { - for input in &sig.inputs { - if let FnArg::Typed(pat_type) = input { - if let Pat::Ident(pat_ident) = &*pat_type.pat { - if pat_ident.ident == "self" { - continue; +fn extract_parameter_types(sig: &syn::Signature) -> Vec { + sig.inputs + .iter() + .filter_map(|input| match input { + FnArg::Typed(pat_type) => { + if matches!(&*pat_type.pat, Pat::Ident(pat) if pat.ident == "self") { + None + } else { + Some((*pat_type.ty).clone()) } } - return Some((*pat_type.ty).clone()); - } - } - None + FnArg::Receiver(_) => None, + }) + .collect() } /// Extract the full return type (e.g. `Result`). diff --git a/crates/goose-cli/Cargo.toml b/crates/goose-cli/Cargo.toml index 8553a52e97..200ff893c9 100644 --- a/crates/goose-cli/Cargo.toml +++ b/crates/goose-cli/Cargo.toml @@ -100,11 +100,13 @@ default = [ "nostr", "otel", "rustls-tls", + "live-voice", "system-keyring", "update", ] roaming = ["dep:goose-roaming", "dep:qrcode", "dep:fs2", "dep:agent-client-protocol"] code-mode = ["goose/code-mode"] +live-voice = ["goose/live-voice"] local-inference = ["goose/local-inference"] aws-providers = ["goose/aws-providers"] cuda = ["goose/cuda", "local-inference"] @@ -115,7 +117,7 @@ nostr = ["goose/nostr"] otel = ["goose/otel"] system-keyring = ["goose/system-keyring"] update = ["dep:sigstore-verify", "dep:snap"] -portable-default = ["rustls-tls", "aws-providers", "telemetry", "otel"] +portable-default = ["rustls-tls", "live-voice", "aws-providers", "telemetry", "otel"] # disables the update command disable-update = [] rustls-tls = [ diff --git a/crates/goose-providers/Cargo.toml b/crates/goose-providers/Cargo.toml index 7cb33f37e6..7bc49299f5 100644 --- a/crates/goose-providers/Cargo.toml +++ b/crates/goose-providers/Cargo.toml @@ -13,7 +13,6 @@ workspace = true [features] default = [] -test-utils = [] local-inference = ["dep:goose-local-inference"] local-inference-hf-hub = ["local-inference", "goose-local-inference/hf-hub"] cuda = ["local-inference", "goose-local-inference/cuda"] diff --git a/crates/goose-providers/src/live_voice_provider.rs b/crates/goose-providers/src/live_voice_provider.rs index 0d3f0940d0..9b1ef56cb5 100644 --- a/crates/goose-providers/src/live_voice_provider.rs +++ b/crates/goose-providers/src/live_voice_provider.rs @@ -38,38 +38,51 @@ fn valid_sdp(sdp: &str) -> bool { #[async_trait] pub trait ProviderConnection: Send { async fn next_event(&mut self) -> ProviderConnectionEvent; + async fn send_delegation_update( + &mut self, + update: DelegationUpdate, + ) -> Result; async fn stop(&mut self) -> Result<()>; } +pub enum DelegationUpdateDelivery { + Delivered, + Undelivered, +} + #[derive(Debug, PartialEq)] pub enum ProviderConnectionEvent { TranscriptDelta { event_id: String, role: rmcp::model::Role, text: String, + start_ms: u64, + end_ms: u64, + }, + DelegationRequested { + event_id: String, + delegation_id: String, + offset_ms: u64, }, ReceiverLagged, Closed, Failed, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DelegationUpdate { + pub provider_delegation_id: String, + pub text: String, +} + #[derive(Clone, Debug, PartialEq)] pub struct LiveVoiceInputMessage { pub role: rmcp::model::Role, pub text: String, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum LiveVoiceProviderAvailability { - Ready, - Disabled, - Unavailable, -} - #[async_trait] pub trait LiveVoiceProvider: Send + Sync { - fn availability(&self) -> LiveVoiceProviderAvailability; - async fn start( &self, offer: WebRtcOffer, @@ -89,137 +102,3 @@ mod tests { assert!(WebRtcAnswer::new("answer".into()).is_some()); } } - -#[cfg(any(test, feature = "test-utils"))] -pub mod fake { - use super::*; - use tokio::sync::{mpsc, oneshot}; - - pub fn provider_channel() -> ( - std::sync::Arc, - mpsc::UnboundedReceiver, - ) { - provider_channel_with_availability(LiveVoiceProviderAvailability::Ready) - } - - pub fn provider_channel_with_availability( - availability: LiveVoiceProviderAvailability, - ) -> ( - std::sync::Arc, - mpsc::UnboundedReceiver, - ) { - let (start_tx, start_rx) = mpsc::unbounded_channel(); - ( - std::sync::Arc::new(FakeLiveVoiceProvider { - availability, - start_tx, - }), - start_rx, - ) - } - - pub struct FakeLiveVoiceProvider { - availability: LiveVoiceProviderAvailability, - start_tx: mpsc::UnboundedSender, - } - - #[async_trait] - impl LiveVoiceProvider for FakeLiveVoiceProvider { - fn availability(&self) -> LiveVoiceProviderAvailability { - self.availability - } - - async fn start( - &self, - offer: WebRtcOffer, - input_messages: Vec, - ) -> Result<(WebRtcAnswer, Box)> { - let (response_tx, response_rx) = oneshot::channel(); - self.start_tx - .send(FakeStartRequest { - offer, - input_messages, - response_tx, - }) - .map_err(|_| anyhow::anyhow!("fake provider driver dropped"))?; - response_rx - .await - .map_err(|_| anyhow::anyhow!("fake provider start response dropped"))? - } - } - - pub struct FakeStartRequest { - pub offer: WebRtcOffer, - pub input_messages: Vec, - response_tx: oneshot::Sender)>>, - } - - impl FakeStartRequest { - pub fn accept(self, answer: WebRtcAnswer) -> Result { - let (stop_request_tx, stop_request_rx) = mpsc::unbounded_channel(); - let (event_tx, event_rx) = mpsc::unbounded_channel(); - self.response_tx - .send(Ok(( - answer, - Box::new(FakeProviderConnection { - stop_request_tx, - event_rx, - }), - ))) - .map_err(|_| anyhow::anyhow!("fake provider start caller dropped"))?; - Ok(FakeConnectionDriver { - stop_request_rx, - event_tx, - }) - } - - pub fn reject(self, message: impl Into) -> Result<()> { - self.response_tx - .send(Err(anyhow::anyhow!(message.into()))) - .map_err(|_| anyhow::anyhow!("fake provider start caller dropped")) - } - } - - pub struct FakeConnectionDriver { - stop_request_rx: mpsc::UnboundedReceiver>>, - event_tx: mpsc::UnboundedSender, - } - - impl FakeConnectionDriver { - pub async fn next_stop_request(&mut self) -> Option>> { - self.stop_request_rx.recv().await - } - - pub fn send_event(&self, event: ProviderConnectionEvent) -> Result<()> { - self.event_tx - .send(event) - .map_err(|_| anyhow::anyhow!("fake provider event receiver dropped")) - } - } - - struct FakeProviderConnection { - stop_request_tx: mpsc::UnboundedSender>>, - event_rx: mpsc::UnboundedReceiver, - } - - #[async_trait] - impl ProviderConnection for FakeProviderConnection { - async fn next_event(&mut self) -> ProviderConnectionEvent { - self.event_rx - .recv() - .await - .unwrap_or(ProviderConnectionEvent::Failed) - } - - async fn stop(&mut self) -> Result<()> { - let (response_tx, response_rx) = oneshot::channel(); - self.stop_request_tx - .send(response_tx) - .map_err(|_| anyhow::anyhow!("fake provider driver dropped"))?; - response_rx - .await - .map_err(|_| anyhow::anyhow!("fake provider stop response dropped"))? - .map_err(anyhow::Error::msg) - } - } -} diff --git a/crates/goose-providers/src/openai_live_voice_provider.rs b/crates/goose-providers/src/openai_live_voice_provider.rs index a500aa34e9..e4257744d0 100644 --- a/crates/goose-providers/src/openai_live_voice_provider.rs +++ b/crates/goose-providers/src/openai_live_voice_provider.rs @@ -3,108 +3,69 @@ use crate::{ live::{LiveSessionEndReason, LiveSessionEvent}, live_voice_provider::{ - LiveVoiceInputMessage, LiveVoiceProvider, LiveVoiceProviderAvailability, + DelegationUpdate, DelegationUpdateDelivery, LiveVoiceInputMessage, LiveVoiceProvider, ProviderConnection, ProviderConnectionEvent, WebRtcAnswer, WebRtcOffer, }, openai_live::{ - ConnectedOpenAiLiveSession, OpenAiLiveClient, OpenAiLiveEvent, OpenAiLiveEventKind, - OpenAiLiveMessage, OpenAiLiveMessageRole, OpenAiLiveSessionConfig, OpenAiLiveSessionId, + ConnectedOpenAiLiveSession, OpenAiLiveClient, OpenAiLiveContext, OpenAiLiveContextChannel, + OpenAiLiveDelegationId, OpenAiLiveEvent, OpenAiLiveEventKind, OpenAiLiveMessage, + OpenAiLiveMessageRole, OpenAiLiveSessionConfig, OpenAiLiveSessionId, }, }; use anyhow::{bail, Result}; use async_trait::async_trait; -use std::time::Duration; +use std::{collections::VecDeque, time::Duration}; use tokio::{ sync::broadcast::error::RecvError, time::{sleep_until, timeout, timeout_at, Instant}, }; -pub const OPENAI_LIVE_VOICE_GATE_ENV: &str = "GOOSE_LIVE_VOICE_ENABLED"; -pub const OPENAI_LIVE_MODEL_ENV: &str = "GOOSE_LIVE_VOICE_MODEL"; -pub const OPENAI_LIVE_VOICE_ENV: &str = "GOOSE_LIVE_VOICE"; -pub const OPENAI_LIVE_API_KEY_ENV: &str = "OPENAI_API_KEY"; -pub const DEFAULT_OPENAI_LIVE_MODEL: &str = "gpt-live-1"; -pub const DEFAULT_OPENAI_LIVE_VOICE: &str = "marin"; +const OPENAI_LIVE_MODEL: &str = "gpt-live-1"; const HTTP_SETUP_TIMEOUT: Duration = Duration::from_secs(15); const SIDEBAND_ATTACH_TIMEOUT: Duration = Duration::from_secs(10); - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct OpenAiLiveVoiceConfig { - pub enabled: bool, - pub model: String, - pub voice: String, -} - -impl OpenAiLiveVoiceConfig { - pub fn from_env() -> Self { - Self { - enabled: std::env::var(OPENAI_LIVE_VOICE_GATE_ENV) - .is_ok_and(|value| value.eq_ignore_ascii_case("true") || value == "1"), - model: std::env::var(OPENAI_LIVE_MODEL_ENV) - .unwrap_or_else(|_| DEFAULT_OPENAI_LIVE_MODEL.into()), - voice: std::env::var(OPENAI_LIVE_VOICE_ENV) - .unwrap_or_else(|_| DEFAULT_OPENAI_LIVE_VOICE.into()), - } - } -} - +const DELEGATION_UPDATE_ACK_TIMEOUT: Duration = Duration::from_secs(10); pub struct OpenAiLiveVoiceProvider { - client: Option, - config: OpenAiLiveVoiceConfig, + client: OpenAiLiveClient, + voice: String, + instructions: String, + delivery_failure_instructions: String, } impl OpenAiLiveVoiceProvider { - pub fn from_env() -> Self { - let config = OpenAiLiveVoiceConfig::from_env(); - let client = std::env::var(OPENAI_LIVE_API_KEY_ENV) - .ok() - .filter(|key| !key.trim().is_empty()) - .map(OpenAiLiveClient::new); - Self { client, config } - } - - pub fn new(api_key: impl Into, config: OpenAiLiveVoiceConfig) -> Result { + pub fn new( + api_key: impl Into, + voice: String, + instructions: String, + delivery_failure_instructions: String, + ) -> Result { let api_key = api_key.into(); if api_key.trim().is_empty() { bail!("OpenAI API key is empty"); } + if voice.trim().is_empty() { + bail!("OpenAI Live voice is empty"); + } Ok(Self { - client: Some(OpenAiLiveClient::new(api_key)), - config, + client: OpenAiLiveClient::new(api_key), + voice, + instructions, + delivery_failure_instructions, }) } } #[async_trait] impl LiveVoiceProvider for OpenAiLiveVoiceProvider { - fn availability(&self) -> LiveVoiceProviderAvailability { - if !self.config.enabled { - LiveVoiceProviderAvailability::Disabled - } else if self.client.is_none() { - LiveVoiceProviderAvailability::Unavailable - } else { - LiveVoiceProviderAvailability::Ready - } - } - async fn start( &self, offer: WebRtcOffer, input_messages: Vec, ) -> Result<(WebRtcAnswer, Box)> { - if !self.config.enabled { - bail!("OpenAI Live voice is disabled"); - } - let client = self - .client - .as_ref() - .ok_or_else(|| anyhow::anyhow!("OpenAI Live credentials are unavailable"))? - .clone(); let config = OpenAiLiveSessionConfig { - model: self.config.model.clone(), - instructions: String::new(), - voice: Some(self.config.voice.clone()), + model: OPENAI_LIVE_MODEL.into(), + instructions: self.instructions.clone(), + voice: Some(self.voice.clone()), input_messages: input_messages .into_iter() .map(|message| OpenAiLiveMessage { @@ -119,25 +80,44 @@ impl LiveVoiceProvider for OpenAiLiveVoiceProvider { }; let negotiation = timeout( HTTP_SETUP_TIMEOUT, - client.webrtc(config).negotiate(offer.into_sdp()), + self.client.webrtc(config).negotiate(offer.into_sdp()), ) .await .map_err(|_| anyhow::anyhow!("OpenAI Live HTTP setup timed out"))??; let session_id = negotiation.session_id; let answer = WebRtcAnswer::new(negotiation.answer_sdp) .ok_or_else(|| anyhow::anyhow!("OpenAI Live returned an invalid WebRTC answer"))?; - let sideband = connect_sideband(&client, session_id).await?; - Ok((answer, Box::new(OpenAiProviderConnection { sideband }))) + let sideband = connect_sideband(&self.client, session_id).await?; + Ok(( + answer, + Box::new(OpenAiProviderConnection { + sideband, + pending_events: VecDeque::new(), + delivery_failure_instructions: self.delivery_failure_instructions.clone(), + }), + )) } } struct OpenAiProviderConnection { sideband: ConnectedOpenAiLiveSession, + pending_events: VecDeque, + delivery_failure_instructions: String, +} + +#[derive(Debug, PartialEq)] +enum AppendContextOutcome { + Accepted, + Rejected, + TimedOut, } #[async_trait] impl ProviderConnection for OpenAiProviderConnection { async fn next_event(&mut self) -> ProviderConnectionEvent { + if let Some(event) = self.pending_events.pop_front() { + return event; + } loop { if let Some(event) = provider_connection_event(self.sideband.recv().await) { return event; @@ -145,11 +125,103 @@ impl ProviderConnection for OpenAiProviderConnection { } } + async fn send_delegation_update( + &mut self, + update: DelegationUpdate, + ) -> Result { + match self + .append_context( + Some(OpenAiLiveDelegationId(update.provider_delegation_id)), + update.text, + ) + .await? + { + AppendContextOutcome::Accepted => Ok(DelegationUpdateDelivery::Delivered), + AppendContextOutcome::Rejected => { + let _ = self + .append_context(None, self.delivery_failure_instructions.clone()) + .await?; + Ok(DelegationUpdateDelivery::Undelivered) + } + AppendContextOutcome::TimedOut => Ok(DelegationUpdateDelivery::Undelivered), + } + } + async fn stop(&mut self) -> Result<()> { self.sideband.close().await } } +impl OpenAiProviderConnection { + async fn append_context( + &mut self, + delegation_id: Option, + text: String, + ) -> Result { + let event_id = format!("event_{}", uuid::Uuid::new_v4()); + self.sideband + .send(crate::openai_live::OpenAiLiveCommand::AppendContext { + event_id: event_id.clone(), + delegation_id, + context: OpenAiLiveContext { + text, + channel: OpenAiLiveContextChannel::Commentary, + }, + }) + .await?; + + match timeout(DELEGATION_UPDATE_ACK_TIMEOUT, async { + loop { + let event = self.sideband.recv().await; + match append_context_response(&event, &event_id) { + Some(outcome) => return Ok(outcome), + None => match provider_connection_event(event) { + Some( + ProviderConnectionEvent::Closed + | ProviderConnectionEvent::Failed + | ProviderConnectionEvent::ReceiverLagged, + ) => { + bail!("OpenAI Live session ended while delivering a delegation update") + } + Some(event) => self.pending_events.push_back(event), + None => {} + }, + } + } + }) + .await + { + Ok(outcome) => outcome, + Err(_) => Ok(AppendContextOutcome::TimedOut), + } + } +} + +fn append_context_response( + event: &std::result::Result, RecvError>, + event_id: &str, +) -> Option { + match event { + Ok(LiveSessionEvent::Message(OpenAiLiveEvent { + kind: + OpenAiLiveEventKind::ContextAppended { + client_event_id: Some(client_event_id), + .. + }, + .. + })) if client_event_id == event_id => Some(AppendContextOutcome::Accepted), + Ok(LiveSessionEvent::Message(OpenAiLiveEvent { + kind: + OpenAiLiveEventKind::Error { + client_event_id: Some(client_event_id), + .. + }, + .. + })) if client_event_id == event_id => Some(AppendContextOutcome::Rejected), + _ => None, + } +} + fn provider_connection_event( event: std::result::Result, RecvError>, ) -> Option { @@ -159,12 +231,31 @@ fn provider_connection_event( event_id, role, delta, + start_ms, + end_ms, .. } => Some(ProviderConnectionEvent::TranscriptDelta { event_id, role, text: delta, + start_ms, + end_ms, }), + OpenAiLiveEventKind::DelegationCreated { + event_id, + delegation, + .. + } if matches!( + &delegation.target, + crate::openai_live::OpenAiLiveDelegationTarget::Client + ) => + { + Some(ProviderConnectionEvent::DelegationRequested { + event_id, + delegation_id: delegation.id.0, + offset_ms: delegation.offset_ms, + }) + } OpenAiLiveEventKind::SessionClosed { .. } => Some(ProviderConnectionEvent::Closed), _ => None, }, @@ -209,20 +300,9 @@ mod tests { use super::*; #[test] - fn configuration_is_gated() { - let provider = OpenAiLiveVoiceProvider::new( - "key", - OpenAiLiveVoiceConfig { - enabled: false, - model: DEFAULT_OPENAI_LIVE_MODEL.into(), - voice: DEFAULT_OPENAI_LIVE_VOICE.into(), - }, - ) - .unwrap(); - assert_eq!( - provider.availability(), - LiveVoiceProviderAvailability::Disabled - ); + fn configuration_must_be_valid() { + assert!(OpenAiLiveVoiceProvider::new("", "voice".into(), "".into(), "".into()).is_err()); + assert!(OpenAiLiveVoiceProvider::new("key", String::new(), "".into(), "".into()).is_err()); } #[test] @@ -247,6 +327,8 @@ mod tests { event_id: "event_transcript_1".into(), role: rmcp::model::Role::Assistant, text: "hello".into(), + start_ms: 10, + end_ms: 20, }), ), ( @@ -305,4 +387,39 @@ mod tests { assert_eq!(provider_connection_event(event), expected); } } + + #[test] + fn correlates_delegation_update_responses() { + let message = |kind| { + Ok(LiveSessionEvent::Message(OpenAiLiveEvent { + kind, + raw: None, + })) + }; + let accepted = message(OpenAiLiveEventKind::ContextAppended { + event_id: "event_accepted".into(), + channel: OpenAiLiveContextChannel::Commentary, + client_event_id: Some("client_event".into()), + start_ms: 10, + end_ms: 20, + }); + let rejected = message(OpenAiLiveEventKind::Error { + event_id: "event_rejected".into(), + error_type: "invalid_request_error".into(), + code: "invalid_delegation".into(), + message: "delegation is stale".into(), + parameter: Some("delegation_id".into()), + client_event_id: Some("client_event".into()), + }); + + assert_eq!( + append_context_response(&accepted, "client_event"), + Some(AppendContextOutcome::Accepted) + ); + assert_eq!( + append_context_response(&rejected, "client_event"), + Some(AppendContextOutcome::Rejected) + ); + assert_eq!(append_context_response(&accepted, "other_event"), None); + } } diff --git a/crates/goose-sdk-types/src/custom_notifications.rs b/crates/goose-sdk-types/src/custom_notifications.rs index 49da57ac2d..d850a67d94 100644 --- a/crates/goose-sdk-types/src/custom_notifications.rs +++ b/crates/goose-sdk-types/src/custom_notifications.rs @@ -26,13 +26,29 @@ pub struct GooseSessionNotification { "mapping": { "usage_update": "#/$defs/SessionUsageUpdate", "status_message": "#/$defs/StatusMessageUpdate", - "message_usage": "#/$defs/MessageUsageUpdate" + "message_usage": "#/$defs/MessageUsageUpdate", + "live_voice_interaction_ended": "#/$defs/LiveVoiceInteractionEndedUpdate" } }))] pub enum GooseSessionUpdate { UsageUpdate(SessionUsageUpdate), StatusMessage(StatusMessageUpdate), MessageUsage(MessageUsageUpdate), + LiveVoiceInteractionEnded(LiveVoiceInteractionEndedUpdate), +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct LiveVoiceInteractionEndedUpdate { + pub interaction_id: String, + pub outcome: LiveVoiceInteractionOutcome, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LiveVoiceInteractionOutcome { + Stopped, + Failed, } /// Dedicated provider notification for OAuth device-code flow. @@ -194,6 +210,33 @@ mod tests { ); } + #[test] + fn live_voice_interaction_ended_serializes_to_expected_wire_shape() { + let notification = GooseSessionNotification { + session_id: "s1".to_string(), + update: GooseSessionUpdate::LiveVoiceInteractionEnded( + LiveVoiceInteractionEndedUpdate { + interaction_id: "live_opaque".to_string(), + outcome: LiveVoiceInteractionOutcome::Failed, + }, + ), + }; + + let value = serde_json::to_value(notification).unwrap(); + + assert_eq!( + value, + json!({ + "sessionId": "s1", + "update": { + "sessionUpdate": "live_voice_interaction_ended", + "interactionId": "live_opaque", + "outcome": "failed" + } + }) + ); + } + #[test] fn message_usage_serializes_to_expected_wire_shape() { let notification = GooseSessionNotification { diff --git a/crates/goose-sdk-types/src/custom_requests.rs b/crates/goose-sdk-types/src/custom_requests.rs index 50ddc13283..9f2cd902a1 100644 --- a/crates/goose-sdk-types/src/custom_requests.rs +++ b/crates/goose-sdk-types/src/custom_requests.rs @@ -1,4 +1,6 @@ -use agent_client_protocol::schema::v1::{AvailableCommand, ContentBlock, McpServer, SessionInfo}; +use agent_client_protocol::schema::v1::{ + AvailableCommand, ContentBlock, McpServer, Meta, SessionInfo, +}; use agent_client_protocol::{JsonRpcRequest, JsonRpcResponse}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -239,7 +241,65 @@ pub struct SteerSessionResponse { pub message_id: String, } +/// Ask whether a new or existing chat can enter direct Live voice. +#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)] +#[request( + method = "_goose/unstable/session/live-voice/availability", + response = LiveVoiceAvailabilityResponse +)] +#[serde(rename_all = "camelCase")] +pub struct LiveVoiceAvailabilityRequest { + pub session_id: Option, + #[serde(default, rename = "_meta", skip_serializing_if = "Option::is_none")] + pub meta: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LiveVoiceStatus { + Ready, + Unavailable, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)] +#[serde(rename_all = "camelCase")] +pub struct LiveVoiceAvailabilityResponse { + pub status: LiveVoiceStatus, + pub message: String, +} + /// Get a diagnostic report for a session. +#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)] +#[request( + method = "_goose/unstable/session/live-voice/start", + response = LiveVoiceStartResponse +)] +#[serde(rename_all = "camelCase")] +pub struct LiveVoiceStartRequest { + pub session_id: String, + pub offer_sdp: String, + #[serde(default, rename = "_meta", skip_serializing_if = "Option::is_none")] + pub meta: Option, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)] +#[serde(rename_all = "camelCase")] +pub struct LiveVoiceStartResponse { + pub interaction_id: String, + pub answer_sdp: String, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)] +#[request( + method = "_goose/unstable/session/live-voice/stop", + response = EmptyResponse +)] +#[serde(rename_all = "camelCase")] +pub struct LiveVoiceStopRequest { + pub session_id: String, + pub interaction_id: String, +} + #[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)] #[request( method = "_goose/unstable/diagnostics/get", diff --git a/crates/goose/Cargo.toml b/crates/goose/Cargo.toml index 04b043bb45..c6236680c5 100644 --- a/crates/goose/Cargo.toml +++ b/crates/goose/Cargo.toml @@ -20,6 +20,7 @@ otel = [ "dep:opentelemetry-stdout", ] code-mode = ["dep:pctx_code_mode"] +live-voice = ["goose-providers/live-websocket"] local-inference = [ "dep:candle-core", "dep:candle-nn", @@ -77,7 +78,7 @@ native-tls = [ "goose-providers/native-tls", ] system-keyring = ["dep:keyring"] -portable-default = ["rustls-tls", "aws-providers", "telemetry", "otel"] +portable-default = ["rustls-tls", "live-voice", "aws-providers", "telemetry", "otel"] nostr = ["dep:nostr", "dep:nostr-sdk"] [lints] @@ -256,7 +257,7 @@ libc = { version = "0.2.182", default-features = false, features = ["std"] } serial_test = { workspace = true } mockall = { version = "0.15", default-features = false } wiremock = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } tokio-util = { workspace = true, features = ["compat"] } dotenvy = { workspace = true } ctor = { version = "1.0", default-features = false, features = ["proc_macro"] } diff --git a/crates/goose/acp-meta.json b/crates/goose/acp-meta.json index d1c3a75006..ec89d600a8 100644 --- a/crates/goose/acp-meta.json +++ b/crates/goose/acp-meta.json @@ -65,6 +65,21 @@ "requestType": "SteerSessionRequest_unstable", "responseType": "SteerSessionResponse_unstable" }, + { + "method": "_goose/unstable/session/live-voice/availability", + "requestType": "LiveVoiceAvailabilityRequest_unstable", + "responseType": "LiveVoiceAvailabilityResponse_unstable" + }, + { + "method": "_goose/unstable/session/live-voice/start", + "requestType": "LiveVoiceStartRequest_unstable", + "responseType": "LiveVoiceStartResponse_unstable" + }, + { + "method": "_goose/unstable/session/live-voice/stop", + "requestType": "LiveVoiceStopRequest_unstable", + "responseType": "EmptyResponse" + }, { "method": "_goose/unstable/diagnostics/get", "requestType": "DiagnosticsGetRequest_unstable", diff --git a/crates/goose/acp-schema.json b/crates/goose/acp-schema.json index 486596aa31..363687391d 100644 --- a/crates/goose/acp-schema.json +++ b/crates/goose/acp-schema.json @@ -1299,6 +1299,110 @@ "x-side": "agent", "x-method": "_goose/unstable/session/steer" }, + "LiveVoiceAvailabilityRequest_unstable": { + "type": "object", + "properties": { + "sessionId": { + "type": [ + "string", + "null" + ] + }, + "_meta": { + "type": [ + "object", + "null" + ], + "additionalProperties": {} + } + }, + "description": "Ask whether a new or existing chat can enter direct Live voice.", + "x-side": "agent", + "x-method": "_goose/unstable/session/live-voice/availability" + }, + "LiveVoiceAvailabilityResponse_unstable": { + "type": "object", + "properties": { + "status": { + "$ref": "#/$defs/LiveVoiceStatus" + }, + "message": { + "type": "string" + } + }, + "required": [ + "status", + "message" + ], + "x-side": "agent", + "x-method": "_goose/unstable/session/live-voice/availability" + }, + "LiveVoiceStatus": { + "type": "string", + "enum": [ + "ready", + "unavailable" + ] + }, + "LiveVoiceStartRequest_unstable": { + "type": "object", + "properties": { + "sessionId": { + "type": "string" + }, + "offerSdp": { + "type": "string" + }, + "_meta": { + "type": [ + "object", + "null" + ], + "additionalProperties": {} + } + }, + "required": [ + "sessionId", + "offerSdp" + ], + "description": "Get a diagnostic report for a session.", + "x-side": "agent", + "x-method": "_goose/unstable/session/live-voice/start" + }, + "LiveVoiceStartResponse_unstable": { + "type": "object", + "properties": { + "interactionId": { + "type": "string" + }, + "answerSdp": { + "type": "string" + } + }, + "required": [ + "interactionId", + "answerSdp" + ], + "x-side": "agent", + "x-method": "_goose/unstable/session/live-voice/start" + }, + "LiveVoiceStopRequest_unstable": { + "type": "object", + "properties": { + "sessionId": { + "type": "string" + }, + "interactionId": { + "type": "string" + } + }, + "required": [ + "sessionId", + "interactionId" + ], + "x-side": "agent", + "x-method": "_goose/unstable/session/live-voice/stop" + }, "DiagnosticsGetRequest_unstable": { "type": "object", "properties": { @@ -1313,7 +1417,6 @@ "required": [ "sessionId" ], - "description": "Get a diagnostic report for a session.", "x-side": "agent", "x-method": "_goose/unstable/diagnostics/get" }, @@ -6689,6 +6792,19 @@ "required": [ "sessionUpdate" ] + }, + { + "$ref": "#/$defs/LiveVoiceInteractionEndedUpdate", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "live_voice_interaction_ended" + } + }, + "required": [ + "sessionUpdate" + ] } ], "description": "Discriminated union of goose-specific session update payloads.\nVariant tag matches ACP's convention (`sessionUpdate: \"\"`).\n\n`discriminator.mapping` is what makes TS codegen (`@hey-api/openapi-ts`)\nemit the correct snake_case tag value even when this enum has a single\nvariant. Add a mapping entry per variant.", @@ -6697,7 +6813,8 @@ "mapping": { "usage_update": "#/$defs/SessionUsageUpdate", "status_message": "#/$defs/StatusMessageUpdate", - "message_usage": "#/$defs/MessageUsageUpdate" + "message_usage": "#/$defs/MessageUsageUpdate", + "live_voice_interaction_ended": "#/$defs/LiveVoiceInteractionEndedUpdate" } } }, @@ -6890,6 +7007,28 @@ ], "description": "Per-message token usage/cost/timing, keyed by the message id used for\nchunk matching. Sent live after a turn's messages and on replay." }, + "LiveVoiceInteractionOutcome": { + "type": "string", + "enum": [ + "stopped", + "failed" + ] + }, + "LiveVoiceInteractionEndedUpdate": { + "type": "object", + "properties": { + "interactionId": { + "type": "string" + }, + "outcome": { + "$ref": "#/$defs/LiveVoiceInteractionOutcome" + } + }, + "required": [ + "interactionId", + "outcome" + ] + }, "ProviderDeviceCodeNotification_unstable": { "type": "object", "properties": { @@ -7098,6 +7237,33 @@ "description": "Params for _goose/unstable/session/steer", "title": "SteerSessionRequest_unstable" }, + { + "allOf": [ + { + "$ref": "#/$defs/LiveVoiceAvailabilityRequest_unstable" + } + ], + "description": "Params for _goose/unstable/session/live-voice/availability", + "title": "LiveVoiceAvailabilityRequest_unstable" + }, + { + "allOf": [ + { + "$ref": "#/$defs/LiveVoiceStartRequest_unstable" + } + ], + "description": "Params for _goose/unstable/session/live-voice/start", + "title": "LiveVoiceStartRequest_unstable" + }, + { + "allOf": [ + { + "$ref": "#/$defs/LiveVoiceStopRequest_unstable" + } + ], + "description": "Params for _goose/unstable/session/live-voice/stop", + "title": "LiveVoiceStopRequest_unstable" + }, { "allOf": [ { @@ -8063,6 +8229,22 @@ ], "title": "SteerSessionResponse_unstable" }, + { + "allOf": [ + { + "$ref": "#/$defs/LiveVoiceAvailabilityResponse_unstable" + } + ], + "title": "LiveVoiceAvailabilityResponse_unstable" + }, + { + "allOf": [ + { + "$ref": "#/$defs/LiveVoiceStartResponse_unstable" + } + ], + "title": "LiveVoiceStartResponse_unstable" + }, { "allOf": [ { diff --git a/crates/goose/src/acp/server.rs b/crates/goose/src/acp/server.rs index 305356e805..e6e4038bfb 100644 --- a/crates/goose/src/acp/server.rs +++ b/crates/goose/src/acp/server.rs @@ -98,6 +98,8 @@ use self::tool_calls::enrichment::{spawn_chain_summary_enrichment, spawn_tool_ti mod agent_requests; pub use agent_requests::agent_request_schemas; mod agent_mentions; +pub use crate::execution::ActiveRunRegistry; +use crate::execution::StartRunError; mod apps; mod config; mod custom_dispatch; @@ -108,8 +110,10 @@ mod elicitation; mod extensions; mod fork_session; mod list_sessions; +mod live_voice; mod load_session; mod local_inference; +pub use crate::live_voice::LiveVoiceService; mod manage_sessions; mod message_meta; mod new_session; @@ -237,28 +241,11 @@ struct GooseAcpSession { agent: Arc, } -pub struct ActivePromptRun { - run_id: String, - cancel_token: CancellationToken, - /// The agent actually running this prompt. Roaming gives each connection - /// its own agent, so a steer arriving on a second connection must be - /// routed here rather than to the caller's connection-local agent. - agent: Arc, -} - struct AgentStreamOutcome { was_cancelled: bool, output_token_limit_reached: bool, } -/// Per-session active-run registry, shared by every `GooseAcpAgent` created -/// from one `AcpServer`. Roaming spawns a fresh agent per connection, so two -/// paired clients loading the same session get distinct agents; sharing this -/// map across them is what makes the "session already has an active run" guard -/// fire between connections instead of letting two loops interleave writes on -/// one session. -pub type ActiveRunRegistry = Arc>>; - /// Releases a registry entry if the task consuming an agent stream is dropped /// without reaching its explicit `clear_active_run` — e.g. a roaming /// connection is revoked or lost mid-turn. Without this, the shared registry @@ -268,7 +255,7 @@ pub type ActiveRunRegistry = Arc>>; /// The explicit clear still runs on normal paths; this drop is then a no-op /// because the entry (matched by run id) is already gone. struct ActiveRunDropGuard { - registry: ActiveRunRegistry, + registry: Arc, session_id: String, run_id: String, cancel_token: CancellationToken, @@ -277,24 +264,15 @@ struct ActiveRunDropGuard { impl Drop for ActiveRunDropGuard { fn drop(&mut self) { self.cancel_token.cancel(); - let registry = self.registry.clone(); let session_id = std::mem::take(&mut self.session_id); let run_id = std::mem::take(&mut self.run_id); + let agent = self.registry.remove_agent_run(&session_id, &run_id); if let Ok(handle) = tokio::runtime::Handle::try_current() { - handle.spawn(async move { - let agent = { - let mut runs = registry.lock().await; - match runs.get(&session_id) { - Some(run) if run.run_id == run_id => { - runs.remove(&session_id).map(|run| run.agent) - } - _ => None, - } - }; - if let Some(agent) = agent { + if let Some(agent) = agent { + handle.spawn(async move { agent.discard_pending_steers(&session_id).await; - } - }); + }); + } } } } @@ -333,15 +311,16 @@ pub struct GooseAcpAgentOptions { /// When set, new sessions use this host-controlled working directory instead /// of the `cwd` the connecting client sends (see `AcpServerFactoryConfig`). pub session_cwd: Option, - /// Active-run registry shared across all agents from one `AcpServer`, so the - /// active-run guard holds across roaming connections that each get a fresh - /// agent for the same session. - pub active_prompt_runs: ActiveRunRegistry, + /// Shared across roaming connections to coordinate prompt, Live, and + /// delegated agent runs for each session. + pub active_runs: Arc, + pub live_voice: Arc, } pub struct GooseAcpAgent { sessions: Arc>>, - active_prompt_runs: Arc>>, + active_runs: Arc, + live_voice: Arc, closed_session_ids: Arc>>, agent_manager: Arc, provider_factory: AcpProviderFactory, @@ -439,6 +418,13 @@ fn extract_timeout_from_meta(meta: &Option) -> Option { .and_then(|v| v.as_u64()) } +fn use_state_machine_from_meta(meta: Option<&Meta>) -> bool { + meta.and_then(|meta| meta.get("goose")) + .and_then(|goose| goose.get("unrolledAgentLoop")) + .and_then(|value| value.as_bool()) + .unwrap_or_else(crate::agents::state_machine::enabled) +} + #[derive(Debug, Default, Deserialize)] struct ClientCapabilitiesMeta { #[serde(default)] @@ -836,8 +822,8 @@ pub(super) fn validate_absolute_cwd(cwd: &Path) -> Result<(), agent_client_proto impl GooseAcpAgent { #[cfg(test)] - pub(crate) fn active_run_registry(&self) -> &ActiveRunRegistry { - &self.active_prompt_runs + pub(crate) fn active_run_registry(&self) -> &Arc { + &self.active_runs } #[cfg(test)] @@ -854,7 +840,7 @@ impl GooseAcpAgent { #[cfg(test)] pub(crate) fn test_drop_active_run_guard(&self, session_id: &str, run_id: &str) { drop(ActiveRunDropGuard { - registry: self.active_prompt_runs.clone(), + registry: self.active_runs.clone(), session_id: session_id.to_string(), run_id: run_id.to_string(), cancel_token: CancellationToken::new(), @@ -983,7 +969,8 @@ impl GooseAcpAgent { Ok(Self { sessions: Arc::new(Mutex::new(HashMap::new())), - active_prompt_runs: options.active_prompt_runs, + active_runs: options.active_runs, + live_voice: options.live_voice, closed_session_ids: Arc::new(Mutex::new(HashSet::new())), agent_manager, provider_factory: options.provider_factory, @@ -1917,40 +1904,24 @@ impl GooseAcpAgent { .data(format!("Session not found: {}", session_id))); } - let mut active_prompt_runs = self.active_prompt_runs.lock().await; - if let Some(active_run) = active_prompt_runs.get(session_id) { - return Err(agent_client_protocol::Error::invalid_params().data(format!( - "session already has active run `{}`; use _goose/unstable/session/steer", - active_run.run_id.as_str() - ))); - } - - active_prompt_runs.insert( - session_id.to_string(), - ActivePromptRun { - run_id, - cancel_token, - agent, - }, - ); + self.active_runs + .start_prompt_run(session_id, run_id, cancel_token, agent) + .map_err(|error| match error { + StartRunError::AgentRunExists { run_id } => { + let message = format!( + "session already has active run `{run_id}`; use _goose/unstable/session/steer" + ); + agent_client_protocol::Error::invalid_params().data(message) + } + StartRunError::LiveVoiceInteractionExists => agent_client_protocol::Error::invalid_params() + .data("session already has an active Live run"), + StartRunError::LiveVoiceInteractionMissing => unreachable!("prompt runs do not require Live"), + })?; Ok(()) } async fn clear_active_run(&self, session_id: &str, run_id: &str) { - let agent = { - let mut active_prompt_runs = self.active_prompt_runs.lock().await; - let Some(active_run) = active_prompt_runs.get(session_id) else { - return; - }; - - if active_run.run_id != run_id { - return; - } - - active_prompt_runs - .remove(session_id) - .map(|active_run| active_run.agent) - }; + let agent = self.active_runs.remove_agent_run(session_id, run_id); // Discard steers on the agent that owned the run; under roaming it may // not be this connection's agent. @@ -1984,23 +1955,22 @@ impl GooseAcpAgent { .data("expectedRunId must not be empty")); } - let active_prompt_runs = self.active_prompt_runs.lock().await; - let active_run = active_prompt_runs.get(session_id).ok_or_else(|| { + let (active_run_id, agent) = self.active_runs.agent_run(session_id).ok_or_else(|| { agent_client_protocol::Error::invalid_params().data("no active run to steer") })?; - if active_run.run_id != expected_run_id { + if active_run_id != expected_run_id { return Err( agent_client_protocol::Error::invalid_params().data(serde_json::json!({ "message": format!( "expected active run id `{expected_run_id}` but found `{}`", - active_run.run_id.as_str() + active_run_id.as_str() ), "expectedRunId": expected_run_id, - "actualRunId": active_run.run_id.as_str(), + "actualRunId": active_run_id.as_str(), })), ); } - Ok((active_run.run_id.clone(), active_run.agent.clone())) + Ok((active_run_id, agent)) } fn active_run_meta(active_run_id: Option<&str>) -> Meta { @@ -2317,7 +2287,7 @@ impl GooseAcpAgent { // connection carrying it is revoked or lost); a normal completion's // explicit clear wins and makes the guard's cleanup a no-op. let _run_guard = ActiveRunDropGuard { - registry: self.active_prompt_runs.clone(), + registry: self.active_runs.clone(), session_id: session_id.clone(), run_id: run_id.clone(), cancel_token: cancel_token.clone(), @@ -2344,13 +2314,7 @@ impl GooseAcpAgent { } let user_message = Self::convert_acp_prompt_to_message(&args.prompt); - let use_state_machine = args - .meta - .as_ref() - .and_then(|meta| meta.get("goose")) - .and_then(|goose| goose.get("unrolledAgentLoop")) - .and_then(|value| value.as_bool()) - .unwrap_or_else(crate::agents::state_machine::enabled); + let use_state_machine = use_state_machine_from_meta(args.meta.as_ref()); let session_config = SessionConfig { id: session_id.clone(), schedule_id: None, @@ -2452,12 +2416,7 @@ impl GooseAcpAgent { debug!(?args, "cancel request"); let session_id = args.session_id.0.to_string(); - let token = { - let active_prompt_runs = self.active_prompt_runs.lock().await; - active_prompt_runs - .get(&session_id) - .map(|active_run| active_run.cancel_token.clone()) - }; + let token = self.active_runs.agent_cancel_token(&session_id); if let Some(token) = token { info!(session_id = %session_id, "prompt cancelled"); @@ -2669,16 +2628,8 @@ impl GooseAcpAgent { .await .insert(session_id.to_string()); - let active_run_token = { - let active_prompt_runs = self.active_prompt_runs.lock().await; - active_prompt_runs - .get(session_id) - .map(|active_run| active_run.cancel_token.clone()) - }; - - if let Some(token) = active_run_token { - token.cancel(); - } + self.active_runs.cancel_agent_run(session_id); + self.live_voice.stop_session_interaction(session_id).await; let mut sessions = self.sessions.lock().await; sessions.remove(session_id); @@ -3663,6 +3614,8 @@ print(\"hello, world\") #[tokio::test] async fn asynchronous_provider_effort_update_is_forwarded_to_client() { let root = tempfile::tempdir().unwrap(); + let active_runs = Arc::new(ActiveRunRegistry::default()); + let live_voice = Arc::new(LiveVoiceService::from_config(active_runs.clone())); let provider_factory: AcpProviderFactory = Arc::new( |_provider_name, _extensions, _working_dir, _use_default_model| { Box::pin(async { Err(anyhow::anyhow!("unused provider factory")) }) @@ -3679,7 +3632,8 @@ print(\"hello, world\") additional_source_roots: Vec::new(), scheduler: None, session_cwd: None, - active_prompt_runs: Default::default(), + active_runs, + live_voice, }) .await .unwrap(), diff --git a/crates/goose/src/acp/server/custom_dispatch.rs b/crates/goose/src/acp/server/custom_dispatch.rs index e7cdf29f9d..86aeda17f5 100644 --- a/crates/goose/src/acp/server/custom_dispatch.rs +++ b/crates/goose/src/acp/server/custom_dispatch.rs @@ -4,7 +4,8 @@ use goose_acp_macros::custom_methods; #[custom_methods] impl GooseAcpAgent { pub async fn dispatch_custom_request( - &self, + self: &Arc, + cx: &ConnectionTo, method: &str, params: serde_json::Value, ) -> Result { @@ -18,7 +19,7 @@ impl GooseAcpAgent { }); } - self.handle_custom_request(method, params).await + self.handle_custom_request(cx, method, params).await } .await; @@ -133,6 +134,31 @@ impl GooseAcpAgent { self.on_steer_session(req).await } + #[custom_method(LiveVoiceAvailabilityRequest)] + async fn dispatch_live_voice_availability( + &self, + req: LiveVoiceAvailabilityRequest, + ) -> Result { + self.on_live_voice_availability(req).await + } + + #[custom_method(LiveVoiceStartRequest)] + async fn dispatch_live_voice_start( + self: &Arc, + cx: &ConnectionTo, + req: LiveVoiceStartRequest, + ) -> Result { + self.on_live_voice_start(cx, req).await + } + + #[custom_method(LiveVoiceStopRequest)] + async fn dispatch_live_voice_stop( + &self, + req: LiveVoiceStopRequest, + ) -> Result { + self.on_live_voice_stop(req).await + } + #[custom_method(DiagnosticsGetRequest)] async fn dispatch_get_diagnostics( &self, diff --git a/crates/goose/src/acp/server/dispatch.rs b/crates/goose/src/acp/server/dispatch.rs index 8623559eb0..f4c348a45c 100644 --- a/crates/goose/src/acp/server/dispatch.rs +++ b/crates/goose/src/acp/server/dispatch.rs @@ -475,8 +475,16 @@ impl HandleDispatchFrom for GooseAcpHandler { |message: Dispatch| async move { match message { Dispatch::Request(req, responder) => { + let request_cx = cx.clone(); cx.spawn(async move { - match agent.dispatch_custom_request(&req.method, req.params).await { + match agent + .dispatch_custom_request( + &request_cx, + &req.method, + req.params, + ) + .await + { Ok(json) => responder.respond(json)?, Err(e) => responder.respond_with_error(e)?, } diff --git a/crates/goose/src/acp/server/live_voice.rs b/crates/goose/src/acp/server/live_voice.rs new file mode 100644 index 0000000000..2f14f614cc --- /dev/null +++ b/crates/goose/src/acp/server/live_voice.rs @@ -0,0 +1,412 @@ +use super::*; +use crate::live_voice::{ + wait_for_completion, LiveMainAgent, LiveVoiceError, LiveVoiceInteractionCompletion, + LiveVoiceInteractionId, LiveVoiceTranscriptPublisher, StartLiveVoiceInteractionResult, + WebRtcOffer, +}; +use futures::FutureExt; + +const LIVE_DELEGATION_INSTRUCTION: &str = concat!( + "You support a live conversation. Treat the latest delegated input as an update to this session's earlier context. ", + "Voice transcripts may be incomplete. If required information is missing, ask a brief clarification instead of guessing. ", + "End every run with a concise, user-facing result below 400 tokens. State the outcome, what changed, validation performed, ", + "failures, and any required user action. Do not include raw tool output." +); + +impl GooseAcpAgent { + async fn load_live_voice_session( + &self, + session_id: &str, + ) -> Result { + self.session_manager + .get_session(session_id, false) + .await + .map_err(|_| { + agent_client_protocol::Error::resource_not_found(None) + .data("Session is not accessible") + }) + } + + pub(super) async fn on_live_voice_availability( + &self, + req: LiveVoiceAvailabilityRequest, + ) -> Result { + if !use_state_machine_from_meta(req.meta.as_ref()) { + return Ok(LiveVoiceAvailabilityResponse { + status: LiveVoiceStatus::Unavailable, + message: "Live voice is unavailable while Use Legacy Agent Loop is enabled".into(), + }); + } + let mode = match req.session_id.as_deref() { + Some(session_id) => self.load_live_voice_session(session_id).await?.goose_mode, + None => crate::config::Config::global() + .get_goose_mode() + .unwrap_or_default(), + }; + Ok( + match self + .live_voice + .availability(req.session_id.as_deref(), mode) + { + Ok(()) => LiveVoiceAvailabilityResponse { + status: LiveVoiceStatus::Ready, + message: "Start Live voice".into(), + }, + Err(message) => LiveVoiceAvailabilityResponse { + status: LiveVoiceStatus::Unavailable, + message: message.into(), + }, + }, + ) + } + + pub(super) async fn on_live_voice_start( + self: &Arc, + cx: &ConnectionTo, + req: LiveVoiceStartRequest, + ) -> Result { + if !use_state_machine_from_meta(req.meta.as_ref()) { + return Err(map_live_voice_error(LiveVoiceError::Unavailable)); + } + let offer = WebRtcOffer::new(req.offer_sdp) + .ok_or_else(agent_client_protocol::Error::invalid_params)?; + let session_id = req.session_id.clone(); + let session = self.load_live_voice_session(&session_id).await?; + let reservation = self + .live_voice + .reserve_interaction(&session_id, session.goose_mode) + .map_err(map_live_voice_error)?; + let transcript_publisher = Self::live_transcript_publisher(cx, &req.session_id); + let agent = self + .prepare_live_agent(&req.session_id) + .await + .map_err(|error| agent_client_protocol::Error::internal_error().data(error))?; + let main_agent = self.live_main_agent(cx, agent); + let start = self.live_voice.start_interaction( + reservation, + offer, + self.session_manager.clone(), + transcript_publisher, + main_agent, + ); + tokio::pin!(start); + let interaction = tokio::select! { + result = &mut start => result.map_err(map_live_voice_error)?, + _ = cx.incoming_closed() => { + return Err(agent_client_protocol::Error::internal_error() + .data("ACP connection closed while Live voice was starting")); + } + }; + + self.watch_live_interaction(cx, session_id, &interaction); + + Ok(LiveVoiceStartResponse { + interaction_id: interaction.interaction_id.0, + answer_sdp: interaction.answer.into_sdp(), + }) + } + + fn live_transcript_publisher( + cx: &ConnectionTo, + session_id: &str, + ) -> LiveVoiceTranscriptPublisher { + let connection = cx.clone(); + let session_id = session_id.to_string(); + Arc::new(move |message| { + let [MessageContent::Text(text)] = message.content.as_slice() else { + return; + }; + let chunk = content_chunk_for_message( + &message, + ContentBlock::Text(TextContent::new(text.text.clone())), + ); + let update = match message.role { + Role::User => SessionUpdate::UserMessageChunk(chunk), + Role::Assistant => SessionUpdate::AgentMessageChunk(chunk), + }; + let _ = connection.send_notification(SessionNotification::new( + SessionId::new(session_id.clone()), + update, + )); + }) + } + + fn live_main_agent( + self: &Arc, + cx: &ConnectionTo, + agent: Arc, + ) -> LiveMainAgent { + let start_owner = Arc::clone(self); + let start_connection = cx.clone(); + let steer_owner = Arc::clone(self); + LiveMainAgent::new( + move |session_id, input| { + start_owner.clone().start_live_delegation( + session_id, + input, + start_connection.clone(), + agent.clone(), + ) + }, + move |session_id, input| { + let owner = steer_owner.clone(); + async move { owner.steer_live_delegation(&session_id, input).await }.boxed() + }, + ) + } + + fn watch_live_interaction( + &self, + cx: &ConnectionTo, + session_id: String, + interaction: &StartLiveVoiceInteractionResult, + ) { + let interaction_id = interaction.interaction_id.clone(); + let completion_rx = interaction.completion_rx.clone(); + let live_voice = self.live_voice.clone(); + let connection = cx.clone(); + let notify_interaction_ended = self.supports_goose_custom_notifications(); + tokio::spawn(async move { + tokio::select! { + biased; + _ = connection.incoming_closed() => { + let _ = live_voice.stop_interaction(&session_id, &interaction_id).await; + } + completion = wait_for_completion(completion_rx) => { + if notify_interaction_ended { + if let Ok(completion) = completion { + let outcome = match completion { + LiveVoiceInteractionCompletion::Stopped => LiveVoiceInteractionOutcome::Stopped, + LiveVoiceInteractionCompletion::Failed => LiveVoiceInteractionOutcome::Failed, + }; + let _ = connection.send_notification(GooseSessionNotification { + session_id, + update: GooseSessionUpdate::LiveVoiceInteractionEnded(LiveVoiceInteractionEndedUpdate { + interaction_id: interaction_id.0, + outcome, + }), + }); + } + } + } + } + }); + } + + pub(super) async fn on_live_voice_stop( + &self, + req: LiveVoiceStopRequest, + ) -> Result { + self.load_live_voice_session(&req.session_id).await?; + let interaction_id = LiveVoiceInteractionId(req.interaction_id); + self.live_voice + .stop_interaction(&req.session_id, &interaction_id) + .await + .map_err(map_live_voice_error)?; + Ok(EmptyResponse {}) + } + + fn start_live_delegation( + self: Arc, + session_id: String, + input: String, + cx: ConnectionTo, + agent: Arc, + ) -> Result, String> { + let cancel_token = CancellationToken::new(); + let run_id = format!("run_{}", Uuid::new_v4()); + if self + .active_runs + .start_live_delegation( + &session_id, + run_id.clone(), + cancel_token.clone(), + agent.clone(), + ) + .is_err() + { + return Err("Goose is already working on a task.".into()); + } + let run_guard = ActiveRunDropGuard { + registry: self.active_runs.clone(), + session_id: session_id.clone(), + run_id: run_id.clone(), + cancel_token: cancel_token.clone(), + }; + + Ok(async move { + let _run_guard = run_guard; + self.run_live_delegation(session_id, input, cancel_token, cx, agent, run_id) + .await + } + .boxed()) + } + + async fn run_live_delegation( + &self, + session_id: String, + input: String, + cancel_token: CancellationToken, + cx: ConnectionTo, + agent: Arc, + run_id: String, + ) -> String { + let acp_session_id = SessionId::new(session_id.clone()); + let _ = Self::send_active_run_update(&cx, &acp_session_id, Some(&run_id)); + + let session_config = SessionConfig { + id: session_id.clone(), + schedule_id: None, + max_turns: None, + retry_config: None, + }; + let input_message = + Message::user().with_text(format!("{LIVE_DELEGATION_INSTRUCTION}\n\n{input}")); + let mut stream = match agent + .reply_live_delegation(input_message, session_config, cancel_token.clone()) + .await + { + Ok(stream) => stream, + Err(_) => { + self.clear_active_run(&session_id, &run_id).await; + let _ = Self::send_active_run_update(&cx, &acp_session_id, None); + return "Goose couldn't start the task.".into(); + } + }; + + let mut tool_requests = HashMap::new(); + let mut outcome_message_id: Option = None; + let mut outcome = String::new(); + let target = SessionAgentTarget { + agent: agent.clone(), + session_id: session_id.clone(), + cancel_token: Some(cancel_token.clone()), + }; + while let Some(event) = stream.next().await { + if cancel_token.is_cancelled() { + self.clear_active_run(&session_id, &run_id).await; + let _ = Self::send_active_run_update(&cx, &acp_session_id, None); + return "The task was cancelled.".into(); + } + match event { + Ok(crate::agents::AgentEvent::Message(message)) => { + let is_user_visible = message.is_user_visible(); + let is_visible_assistant = message.role == Role::Assistant && is_user_visible; + let projected_message = message.user_visible_content(); + for content in &projected_message.content { + if let MessageContent::ToolRequest(tool_request) = content { + tool_requests.insert(tool_request.id.clone(), tool_request.clone()); + } + let should_project = match content { + MessageContent::ToolRequest(_) + | MessageContent::ToolResponse(_) + | MessageContent::Thinking(_) + | MessageContent::ActionRequired(_) => true, + _ => is_user_visible, + }; + if !should_project { + continue; + } + let _ = self + .handle_message_content( + content, + &projected_message, + &acp_session_id, + &target, + &tool_requests, + &cx, + ) + .await; + } + if is_visible_assistant { + let text = projected_message + .content + .iter() + .filter_map(MessageContent::as_text) + .collect::(); + if !text.trim().is_empty() { + if outcome_message_id.is_some() + && outcome_message_id == projected_message.id + { + outcome.push_str(&text); + } else { + outcome_message_id = projected_message.id; + outcome = text; + } + } + } + } + Ok(crate::agents::AgentEvent::McpNotification((request_id, notification))) => { + if let Some(update) = + tool_notifications::tool_notification_update(request_id, notification) + { + let _ = ToolCallNotifier::new(&cx, &acp_session_id).send_update(update); + } + } + Ok(crate::agents::AgentEvent::MessageUsage { message_id, usage }) => { + if self.supports_goose_custom_notifications() { + let _ = cx.send_notification(GooseSessionNotification { + session_id: session_id.clone(), + update: GooseSessionUpdate::MessageUsage(message_usage_update( + message_id, &usage, + )), + }); + } + } + Ok(_) => {} + Err(_) => { + self.clear_active_run(&session_id, &run_id).await; + let _ = Self::send_active_run_update(&cx, &acp_session_id, None); + return "The task failed before it could complete.".into(); + } + } + } + + self.clear_active_run(&session_id, &run_id).await; + let _ = Self::send_active_run_update(&cx, &acp_session_id, None); + if cancel_token.is_cancelled() { + return "The task was cancelled.".into(); + } + if outcome.is_empty() { + return "The task finished without a result to share.".into(); + } + outcome + } + + async fn steer_live_delegation( + &self, + session_id: &str, + input: String, + ) -> Result { + let Some((_, agent)) = self.active_runs.agent_run(session_id) else { + return Err("The task could not receive the latest instruction.".into()); + }; + agent + .steer(session_id, Message::user().with_text(input).agent_only()) + .await; + Ok( + "The latest instruction was added to the work in progress. Wait for its updated result." + .into(), + ) + } + + async fn prepare_live_agent(&self, session_id: &str) -> Result, String> { + self.get_session_agent(session_id) + .await + .map_err(|_| "Goose could not activate the coding agent.".to_string()) + } +} + +fn map_live_voice_error(error: LiveVoiceError) -> agent_client_protocol::Error { + match error { + LiveVoiceError::Unavailable => { + agent_client_protocol::Error::invalid_params().data("Live voice is unavailable") + } + LiveVoiceError::StartFailed => { + agent_client_protocol::Error::internal_error().data("Live voice could not start") + } + LiveVoiceError::StopFailed => { + agent_client_protocol::Error::internal_error().data("Live voice could not stop cleanly") + } + } +} diff --git a/crates/goose/src/acp/server/load_session.rs b/crates/goose/src/acp/server/load_session.rs index 6837347a3c..1db8852edc 100644 --- a/crates/goose/src/acp/server/load_session.rs +++ b/crates/goose/src/acp/server/load_session.rs @@ -305,7 +305,7 @@ impl GooseAcpAgent { let task_acp_session_id = acp_session_id.clone(); if let Err(error) = cx.spawn(async move { let _run_guard = ActiveRunDropGuard { - registry: server.active_prompt_runs.clone(), + registry: server.active_runs.clone(), session_id: task_session_id.clone(), run_id: task_run_id.clone(), cancel_token: task_cancel_token.clone(), diff --git a/crates/goose/src/acp/server/manage_sessions.rs b/crates/goose/src/acp/server/manage_sessions.rs index cb1e712ca4..a7d3c27506 100644 --- a/crates/goose/src/acp/server/manage_sessions.rs +++ b/crates/goose/src/acp/server/manage_sessions.rs @@ -100,6 +100,8 @@ impl GooseAcpAgent { req: DeleteSessionRequest, ) -> Result { let session_id = req.session_id.0.to_string(); + self.active_runs.cancel_agent_run(&session_id); + self.live_voice.stop_session_interaction(&session_id).await; self.session_manager .delete_session(&session_id) .await diff --git a/crates/goose/src/acp/server_factory.rs b/crates/goose/src/acp/server_factory.rs index 073948bea8..c0dd707594 100644 --- a/crates/goose/src/acp/server_factory.rs +++ b/crates/goose/src/acp/server_factory.rs @@ -1,5 +1,6 @@ use crate::acp::server::{ - AcpBuiltinSelection, AcpProviderFactory, ActiveRunRegistry, GooseAcpAgent, GooseAcpAgentOptions, + AcpBuiltinSelection, AcpProviderFactory, ActiveRunRegistry, GooseAcpAgent, + GooseAcpAgentOptions, LiveVoiceService, }; use crate::agents::GoosePlatform; use crate::scheduler_trait::SchedulerTrait; @@ -26,15 +27,19 @@ pub struct AcpServerFactoryConfig { pub struct AcpServer { config: AcpServerFactoryConfig, scheduler: OnceCell>, - active_prompt_runs: ActiveRunRegistry, + active_runs: Arc, + live_voice: Arc, } impl AcpServer { pub fn new(config: AcpServerFactoryConfig) -> Self { + let active_runs = Arc::new(ActiveRunRegistry::default()); + let live_voice = Arc::new(LiveVoiceService::from_config(active_runs.clone())); Self { config, scheduler: OnceCell::new(), - active_prompt_runs: ActiveRunRegistry::default(), + active_runs, + live_voice, } } @@ -123,7 +128,8 @@ impl AcpServer { additional_source_roots: self.config.additional_source_roots.clone(), session_cwd, scheduler, - active_prompt_runs: self.active_prompt_runs.clone(), + active_runs: self.active_runs.clone(), + live_voice: self.live_voice.clone(), }) .await?; info!("Created new ACP agent"); @@ -175,8 +181,7 @@ mod tests { assert!( Arc::ptr_eq(a.active_run_registry(), b.active_run_registry()), - "each connection's agent must share one per-session run registry so \ - the active-run guard holds across roaming connections" + "each connection's agent must share one per-session run registry" ); } diff --git a/crates/goose/src/agents/agent.rs b/crates/goose/src/agents/agent.rs index 8139fb14e1..370662ea17 100644 --- a/crates/goose/src/agents/agent.rs +++ b/crates/goose/src/agents/agent.rs @@ -2013,6 +2013,19 @@ impl Agent { )) } + pub(crate) async fn reply_live_delegation( + &self, + user_message: Message, + session_config: SessionConfig, + cancel_token: CancellationToken, + ) -> Result>> { + let user_message = user_message.agent_only(); + let events = self + .reply_with_state_machine(user_message, session_config, Some(cancel_token)) + .await?; + Ok(Box::pin(events.map_ok(ensure_message_event_id))) + } + #[instrument( skip(self, user_message, session_config, use_state_machine, cancel_token), fields( diff --git a/crates/goose/src/agents/state_machine/effects.rs b/crates/goose/src/agents/state_machine/effects.rs index bc2dbbdd5d..f832917431 100644 --- a/crates/goose/src/agents/state_machine/effects.rs +++ b/crates/goose/src/agents/state_machine/effects.rs @@ -7,7 +7,7 @@ use goose_agent::operation::{ConversationEffect, MachineEffect}; pub enum GooseEffect { Conversation(ConversationEffect), - ReplaceConversation { + CompactConversation { conversation: Conversation, usage: Option, }, @@ -20,7 +20,7 @@ impl MachineEffect for GooseEffect { fn ensure_message_ids(&mut self) { match self { GooseEffect::Conversation(effect) => effect.ensure_message_ids(), - GooseEffect::ReplaceConversation { conversation, .. } => { + GooseEffect::CompactConversation { conversation, .. } => { for message in conversation.messages_mut() { if message.id.is_none() { message.id = Some(format!("msg_{}", uuid::Uuid::new_v4())); diff --git a/crates/goose/src/agents/state_machine/ops_compaction.rs b/crates/goose/src/agents/state_machine/ops_compaction.rs index 08166b2a9d..a86a9debd9 100644 --- a/crates/goose/src/agents/state_machine/ops_compaction.rs +++ b/crates/goose/src/agents/state_machine/ops_compaction.rs @@ -191,7 +191,7 @@ impl Operation for CompactionOperation { emit.message(command).await; let response = emit.message(response).await; yielded_with([ - GooseEffect::ReplaceConversation { + GooseEffect::CompactConversation { conversation: compacted, usage: Some(usage), }, @@ -306,7 +306,7 @@ impl Operation for CompactionOperation { "Compaction complete", )) .await; - applied([GooseEffect::ReplaceConversation { + applied([GooseEffect::CompactConversation { conversation: compacted, usage: Some(usage), }]) diff --git a/crates/goose/src/agents/state_machine/session.rs b/crates/goose/src/agents/state_machine/session.rs index 2e9fa5f95a..c1a570656a 100644 --- a/crates/goose/src/agents/state_machine/session.rs +++ b/crates/goose/src/agents/state_machine/session.rs @@ -19,7 +19,6 @@ fn contains_tool_confirmation_request(message: &Message) -> bool { ) }) } - impl MachineSession for Session { fn id(&self) -> &str { &self.id @@ -63,14 +62,15 @@ impl EffectHandler for SessionManager { .apply() .await?; } - GooseEffect::ReplaceConversation { + GooseEffect::CompactConversation { conversation, usage: replacement_usage, } => { if let Some(provider_usage) = replacement_usage { usage::record(self, session, provider_usage, true).await?; } - self.replace_conversation(&session.id, conversation).await?; + self.save_compacted_conversation(&session.id, conversation) + .await?; self.update(&session.id) .usage(usage::estimate_context(conversation).await?) .apply() @@ -137,7 +137,7 @@ impl EffectHandler for SessionManager { GooseEffect::Conversation(ConversationEffect::ReplaceConversation( conversation, )) - | GooseEffect::ReplaceConversation { conversation, .. } => { + | GooseEffect::CompactConversation { conversation, .. } => { emit.emit(AgentEvent::HistoryReplaced(conversation.clone())) .await; } @@ -158,7 +158,7 @@ impl EffectUsage for SessionManager { ) -> Option { match effect { GooseEffect::RecordUsage(usage) - | GooseEffect::ReplaceConversation { + | GooseEffect::CompactConversation { usage: Some(usage), .. } => Some(usage.usage), _ => None, @@ -237,3 +237,76 @@ pub(crate) async fn run( crate::agents::gen_ai_telemetry::record_usage(&tracing::Span::current(), &turn_usage); Ok(session) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::GooseMode; + use crate::conversation::message::Message; + use crate::session::session_manager::SessionType; + + #[tokio::test] + async fn live_replacement_preserves_concurrent_transcript_and_hidden_handoff() { + let manager = SessionManager::new(tempfile::tempdir().unwrap().keep()); + let session = manager + .create_session( + "/tmp".into(), + "concurrent compaction".into(), + SessionType::User, + GooseMode::Auto, + ) + .await + .unwrap(); + let kickoff = Message::user() + .with_id("kickoff") + .with_text("delegated request") + .agent_only(); + manager.add_message(&session.id, &kickoff).await.unwrap(); + + let snapshot = manager.get_session(&session.id, true).await.unwrap(); + + manager + .add_message( + &session.id, + &Message::user() + .with_id("concurrent-live") + .with_text("still speaking") + .user_only(), + ) + .await + .unwrap(); + + let mut compacted = snapshot.conversation.clone().unwrap(); + for message in compacted.messages_mut() { + message.metadata.agent_visible = false; + } + compacted.messages_mut().push( + Message::assistant() + .with_id("summary") + .with_text("summary") + .agent_only(), + ); + manager + .save_compacted_conversation(&session.id, &compacted) + .await + .unwrap(); + + let reloaded = manager.get_session(&session.id, true).await.unwrap(); + let messages = reloaded.conversation.unwrap(); + let kickoff = messages + .messages() + .iter() + .find(|message| message.id.as_deref() == Some("kickoff")) + .unwrap(); + assert!(!kickoff.is_user_visible()); + assert!(!kickoff.is_agent_visible()); + assert!(messages + .messages() + .iter() + .any(|message| message.id.as_deref() == Some("concurrent-live"))); + assert!(messages + .messages() + .iter() + .any(|message| message.id.as_deref() == Some("summary"))); + } +} diff --git a/crates/goose/src/agents/state_machine/usage.rs b/crates/goose/src/agents/state_machine/usage.rs index 3254540c16..04fb4e3a09 100644 --- a/crates/goose/src/agents/state_machine/usage.rs +++ b/crates/goose/src/agents/state_machine/usage.rs @@ -24,7 +24,7 @@ pub(super) fn enrich(session: &Session, effects: &mut [GooseEffect]) { for index in 0..effects.len() { let (usage, replaces_conversation) = match &effects[index] { GooseEffect::RecordUsage(usage) => (usage.clone(), false), - GooseEffect::ReplaceConversation { + GooseEffect::CompactConversation { usage: Some(usage), .. } => (usage.clone(), true), _ => continue, @@ -53,7 +53,7 @@ pub(super) fn enrich(session: &Session, effects: &mut [GooseEffect]) { } match &mut effects[index] { GooseEffect::RecordUsage(usage) => *usage = enriched, - GooseEffect::ReplaceConversation { usage, .. } => *usage = Some(enriched), + GooseEffect::CompactConversation { usage, .. } => *usage = Some(enriched), _ => {} } } diff --git a/crates/goose/src/execution/active_run.rs b/crates/goose/src/execution/active_run.rs new file mode 100644 index 0000000000..2ffbf5a8f2 --- /dev/null +++ b/crates/goose/src/execution/active_run.rs @@ -0,0 +1,244 @@ +use crate::agents::Agent; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; +use tokio_util::sync::CancellationToken; + +struct ActiveRun { + run_id: String, + cancel_token: CancellationToken, + /// Routes steering from another roaming connection to the run owner. + agent: Arc, +} + +struct SessionRunState { + agent_run: Option, + live_active: bool, +} + +pub(crate) enum StartRunError { + AgentRunExists { run_id: String }, + LiveVoiceInteractionExists, + LiveVoiceInteractionMissing, +} + +#[derive(Default)] +pub struct ActiveRunRegistry { + runs_by_session: Mutex>, +} + +impl ActiveRunRegistry { + pub(crate) fn start_prompt_run( + &self, + session_id: &str, + run_id: String, + cancel_token: CancellationToken, + agent: Arc, + ) -> Result<(), StartRunError> { + let mut runs = self + .runs_by_session + .lock() + .expect("active run lock poisoned"); + if let Some(state) = runs.get(session_id) { + if let Some(agent_run) = &state.agent_run { + return Err(StartRunError::AgentRunExists { + run_id: agent_run.run_id.clone(), + }); + } + if state.live_active { + return Err(StartRunError::LiveVoiceInteractionExists); + } + } + runs.insert( + session_id.to_string(), + SessionRunState { + agent_run: Some(ActiveRun { + run_id, + cancel_token, + agent, + }), + live_active: false, + }, + ); + Ok(()) + } + + pub(crate) fn start_live_delegation( + &self, + session_id: &str, + run_id: String, + cancel_token: CancellationToken, + agent: Arc, + ) -> Result<(), StartRunError> { + let mut runs = self + .runs_by_session + .lock() + .expect("active run lock poisoned"); + let Some(state) = runs.get_mut(session_id) else { + return Err(StartRunError::LiveVoiceInteractionMissing); + }; + if let Some(agent_run) = &state.agent_run { + return Err(StartRunError::AgentRunExists { + run_id: agent_run.run_id.clone(), + }); + } + state.agent_run = Some(ActiveRun { + run_id, + cancel_token, + agent, + }); + Ok(()) + } + + pub(crate) fn agent_run(&self, session_id: &str) -> Option<(String, Arc)> { + self.runs_by_session + .lock() + .expect("active run lock poisoned") + .get(session_id) + .and_then(|state| state.agent_run.as_ref()) + .map(|run| (run.run_id.clone(), run.agent.clone())) + } + + pub(crate) fn agent_cancel_token(&self, session_id: &str) -> Option { + self.runs_by_session + .lock() + .expect("active run lock poisoned") + .get(session_id) + .and_then(|state| state.agent_run.as_ref()) + .map(|run| run.cancel_token.clone()) + } + + pub(crate) fn cancel_agent_run(&self, session_id: &str) { + if let Some(cancel_token) = self.agent_cancel_token(session_id) { + cancel_token.cancel(); + } + } + + pub(crate) fn remove_agent_run(&self, session_id: &str, run_id: &str) -> Option> { + let mut runs = self + .runs_by_session + .lock() + .expect("active run lock poisoned"); + let state = runs.get_mut(session_id)?; + if state.agent_run.as_ref()?.run_id != run_id { + return None; + } + let agent = state.agent_run.take()?.agent; + if !state.live_active { + runs.remove(session_id); + } + Some(agent) + } + + pub(crate) fn start_live(&self, session_id: &str) -> bool { + let mut runs = self + .runs_by_session + .lock() + .expect("active run lock poisoned"); + if runs.contains_key(session_id) { + return false; + } + runs.insert( + session_id.to_string(), + SessionRunState { + agent_run: None, + live_active: true, + }, + ); + true + } + + pub(crate) fn finish_live(&self, session_id: &str) { + let mut runs = self + .runs_by_session + .lock() + .expect("active run lock poisoned"); + if let Some(state) = runs.get_mut(session_id) { + state.live_active = false; + if state.agent_run.is_none() { + runs.remove(session_id); + } + } + } + + pub(crate) fn is_active(&self, session_id: &str) -> bool { + self.runs_by_session + .lock() + .expect("active run lock poisoned") + .contains_key(session_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn live_runs_are_scoped_by_session() { + let registry = ActiveRunRegistry::default(); + + assert!(registry.start_live("one")); + assert!(!registry.start_live("one")); + assert!(registry.start_live("two")); + + registry.finish_live("one"); + assert!(registry.start_live("one")); + } + + #[tokio::test] + async fn prompt_and_live_runs_conflict() { + let registry = ActiveRunRegistry::default(); + + assert!(registry + .start_prompt_run( + "session", + "run".into(), + CancellationToken::new(), + Arc::new(Agent::new()), + ) + .is_ok()); + assert!(!registry.start_live("session")); + + registry.remove_agent_run("session", "run"); + assert!(registry.start_live("session")); + assert!(matches!( + registry.start_prompt_run( + "session", + "run".into(), + CancellationToken::new(), + Arc::new(Agent::new()), + ), + Err(StartRunError::LiveVoiceInteractionExists) + )); + } + + #[tokio::test] + async fn live_delegation_shares_the_live_session_without_admitting_another_prompt() { + let registry = ActiveRunRegistry::default(); + assert!(registry.start_live("session")); + assert!(registry + .start_live_delegation( + "session", + "delegated".into(), + CancellationToken::new(), + Arc::new(Agent::new()), + ) + .is_ok()); + assert!(matches!( + registry.start_prompt_run( + "session", + "prompt".into(), + CancellationToken::new(), + Arc::new(Agent::new()), + ), + Err(StartRunError::AgentRunExists { .. }) + )); + + registry.finish_live("session"); + assert_eq!( + registry.agent_run("session").map(|(run_id, _)| run_id), + Some("delegated".into()) + ); + } +} diff --git a/crates/goose/src/execution/mod.rs b/crates/goose/src/execution/mod.rs index beff687ab8..06563b68de 100644 --- a/crates/goose/src/execution/mod.rs +++ b/crates/goose/src/execution/mod.rs @@ -3,8 +3,12 @@ //! This module provides centralized agent lifecycle management with session isolation, //! enabling multiple concurrent sessions with independent agents, extensions, and providers. +mod active_run; pub mod manager; +pub use active_run::ActiveRunRegistry; +pub(crate) use active_run::StartRunError; + use serde::{Deserialize, Serialize}; use std::fmt; diff --git a/crates/goose/src/lib.rs b/crates/goose/src/lib.rs index c3f32c4bd5..b124725280 100644 --- a/crates/goose/src/lib.rs +++ b/crates/goose/src/lib.rs @@ -25,6 +25,7 @@ pub mod goose_apps; pub mod hints; pub mod hooks; pub mod instance_id; +mod live_voice; pub mod logging; pub mod mcp_utils; pub mod model_config; diff --git a/crates/goose/src/live_voice/interaction.rs b/crates/goose/src/live_voice/interaction.rs new file mode 100644 index 0000000000..f1c5992c79 --- /dev/null +++ b/crates/goose/src/live_voice/interaction.rs @@ -0,0 +1,474 @@ +use super::service::{ + LiveVoiceInteractionCompletion, LiveVoiceInteractionGuard, LiveVoiceTranscriptPublisher, +}; +use super::transcript::{DelegationContext, LiveTranscript}; +use crate::{conversation::message::Message, session::SessionManager, token_counter::TokenCounter}; +use futures::future::BoxFuture; +use goose_providers::live_voice_provider::{ + DelegationUpdate, DelegationUpdateDelivery, ProviderConnection, ProviderConnectionEvent, +}; +use rmcp::model::Role; +use std::{collections::HashSet, sync::Arc, time::Duration}; +use tokio::time::timeout; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +pub(super) const PROVIDER_CLEANUP_TIMEOUT: Duration = Duration::from_secs(20); +pub(super) const DELEGATION_INSTRUCTION: &str = + "Based on this conversation, identify and complete the user's request."; +const DELEGATION_UPDATE_TOKEN_LIMIT: usize = 500; +const UNDELIVERED_DELEGATION_UPDATE_NOTICE: &str = + "I couldn't confirm that the latest delegated update reached this voice conversation. Please ask me to share it again."; +const SAVED_RESULT_NOTICE: &str = "\n\nThe full result is saved in Goose."; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct LiveVoiceInteractionId(pub(crate) String); + +impl LiveVoiceInteractionId { + pub(super) fn new() -> Self { + Self(format!("live_{}", Uuid::now_v7())) + } +} + +pub(super) struct LiveVoiceInteraction { + provider_connection: Box, + provider_event_ids: HashSet, + provider_delegation_ids: HashSet, + transcript: LiveTranscript, + delegated_main_agent_run: Option, + session_manager: Arc, + transcript_publisher: LiveVoiceTranscriptPublisher, + main_agent: LiveMainAgent, + interaction_guard: LiveVoiceInteractionGuard, +} + +struct DelegatedMainAgentRun { + // GPT-Live expects the final result on the delegation that started this run. + provider_delegation_id: String, + result_future: BoxFuture<'static, String>, +} + +enum LiveVoiceInteractionEvent { + StopRequested, + MainAgentFinished(String), + Provider(ProviderConnectionEvent), +} + +enum DelegationDecision { + Ignore, + Reject(String), + Accept(String), +} + +impl LiveVoiceInteraction { + pub(super) fn new( + provider_connection: Box, + session_manager: Arc, + transcript_publisher: LiveVoiceTranscriptPublisher, + main_agent: LiveMainAgent, + interaction_guard: LiveVoiceInteractionGuard, + ) -> Self { + Self { + provider_connection, + provider_event_ids: HashSet::new(), + provider_delegation_ids: HashSet::new(), + transcript: LiveTranscript::default(), + delegated_main_agent_run: None, + session_manager, + transcript_publisher, + main_agent, + interaction_guard, + } + } + + pub(super) async fn run(mut self) { + let stop_requested = self.interaction_guard.stop_requested().clone(); + let mut stopping = false; + let completion = loop { + match self.next_event(&stop_requested, stopping).await { + LiveVoiceInteractionEvent::StopRequested => { + match timeout(PROVIDER_CLEANUP_TIMEOUT, self.cleanup_provider()).await { + Ok(Ok(())) => stopping = true, + _ => break LiveVoiceInteractionCompletion::Failed, + } + } + LiveVoiceInteractionEvent::MainAgentFinished(result) => { + if self.handle_main_agent_finished(result).await.is_err() { + self.stop_provider_after_error(stopping).await; + break LiveVoiceInteractionCompletion::Failed; + } + } + LiveVoiceInteractionEvent::Provider(ProviderConnectionEvent::TranscriptDelta { + event_id, + role, + text, + end_ms, + .. + }) => { + if self + .receive_transcript(event_id, role, text, end_ms) + .await + .is_err() + { + self.stop_provider_after_error(stopping).await; + break LiveVoiceInteractionCompletion::Failed; + } + } + LiveVoiceInteractionEvent::Provider( + ProviderConnectionEvent::DelegationRequested { + event_id, + delegation_id, + offset_ms, + }, + ) => { + if stopping { + continue; + } + if self + .receive_delegation(event_id, delegation_id, offset_ms) + .await + .is_err() + { + self.stop_provider_after_error(stopping).await; + break LiveVoiceInteractionCompletion::Failed; + } + } + LiveVoiceInteractionEvent::Provider(ProviderConnectionEvent::Closed) => { + break if stopping { + LiveVoiceInteractionCompletion::Stopped + } else { + LiveVoiceInteractionCompletion::Failed + }; + } + LiveVoiceInteractionEvent::Provider( + ProviderConnectionEvent::ReceiverLagged | ProviderConnectionEvent::Failed, + ) => { + self.stop_provider_after_error(stopping).await; + break LiveVoiceInteractionCompletion::Failed; + } + } + }; + + self.finish_interaction(completion).await; + } + + async fn next_event( + &mut self, + stop_requested: &CancellationToken, + stopping: bool, + ) -> LiveVoiceInteractionEvent { + if stopping { + return LiveVoiceInteractionEvent::Provider( + self.provider_connection.next_event().await, + ); + } + + let delegated_run = &mut self.delegated_main_agent_run; + let provider_connection = &mut self.provider_connection; + tokio::select! { + biased; + _ = stop_requested.cancelled() => LiveVoiceInteractionEvent::StopRequested, + result = async { + delegated_run + .as_mut() + .expect("main agent is running") + .result_future + .as_mut() + .await + }, if delegated_run.is_some() => LiveVoiceInteractionEvent::MainAgentFinished(result), + event = provider_connection.next_event() => LiveVoiceInteractionEvent::Provider(event), + } + } + + async fn receive_transcript( + &mut self, + event_id: String, + role: Role, + text: String, + end_ms: u64, + ) -> anyhow::Result<()> { + let Some(transcript_delta_to_display) = + self.record_transcript(event_id, role, &text, end_ms) + else { + return Ok(()); + }; + + (self.transcript_publisher)(transcript_delta_to_display); + save_raw_transcript_entries( + &self.session_manager, + self.interaction_guard.session_id(), + &mut self.transcript, + ) + .await?; + Ok(()) + } + + fn record_transcript( + &mut self, + event_id: String, + role: Role, + delta_text: &str, + end_ms: u64, + ) -> Option { + if !self.provider_event_ids.insert(event_id) { + return None; + } + self.transcript.append(role, delta_text, end_ms) + } + + async fn receive_delegation( + &mut self, + event_id: String, + delegation_id: String, + offset_ms: u64, + ) -> anyhow::Result<()> { + match self.handle_delegation_request(event_id, delegation_id.clone(), offset_ms) { + DelegationDecision::Ignore => {} + DelegationDecision::Reject(text) => { + self.send_delegation_update(delegation_id, bound_delegation_update(text).await) + .await?; + } + DelegationDecision::Accept(context) => { + let session_id = self.interaction_guard.session_id().to_string(); + if self.delegated_main_agent_run.is_some() { + let input = format!("{context}\n{DELEGATION_INSTRUCTION}"); + let response = match self.main_agent.steer(session_id, input).await { + Ok(response) => { + self.transcript + .mark_context_sent_to_main_agent_through(offset_ms); + response + } + Err(response) => response, + }; + self.send_delegation_update( + delegation_id, + bound_delegation_update(response).await, + ) + .await?; + } else { + self.transcript.finish_transcript_entry_being_built(); + save_raw_transcript_entries( + &self.session_manager, + &session_id, + &mut self.transcript, + ) + .await?; + let input = format!("{context}\n{DELEGATION_INSTRUCTION}"); + match self.main_agent.start(session_id, input) { + Ok(completion) => { + self.delegated_main_agent_run = Some(DelegatedMainAgentRun { + provider_delegation_id: delegation_id, + result_future: completion, + }); + self.transcript + .mark_context_sent_to_main_agent_through(offset_ms); + } + Err(response) => { + self.send_delegation_update( + delegation_id, + bound_delegation_update(response).await, + ) + .await?; + } + } + } + } + } + Ok(()) + } + + fn handle_delegation_request( + &mut self, + event_id: String, + delegation_id: String, + offset_ms: u64, + ) -> DelegationDecision { + if !self.provider_event_ids.insert(event_id) + || !self.provider_delegation_ids.insert(delegation_id) + { + return DelegationDecision::Ignore; + } + match self.transcript.context_for_delegation(offset_ms) { + DelegationContext::Stale => { + DelegationDecision::Reject("The delegated conversation position is stale.".into()) + } + DelegationContext::MissingUserInput => { + DelegationDecision::Reject("I couldn't identify a request to complete.".into()) + } + DelegationContext::Available(context) => DelegationDecision::Accept(context), + } + } + + async fn handle_main_agent_finished(&mut self, result: String) -> anyhow::Result<()> { + let run = self + .delegated_main_agent_run + .take() + .expect("main agent is running"); + self.transcript.finish_transcript_entry_being_built(); + save_raw_transcript_entries( + &self.session_manager, + self.interaction_guard.session_id(), + &mut self.transcript, + ) + .await?; + self.send_delegation_update( + run.provider_delegation_id, + bound_delegation_update(result).await, + ) + .await + } + + async fn finish_interaction(mut self, mut completion: LiveVoiceInteractionCompletion) { + match self.delegated_main_agent_run.take() { + None => { + if save_transcript_and_context_waiting_for_main_agent( + &self.session_manager, + self.interaction_guard.session_id(), + &mut self.transcript, + ) + .await + .is_err() + { + completion = LiveVoiceInteractionCompletion::Failed; + } + self.interaction_guard + .publish_completion_after_cleanup(completion); + } + Some(run) => { + self.interaction_guard.publish_completion(completion); + let _ = run.result_future.await; + let _ = save_transcript_and_context_waiting_for_main_agent( + &self.session_manager, + self.interaction_guard.session_id(), + &mut self.transcript, + ) + .await; + } + } + } + + async fn stop_provider_after_error(&mut self, stopping: bool) { + if !stopping { + let _ = timeout(PROVIDER_CLEANUP_TIMEOUT, self.cleanup_provider()).await; + } + } + + async fn send_delegation_update( + &mut self, + provider_delegation_id: String, + text: String, + ) -> anyhow::Result<()> { + let delivery = self + .provider_connection + .send_delegation_update(DelegationUpdate { + provider_delegation_id, + text, + }) + .await?; + match delivery { + DelegationUpdateDelivery::Delivered => {} + DelegationUpdateDelivery::Undelivered => (self.transcript_publisher)( + Message::assistant() + .with_id(format!("msg_live_{}", Uuid::now_v7())) + .with_text(UNDELIVERED_DELEGATION_UPDATE_NOTICE) + .user_only(), + ), + } + Ok(()) + } + + async fn cleanup_provider(&mut self) -> anyhow::Result<()> { + self.provider_connection.stop().await + } +} + +pub(crate) struct LiveMainAgent { + start: Box Result + Send + Sync>, + steer: Box MainAgentSteerResponse + Send + Sync>, +} + +type MainAgentRun = BoxFuture<'static, String>; +type MainAgentSteerResponse = BoxFuture<'static, Result>; + +impl LiveMainAgent { + pub(crate) fn new( + start: impl Fn(String, String) -> Result + Send + Sync + 'static, + steer: impl Fn(String, String) -> MainAgentSteerResponse + Send + Sync + 'static, + ) -> Self { + Self { + start: Box::new(start), + steer: Box::new(steer), + } + } + + fn start(&self, session_id: String, input: String) -> Result { + (self.start)(session_id, input) + } + + fn steer(&self, session_id: String, input: String) -> MainAgentSteerResponse { + (self.steer)(session_id, input) + } +} + +async fn save_raw_transcript_entries( + session_manager: &SessionManager, + session_id: &str, + transcript: &mut LiveTranscript, +) -> anyhow::Result<()> { + for message in transcript.raw_transcript_entries_waiting_to_save() { + session_manager.add_message(session_id, message).await?; + } + transcript.mark_raw_transcript_entries_saved(); + Ok(()) +} + +async fn save_transcript_and_context_waiting_for_main_agent( + session_manager: &SessionManager, + session_id: &str, + transcript: &mut LiveTranscript, +) -> anyhow::Result<()> { + transcript.finish_transcript_entry_being_built(); + save_raw_transcript_entries(session_manager, session_id, transcript).await?; + if let Some(context) = transcript.agent_only_context_message_waiting_to_save() { + session_manager.add_message(session_id, &context).await?; + transcript.mark_context_saved_for_main_agent(); + } + Ok(()) +} + +async fn bound_delegation_update(text: String) -> String { + let Ok(counter) = TokenCounter::new().await else { + return text; + }; + if counter.count_tokens(&text) <= DELEGATION_UPDATE_TOKEN_LIMIT { + return text; + } + + let allowed = DELEGATION_UPDATE_TOKEN_LIMIT - counter.count_tokens(SAVED_RESULT_NOTICE); + let boundaries = text + .char_indices() + .map(|(index, _)| index) + .collect::>(); + let mut low = 0; + let mut high = boundaries.len(); + while low < high { + let middle = (low + high).div_ceil(2); + let end = boundaries.get(middle).copied().unwrap_or(text.len()); + let candidate = text + .get(..end) + .expect("delegation update boundary comes from char_indices"); + if counter.count_tokens(candidate) <= allowed { + low = middle; + } else { + high = middle - 1; + } + } + let end = boundaries.get(low).copied().unwrap_or(text.len()); + let truncated = text + .get(..end) + .expect("delegation update boundary comes from char_indices"); + format!("{}{}", truncated.trim_end(), SAVED_RESULT_NOTICE) +} + +#[cfg(test)] +mod tests; diff --git a/crates/goose/src/live_voice/interaction/tests.rs b/crates/goose/src/live_voice/interaction/tests.rs new file mode 100644 index 0000000000..c276242bbe --- /dev/null +++ b/crates/goose/src/live_voice/interaction/tests.rs @@ -0,0 +1,249 @@ +use super::*; +use async_trait::async_trait; +use tokio::sync::oneshot; + +struct TestConnection { + stopped: Option>, +} + +struct UndeliveredConnection; + +fn test_interaction( + provider_connection: impl ProviderConnection + 'static, +) -> LiveVoiceInteraction { + test_interaction_with_publisher(provider_connection, Arc::new(|_| {})) +} + +fn test_interaction_with_publisher( + provider_connection: impl ProviderConnection + 'static, + transcript_publisher: LiveVoiceTranscriptPublisher, +) -> LiveVoiceInteraction { + LiveVoiceInteraction::new( + Box::new(provider_connection), + Arc::new(SessionManager::new(tempfile::tempdir().unwrap().keep())), + transcript_publisher, + LiveMainAgent::new( + |_, _| Err("unused".into()), + |_, _| Box::pin(async { Err("unused".into()) }), + ), + LiveVoiceInteractionGuard::for_test("test-session"), + ) +} + +#[async_trait] +impl ProviderConnection for TestConnection { + async fn next_event(&mut self) -> ProviderConnectionEvent { + std::future::pending().await + } + + async fn send_delegation_update( + &mut self, + _update: DelegationUpdate, + ) -> anyhow::Result { + Ok(DelegationUpdateDelivery::Delivered) + } + + async fn stop(&mut self) -> anyhow::Result<()> { + if let Some(stopped) = self.stopped.take() { + let _ = stopped.send(()); + } + Ok(()) + } +} + +#[async_trait] +impl ProviderConnection for UndeliveredConnection { + async fn next_event(&mut self) -> ProviderConnectionEvent { + std::future::pending().await + } + + async fn send_delegation_update( + &mut self, + _update: DelegationUpdate, + ) -> anyhow::Result { + Ok(DelegationUpdateDelivery::Undelivered) + } + + async fn stop(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} + +#[tokio::test] +async fn a_live_voice_interaction_owns_and_stops_its_provider_connection() { + let (stopped, did_stop) = oneshot::channel(); + let mut interaction = test_interaction(TestConnection { + stopped: Some(stopped), + }); + + interaction.cleanup_provider().await.unwrap(); + did_stop.await.unwrap(); +} + +#[tokio::test] +async fn an_undelivered_delegation_update_notifies_the_user() { + let (notice_tx, mut notice_rx) = tokio::sync::mpsc::unbounded_channel(); + let transcript_publisher: LiveVoiceTranscriptPublisher = Arc::new(move |message| { + notice_tx.send(message).unwrap(); + }); + let mut interaction = + test_interaction_with_publisher(UndeliveredConnection, transcript_publisher); + + interaction + .send_delegation_update("delegation-1".into(), "result".into()) + .await + .unwrap(); + + let notice = notice_rx.recv().await.unwrap(); + assert_eq!(notice.role, Role::Assistant); + assert_eq!( + notice.as_concat_text(), + UNDELIVERED_DELEGATION_UPDATE_NOTICE + ); +} + +#[tokio::test] +async fn delegation_preparation_respects_offset_and_suppresses_duplicates() { + let mut interaction = test_interaction(TestConnection { stopped: None }); + interaction.record_transcript("1".into(), Role::Assistant, "ready", 5); + interaction.record_transcript("2".into(), Role::User, "do ", 10); + interaction.record_transcript("3".into(), Role::User, "this", 20); + interaction.record_transcript("4".into(), Role::Assistant, "crossing", 25); + interaction.record_transcript("5".into(), Role::Assistant, "later", 40); + + let DelegationDecision::Accept(input) = + interaction.handle_delegation_request("event-1".into(), "delegation-1".into(), 20) + else { + panic!("delegation should be accepted"); + }; + assert_eq!( + input, + "Live conversation context:\nVoice assistant: ready\nUser: do this" + ); + interaction + .transcript + .mark_context_sent_to_main_agent_through(20); + assert!(matches!( + interaction.handle_delegation_request("event-1".into(), "delegation-1".into(), 20), + DelegationDecision::Ignore + )); + + interaction + .record_transcript("6".into(), Role::User, " late detail", 20) + .unwrap(); + assert_eq!( + interaction + .transcript + .raw_transcript_entries_waiting_to_save() + .last() + .unwrap() + .as_concat_text(), + "crossinglater" + ); + let DelegationDecision::Accept(continuation) = + interaction.handle_delegation_request("event-2".into(), "delegation-2".into(), 20) + else { + panic!("continuation should be accepted"); + }; + assert_eq!( + continuation, + "Live conversation context:\nUser: late detail" + ); + interaction + .transcript + .mark_context_sent_to_main_agent_through(20); + + assert!(matches!( + interaction.handle_delegation_request("event-3".into(), "delegation-3".into(), 19), + DelegationDecision::Reject(_) + )); + + interaction.record_transcript("7".into(), Role::User, "again", 50); + let DelegationDecision::Accept(next) = + interaction.handle_delegation_request("event-4".into(), "delegation-4".into(), 50) + else { + panic!("later continuation should be accepted"); + }; + assert_eq!( + next, + "Live conversation context:\nVoice assistant: crossinglater\nUser: again" + ); + interaction + .transcript + .mark_context_sent_to_main_agent_through(50); + + let mut missing_user = test_interaction(TestConnection { stopped: None }); + missing_user.record_transcript("1".into(), Role::Assistant, "hello", 10); + assert!(matches!( + missing_user.handle_delegation_request("event-1".into(), "delegation-1".into(), 10), + DelegationDecision::Reject(_) + )); +} + +#[tokio::test] +async fn context_waiting_for_main_agent_excludes_instruction_and_clears_after_handoff() { + let mut interaction = test_interaction(TestConnection { stopped: None }); + interaction.record_transcript("1".into(), Role::Assistant, "Anything else?", 10); + interaction.record_transcript("2".into(), Role::User, "No thanks", 20); + + let context = interaction + .transcript + .context_waiting_for_main_agent() + .unwrap(); + assert_eq!( + context, + "Live conversation context:\nVoice assistant: Anything else?\nUser: No thanks" + ); + assert_eq!( + interaction + .transcript + .context_waiting_for_main_agent() + .unwrap(), + context + ); + + interaction.record_transcript("3".into(), Role::User, "Already shared", 30); + interaction + .transcript + .mark_context_sent_to_main_agent_through(30); + assert!(interaction + .transcript + .context_waiting_for_main_agent() + .is_none()); +} + +#[tokio::test] +async fn transcript_grouping_projects_deltas_and_finalizes_messages() { + let mut interaction = test_interaction(TestConnection { stopped: None }); + let first = interaction + .record_transcript("1".into(), Role::User, "hello", 10) + .unwrap(); + assert!(first.is_user_visible()); + assert!(!first.is_agent_visible()); + let message_id = first.id.clone(); + let second = interaction + .record_transcript("2".into(), Role::User, " world", 20) + .unwrap(); + assert_eq!(second.id, message_id); + assert_eq!(second.as_concat_text(), " world"); + + assert!(interaction + .record_transcript("2".into(), Role::User, " world", 20) + .is_none()); + + let role_change = interaction + .record_transcript("3".into(), Role::Assistant, "hello", 30) + .unwrap(); + assert_eq!( + interaction + .transcript + .raw_transcript_entries_waiting_to_save() + .last() + .unwrap() + .as_concat_text(), + "hello world" + ); + assert_eq!(role_change.role, Role::Assistant); + assert!(role_change.is_user_visible()); + assert!(!role_change.is_agent_visible()); +} diff --git a/crates/goose/src/live_voice/mod.rs b/crates/goose/src/live_voice/mod.rs new file mode 100644 index 0000000000..c45bf01ed7 --- /dev/null +++ b/crates/goose/src/live_voice/mod.rs @@ -0,0 +1,10 @@ +mod interaction; +mod service; +mod transcript; + +pub(crate) use interaction::{LiveMainAgent, LiveVoiceInteractionId}; +pub use service::LiveVoiceService; +pub(crate) use service::{ + wait_for_completion, LiveVoiceError, LiveVoiceInteractionCompletion, + LiveVoiceTranscriptPublisher, StartLiveVoiceInteractionResult, WebRtcOffer, +}; diff --git a/crates/goose/src/live_voice/service.rs b/crates/goose/src/live_voice/service.rs new file mode 100644 index 0000000000..e539511e52 --- /dev/null +++ b/crates/goose/src/live_voice/service.rs @@ -0,0 +1,463 @@ +use super::interaction::{LiveMainAgent, LiveVoiceInteraction, LiveVoiceInteractionId}; +use crate::config::GooseMode; +use crate::conversation::message::{Message, MessageContent}; +use crate::conversation::Conversation; +use crate::execution::ActiveRunRegistry; +use crate::session::SessionManager; +use crate::token_counter::TokenCounter; +use goose_providers::live_voice_provider::{LiveVoiceInputMessage, LiveVoiceProvider}; +pub(crate) use goose_providers::live_voice_provider::{WebRtcAnswer, WebRtcOffer}; +#[cfg(feature = "live-voice")] +use goose_providers::openai_live_voice_provider::OpenAiLiveVoiceProvider; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; +use tokio::sync::watch; +use tokio_util::sync::CancellationToken; + +const LIVE_VOICE_INPUT_MESSAGE_LIMIT: usize = 128; +const LIVE_VOICE_INPUT_TOKEN_LIMIT: usize = 8_192; +#[cfg(feature = "live-voice")] +const LIVE_VOICE_ENABLED_CONFIG_KEY: &str = "GOOSE_LIVE_VOICE_ENABLED"; +#[cfg(feature = "live-voice")] +const LIVE_VOICE_CONFIG_KEY: &str = "GOOSE_LIVE_VOICE"; +#[cfg(feature = "live-voice")] +const DEFAULT_OPENAI_LIVE_VOICE: &str = "marin"; +#[cfg(feature = "live-voice")] +const LIVE_SESSION_INSTRUCTIONS: &str = concat!( + "You are Goose's live voice interface. Keep the conversation natural and concise.\n", + "Interruption policy: Stop speaking when the user interrupts and listen to what they say.\n", + "Delegation policy:\n", + "Backend tools:\n", + "- Goose can use backend reasoning and tools for longer tasks.\n", + "Delegate to Goose when:\n", + "- The user has finished stating a complete request that needs backend tools or reasoning.\n", + "- The user corrects or changes backend work already in progress.\n", + "Do not delegate to Goose when:\n", + "- The request is unfinished or is missing a required detail such as a location, object, ", + "command, or desired outcome. Ask one brief clarification and wait for the answer.\n", + "- The user is greeting you or making conversation that you can answer directly.\n", + "After delegating, briefly say the work is underway. Keep listening and accept corrections ", + "while Goose works. Do not guess the result. Present delegated results directly. Only say ", + "the task stopped or finished after Goose confirms it." +); +#[cfg(feature = "live-voice")] +const DELEGATION_DELIVERY_FAILURE_INSTRUCTIONS: &str = concat!( + "The latest update for the delegated request could not be delivered. Tell the user you ", + "couldn't bring the update into this voice conversation and ask them to try again. Do not ", + "say whether the delegated work succeeded or failed." +); + +type LiveVoiceInteractionControls = Arc>>>; +pub(crate) type LiveVoiceTranscriptPublisher = Arc; +type LiveVoiceResolver = + Arc Result, &'static str> + Send + Sync>; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LiveVoiceInteractionCompletion { + Stopped, + Failed, +} + +pub(crate) struct StartLiveVoiceInteractionResult { + pub(crate) interaction_id: LiveVoiceInteractionId, + pub(crate) answer: WebRtcAnswer, + pub(crate) completion_rx: watch::Receiver>, +} + +#[derive(Debug)] +pub(crate) enum LiveVoiceError { + Unavailable, + StartFailed, + StopFailed, +} + +struct LiveVoiceInteractionControl { + interaction_id: LiveVoiceInteractionId, + stop_requested: CancellationToken, + cleanup_finished: CancellationToken, + completion_tx: watch::Sender>, +} + +impl LiveVoiceInteractionControl { + fn request_stop(&self) -> watch::Receiver> { + self.stop_requested.cancel(); + self.completion_tx.subscribe() + } + + fn request_cleanup(&self) -> CancellationToken { + self.stop_requested.cancel(); + self.cleanup_finished.clone() + } + + fn publish_completion(&self, completion: LiveVoiceInteractionCompletion) { + if self.completion_tx.borrow().is_none() { + self.completion_tx.send_replace(Some(completion)); + } + } +} + +/// Owns the interaction reservation from before agent preparation until cleanup finishes. +pub(crate) struct LiveVoiceInteractionGuard { + active_runs: Arc, + interactions_by_session: LiveVoiceInteractionControls, + session_id: String, + control: Arc, + completion_on_drop: Option, +} + +impl LiveVoiceInteractionGuard { + fn new( + active_runs: Arc, + interactions_by_session: LiveVoiceInteractionControls, + session_id: &str, + control: Arc, + ) -> Self { + Self { + active_runs, + interactions_by_session, + session_id: session_id.to_string(), + control, + completion_on_drop: None, + } + } + + pub(super) fn publish_completion(&self, completion: LiveVoiceInteractionCompletion) { + self.control.publish_completion(completion); + } + + pub(super) fn publish_completion_after_cleanup( + &mut self, + completion: LiveVoiceInteractionCompletion, + ) { + self.completion_on_drop = Some(completion); + } + + pub(super) fn stop_requested(&self) -> &CancellationToken { + &self.control.stop_requested + } + + pub(super) fn session_id(&self) -> &str { + &self.session_id + } + + #[cfg(test)] + pub(super) fn for_test(session_id: &str) -> Self { + let (completion_tx, _) = watch::channel(None); + Self::new( + Arc::new(ActiveRunRegistry::default()), + Arc::new(Mutex::new(HashMap::new())), + session_id, + Arc::new(LiveVoiceInteractionControl { + interaction_id: LiveVoiceInteractionId::new(), + stop_requested: CancellationToken::new(), + cleanup_finished: CancellationToken::new(), + completion_tx, + }), + ) + } +} + +impl Drop for LiveVoiceInteractionGuard { + fn drop(&mut self) { + remove_interaction_if_current( + &self.interactions_by_session, + &self.session_id, + &self.control.interaction_id, + ); + self.active_runs.finish_live(&self.session_id); + self.publish_completion( + self.completion_on_drop + .unwrap_or(LiveVoiceInteractionCompletion::Failed), + ); + self.control.cleanup_finished.cancel(); + } +} + +pub struct LiveVoiceService { + live_voice_resolver: LiveVoiceResolver, + interactions_by_session: LiveVoiceInteractionControls, + active_runs: Arc, +} + +impl LiveVoiceService { + pub fn from_config(active_runs: Arc) -> Self { + Self::new(Arc::new(configured_live_voice), active_runs) + } + + fn new(live_voice_resolver: LiveVoiceResolver, active_runs: Arc) -> Self { + Self { + live_voice_resolver, + interactions_by_session: Arc::new(Mutex::new(HashMap::new())), + active_runs, + } + } + + pub(crate) fn availability( + &self, + session_id: Option<&str>, + mode: GooseMode, + ) -> Result<(), &'static str> { + self.eligible_provider(session_id, mode).map(|_| ()) + } + + fn eligible_provider( + &self, + session_id: Option<&str>, + mode: GooseMode, + ) -> Result, &'static str> { + let provider = self.provider_for_mode(mode)?; + if session_id.is_some_and(|session_id| self.active_runs.is_active(session_id)) { + Err("Live voice is unavailable while this session is busy") + } else { + Ok(provider) + } + } + + fn provider_for_mode( + &self, + mode: GooseMode, + ) -> Result, &'static str> { + let provider = (self.live_voice_resolver)()?; + if mode != GooseMode::Auto { + Err("Live voice requires Autonomous mode") + } else { + Ok(provider) + } + } + + pub(crate) fn reserve_interaction( + &self, + session_id: &str, + mode: GooseMode, + ) -> Result { + self.eligible_provider(Some(session_id), mode) + .map_err(|_| LiveVoiceError::Unavailable)?; + let (completion_tx, _) = watch::channel(None); + let control = Arc::new(LiveVoiceInteractionControl { + interaction_id: LiveVoiceInteractionId::new(), + stop_requested: CancellationToken::new(), + cleanup_finished: CancellationToken::new(), + completion_tx, + }); + { + let mut interactions = self + .interactions_by_session + .lock() + .expect("live voice lock poisoned"); + if !self.active_runs.start_live(session_id) { + return Err(LiveVoiceError::Unavailable); + } + interactions.insert(session_id.to_string(), control.clone()); + } + Ok(LiveVoiceInteractionGuard::new( + self.active_runs.clone(), + self.interactions_by_session.clone(), + session_id, + control, + )) + } + + pub(crate) async fn start_interaction( + &self, + interaction_guard: LiveVoiceInteractionGuard, + offer: WebRtcOffer, + session_manager: Arc, + transcript_publisher: LiveVoiceTranscriptPublisher, + main_agent: LiveMainAgent, + ) -> Result { + let session = session_manager + .get_session(&interaction_guard.session_id, true) + .await + .map_err(|_| LiveVoiceError::Unavailable)?; + let provider = self + .provider_for_mode(session.goose_mode) + .map_err(|_| LiveVoiceError::Unavailable)?; + let input_messages = + live_voice_input_messages(&session.conversation.unwrap_or_default()).await?; + if interaction_guard.stop_requested().is_cancelled() { + return Err(LiveVoiceError::Unavailable); + } + let (answer, provider_connection) = provider + .start(offer, input_messages) + .await + .map_err(|_| LiveVoiceError::StartFailed)?; + + let interaction_id = interaction_guard.control.interaction_id.clone(); + let completion_rx = interaction_guard.control.completion_tx.subscribe(); + let interaction = LiveVoiceInteraction::new( + provider_connection, + session_manager, + transcript_publisher, + main_agent, + interaction_guard, + ); + tokio::spawn(interaction.run()); + Ok(StartLiveVoiceInteractionResult { + interaction_id, + answer, + completion_rx, + }) + } + + pub(crate) async fn stop_interaction( + &self, + session_id: &str, + interaction_id: &LiveVoiceInteractionId, + ) -> Result<(), LiveVoiceError> { + let completion_rx = { + let interactions = self + .interactions_by_session + .lock() + .expect("live voice lock poisoned"); + let Some(control) = interactions.get(session_id) else { + return Ok(()); + }; + if &control.interaction_id != interaction_id { + return Ok(()); + } + control.request_stop() + }; + match wait_for_completion(completion_rx).await? { + LiveVoiceInteractionCompletion::Stopped => Ok(()), + LiveVoiceInteractionCompletion::Failed => Err(LiveVoiceError::StopFailed), + } + } + + pub(crate) async fn stop_session_interaction(&self, session_id: &str) { + let cleanup_finished = { + let interactions = self + .interactions_by_session + .lock() + .expect("live voice lock poisoned"); + interactions + .get(session_id) + .map(|control| control.request_cleanup()) + }; + + if let Some(cleanup_finished) = cleanup_finished { + cleanup_finished.cancelled().await; + } + } +} + +#[cfg(feature = "live-voice")] +fn configured_live_voice_enabled() -> bool { + crate::config::Config::global() + .get_param::(LIVE_VOICE_ENABLED_CONFIG_KEY) + .is_ok_and(|value| match value { + serde_json::Value::Bool(enabled) => enabled, + serde_json::Value::Number(enabled) => enabled.as_u64() == Some(1), + serde_json::Value::String(enabled) => { + enabled == "1" || enabled.eq_ignore_ascii_case("true") + } + _ => false, + }) +} + +#[cfg(feature = "live-voice")] +fn configured_live_voice() -> Result, &'static str> { + if !configured_live_voice_enabled() { + return Err("Live voice is disabled"); + } + + let config = crate::config::Config::global(); + let api_key = config + .get_secret::("OPENAI_API_KEY") + .map_err(|_| "Live voice provider is not configured")?; + let voice = config + .get_param::(LIVE_VOICE_CONFIG_KEY) + .unwrap_or_else(|_| DEFAULT_OPENAI_LIVE_VOICE.into()); + OpenAiLiveVoiceProvider::new( + api_key, + voice, + LIVE_SESSION_INSTRUCTIONS.into(), + DELEGATION_DELIVERY_FAILURE_INSTRUCTIONS.into(), + ) + .map(|provider| Arc::new(provider) as Arc) + .map_err(|_| "Live voice provider is not configured") +} + +#[cfg(not(feature = "live-voice"))] +fn configured_live_voice() -> Result, &'static str> { + Err("Live voice is disabled") +} + +async fn live_voice_input_messages( + conversation: &Conversation, +) -> Result, LiveVoiceError> { + let messages = conversation + .messages() + .iter() + .filter(|message| message.is_user_visible()) + .filter_map(|message| { + let text = message + .user_visible_content() + .content + .iter() + .filter_map(MessageContent::as_text) + .collect::(); + if text.trim().is_empty() { + return None; + } + Some(LiveVoiceInputMessage { + role: message.role.clone(), + text, + }) + }) + .collect::>(); + if messages.is_empty() { + return Ok(messages); + } + let token_counter = TokenCounter::new() + .await + .map_err(|_| LiveVoiceError::StartFailed)?; + let mut start = messages.len(); + let mut tokens = 0; + for (index, message) in messages.iter().enumerate().rev() { + if messages.len() - start == LIVE_VOICE_INPUT_MESSAGE_LIMIT { + break; + } + let message_tokens = token_counter.count_tokens(&message.text); + if message_tokens > LIVE_VOICE_INPUT_TOKEN_LIMIT - tokens { + break; + } + tokens += message_tokens; + start = index; + } + Ok(messages.into_iter().skip(start).collect()) +} + +pub(crate) async fn wait_for_completion( + mut completion_rx: watch::Receiver>, +) -> Result { + loop { + if let Some(completion) = *completion_rx.borrow() { + return Ok(completion); + } + completion_rx + .changed() + .await + .map_err(|_| LiveVoiceError::StopFailed)?; + } +} + +fn remove_interaction_if_current( + interactions_by_session: &LiveVoiceInteractionControls, + session_id: &str, + interaction_id: &LiveVoiceInteractionId, +) { + let mut interactions = interactions_by_session + .lock() + .expect("live voice lock poisoned"); + if matches!( + interactions.get(session_id), + Some(control) if &control.interaction_id == interaction_id + ) { + interactions.remove(session_id); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/goose/src/live_voice/service/tests.rs b/crates/goose/src/live_voice/service/tests.rs new file mode 100644 index 0000000000..b13ea121af --- /dev/null +++ b/crates/goose/src/live_voice/service/tests.rs @@ -0,0 +1,1195 @@ +mod fake_live_voice_provider; + +use super::super::interaction::{LiveMainAgent, DELEGATION_INSTRUCTION, PROVIDER_CLEANUP_TIMEOUT}; +use super::*; +use crate::agents::Agent; +use fake_live_voice_provider::{provider_channel, FakeConnectionDriver}; +use goose_providers::{live_voice_provider::ProviderConnectionEvent, model::ModelConfig}; +use rmcp::model::Role; +use std::time::Duration; +use tokio::task::JoinHandle; + +type StartResult = Result; + +impl LiveVoiceService { + fn for_test(provider: Arc, active_runs: Arc) -> Self { + Self::new(Arc::new(move || Ok(provider.clone())), active_runs) + } +} + +fn ignore_transcript_publisher() -> LiveVoiceTranscriptPublisher { + Arc::new(|_| {}) +} + +fn ignore_main_agent() -> LiveMainAgent { + LiveMainAgent::new( + |_, _| Ok(Box::pin(async { "unused".to_string() })), + |_, _| Box::pin(async { Ok("unused".to_string()) }), + ) +} + +fn controlled_main_agent() -> ( + LiveMainAgent, + tokio::sync::mpsc::UnboundedReceiver, + tokio::sync::mpsc::UnboundedReceiver, + tokio::sync::mpsc::UnboundedSender, +) { + let (start_tx, start_rx) = tokio::sync::mpsc::unbounded_channel(); + let (steer_tx, steer_rx) = tokio::sync::mpsc::unbounded_channel(); + let (finish_tx, finish_rx) = tokio::sync::mpsc::unbounded_channel(); + let finish_rx = Arc::new(tokio::sync::Mutex::new(finish_rx)); + let main_agent = LiveMainAgent::new( + move |_, input| { + start_tx.send(input).unwrap(); + let finish_rx = finish_rx.clone(); + Ok(Box::pin(async move { + finish_rx.lock().await.recv().await.unwrap() + })) + }, + move |_, input| { + steer_tx.send(input).unwrap(); + Box::pin(async { Ok("steered".to_string()) }) + }, + ); + (main_agent, start_rx, steer_rx, finish_tx) +} + +fn session_manager() -> Arc { + Arc::new(SessionManager::new(tempfile::tempdir().unwrap().keep())) +} + +fn service() -> LiveVoiceService { + let (provider, _starts) = provider_channel(); + LiveVoiceService::for_test(provider, Arc::new(ActiveRunRegistry::default())) +} + +async fn live_session( + conversation: impl IntoIterator, +) -> (Arc, String) { + let manager = session_manager(); + let session = manager + .create_session( + std::path::PathBuf::from("/tmp/test"), + "Live voice".into(), + crate::session::session_manager::SessionType::User, + GooseMode::Auto, + ) + .await + .unwrap(); + for message in conversation { + manager.add_message(&session.id, &message).await.unwrap(); + } + (manager, session.id) +} + +fn spawn_start( + service: Arc, + session_id: String, + offer: &'static str, + session_manager: Arc, +) -> JoinHandle { + tokio::spawn(async move { + let reservation = service + .reserve_interaction(&session_id, GooseMode::Auto) + .unwrap(); + service + .start_interaction( + reservation, + WebRtcOffer::new(offer.into()).unwrap(), + session_manager, + ignore_transcript_publisher(), + ignore_main_agent(), + ) + .await + }) +} + +fn assert_availability(service: &LiveVoiceService, expected: Result<(), &'static str>) { + assert_eq!( + service.availability(Some("main-session"), GooseMode::Auto), + expected + ); +} + +async fn establish_interaction() -> ( + Arc, + FakeConnectionDriver, + LiveVoiceInteractionId, + String, +) { + let (service, connection, interaction_id, session_id, _) = + establish_interaction_with(ignore_main_agent(), ignore_transcript_publisher()).await; + (service, connection, interaction_id, session_id) +} + +async fn establish_interaction_with( + main_agent: LiveMainAgent, + transcript_publisher: LiveVoiceTranscriptPublisher, +) -> ( + Arc, + FakeConnectionDriver, + LiveVoiceInteractionId, + String, + Arc, +) { + let (provider, mut starts) = provider_channel(); + let service = Arc::new(LiveVoiceService::for_test( + provider, + Arc::new(ActiveRunRegistry::default()), + )); + let (manager, session_id) = live_session([Message::user().with_text("prior context")]).await; + let start_service = service.clone(); + let start_session_id = session_id.clone(); + let start_manager = manager.clone(); + let start_task = tokio::spawn(async move { + let reservation = start_service + .reserve_interaction(&start_session_id, GooseMode::Auto) + .unwrap(); + start_service + .start_interaction( + reservation, + WebRtcOffer::new("offer".into()).unwrap(), + start_manager, + transcript_publisher, + main_agent, + ) + .await + }); + let connection = starts + .recv() + .await + .unwrap() + .accept(WebRtcAnswer::new("answer".into()).unwrap()) + .unwrap(); + let interaction_id = start_task.await.unwrap().unwrap().interaction_id; + (service, connection, interaction_id, session_id, manager) +} + +fn completion_receiver( + service: &LiveVoiceService, + session_id: &str, +) -> watch::Receiver> { + service + .interactions_by_session + .lock() + .unwrap() + .get(session_id) + .unwrap() + .completion_tx + .subscribe() +} + +#[test] +fn reports_each_eligibility_gate() { + let mut disabled = service(); + disabled.live_voice_resolver = Arc::new(|| Err("Live voice is disabled")); + assert_availability(&disabled, Err("Live voice is disabled")); + let mut unavailable = service(); + unavailable.live_voice_resolver = Arc::new(|| Err("Live voice provider is not configured")); + assert_availability(&unavailable, Err("Live voice provider is not configured")); + let ready = service(); + assert!(ready.active_runs.start_live("main-session")); + assert_eq!( + ready.availability(Some("main-session"), GooseMode::Auto), + Err("Live voice is unavailable while this session is busy") + ); + assert_eq!(ready.availability(None, GooseMode::Auto), Ok(())); + ready.active_runs.finish_live("main-session"); + assert_eq!( + ready.availability(Some("main-session"), GooseMode::Approve), + Err("Live voice requires Autonomous mode") + ); + assert_availability(&ready, Ok(())); +} + +#[test] +#[cfg(not(feature = "live-voice"))] +fn configured_live_voice_is_disabled_without_the_feature() { + assert!(matches!( + configured_live_voice(), + Err("Live voice is disabled") + )); +} + +#[tokio::test] +async fn start_rechecks_requirements_before_contacting_the_provider() { + let (provider, mut starts) = provider_channel(); + let mut service = LiveVoiceService::for_test(provider, Arc::new(ActiveRunRegistry::default())); + service.live_voice_resolver = Arc::new(|| Err("Live voice is disabled")); + let (_manager, session_id) = live_session([]).await; + + let result = service.reserve_interaction(&session_id, GooseMode::Auto); + + assert!(matches!(result, Err(LiveVoiceError::Unavailable))); + assert!(starts.try_recv().is_err()); +} + +#[tokio::test] +async fn input_messages_include_all_visible_non_empty_text_when_it_fits() { + let mut messages = vec![ + Message::user().with_text("hidden").agent_only(), + Message::assistant().with_thinking("internal", "signature"), + Message::user().with_text(" "), + ]; + messages.extend((0..12).map(|index| { + if index % 2 == 0 { + Message::user().with_text(format!("message {index}")) + } else { + Message::assistant().with_text(format!("message {index}")) + } + })); + messages.push(Message::assistant().with_text("delegated result")); + messages.push(Message::assistant().with_text("other hidden").agent_only()); + let conversation = Conversation::new_unvalidated(messages); + + let input_messages = live_voice_input_messages(&conversation).await.unwrap(); + assert_eq!(input_messages.len(), 13); + assert_eq!(input_messages.first().unwrap().text, "message 0"); + assert_eq!(input_messages.first().unwrap().role, Role::User); + assert_eq!(input_messages.last().unwrap().text, "delegated result"); + assert_eq!(input_messages.last().unwrap().role, Role::Assistant); + assert!(!input_messages + .iter() + .any(|message| message.text == "other hidden")); +} + +#[tokio::test] +async fn input_messages_keep_the_newest_complete_messages_within_provider_limits() { + let message_limited = Conversation::new_unvalidated( + (0..LIVE_VOICE_INPUT_MESSAGE_LIMIT + 2) + .map(|index| Message::user().with_text(format!("message {index}"))), + ); + let selected = live_voice_input_messages(&message_limited).await.unwrap(); + assert_eq!(selected.len(), LIVE_VOICE_INPUT_MESSAGE_LIMIT); + assert_eq!(selected.first().unwrap().text, "message 2"); + assert_eq!( + selected.last().unwrap().text, + format!("message {}", LIVE_VOICE_INPUT_MESSAGE_LIMIT + 1) + ); + + let token_limited = Conversation::new_unvalidated([ + Message::user().with_text("earlier"), + Message::assistant().with_text("history ".repeat(LIVE_VOICE_INPUT_TOKEN_LIMIT + 1)), + Message::user().with_text("newest"), + ]); + assert_eq!( + live_voice_input_messages(&token_limited).await.unwrap(), + vec![LiveVoiceInputMessage { + role: Role::User, + text: "newest".into(), + }] + ); +} + +#[tokio::test] +async fn a_start_reserves_the_session_until_it_finishes() { + let (provider, mut starts) = provider_channel(); + let service = Arc::new(LiveVoiceService::for_test( + provider, + Arc::new(ActiveRunRegistry::default()), + )); + let (manager, session_id) = live_session([Message::user().with_text("prior context")]).await; + let first = spawn_start( + service.clone(), + session_id.clone(), + "first-offer", + manager.clone(), + ); + let pending = starts.recv().await.unwrap(); + + assert_eq!( + pending.input_messages, + vec![LiveVoiceInputMessage { + role: Role::User, + text: "prior context".into(), + }] + ); + + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Err("Live voice is unavailable while this session is busy") + ); + + let second = service.reserve_interaction(&session_id, GooseMode::Auto); + assert!(matches!(second, Err(LiveVoiceError::Unavailable))); + + pending.reject("failed").unwrap(); + assert!(matches!( + first.await.unwrap(), + Err(LiveVoiceError::StartFailed) + )); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); +} + +#[tokio::test] +async fn a_cancelled_reservation_does_not_contact_the_provider() { + let (provider, mut starts) = provider_channel(); + let service = LiveVoiceService::for_test(provider, Arc::new(ActiveRunRegistry::default())); + let (manager, session_id) = live_session([]).await; + let reservation = service + .reserve_interaction(&session_id, GooseMode::Auto) + .unwrap(); + reservation.stop_requested().cancel(); + + let result = service + .start_interaction( + reservation, + WebRtcOffer::new("offer".into()).unwrap(), + manager, + ignore_transcript_publisher(), + ignore_main_agent(), + ) + .await; + + assert!(matches!(result, Err(LiveVoiceError::Unavailable))); + assert!(starts.try_recv().is_err()); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); +} + +#[tokio::test] +async fn a_cancelled_start_releases_the_session() { + let (provider, mut starts) = provider_channel(); + let service = Arc::new(LiveVoiceService::for_test( + provider, + Arc::new(ActiveRunRegistry::default()), + )); + let (manager, session_id) = live_session([]).await; + let start_task = spawn_start(service.clone(), session_id.clone(), "offer", manager); + let pending = starts.recv().await.unwrap(); + + start_task.abort(); + assert!(matches!(start_task.await, Err(error) if error.is_cancelled())); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); + drop(pending); +} + +#[tokio::test] +async fn session_stop_tracks_provider_start_until_the_interaction_stops() { + let (provider, mut starts) = provider_channel(); + let service = Arc::new(LiveVoiceService::for_test( + provider, + Arc::new(ActiveRunRegistry::default()), + )); + let (manager, session_id) = live_session([]).await; + let start_task = spawn_start(service.clone(), session_id.clone(), "offer", manager); + let pending = starts.recv().await.unwrap(); + + let stop = service.stop_session_interaction(&session_id); + let provider = async move { + let mut connection = pending + .accept(WebRtcAnswer::new("answer".into()).unwrap()) + .unwrap(); + start_task.await.unwrap().unwrap(); + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + }; + let ((), ()) = tokio::join!(stop, provider); + + assert!(!service + .interactions_by_session + .lock() + .unwrap() + .contains_key(&session_id)); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); +} + +#[tokio::test] +async fn session_stop_waits_for_cleanup_after_interaction_completion() { + let (main_agent, mut starts, _, finish_run) = controlled_main_agent(); + let (service, mut connection, interaction_id, session_id, _) = + establish_interaction_with(main_agent, ignore_transcript_publisher()).await; + let active_runs = service.active_runs.clone(); + let cancel_token = CancellationToken::new(); + assert!(active_runs + .start_live_delegation( + &session_id, + "delegated".into(), + cancel_token.clone(), + Arc::new(Agent::new()), + ) + .is_ok()); + connection + .send_event(ProviderConnectionEvent::TranscriptDelta { + event_id: "transcript".into(), + role: Role::User, + text: "do work".into(), + start_ms: 0, + end_ms: 1, + }) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::DelegationRequested { + event_id: "delegation-event".into(), + delegation_id: "delegation".into(), + offset_ms: 1, + }) + .unwrap(); + starts.recv().await.unwrap(); + + let stop_service = service.clone(); + let stop_session_id = session_id.clone(); + let stop = tokio::spawn(async move { + stop_service + .stop_interaction(&stop_session_id, &interaction_id) + .await + }); + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + assert!(matches!(stop.await.unwrap(), Ok(()))); + + active_runs.cancel_agent_run(&session_id); + cancel_token.cancelled().await; + let cleanup_service = service.clone(); + let cleanup_session_id = session_id.clone(); + let cleanup = tokio::spawn(async move { + cleanup_service + .stop_session_interaction(&cleanup_session_id) + .await; + }); + tokio::task::yield_now().await; + assert!(!cleanup.is_finished()); + finish_run.send("cancelled".into()).unwrap(); + cleanup.await.unwrap(); + active_runs.remove_agent_run(&session_id, "delegated"); + assert!(!active_runs.is_active(&session_id)); +} + +#[tokio::test] +async fn stop_failure_releases_the_session() { + let (service, mut connection, interaction_id, session_id) = establish_interaction().await; + let stop_service = service.clone(); + let stop_session_id = session_id.clone(); + let stop = tokio::spawn(async move { + stop_service + .stop_interaction(&stop_session_id, &interaction_id) + .await + }); + connection + .next_stop_request() + .await + .unwrap() + .send(Err("failed".into())) + .unwrap(); + + assert!(matches!( + stop.await.unwrap(), + Err(LiveVoiceError::StopFailed) + )); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); +} + +#[tokio::test] +async fn repeated_stop_uses_one_provider_shutdown() { + let (service, mut connection, interaction_id, session_id) = establish_interaction().await; + let first = service.stop_interaction(&session_id, &interaction_id); + let second = service.stop_interaction(&session_id, &interaction_id); + let provider = async move { + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + assert!(connection.next_stop_request().await.is_none()); + }; + + let (first, second, ()) = tokio::join!(first, second, provider); + + assert!(first.is_ok()); + assert!(second.is_ok()); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); +} + +#[tokio::test] +async fn stale_stop_does_not_stop_the_current_interaction() { + let service = service(); + let current_id = LiveVoiceInteractionId("current".into()); + let (completion_tx, _) = watch::channel(None); + let control = Arc::new(LiveVoiceInteractionControl { + interaction_id: current_id, + stop_requested: CancellationToken::new(), + cleanup_finished: CancellationToken::new(), + completion_tx, + }); + service + .interactions_by_session + .lock() + .unwrap() + .insert("main-session".into(), control.clone()); + + assert!(service + .stop_interaction("main-session", &LiveVoiceInteractionId("stale".into())) + .await + .is_ok()); + assert!(!control.stop_requested.is_cancelled()); +} + +#[tokio::test] +async fn transcript_without_delegation_saves_raw_and_main_agent_context_after_stop() { + let (service, mut connection, interaction_id, session_id, manager) = + establish_interaction_with(ignore_main_agent(), ignore_transcript_publisher()).await; + connection + .send_event(ProviderConnectionEvent::TranscriptDelta { + event_id: "transcript".into(), + role: Role::User, + text: "spoken context".into(), + start_ms: 0, + end_ms: 1, + }) + .unwrap(); + + let stop = service.stop_interaction(&session_id, &interaction_id); + let provider = async move { + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + }; + let (stop, ()) = tokio::join!(stop, provider); + assert!(stop.is_ok()); + + let session = manager.get_session(&session_id, true).await.unwrap(); + let conversation = session.conversation.unwrap(); + let transcript = conversation + .messages() + .iter() + .find(|message| message.as_concat_text() == "spoken context") + .cloned() + .expect("Live transcript should be persisted"); + assert!(transcript.is_user_visible()); + assert!(!transcript.is_agent_visible()); + let context = conversation + .messages() + .iter() + .find(|message| message.as_concat_text().contains("User: spoken context")) + .expect("Live context should be persisted"); + assert!(!context.is_user_visible()); + assert!(context.is_agent_visible()); +} + +#[tokio::test] +async fn rejected_delegation_keeps_context_for_live_completion() { + let main_agent = LiveMainAgent::new( + |_, _| Err("main agent unavailable".into()), + |_, _| Box::pin(async { Ok("unused".into()) }), + ); + let (service, mut connection, interaction_id, session_id, manager) = + establish_interaction_with(main_agent, ignore_transcript_publisher()).await; + connection + .send_event(ProviderConnectionEvent::TranscriptDelta { + event_id: "transcript".into(), + role: Role::User, + text: "keep this context".into(), + start_ms: 0, + end_ms: 1, + }) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::DelegationRequested { + event_id: "delegation-event".into(), + delegation_id: "delegation".into(), + offset_ms: 1, + }) + .unwrap(); + assert_eq!( + connection.next_delegation_update().await.unwrap().text, + "main agent unavailable" + ); + + let stop = service.stop_interaction(&session_id, &interaction_id); + let provider = async move { + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + }; + let (stop, ()) = tokio::join!(stop, provider); + assert!(stop.is_ok()); + + let session = manager.get_session(&session_id, true).await.unwrap(); + let conversation = session.conversation.unwrap(); + assert!(conversation.messages().iter().any(|message| { + message.is_agent_visible() + && !message.is_user_visible() + && message.as_concat_text().contains("User: keep this context") + })); +} + +#[tokio::test] +async fn completed_run_keeps_context_for_a_queued_delegation() { + let (main_agent, mut starts, _, finish_run) = controlled_main_agent(); + let (transcript_tx, mut transcript_rx) = tokio::sync::mpsc::unbounded_channel(); + let transcript_publisher: LiveVoiceTranscriptPublisher = Arc::new(move |message| { + transcript_tx.send(message).unwrap(); + }); + let (service, mut connection, interaction_id, session_id, _) = + establish_interaction_with(main_agent, transcript_publisher).await; + let delta = |event_id: &str, text: &str, end_ms| ProviderConnectionEvent::TranscriptDelta { + event_id: event_id.into(), + role: Role::User, + text: text.into(), + start_ms: 0, + end_ms, + }; + + connection + .send_event(delta("initial-transcript", "start work", 10)) + .unwrap(); + transcript_rx.recv().await.unwrap(); + connection + .send_event(ProviderConnectionEvent::DelegationRequested { + event_id: "initial-delegation-event".into(), + delegation_id: "initial-delegation".into(), + offset_ms: 10, + }) + .unwrap(); + assert!(starts.recv().await.unwrap().contains("User: start work")); + + connection + .send_event(delta("correction-transcript", "change the request", 20)) + .unwrap(); + transcript_rx.recv().await.unwrap(); + connection + .send_event(ProviderConnectionEvent::DelegationRequested { + event_id: "correction-delegation-event".into(), + delegation_id: "correction-delegation".into(), + offset_ms: 20, + }) + .unwrap(); + finish_run.send("first result".into()).unwrap(); + + assert_eq!( + connection.next_delegation_update().await.unwrap().text, + "first result" + ); + let correction = tokio::select! { + Some(correction) = starts.recv() => correction, + Some(update) = connection.next_delegation_update() => + panic!("queued delegation was rejected: {}", update.text), + }; + assert!(correction.contains("User: change the request")); + assert!(!correction.contains("start work")); + finish_run.send("corrected result".into()).unwrap(); + assert_eq!( + connection.next_delegation_update().await.unwrap().text, + "corrected result" + ); + + let stop = service.stop_interaction(&session_id, &interaction_id); + let provider = async move { + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + }; + let (stop, ()) = tokio::join!(stop, provider); + assert!(stop.is_ok()); +} + +#[tokio::test] +async fn transcript_is_projected_and_flushed_before_delegation() { + let temp_dir = tempfile::tempdir().unwrap(); + let session_manager = Arc::new(SessionManager::new(temp_dir.path().to_path_buf())); + let session = session_manager + .create_session( + std::path::PathBuf::from("/tmp/test"), + "Live transcript".into(), + crate::session::session_manager::SessionType::User, + GooseMode::Auto, + ) + .await + .unwrap(); + session_manager + .update(&session.id) + .provider_name("test") + .model_config(ModelConfig::new("test-model")) + .apply() + .await + .unwrap(); + let (provider, mut starts) = provider_channel(); + let service = Arc::new(LiveVoiceService::for_test( + provider, + Arc::new(ActiveRunRegistry::default()), + )); + let (transcript_tx, mut transcript_rx) = tokio::sync::mpsc::unbounded_channel(); + let transcript_publisher: LiveVoiceTranscriptPublisher = Arc::new(move |message| { + transcript_tx.send(message).unwrap(); + }); + let (start_tx, mut start_rx) = tokio::sync::mpsc::unbounded_channel(); + let main_agent = LiveMainAgent::new( + move |_, input| { + start_tx.send(input).unwrap(); + Ok(Box::pin(async { "unused".to_string() })) + }, + |_, _| Box::pin(async { Ok("unused".to_string()) }), + ); + let start_service = service.clone(); + let start_session_id = session.id.clone(); + let start_manager = session_manager.clone(); + let start = tokio::spawn(async move { + let reservation = start_service + .reserve_interaction(&start_session_id, GooseMode::Auto) + .unwrap(); + start_service + .start_interaction( + reservation, + WebRtcOffer::new("offer".into()).unwrap(), + start_manager, + transcript_publisher, + main_agent, + ) + .await + }); + let mut connection = starts + .recv() + .await + .unwrap() + .accept(WebRtcAnswer::new("answer".into()).unwrap()) + .unwrap(); + let interaction_id = start.await.unwrap().unwrap().interaction_id; + let delta = |event_id: &str, text: &str| ProviderConnectionEvent::TranscriptDelta { + event_id: event_id.into(), + role: Role::User, + text: text.into(), + start_ms: 0, + end_ms: 1, + }; + + connection.send_event(delta("1", "hello")).unwrap(); + assert_eq!( + transcript_rx.recv().await.unwrap().as_concat_text(), + "hello" + ); + connection + .send_event(ProviderConnectionEvent::DelegationRequested { + event_id: "delegation-event".into(), + delegation_id: "delegation".into(), + offset_ms: 1, + }) + .unwrap(); + let input = start_rx.recv().await.unwrap(); + assert_eq!( + input, + format!("Live conversation context:\nUser: hello\n{DELEGATION_INSTRUCTION}") + ); + assert_eq!( + connection.next_delegation_update().await.unwrap().text, + "unused" + ); + let stored = session_manager + .get_session(&session.id, true) + .await + .unwrap() + .conversation + .unwrap(); + assert_eq!(stored.messages().len(), 1); + assert_eq!(stored.messages()[0].as_concat_text(), "hello"); + assert!(!stored.messages()[0].is_agent_visible()); + + let stop_service = service.clone(); + let stop_session_id = session.id.clone(); + let stop = tokio::spawn(async move { + stop_service + .stop_interaction(&stop_session_id, &interaction_id) + .await + }); + let stop_response = connection.next_stop_request().await.unwrap(); + connection.send_event(delta("2", " world")).unwrap(); + connection + .send_event(ProviderConnectionEvent::DelegationRequested { + event_id: "stopping-delegation-event".into(), + delegation_id: "stopping-delegation".into(), + offset_ms: 1, + }) + .unwrap(); + stop_response.send(Ok(())).unwrap(); + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + + let revised = transcript_rx.recv().await.unwrap(); + assert_eq!(revised.as_concat_text(), " world"); + assert!(stop.await.unwrap().is_ok()); + assert!(start_rx.try_recv().is_err()); + + let stored = session_manager + .get_session(&session.id, true) + .await + .unwrap(); + let messages = stored.conversation.unwrap(); + assert_eq!(messages.messages().len(), 3); + assert_eq!(messages.messages()[0].as_concat_text(), "hello"); + assert_eq!(messages.messages()[1].as_concat_text(), " world"); + assert!(!messages.messages()[0].is_agent_visible()); + assert!(!messages.messages()[1].is_agent_visible()); + assert!(messages.messages()[2].is_agent_visible()); + assert!(!messages.messages()[2].is_user_visible()); + assert!(messages.messages()[2] + .as_concat_text() + .contains("User: world")); + assert!(!messages.messages()[2].as_concat_text().contains("hello")); +} + +#[tokio::test] +async fn running_transcript_is_saved_user_only_and_steered_to_the_main_agent() { + let (main_agent, mut starts, mut steers, finish_run) = controlled_main_agent(); + let (transcript_tx, mut transcript_rx) = tokio::sync::mpsc::unbounded_channel(); + let transcript_publisher: LiveVoiceTranscriptPublisher = Arc::new(move |message| { + transcript_tx.send(message).unwrap(); + }); + let (service, mut connection, interaction_id, session_id, manager) = + establish_interaction_with(main_agent, transcript_publisher).await; + let delta = + |event_id: &str, role: Role, text: &str, end_ms| ProviderConnectionEvent::TranscriptDelta { + event_id: event_id.into(), + role, + text: text.into(), + start_ms: 0, + end_ms, + }; + + connection + .send_event(delta("idle", Role::User, "start work", 10)) + .unwrap(); + assert_eq!( + transcript_rx.recv().await.unwrap().as_concat_text(), + "start work" + ); + connection + .send_event(ProviderConnectionEvent::DelegationRequested { + event_id: "start-event".into(), + delegation_id: "start".into(), + offset_ms: 10, + }) + .unwrap(); + let input = starts.recv().await.unwrap(); + assert_eq!( + input, + format!("Live conversation context:\nUser: start work\n{DELEGATION_INSTRUCTION}") + ); + + connection + .send_event(delta("working", Role::Assistant, "working", 20)) + .unwrap(); + connection + .send_event(delta("detail", Role::User, "also run tests", 30)) + .unwrap(); + for expected in ["working", "also run tests"] { + assert_eq!( + transcript_rx.recv().await.unwrap().as_concat_text(), + expected + ); + } + connection + .send_event(ProviderConnectionEvent::DelegationRequested { + event_id: "steer-event".into(), + delegation_id: "steer".into(), + offset_ms: 30, + }) + .unwrap(); + let input = steers.recv().await.unwrap(); + assert!(input.contains("Voice assistant: working\nUser: also run tests\n")); + assert!(!input.contains("start work")); + assert_eq!( + connection.next_delegation_update().await.unwrap().text, + "steered" + ); + + let stored = manager.get_session(&session_id, true).await.unwrap(); + let stored_messages = stored.conversation.unwrap(); + assert_eq!(stored_messages.messages().len(), 3); + assert_eq!(stored_messages.messages()[2].as_concat_text(), "working"); + assert!(stored_messages.messages()[2].is_user_visible()); + assert!(!stored_messages.messages()[2].is_agent_visible()); + + connection + .send_event(delta("question", Role::Assistant, "Anything else?", 40)) + .unwrap(); + connection + .send_event(delta("tail", Role::User, "Document it", 50)) + .unwrap(); + for expected in ["Anything else?", "Document it"] { + assert_eq!( + transcript_rx.recv().await.unwrap().as_concat_text(), + expected + ); + } + let stop = service.stop_interaction(&session_id, &interaction_id); + let provider = async move { + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + }; + let (stop, ()) = tokio::join!(stop, provider); + assert!(stop.is_ok()); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Err("Live voice is unavailable while this session is busy") + ); + assert_eq!( + manager + .get_session(&session_id, true) + .await + .unwrap() + .conversation + .unwrap() + .messages() + .len(), + 5 + ); + + finish_run.send("done".into()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while service + .availability(Some(&session_id), GooseMode::Auto) + .is_err() + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + let stored = manager + .get_session(&session_id, true) + .await + .unwrap() + .conversation + .unwrap(); + let messages = stored.messages(); + assert_eq!(messages.len(), 7); + assert_eq!(messages[1].as_concat_text(), "start work"); + assert!(!messages[1].is_agent_visible()); + for (message, text) in + messages[2..6] + .iter() + .zip(["working", "also run tests", "Anything else?", "Document it"]) + { + assert_eq!(message.as_concat_text(), text); + assert!(message.is_user_visible()); + assert!(!message.is_agent_visible()); + } + assert!(!messages[6].is_user_visible()); + assert!(messages[6].is_agent_visible()); + assert!(messages[6] + .as_concat_text() + .contains("Voice assistant: Anything else?\nUser: Document it")); + assert!(!messages[6].as_concat_text().contains("also run tests")); + assert!(!messages[6] + .as_concat_text() + .contains(DELEGATION_INSTRUCTION)); +} + +#[tokio::test] +async fn provider_terminal_events_fail_and_release_the_session() { + for event in [ + ProviderConnectionEvent::Closed, + ProviderConnectionEvent::Failed, + ] { + let (service, mut connection, interaction_id, session_id) = establish_interaction().await; + let completion_rx = completion_receiver(&service, &session_id); + let requires_cleanup = event == ProviderConnectionEvent::Failed; + connection.send_event(event).unwrap(); + if requires_cleanup { + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + } + + assert_eq!( + wait_for_completion(completion_rx).await.unwrap(), + LiveVoiceInteractionCompletion::Failed + ); + assert!(service + .stop_interaction(&session_id, &interaction_id) + .await + .is_ok()); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); + } +} + +#[tokio::test] +async fn provider_terminal_publishes_completion_after_release() { + let (provider, mut starts) = provider_channel(); + let service = Arc::new(LiveVoiceService::for_test( + provider, + Arc::new(ActiveRunRegistry::default()), + )); + let start_service = service.clone(); + let (manager, session_id) = live_session([]).await; + let start_session_id = session_id.clone(); + let start_task = tokio::spawn(async move { + let reservation = start_service + .reserve_interaction(&start_session_id, GooseMode::Auto) + .unwrap(); + start_service + .start_interaction( + reservation, + WebRtcOffer::new("offer".into()).unwrap(), + manager, + ignore_transcript_publisher(), + ignore_main_agent(), + ) + .await + }); + let connection = starts + .recv() + .await + .unwrap() + .accept(WebRtcAnswer::new("answer".into()).unwrap()) + .unwrap(); + let started = start_task.await.unwrap().unwrap(); + + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + let completion = wait_for_completion(started.completion_rx).await.unwrap(); + + assert_eq!(completion, LiveVoiceInteractionCompletion::Failed); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); +} + +#[tokio::test] +async fn cleanup_timeout_fails_and_releases_the_session() { + let (service, mut connection, interaction_id, session_id) = establish_interaction().await; + tokio::time::pause(); + let stop_service = service.clone(); + let stop_session_id = session_id.clone(); + let stop = tokio::spawn(async move { + stop_service + .stop_interaction(&stop_session_id, &interaction_id) + .await + }); + let pending_response = connection.next_stop_request().await.unwrap(); + tokio::time::advance(PROVIDER_CLEANUP_TIMEOUT).await; + + assert!(matches!( + stop.await.unwrap(), + Err(LiveVoiceError::StopFailed) + )); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); + drop(pending_response); +} + +#[tokio::test] +async fn queued_stop_wins_a_provider_close_race() { + let (service, mut connection, interaction_id, session_id) = establish_interaction().await; + connection + .send_event(ProviderConnectionEvent::Closed) + .unwrap(); + + let stop = service.stop_interaction(&session_id, &interaction_id); + let provider = async move { + connection + .next_stop_request() + .await + .unwrap() + .send(Ok(())) + .unwrap(); + }; + let (stop, ()) = tokio::join!(stop, provider); + + assert!(stop.is_ok()); + assert_eq!( + service.availability(Some(&session_id), GooseMode::Auto), + Ok(()) + ); +} + +#[test] +fn stale_cleanup_cannot_remove_a_later_call() { + let active_runs = Arc::new(ActiveRunRegistry::default()); + assert!(active_runs.start_live("main-session")); + let interactions = Arc::new(Mutex::new(HashMap::new())); + let current_id = LiveVoiceInteractionId("current".into()); + let (completion_tx, _) = watch::channel(None); + interactions.lock().unwrap().insert( + "main-session".into(), + Arc::new(LiveVoiceInteractionControl { + interaction_id: current_id.clone(), + stop_requested: CancellationToken::new(), + cleanup_finished: CancellationToken::new(), + completion_tx, + }), + ); + + remove_interaction_if_current( + &interactions, + "main-session", + &LiveVoiceInteractionId("stale".into()), + ); + + assert!(matches!( + interactions.lock().unwrap().get("main-session"), + Some(control) if control.interaction_id == current_id + )); + assert!(active_runs.is_active("main-session")); +} diff --git a/crates/goose/src/live_voice/service/tests/fake_live_voice_provider.rs b/crates/goose/src/live_voice/service/tests/fake_live_voice_provider.rs new file mode 100644 index 0000000000..47eec116dc --- /dev/null +++ b/crates/goose/src/live_voice/service/tests/fake_live_voice_provider.rs @@ -0,0 +1,135 @@ +use anyhow::{anyhow, Error, Result}; +use async_trait::async_trait; +use goose_providers::live_voice_provider::{ + DelegationUpdate, DelegationUpdateDelivery, LiveVoiceInputMessage, LiveVoiceProvider, + ProviderConnection, ProviderConnectionEvent, WebRtcAnswer, WebRtcOffer, +}; +use std::sync::Arc; +use tokio::sync::{mpsc, oneshot}; + +pub(super) fn provider_channel() -> ( + Arc, + mpsc::UnboundedReceiver, +) { + let (start_tx, start_rx) = mpsc::unbounded_channel(); + (Arc::new(FakeLiveVoiceProvider { start_tx }), start_rx) +} + +pub(super) struct FakeLiveVoiceProvider { + start_tx: mpsc::UnboundedSender, +} + +#[async_trait] +impl LiveVoiceProvider for FakeLiveVoiceProvider { + async fn start( + &self, + _offer: WebRtcOffer, + input_messages: Vec, + ) -> Result<(WebRtcAnswer, Box)> { + let (response_tx, response_rx) = oneshot::channel(); + self.start_tx + .send(FakeStartRequest { + input_messages, + response_tx, + }) + .map_err(|_| anyhow!("fake provider driver dropped"))?; + response_rx + .await + .map_err(|_| anyhow!("fake provider start response dropped"))? + } +} + +pub(super) struct FakeStartRequest { + pub(super) input_messages: Vec, + response_tx: oneshot::Sender)>>, +} + +impl FakeStartRequest { + pub(super) fn accept(self, answer: WebRtcAnswer) -> Result { + let (stop_request_tx, stop_request_rx) = mpsc::unbounded_channel(); + let (event_tx, event_rx) = mpsc::unbounded_channel(); + let (delegation_update_tx, delegation_update_rx) = mpsc::unbounded_channel(); + self.response_tx + .send(Ok(( + answer, + Box::new(FakeProviderConnection { + stop_request_tx, + event_rx, + delegation_update_tx, + }), + ))) + .map_err(|_| anyhow!("fake provider start caller dropped"))?; + Ok(FakeConnectionDriver { + stop_request_rx, + event_tx, + delegation_update_rx, + }) + } + + pub(super) fn reject(self, message: impl Into) -> Result<()> { + self.response_tx + .send(Err(anyhow!(message.into()))) + .map_err(|_| anyhow!("fake provider start caller dropped")) + } +} + +pub(super) struct FakeConnectionDriver { + stop_request_rx: mpsc::UnboundedReceiver>>, + event_tx: mpsc::UnboundedSender, + delegation_update_rx: mpsc::UnboundedReceiver, +} + +impl FakeConnectionDriver { + pub(super) async fn next_stop_request( + &mut self, + ) -> Option>> { + self.stop_request_rx.recv().await + } + + pub(super) fn send_event(&self, event: ProviderConnectionEvent) -> Result<()> { + self.event_tx + .send(event) + .map_err(|_| anyhow!("fake provider event receiver dropped")) + } + + pub(super) async fn next_delegation_update(&mut self) -> Option { + self.delegation_update_rx.recv().await + } +} + +struct FakeProviderConnection { + stop_request_tx: mpsc::UnboundedSender>>, + event_rx: mpsc::UnboundedReceiver, + delegation_update_tx: mpsc::UnboundedSender, +} + +#[async_trait] +impl ProviderConnection for FakeProviderConnection { + async fn next_event(&mut self) -> ProviderConnectionEvent { + self.event_rx + .recv() + .await + .unwrap_or(ProviderConnectionEvent::Failed) + } + + async fn send_delegation_update( + &mut self, + update: DelegationUpdate, + ) -> Result { + self.delegation_update_tx + .send(update) + .map_err(|_| anyhow!("fake provider driver dropped"))?; + Ok(DelegationUpdateDelivery::Delivered) + } + + async fn stop(&mut self) -> Result<()> { + let (response_tx, response_rx) = oneshot::channel(); + self.stop_request_tx + .send(response_tx) + .map_err(|_| anyhow!("fake provider driver dropped"))?; + response_rx + .await + .map_err(|_| anyhow!("fake provider stop response dropped"))? + .map_err(Error::msg) + } +} diff --git a/crates/goose/src/live_voice/transcript.rs b/crates/goose/src/live_voice/transcript.rs new file mode 100644 index 0000000000..ecc4c8ec58 --- /dev/null +++ b/crates/goose/src/live_voice/transcript.rs @@ -0,0 +1,171 @@ +use crate::conversation::message::{Message, MessageContent}; +use chrono::Utc; +use rmcp::model::Role; +use uuid::Uuid; + +#[derive(Default)] +pub(super) struct LiveTranscript { + // Consecutive deltas with the same role form one saved transcript entry. + transcript_entry_being_built: Option, + raw_transcript_entries_waiting_to_save: Vec, + // Delegation offsets address individual provider deltas, not grouped messages. + context_waiting_for_main_agent: Vec, + // A delegation before this prior position would resend old context. + last_delegation_position_ms: Option, +} + +pub(super) enum DelegationContext { + Stale, + MissingUserInput, + Available(String), +} + +struct ContextDelta { + role: Role, + text: String, + end_ms: u64, +} + +impl LiveTranscript { + pub(super) fn append(&mut self, role: Role, delta_text: &str, end_ms: u64) -> Option { + if delta_text.is_empty() { + return None; + } + + self.context_waiting_for_main_agent.push(ContextDelta { + role: role.clone(), + text: delta_text.to_string(), + end_ms, + }); + + if self + .transcript_entry_being_built + .as_ref() + .is_some_and(|message| message.role == role) + { + let message = self + .transcript_entry_being_built + .as_mut() + .expect("message exists"); + let [MessageContent::Text(content)] = message.content.as_mut_slice() else { + unreachable!("Live transcript messages contain one text block"); + }; + content.text.push_str(delta_text); + return Some(transcript_delta(message, delta_text)); + } + + if let Some(message) = self.transcript_entry_being_built.take() { + self.raw_transcript_entries_waiting_to_save.push(message); + } + let message = transcript_message(role, delta_text.to_string()); + let transcript_delta_to_display = transcript_delta(&message, delta_text); + self.transcript_entry_being_built = Some(message); + Some(transcript_delta_to_display) + } + + pub(super) fn finish_transcript_entry_being_built(&mut self) { + if let Some(message) = self.transcript_entry_being_built.take() { + self.raw_transcript_entries_waiting_to_save.push(message); + } + } + + pub(super) fn raw_transcript_entries_waiting_to_save(&self) -> &[Message] { + &self.raw_transcript_entries_waiting_to_save + } + + pub(super) fn mark_raw_transcript_entries_saved(&mut self) { + self.raw_transcript_entries_waiting_to_save.clear(); + } + + pub(super) fn context_for_delegation(&self, offset_ms: u64) -> DelegationContext { + if self + .last_delegation_position_ms + .is_some_and(|last_position| offset_ms < last_position) + { + return DelegationContext::Stale; + } + + let deltas = self + .context_waiting_for_main_agent + .iter() + .filter(|delta| delta.end_ms <= offset_ms) + .collect::>(); + if !deltas + .iter() + .any(|delta| delta.role == Role::User && !delta.text.trim().is_empty()) + { + return DelegationContext::MissingUserInput; + } + + DelegationContext::Available( + format_context(deltas).expect("delegation context contains transcript"), + ) + } + + pub(super) fn mark_context_sent_to_main_agent_through(&mut self, offset_ms: u64) { + self.context_waiting_for_main_agent + .retain(|delta| delta.end_ms > offset_ms); + self.last_delegation_position_ms = Some(offset_ms); + } + + pub(super) fn context_waiting_for_main_agent(&self) -> Option { + format_context(&self.context_waiting_for_main_agent) + } + + pub(super) fn agent_only_context_message_waiting_to_save(&self) -> Option { + self.context_waiting_for_main_agent().map(|context| { + Message::user() + .with_id(format!("msg_live_context_{}", Uuid::now_v7())) + .with_text(context) + .agent_only() + }) + } + + pub(super) fn mark_context_saved_for_main_agent(&mut self) { + self.context_waiting_for_main_agent.clear(); + } +} + +fn format_context<'a>(deltas: impl IntoIterator) -> Option { + let mut input = String::from("Live conversation context:\n"); + let mut previous_role = None; + for delta in deltas { + if previous_role.as_ref() == Some(&delta.role) { + input.push_str(&delta.text); + } else { + if previous_role.is_some() { + input.push('\n'); + } + input.push_str(speaker(&delta.role)); + input.push_str(": "); + input.push_str(delta.text.trim_start()); + previous_role = Some(delta.role.clone()); + } + } + previous_role?; + Some(input) +} + +fn speaker(role: &Role) -> &'static str { + match role { + Role::User => "User", + Role::Assistant => "Voice assistant", + } +} + +fn transcript_message(role: Role, text: String) -> Message { + Message::new(role, Utc::now().timestamp(), vec![]) + .with_id(format!("msg_live_{}", Uuid::now_v7())) + .with_text(text) + .user_only() +} + +fn transcript_delta(message: &Message, text: &str) -> Message { + Message { + id: message.id.clone(), + role: message.role.clone(), + created: message.created, + content: vec![MessageContent::text(text)], + metadata: message.metadata.clone(), + } +} diff --git a/crates/goose/src/session/session_manager.rs b/crates/goose/src/session/session_manager.rs index c2d3ef7938..c1320cbc00 100644 --- a/crates/goose/src/session/session_manager.rs +++ b/crates/goose/src/session/session_manager.rs @@ -1,6 +1,6 @@ use crate::config::paths::Paths; use crate::config::GooseMode; -use crate::conversation::message::{Message, MessageUsage, TokenState}; +use crate::conversation::message::{Message, MessageMetadata, MessageUsage, TokenState}; use crate::conversation::Conversation; use crate::providers::base::CostSource; use crate::providers::base::Provider; @@ -460,6 +460,16 @@ impl SessionManager { self.storage.replace_conversation(id, conversation).await } + pub(crate) async fn save_compacted_conversation( + &self, + id: &str, + conversation: &Conversation, + ) -> Result<()> { + self.storage + .save_compacted_conversation(id, conversation) + .await + } + pub async fn list_sessions(&self) -> Result> { self.storage.list_sessions().await } @@ -2000,6 +2010,62 @@ impl SessionStorage { Self::replace_conversation_inner(pool, session_id, conversation).await } + async fn save_compacted_conversation( + &self, + session_id: &str, + conversation: &Conversation, + ) -> Result<()> { + let pool = self.pool().await?; + let mut tx = pool.begin_with("BEGIN IMMEDIATE").await?; + + for message in conversation.messages() { + let message_id = message + .id + .as_ref() + .ok_or_else(|| anyhow::anyhow!("compacted conversation message has no id"))?; + let stored_metadata_json = sqlx::query_scalar::<_, Option>( + "SELECT metadata_json FROM messages WHERE session_id = ? AND message_id = ?", + ) + .bind(session_id) + .bind(message_id) + .fetch_optional(&mut *tx) + .await?; + if let Some(stored_metadata_json) = stored_metadata_json { + let mut metadata = stored_metadata_json + .and_then(|json| serde_json::from_str::(&json).ok()) + .unwrap_or_default(); + metadata.agent_visible = message.metadata.agent_visible; + sqlx::query( + "UPDATE messages SET metadata_json = ? WHERE session_id = ? AND message_id = ?", + ) + .bind(serde_json::to_string(&metadata)?) + .bind(session_id) + .bind(message_id) + .execute(&mut *tx) + .await?; + } else { + sqlx::query( + "INSERT INTO messages (message_id, session_id, role, content_json, created_timestamp, metadata_json) VALUES (?, ?, ?, ?, ?, ?)", + ) + .bind(message_id) + .bind(session_id) + .bind(role_to_string(&message.role)) + .bind(serde_json::to_string(&message.content)?) + .bind(message.created) + .bind(serde_json::to_string(&message.metadata)?) + .execute(&mut *tx) + .await?; + } + } + + sqlx::query("UPDATE sessions SET updated_at = datetime('now') WHERE id = ?") + .bind(session_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + async fn list_sessions_matching(&self, query: SessionListQuery<'_>) -> Result> { let filters = &query.filters; if matches!(filters.types, Some(types) if types.is_empty()) { diff --git a/crates/goose/tests/acp_custom_requests_test.rs b/crates/goose/tests/acp_custom_requests_test.rs index ba73d3cbfb..4a31d2e36c 100644 --- a/crates/goose/tests/acp_custom_requests_test.rs +++ b/crates/goose/tests/acp_custom_requests_test.rs @@ -187,6 +187,98 @@ fn test_custom_get_tools() { }); } +#[test] +#[serial] +#[cfg(feature = "live-voice")] +fn test_live_voice_availability_is_bound_to_an_accessible_main_session() { + let _guard = env_lock::lock_env([ + ("GOOSE_LIVE_VOICE_ENABLED", None::<&str>), + ("OPENAI_API_KEY", None::<&str>), + ]); + write_acp_global_config(DEFAULT_ACP_TEST_CONFIG); + + run_test(async move { + let openai = OpenAiFixture::new(vec![], Arc::new(EnforceSessionId::default())).await; + let mut conn = AcpServerConnection::new(TestConnectionConfig::default(), openai).await; + let SessionData { session, .. } = conn.new_session().await.unwrap(); + let disabled = send_custom( + conn.cx(), + "_goose/unstable/session/live-voice/availability", + serde_json::json!({ + "sessionId": session.session_id().0, + "_meta": { "goose": { "unrolledAgentLoop": true } } + }), + ) + .await + .unwrap(); + assert_eq!(disabled["status"], "unavailable"); + assert_eq!(disabled["message"], "Live voice is disabled"); + + send_custom( + conn.cx(), + "_goose/unstable/config/upsert", + serde_json::json!({ + "key": "GOOSE_LIVE_VOICE_ENABLED", + "value": true, + "isSecret": false + }), + ) + .await + .unwrap(); + send_custom( + conn.cx(), + "_goose/unstable/config/upsert", + serde_json::json!({ + "key": "OPENAI_API_KEY", + "value": "test-live-key", + "isSecret": true + }), + ) + .await + .unwrap(); + + let legacy_loop = send_custom( + conn.cx(), + "_goose/unstable/session/live-voice/availability", + serde_json::json!({ + "sessionId": session.session_id().0, + "_meta": { "goose": { "unrolledAgentLoop": false } } + }), + ) + .await + .unwrap(); + assert_eq!(legacy_loop["status"], "unavailable"); + assert_eq!( + legacy_loop["message"], + "Live voice is unavailable while Use Legacy Agent Loop is enabled" + ); + + let response = send_custom( + conn.cx(), + "_goose/unstable/session/live-voice/availability", + serde_json::json!({ + "sessionId": session.session_id().0, + "_meta": { "goose": { "unrolledAgentLoop": true } } + }), + ) + .await + .unwrap(); + assert_eq!(response["status"], "ready"); + assert_eq!(response["message"], "Start Live voice"); + + let inaccessible = send_custom( + conn.cx(), + "_goose/unstable/session/live-voice/availability", + serde_json::json!({ + "sessionId": "not-loaded-on-this-connection", + "_meta": { "goose": { "unrolledAgentLoop": true } } + }), + ) + .await; + assert!(inaccessible.is_err()); + }); +} + #[test] #[serial] fn test_custom_get_extensions() { diff --git a/crates/goose/tests/acp_fixtures/mod.rs b/crates/goose/tests/acp_fixtures/mod.rs index ec7e59bdf3..e486a5e271 100644 --- a/crates/goose/tests/acp_fixtures/mod.rs +++ b/crates/goose/tests/acp_fixtures/mod.rs @@ -389,6 +389,10 @@ pub async fn spawn_acp_server_in_process( ) }); + let active_runs = Arc::new(goose::acp::server::ActiveRunRegistry::default()); + let live_voice = Arc::new(goose::acp::server::LiveVoiceService::from_config( + active_runs.clone(), + )); let agent = GooseAcpAgent::new(GooseAcpAgentOptions { provider_factory, builtin_selection: goose::acp::server::AcpBuiltinSelection { @@ -402,7 +406,8 @@ pub async fn spawn_acp_server_in_process( additional_source_roots: Vec::new(), session_cwd: None, scheduler: Some(Arc::new(FixtureScheduler::new())), - active_prompt_runs: Default::default(), + active_runs, + live_voice, }) .await .unwrap(); diff --git a/ui/desktop/eslint.config.js b/ui/desktop/eslint.config.js index f6fafd6df2..f97af1a41e 100644 --- a/ui/desktop/eslint.config.js +++ b/ui/desktop/eslint.config.js @@ -69,8 +69,10 @@ module.exports = [ setInterval: 'readonly', clearTimeout: 'readonly', CustomEvent: 'readonly', + EventTarget: 'readonly', Element: 'readonly', HTMLElement: 'readonly', + HTMLAudioElement: 'readonly', HTMLInputElement: 'readonly', HTMLSelectElement: 'readonly', HTMLTextAreaElement: 'readonly', @@ -110,6 +112,13 @@ module.exports = [ AnalyserNode: 'readonly', MediaRecorder: 'readonly', MediaStream: 'readonly', + RTCDataChannel: 'readonly', + RTCDataChannelState: 'readonly', + RTCIceGatheringState: 'readonly', + RTCPeerConnection: 'readonly', + RTCPeerConnectionState: 'readonly', + RTCSessionDescriptionInit: 'readonly', + RTCTrackEvent: 'readonly', AudioWorkletNode: 'readonly', DOMException: 'readonly', MediaDeviceInfo: 'readonly', diff --git a/ui/desktop/src/App.tsx b/ui/desktop/src/App.tsx index e32c5a7cd9..313d3d65a5 100644 --- a/ui/desktop/src/App.tsx +++ b/ui/desktop/src/App.tsx @@ -57,6 +57,7 @@ import { trackErrorWithContext } from './utils/analytics'; import { AppEvents } from './constants/events'; import { registerPlatformEventHandlers } from './utils/platform_events'; import { reconnectAcpAfterSystemResume } from './acp/acpConnection'; +import { useLiveVoice, type LiveVoiceController } from './liveVoice/useLiveVoice'; function PageViewTracker() { usePageViewTracking(); @@ -64,9 +65,15 @@ function PageViewTracker() { } // Route Components -const HubRouteWrapper = ({ draftRef }: { draftRef: RefObject }) => { +const HubRouteWrapper = ({ + draftRef, + liveVoice, +}: { + draftRef: RefObject; + liveVoice: LiveVoiceController; +}) => { const setView = useNavigation(); - return ; + return ; }; export function resolveSessionInitialMessage( @@ -328,7 +335,17 @@ export function AppInner() { const nostrImportInFlight = useRef(null); const navigate = useNavigate(); + const location = useLocation(); const setView = useNavigation(); + const liveVoice = useLiveVoice(); + const { activeSessionId: activeLiveVoiceSessionId, stop: stopLiveVoice } = liveVoice; + + useEffect(() => { + const hasLiveVoiceEntryPoint = location.pathname === '/' || location.pathname === '/pair'; + if (!hasLiveVoiceEntryPoint && activeLiveVoiceSessionId) { + void stopLiveVoice(); + } + }, [activeLiveVoiceSessionId, location.pathname, stopLiveVoice]); const [chat, setChat] = useState({ sessionId: '', @@ -662,12 +679,15 @@ export function AppInner() { element={ - + } > - } /> + } + /> ({ acpChatSessionStore: { @@ -184,4 +188,29 @@ describe('handleAcpSessionNotification', () => { expect(maybeHandlePlatformEvent).not.toHaveBeenCalled(); }); + + it('routes Live voice interaction endings outside the conversation store', async () => { + const listener = vi.fn(); + const unsubscribe = subscribeToLiveVoiceInteractionEnded(listener); + + await handleAcpGooseSessionNotification({ + sessionId: SESSION_ID, + update: { + sessionUpdate: 'live_voice_interaction_ended', + interactionId: 'live-opaque', + outcome: 'failed', + }, + }); + + expect(listener).toHaveBeenCalledWith({ + sessionId: SESSION_ID, + update: { + sessionUpdate: 'live_voice_interaction_ended', + interactionId: 'live-opaque', + outcome: 'failed', + }, + }); + expect(acpChatSessionActions.applyAcpGooseSessionNotification).not.toHaveBeenCalled(); + unsubscribe(); + }); }); diff --git a/ui/desktop/src/acp/__tests__/liveVoice.test.ts b/ui/desktop/src/acp/__tests__/liveVoice.test.ts new file mode 100644 index 0000000000..8446a0ba77 --- /dev/null +++ b/ui/desktop/src/acp/__tests__/liveVoice.test.ts @@ -0,0 +1,102 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { getAcpClient } from '../acpConnection'; +import { acpGetLiveVoiceAvailability, acpStartLiveVoice, acpStopLiveVoice } from '../liveVoice'; + +vi.mock('../acpConnection', () => ({ getAcpClient: vi.fn() })); + +describe('ACP Live voice', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(window.electron.getSetting).mockResolvedValue(false); + }); + + it('uses the generated availability client for the displayed session', async () => { + const sessionLiveVoiceAvailability = vi.fn().mockResolvedValue({ + status: 'ready', + message: 'Start Live voice', + }); + vi.mocked(getAcpClient).mockResolvedValue({ + goose: { + sessionLiveVoiceAvailability_unstable: sessionLiveVoiceAvailability, + }, + } as unknown as Awaited>); + + await expect(acpGetLiveVoiceAvailability('main-session')).resolves.toEqual({ + status: 'ready', + message: 'Start Live voice', + }); + expect(sessionLiveVoiceAvailability).toHaveBeenCalledWith({ + sessionId: 'main-session', + _meta: { goose: { unrolledAgentLoop: true } }, + }); + }); + + it('checks availability without a session for a new chat', async () => { + const sessionLiveVoiceAvailability = vi.fn().mockResolvedValue({ + status: 'unavailable', + message: 'Live voice is disabled', + }); + vi.mocked(getAcpClient).mockResolvedValue({ + goose: { + sessionLiveVoiceAvailability_unstable: sessionLiveVoiceAvailability, + }, + } as unknown as Awaited>); + + await expect(acpGetLiveVoiceAvailability()).resolves.toEqual({ + status: 'unavailable', + message: 'Live voice is disabled', + }); + expect(sessionLiveVoiceAvailability).toHaveBeenCalledWith({ + _meta: { goose: { unrolledAgentLoop: true } }, + }); + }); + + it('passes the legacy loop selection to availability', async () => { + vi.mocked(window.electron.getSetting).mockResolvedValue(true); + const sessionLiveVoiceAvailability = vi.fn().mockResolvedValue({ + status: 'unavailable', + message: 'Live voice is unavailable while Use Legacy Agent Loop is enabled', + }); + vi.mocked(getAcpClient).mockResolvedValue({ + goose: { + sessionLiveVoiceAvailability_unstable: sessionLiveVoiceAvailability, + }, + } as unknown as Awaited>); + + await acpGetLiveVoiceAvailability('main-session'); + + expect(sessionLiveVoiceAvailability).toHaveBeenCalledWith({ + sessionId: 'main-session', + _meta: { goose: { unrolledAgentLoop: false } }, + }); + }); + + it('uses generated start and stop clients with the interaction ID', async () => { + const start = vi.fn().mockResolvedValue({ + interactionId: 'live-opaque', + answerSdp: 'answer', + }); + const stop = vi.fn().mockResolvedValue({}); + vi.mocked(getAcpClient).mockResolvedValue({ + goose: { + sessionLiveVoiceStart_unstable: start, + sessionLiveVoiceStop_unstable: stop, + }, + } as unknown as Awaited>); + + await expect(acpStartLiveVoice('main-session', 'offer')).resolves.toMatchObject({ + interactionId: 'live-opaque', + }); + await acpStopLiveVoice('main-session', 'live-opaque'); + + expect(start).toHaveBeenCalledWith({ + sessionId: 'main-session', + offerSdp: 'offer', + _meta: { goose: { unrolledAgentLoop: true } }, + }); + expect(stop).toHaveBeenCalledWith({ + sessionId: 'main-session', + interactionId: 'live-opaque', + }); + }); +}); diff --git a/ui/desktop/src/acp/chatNotifications.ts b/ui/desktop/src/acp/chatNotifications.ts index 5a0e5ba9e0..56157f8708 100644 --- a/ui/desktop/src/acp/chatNotifications.ts +++ b/ui/desktop/src/acp/chatNotifications.ts @@ -7,11 +7,11 @@ import { AppEvents } from '../constants/events'; import { maybeHandlePlatformEvent } from '../utils/platform_events'; import { toolNotificationEvent } from './adapter/toolNotifications'; import { acpChatSessionActions, acpChatSessionStore } from './chatSessionStore'; +import { publishLiveVoiceInteractionEnded } from './liveVoiceNotifications'; export function handleAcpSessionNotification(notification: SessionNotification): Promise { - const sessionNameBeforeNotification = acpChatSessionStore.getSnapshot( - notification.sessionId - )?.session?.name; + const sessionNameBeforeNotification = acpChatSessionStore.getSnapshot(notification.sessionId) + ?.session?.name; const updatedName = notification.update.sessionUpdate === 'session_info_update' ? notification.update.title @@ -49,6 +49,14 @@ function maybeHandleLivePlatformEvent(notification: SessionNotification): void { export function handleAcpGooseSessionNotification( notification: GooseSessionNotification_unstable ): Promise { + if (notification.update.sessionUpdate === 'live_voice_interaction_ended') { + publishLiveVoiceInteractionEnded({ + sessionId: notification.sessionId, + update: notification.update, + }); + return Promise.resolve(); + } + acpChatSessionActions.applyAcpGooseSessionNotification(notification); return Promise.resolve(); } diff --git a/ui/desktop/src/acp/liveVoice.ts b/ui/desktop/src/acp/liveVoice.ts new file mode 100644 index 0000000000..98780ef5fe --- /dev/null +++ b/ui/desktop/src/acp/liveVoice.ts @@ -0,0 +1,34 @@ +import type { + LiveVoiceAvailabilityResponse_unstable, + LiveVoiceStartResponse_unstable, +} from '@aaif/goose-acp-client'; +import { getAcpClient } from './acpConnection'; + +export async function acpGetLiveVoiceAvailability( + sessionId?: string +): Promise { + const { goose } = await getAcpClient(); + const useLegacyAgentLoop = await window.electron.getSetting('useLegacyAgentLoop'); + return goose.sessionLiveVoiceAvailability_unstable({ + ...(sessionId ? { sessionId } : {}), + _meta: { goose: { unrolledAgentLoop: !useLegacyAgentLoop } }, + }); +} + +export async function acpStartLiveVoice( + sessionId: string, + offerSdp: string +): Promise { + const { goose } = await getAcpClient(); + const useLegacyAgentLoop = await window.electron.getSetting('useLegacyAgentLoop'); + return goose.sessionLiveVoiceStart_unstable({ + sessionId, + offerSdp, + _meta: { goose: { unrolledAgentLoop: !useLegacyAgentLoop } }, + }); +} + +export async function acpStopLiveVoice(sessionId: string, interactionId: string): Promise { + const { goose } = await getAcpClient(); + await goose.sessionLiveVoiceStop_unstable({ sessionId, interactionId }); +} diff --git a/ui/desktop/src/acp/liveVoiceNotifications.ts b/ui/desktop/src/acp/liveVoiceNotifications.ts new file mode 100644 index 0000000000..aa811aa898 --- /dev/null +++ b/ui/desktop/src/acp/liveVoiceNotifications.ts @@ -0,0 +1,30 @@ +import type { GooseSessionNotification_unstable } from '@aaif/goose-acp-client'; + +export type LiveVoiceInteractionEndedNotification = { + sessionId: string; + update: Extract< + GooseSessionNotification_unstable['update'], + { sessionUpdate: 'live_voice_interaction_ended' } + >; +}; + +type LiveVoiceInteractionEndedListener = ( + notification: LiveVoiceInteractionEndedNotification +) => void; + +const listeners = new Set(); + +export function subscribeToLiveVoiceInteractionEnded( + listener: LiveVoiceInteractionEndedListener +): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function publishLiveVoiceInteractionEnded( + notification: LiveVoiceInteractionEndedNotification +): void { + for (const listener of listeners) { + listener(notification); + } +} diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index 98015c7a65..ce93cb046b 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -34,6 +34,11 @@ import { Goose } from './icons'; import EnvironmentBadge from './GooseSidebar/EnvironmentBadge'; import SessionActionsHeader from './SessionActionsHeader'; import { isAcpRecovering, subscribeToAcpRecovery } from '../acp/acpConnection'; +import type { LiveVoiceAvailabilityResponse_unstable } from '@aaif/goose-acp-client'; +import { acpGetLiveVoiceAvailability } from '../acp/liveVoice'; +import type { LiveVoiceController } from '../liveVoice/useLiveVoice'; + +const NEW_LIVE_VOICE_GREETING = 'Hello! What can I help you with?'; const i18n = defineMessages({ failedToLoadSession: { @@ -69,6 +74,7 @@ interface BaseChatProps { isActiveSession: boolean; initialMessage?: UserInput; noAutoSubmit?: boolean; + liveVoice: LiveVoiceController; } export default function BaseChat({ @@ -80,6 +86,7 @@ export default function BaseChat({ initialMessage, noAutoSubmit, isActiveSession, + liveVoice, }: BaseChatProps) { const intl = useIntl(); const location = useLocation(); @@ -87,11 +94,32 @@ export default function BaseChat({ const scrollRef = useRef(null); const chatInputRef = useRef(null); const disableAnimation = location.state?.disableAnimation || false; + const shouldStartLiveVoice = location.state?.startLiveVoice === true; const [hasStartedUsingRecipe, setHasStartedUsingRecipe] = React.useState(false); const [acpRecovering, setAcpRecovering] = useState(isAcpRecovering); + const [liveVoiceAvailability, setLiveVoiceAvailability] = + useState(null); const isMobile = useIsMobile(); const navContext = useNavigationContextSafe(); const setView = useNavigation(); + const { + activeSessionId: activeLiveVoiceSessionId, + liveVoiceSessionId, + start: startSharedLiveVoice, + } = liveVoice; + const ownsLiveVoice = liveVoiceSessionId === sessionId; + const liveVoiceActiveInAnotherSession = + activeLiveVoiceSessionId !== null && activeLiveVoiceSessionId !== sessionId; + const startLiveVoice = useCallback( + async (initialCommentary?: string) => { + if (activeLiveVoiceSessionId && activeLiveVoiceSessionId !== sessionId) { + setView('pair', { resumeSessionId: activeLiveVoiceSessionId }); + return; + } + await startSharedLiveVoice(sessionId, initialCommentary); + }, + [activeLiveVoiceSessionId, sessionId, setView, startSharedLiveVoice] + ); const isNavCollapsed = !navContext?.isNavExpanded; const contentClassName = cn('pr-1 pb-10 pt-12', (isMobile || isNavCollapsed) && 'pt-16'); const { droppedFiles, setDroppedFiles, handleDrop, handleDragOver } = useFileDrop(); @@ -115,6 +143,7 @@ export default function BaseChat({ notifications: toolCallNotifications, pauseQueueOnStop, queueProcessingBlocked, + hasActiveRun, onMessageUpdate, } = useChatSession({ sessionId, @@ -125,6 +154,66 @@ export default function BaseChat({ [handleSubmit] ); + const sessionLoaded = session !== undefined; + const liveVoiceChatBusy = chatState !== ChatState.Idle; + + useEffect(() => { + if (!isActiveSession || !shouldStartLiveVoice || liveVoiceAvailability === null) { + return; + } + + if (liveVoiceAvailability.status === 'ready') { + void startLiveVoice(NEW_LIVE_VOICE_GREETING); + } + + navigate(location, { + replace: true, + state: { ...location.state, startLiveVoice: undefined }, + }); + }, [ + isActiveSession, + shouldStartLiveVoice, + liveVoiceAvailability, + startLiveVoice, + location, + navigate, + ]); + + useEffect(() => { + if (!isActiveSession || !sessionLoaded || acpRecovering || liveVoiceActiveInAnotherSession) { + setLiveVoiceAvailability(null); + return; + } + + let current = true; + setLiveVoiceAvailability(null); + void acpGetLiveVoiceAvailability(sessionId).then( + (response) => { + if (current) { + setLiveVoiceAvailability(response); + } + }, + () => { + if (current) { + setLiveVoiceAvailability(null); + } + } + ); + return () => { + current = false; + }; + }, [ + acpRecovering, + hasActiveRun, + isActiveSession, + liveVoice.phase, + liveVoiceActiveInAnotherSession, + liveVoiceChatBusy, + session?.goose_mode, + sessionId, + sessionLoaded, + ]); + const handleWorkingDirChange = useCallback( async (newDir: string) => { if (!session) { @@ -214,7 +303,6 @@ export default function BaseChat({ const sessionModel = session?.model_config?.model_name ?? null; const sessionProvider = session?.provider_name ?? null; - const sessionLoaded = session !== undefined; const latestInference = useMemo(() => { for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; @@ -475,6 +563,7 @@ export default function BaseChat({ sessionId={sessionId} handleSubmit={chatInputSubmit} chatState={chatState} + hasActiveRun={hasActiveRun} onStop={stopStreaming} onSteerQueuedMessage={onSteerQueuedMessage} pauseQueueOnStop={pauseQueueOnStop} @@ -507,6 +596,15 @@ export default function BaseChat({ workingDir={session?.working_dir} onWorkingDirChange={handleWorkingDirChange} latestInference={latestInference} + liveVoice={{ + availability: liveVoiceAvailability, + phase: ownsLiveVoice ? liveVoice.phase : 'idle', + muted: ownsLiveVoice ? liveVoice.muted : false, + activeInAnotherSession: liveVoiceActiveInAnotherSession, + start: () => startLiveVoice(), + stop: liveVoice.stop, + toggleMute: liveVoice.toggleMute, + }} {...customChatInputProps} /> diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index 0aae1875bc..ca9c95bb86 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -41,6 +41,9 @@ import { getTextDirection } from '../utils/textDirection'; import { defineMessages, useIntl } from '../i18n'; import TurndownService from 'turndown'; import type { NextChatExtensionDraft } from '../utils/nextChatExtensions'; +import { LiveVoiceButton } from './LiveVoiceButton'; +import type { LiveVoiceAvailabilityResponse_unstable } from '@aaif/goose-acp-client'; +import { isLiveVoiceActive, type LiveVoiceController } from '../liveVoice/useLiveVoice'; const turndown = new TurndownService({ headingStyle: 'atx', @@ -69,6 +72,15 @@ interface PastedImage { error?: string; } +type ChatInputLiveVoice = Pick< + LiveVoiceController, + 'phase' | 'muted' | 'stop' | 'toggleMute' +> & { + availability: LiveVoiceAvailabilityResponse_unstable | null; + activeInAnotherSession: boolean; + start: () => Promise; +}; + const moveQueuedMessageToFront = ( messages: QueuedMessage[], messageId: string @@ -163,6 +175,7 @@ interface ChatInputProps { sessionId: string | null; handleSubmit: (input: UserInput) => void; chatState: ChatState; + hasActiveRun: boolean; onStop?: () => void; onSteerQueuedMessage?: (input: UserInput) => Promise; pauseQueueOnStop?: boolean; @@ -198,12 +211,14 @@ interface ChatInputProps { latestInference?: Message['metadata']['inference'] | null; nextChatExtensionDraft?: NextChatExtensionDraft; onNextChatExtensionDraftChange?: (draft: NextChatExtensionDraft) => void; + liveVoice?: ChatInputLiveVoice; } export default function ChatInput({ sessionId, handleSubmit, chatState = ChatState.Idle, + hasActiveRun, onStop, onSteerQueuedMessage, pauseQueueOnStop = false, @@ -234,6 +249,7 @@ export default function ChatInput({ latestInference, nextChatExtensionDraft, onNextChatExtensionDraftChange, + liveVoice, }: ChatInputProps) { const [_value, setValue] = useState(initialValue); const [displayValue, setDisplayValue] = useState(initialValue); // For immediate visual feedback @@ -256,13 +272,14 @@ export default function ChatInput({ // Derived state - chatState != Idle means we're in some form of loading state const isLoading = chatState !== ChatState.Idle; - const isLoadingRef = useRef(isLoading); - + const liveVoiceBlocksSubmission = liveVoice ? isLiveVoiceActive(liveVoice.phase) : false; + const isSubmissionBusy = isLoading || hasActiveRun || liveVoiceBlocksSubmission; + const isSubmissionBusyRef = useRef(isSubmissionBusy); const composerDir = useMemo(() => getTextDirection(displayValue) ?? undefined, [displayValue]); const queueProcessingBlockedRef = useRef(queueProcessingBlocked); - const wasLoadingRef = useRef(isLoading); + const wasSubmissionBusyRef = useRef(isSubmissionBusy); const wasQueueProcessingBlockedRef = useRef(queueProcessingBlocked); - isLoadingRef.current = isLoading; + isSubmissionBusyRef.current = isSubmissionBusy; queueProcessingBlockedRef.current = queueProcessingBlocked; // Queue functionality - ephemeral, only exists in memory for this chat instance @@ -393,12 +410,12 @@ export default function ChatInput({ // Queue processing useEffect(() => { - const becameIdle = wasLoadingRef.current && !isLoading; + const becameAvailable = wasSubmissionBusyRef.current && !isSubmissionBusy; const becameUnblocked = wasQueueProcessingBlockedRef.current && !queueProcessingBlocked; const hasSendNowInFlight = sendNowInFlightMessageIdsRef.current.size > 0; if ( - (becameIdle || (becameUnblocked && !isLoading)) && + (becameAvailable || (becameUnblocked && !isSubmissionBusy)) && !queueProcessingBlocked && !hasSendNowInFlight && queuedMessages.length > 0 @@ -410,13 +427,13 @@ export default function ChatInput({ if (pendingSendAfterStopId && !messageToSend) { clearPendingSendAfterStop(pendingSendAfterStopId); - wasLoadingRef.current = isLoading; + wasSubmissionBusyRef.current = isSubmissionBusy; wasQueueProcessingBlockedRef.current = queueProcessingBlocked; return; } if (!messageToSend) { - wasLoadingRef.current = isLoading; + wasSubmissionBusyRef.current = isSubmissionBusy; wasQueueProcessingBlockedRef.current = queueProcessingBlocked; return; } @@ -452,10 +469,10 @@ export default function ChatInput({ } } } - wasLoadingRef.current = isLoading; + wasSubmissionBusyRef.current = isSubmissionBusy; wasQueueProcessingBlockedRef.current = queueProcessingBlocked; }, [ - isLoading, + isSubmissionBusy, queueProcessingBlocked, queuedMessages, handleSubmit, @@ -1072,14 +1089,14 @@ export default function ChatInput({ }; const handleInterruptionAndQueue = () => { - if (!isLoading || !hasSubmittableContent) { + if (!isSubmissionBusy || !hasSubmittableContent) { return false; } const imageData = convertImagesToImageData(); const contentToQueue = appendDroppedFilePaths(displayValue.trim()); - const interruptionMatch = detectInterruption(displayValue.trim()); + const interruptionMatch = isLoading ? detectInterruption(displayValue.trim()) : null; if (interruptionMatch && interruptionMatch.shouldInterrupt) { setLastInterruption(interruptionMatch.matchedText); @@ -1122,7 +1139,7 @@ export default function ChatInput({ }; const canSubmit = - !isLoading && + !isSubmissionBusy && !queueProcessingBlocked && (displayValue.trim() || pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) || @@ -1244,12 +1261,12 @@ export default function ChatInput({ if (queueProcessingBlocked) { return; } - if (isLoading && hasSubmittableContent) { + if (isSubmissionBusy && hasSubmittableContent) { handleInterruptionAndQueue(); return; } const canSubmit = - !isLoading && + !isSubmissionBusy && !queueProcessingBlocked && (displayValue.trim() || pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) || @@ -1359,7 +1376,6 @@ export default function ChatInput({ allDroppedFiles.some((file) => !file.error && !file.isLoading); const isAnyImageLoading = pastedImages.some((img) => img.isLoading); const isAnyDroppedFileLoading = allDroppedFiles.some((file) => file.isLoading); - const isSubmitButtonDisabled = !hasSubmittableContent || isAnyImageLoading || @@ -1412,7 +1428,7 @@ export default function ChatInput({ if (!messageToSend) return; if (queueProcessingBlocked) return; - if (!isLoading) { + if (!isSubmissionBusy) { setQueuedMessages((prev) => removeQueuedMessage(prev, messageId)); LocalMessageStorage.addMessage(messageToSend.content); handleSubmit({ msg: messageToSend.content, images: messageToSend.images }); @@ -1451,7 +1467,7 @@ export default function ChatInput({ setSendNowInFlightMessage(messageId, false); } - if (!isLoadingRef.current && !queueProcessingBlockedRef.current) { + if (!isSubmissionBusyRef.current && !queueProcessingBlockedRef.current) { queuePausedRef.current = wasQueuePausedBeforeSteer; setQueuedMessages((prev) => { const newQueue = removeQueuedMessage(prev, messageId); @@ -1482,7 +1498,7 @@ export default function ChatInput({ const handleResumeQueue = () => { queuePausedRef.current = false; setLastInterruption(null); - if (!isLoading && !queueProcessingBlocked && queuedMessages.length > 0) { + if (!isSubmissionBusy && !queueProcessingBlocked && queuedMessages.length > 0) { const nextMessage = queuedMessages[0]; LocalMessageStorage.addMessage(nextMessage.content); handleSubmit({ msg: nextMessage.content, images: nextMessage.images }); @@ -1553,7 +1569,7 @@ export default function ChatInput({ onBlur={() => setIsFocused(false)} ref={textAreaRef} rows={1} - readOnly={isRecording} + readOnly={isRecording || liveVoiceBlocksSubmission} style={{ minHeight: `${minTextareaHeight}px`, maxHeight: `${maxHeight}px`, @@ -1800,6 +1816,23 @@ export default function ChatInput({ )} + {liveVoice && ( + void liveVoice.start()} + onStop={() => void liveVoice.stop()} + onToggleMute={liveVoice.toggleMute} + composerEmpty={ + displayValue.trim().length === 0 && + pastedImages.length === 0 && + allDroppedFiles.length === 0 + } + /> + )} + {/* Right: mic — ghost icon, no background when idle */} {dictationProvider && ( diff --git a/ui/desktop/src/components/ChatSessionsContainer.test.tsx b/ui/desktop/src/components/ChatSessionsContainer.test.tsx index 1be05c2864..f52c431a7d 100644 --- a/ui/desktop/src/components/ChatSessionsContainer.test.tsx +++ b/ui/desktop/src/components/ChatSessionsContainer.test.tsx @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import ChatSessionsContainer from './ChatSessionsContainer'; import { subscribeToAcpRecovery } from '../acp/acpConnection'; import { acpChatSessionController } from '../acp/chatSessionController'; +import type { LiveVoiceController } from '../liveVoice/useLiveVoice'; vi.mock('react-router', () => ({ useSearchParams: () => [new URLSearchParams('resumeSessionId=session-1')], @@ -38,6 +39,17 @@ describe('ChatSessionsContainer', () => { ); diff --git a/ui/desktop/src/components/ChatSessionsContainer.tsx b/ui/desktop/src/components/ChatSessionsContainer.tsx index 6ee1372429..fb8bcf0ce7 100644 --- a/ui/desktop/src/components/ChatSessionsContainer.tsx +++ b/ui/desktop/src/components/ChatSessionsContainer.tsx @@ -5,6 +5,7 @@ import { ChatType } from '../types/chat'; import { UserInput } from '../types/message'; import { subscribeToAcpRecovery } from '../acp/acpConnection'; import { acpChatSessionController } from '../acp/chatSessionController'; +import type { LiveVoiceController } from '../liveVoice/useLiveVoice'; interface ChatSessionsContainerProps { setChat: (chat: ChatType) => void; @@ -13,6 +14,7 @@ interface ChatSessionsContainerProps { initialMessage?: UserInput; noAutoSubmit?: boolean; }>; + liveVoice: LiveVoiceController; } /** @@ -23,6 +25,7 @@ interface ChatSessionsContainerProps { export default function ChatSessionsContainer({ setChat, activeSessions, + liveVoice, }: ChatSessionsContainerProps) { const [searchParams] = useSearchParams(); const currentSessionId = searchParams.get('resumeSessionId') ?? undefined; @@ -72,6 +75,7 @@ export default function ChatSessionsContainer({ noAutoSubmit={session.noAutoSubmit} suppressEmptyState={false} isActiveSession={isVisible} + liveVoice={liveVoice} /> ); diff --git a/ui/desktop/src/components/Hub.test.tsx b/ui/desktop/src/components/Hub.test.tsx index 5b8fbe58a0..fbf930dddd 100644 --- a/ui/desktop/src/components/Hub.test.tsx +++ b/ui/desktop/src/components/Hub.test.tsx @@ -1,21 +1,38 @@ /** * @vitest-environment jsdom */ -import { act, render } from '@testing-library/react'; +import { act, render, waitFor } from '@testing-library/react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import Hub from './Hub'; import { IntlTestWrapper } from '../i18n/test-utils'; import { createSession } from '../sessions'; import { UserInput } from '../types/message'; +import { acpGetLiveVoiceAvailability } from '../acp/liveVoice'; +import { subscribeToAcpRecovery } from '../acp/acpConnection'; +import type { LiveVoiceController } from '../liveVoice/useLiveVoice'; type ChatInputCapture = { draftRef?: { current: string }; handleSubmit: (input: UserInput) => void; + liveVoice?: { + availability: { status: string; message: string } | null; + start: () => Promise; + }; onNextChatExtensionDraftChange?: (draft: { selectedNames: Set }) => void; }; type Session = Awaited>; +const liveVoice: LiveVoiceController = { + activeSessionId: null, + liveVoiceSessionId: null, + phase: 'idle', + muted: false, + start: vi.fn(), + stop: vi.fn(), + toggleMute: vi.fn(), +}; + const captured = vi.hoisted(() => ({ chatInput: null as ChatInputCapture | null })); vi.mock('./ChatInput', () => ({ @@ -47,6 +64,10 @@ vi.mock('../acp/errors', () => ({ formatAcpError: (error: unknown) => String(err vi.mock('../toasts', () => ({ toastError: vi.fn() })); +vi.mock('../acp/liveVoice', () => ({ acpGetLiveVoiceAvailability: vi.fn() })); + +vi.mock('../acp/acpConnection', () => ({ subscribeToAcpRecovery: vi.fn() })); + const DRAFT = 'a half-written thought'; const TYPED_WHILE_STARTING = 'and one more thought'; @@ -63,10 +84,10 @@ function pendingSession() { return settle; } -function renderHub(draftRef: { current: string }) { +function renderHub(draftRef: { current: string }, setView = vi.fn()) { return render( - + ); } @@ -80,7 +101,48 @@ async function submit() { describe('Hub', () => { beforeEach(() => { vi.clearAllMocks(); + liveVoice.activeSessionId = null; + liveVoice.liveVoiceSessionId = null; + liveVoice.phase = 'idle'; captured.chatInput = null; + vi.mocked(acpGetLiveVoiceAvailability).mockRejectedValue(new Error('ACP unavailable')); + vi.mocked(subscribeToAcpRecovery).mockReturnValue(() => undefined); + }); + + it('requests Live voice availability again after ACP recovers', async () => { + let recoveryChanged: ((recovering: boolean) => void) | undefined; + const available = { status: 'ready' as const, message: 'Start Live voice' }; + vi.mocked(acpGetLiveVoiceAvailability) + .mockRejectedValueOnce(new Error('ACP disconnected')) + .mockResolvedValueOnce(available); + vi.mocked(subscribeToAcpRecovery).mockImplementation((listener) => { + recoveryChanged = listener; + return () => undefined; + }); + + renderHub({ current: '' }); + await waitFor(() => expect(acpGetLiveVoiceAvailability).toHaveBeenCalledTimes(1)); + + act(() => recoveryChanged?.(true)); + act(() => recoveryChanged?.(false)); + + await waitFor(() => { + expect(acpGetLiveVoiceAvailability).toHaveBeenCalledTimes(2); + expect(captured.chatInput?.liveVoice?.availability).toEqual(available); + }); + }); + + it('returns to the session with the active Live voice interaction', async () => { + const setView = vi.fn(); + liveVoice.activeSessionId = 'session-with-live-voice'; + renderHub({ current: '' }, setView); + + await act(async () => captured.chatInput?.liveVoice?.start?.()); + + expect(setView).toHaveBeenCalledWith('pair', { + resumeSessionId: 'session-with-live-voice', + }); + expect(createSession).not.toHaveBeenCalled(); }); it('starts a chat with no extensions when the user cleared the picker', async () => { diff --git a/ui/desktop/src/components/Hub.tsx b/ui/desktop/src/components/Hub.tsx index a9b73d6c1c..c84230c7b5 100644 --- a/ui/desktop/src/components/Hub.tsx +++ b/ui/desktop/src/components/Hub.tsx @@ -28,6 +28,10 @@ import { import { formatAcpError } from '../acp/errors'; import { toastError } from '../toasts'; import { formatClockDisplay } from '../utils/timeUtils'; +import { acpGetLiveVoiceAvailability } from '../acp/liveVoice'; +import type { LiveVoiceAvailabilityResponse_unstable } from '@aaif/goose-acp-client'; +import { subscribeToAcpRecovery } from '../acp/acpConnection'; +import type { LiveVoiceController } from '../liveVoice/useLiveVoice'; const i18n = defineMessages({ goodMorning: { id: 'hub.goodMorning', defaultMessage: 'Good morning' }, @@ -48,16 +52,20 @@ function useClock() { export default function Hub({ setView, draftRef, + liveVoice, }: { setView: (view: View, viewOptions?: ViewOptions) => void; /** Unsent input of this screen, kept above the route outlet across the unmount. */ draftRef: RefObject; + liveVoice: LiveVoiceController; }) { const intl = useIntl(); const { extensionsList } = useConfig(); const [workingDir, setWorkingDir] = useState(getInitialWorkingDir()); const userSelectedWorkingDirRef = useRef(false); const [isCreatingSession, setIsCreatingSession] = useState(false); + const [liveVoiceAvailability, setLiveVoiceAvailability] = + useState(null); const [nextChatExtensionDraft, setNextChatExtensionDraft] = useState(null); const inputRef = useRef(null); @@ -75,6 +83,35 @@ export default function Hub({ }; }, []); + useEffect(() => { + let requestGeneration = 0; + const requestAvailability = async () => { + const generation = ++requestGeneration; + try { + const availability = await acpGetLiveVoiceAvailability(); + if (generation === requestGeneration) { + setLiveVoiceAvailability(availability); + } + } catch { + if (generation === requestGeneration) { + setLiveVoiceAvailability(null); + } + } + }; + + void requestAvailability(); + const unsubscribe = subscribeToAcpRecovery((recovering) => { + if (!recovering) { + void requestAvailability(); + } + }); + + return () => { + requestGeneration += 1; + unsubscribe(); + }; + }, []); + const greeting = useMemo(() => { if (hour < 12) return intl.formatMessage(i18n.goodMorning); if (hour < 18) return intl.formatMessage(i18n.goodAfternoon); @@ -103,11 +140,8 @@ export default function Hub({ setWorkingDir(dir); }, []); - const handleSubmit = async (input: UserInput) => { - const { msg: userMessage, images } = input; - if (!(images.length > 0 || userMessage.trim()) || isCreatingSession) return; - - const draftAtSubmit = draftRef.current; + const createHubSession = async () => { + if (isCreatingSession) return null; setIsCreatingSession(true); try { @@ -124,36 +158,68 @@ export default function Hub({ const dir = userSelectedWorkingDirRef.current ? workingDir : await getEffectiveWorkingDir(); const session = await createSession(dir, sessionOptions); setNextChatExtensionDraft(null); - - window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED)); - window.dispatchEvent( - new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, { - detail: { sessionId: session.id, initialMessage: { msg: userMessage, images } }, - }) - ); - - // The draft is this screen's own, so it is dropped once the session exists. - // Comparing it against the value at submit leaves an edit made while the - // session was starting alone, including one that emptied the input. - if (draftRef.current === draftAtSubmit) { - draftRef.current = ''; - } - - setView('pair', { - disableAnimation: true, - resumeSessionId: session.id, - initialMessage: { msg: userMessage, images }, - }); + return session; } catch (error) { console.error('Failed to create session:', error); toastError({ title: "Couldn't start chat", msg: formatAcpError(error) }); setIsCreatingSession(false); + return null; } }; + const handleSubmit = async (input: UserInput) => { + const { msg: userMessage, images } = input; + if (!(images.length > 0 || userMessage.trim())) return; + + const draftAtSubmit = draftRef.current; + const session = await createHubSession(); + if (!session) return; + + window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED)); + window.dispatchEvent( + new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, { + detail: { sessionId: session.id, initialMessage: { msg: userMessage, images } }, + }) + ); + + // Preserve edits made while the session was being created. + if (draftRef.current === draftAtSubmit) { + draftRef.current = ''; + } + + setView('pair', { + disableAnimation: true, + resumeSessionId: session.id, + initialMessage: { msg: userMessage, images }, + }); + }; + + const handleStartLiveVoice = async () => { + if (liveVoice.activeSessionId) { + setView('pair', { resumeSessionId: liveVoice.activeSessionId }); + return; + } + + const session = await createHubSession(); + if (!session) return; + + window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED)); + window.dispatchEvent( + new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, { + detail: { sessionId: session.id }, + }) + ); + + setView('pair', { + disableAnimation: true, + resumeSessionId: session.id, + startLiveVoice: true, + }); + }; + return (
-
+
{time} @@ -170,6 +236,7 @@ export default function Hub({ draftRef={draftRef} handleSubmit={handleSubmit} chatState={isCreatingSession ? ChatState.LoadingConversation : ChatState.Idle} + hasActiveRun={false} onStop={() => {}} initialValue="" setView={setView} @@ -185,6 +252,15 @@ export default function Hub({ inputRef={inputRef} nextChatExtensionDraft={draftForMenu} onNextChatExtensionDraftChange={handleNextChatExtensionDraftChange} + liveVoice={{ + availability: isCreatingSession ? null : liveVoiceAvailability, + phase: 'idle', + muted: false, + activeInAnotherSession: liveVoice.activeSessionId !== null, + start: handleStartLiveVoice, + stop: liveVoice.stop, + toggleMute: liveVoice.toggleMute, + }} />
diff --git a/ui/desktop/src/components/Layout/AppLayout.tsx b/ui/desktop/src/components/Layout/AppLayout.tsx index 158ef5b874..8960e5e4fb 100644 --- a/ui/desktop/src/components/Layout/AppLayout.tsx +++ b/ui/desktop/src/components/Layout/AppLayout.tsx @@ -12,6 +12,7 @@ import { Navigation } from './NavigationPanel'; import { Z_INDEX } from './constants'; import { cn } from '../../utils'; import { UserInput } from '../../types/message'; +import type { LiveVoiceController } from '../../liveVoice/useLiveVoice'; const i18n = defineMessages({ openNavigation: { @@ -30,9 +31,10 @@ interface AppLayoutContentProps { initialMessage?: UserInput; noAutoSubmit?: boolean; }>; + liveVoice: LiveVoiceController; } -const AppLayoutContent: React.FC = ({ activeSessions }) => { +const AppLayoutContent: React.FC = ({ activeSessions, liveVoice }) => { const intl = useIntl(); const location = useLocation(); const safeIsMacOS = (window?.electron?.platform || 'darwin') === 'darwin'; @@ -102,7 +104,11 @@ const AppLayoutContent: React.FC = ({ activeSessions }) =
- +
); @@ -147,7 +153,7 @@ const AppLayoutContent: React.FC = ({ activeSessions }) = className="relative flex-shrink-0 overflow-hidden h-full p-2" >
- +
{isNavExpanded && (
= ({ activeSessions }) = {/* Always render ChatSessionsContainer to keep SSE connections alive. When navigating away from /pair, hide it with CSS */}
- +
@@ -177,12 +187,13 @@ interface AppLayoutProps { initialMessage?: UserInput; noAutoSubmit?: boolean; }>; + liveVoice: LiveVoiceController; } -export const AppLayout: React.FC = ({ activeSessions }) => { +export const AppLayout: React.FC = ({ activeSessions, liveVoice }) => { return ( - + ); }; diff --git a/ui/desktop/src/components/Layout/NavigationPanel.tsx b/ui/desktop/src/components/Layout/NavigationPanel.tsx index 676a1a78b1..4c8749fd1a 100644 --- a/ui/desktop/src/components/Layout/NavigationPanel.tsx +++ b/ui/desktop/src/components/Layout/NavigationPanel.tsx @@ -1,6 +1,6 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useLocation } from 'react-router'; -import { ChevronDown, ChevronRight } from 'lucide-react'; +import { AudioLines, ChevronDown, ChevronRight } from 'lucide-react'; import { motion } from 'framer-motion'; import { useNavigationContext } from './NavigationContext'; import { useConfig } from '../ConfigContext'; @@ -77,6 +77,10 @@ const i18n = defineMessages({ id: 'navigationPanel.statusIdle', defaultMessage: 'Idle', }, + returnToActiveLiveVoice: { + id: 'liveVoice.returnToActive', + defaultMessage: 'Return to active Live voice', + }, }); const navItemClass = (active: boolean) => @@ -111,6 +115,7 @@ const NavRow: React.FC = ({ item, active, onClick }) => { interface SessionRowProps { session: SessionListItem; active: boolean; + isLiveVoiceActive: boolean; status: SessionStatus | undefined; onClick: () => void; onRenamed: () => void; @@ -163,7 +168,14 @@ const SessionTooltipContent: React.FC = ({ session, ); }; -const SessionRow: React.FC = ({ session, active, status, onClick, onRenamed }) => { +const SessionRow: React.FC = ({ + session, + active, + isLiveVoiceActive, + status, + onClick, + onRenamed, +}) => { const intl = useIntl(); const [isEditing, setIsEditing] = useState(false); const [tooltipOpen, setTooltipOpen] = useState(false); @@ -209,6 +221,12 @@ const SessionRow: React.FC = ({ session, active, status, onClic onEditStart={() => setIsEditing(true)} onEditEnd={() => setIsEditing(false)} /> + {isLiveVoiceActive && ( + + )}
@@ -219,7 +237,10 @@ const SessionRow: React.FC = ({ session, active, status, onClic ); }; -export const Navigation: React.FC<{ className?: string }> = ({ className }) => { +export const Navigation: React.FC<{ + className?: string; + activeLiveVoiceSessionId: string | null; +}> = ({ className, activeLiveVoiceSessionId }) => { const intl = useIntl(); const { isNavExpanded } = useNavigationContext(); const location = useLocation(); @@ -369,6 +390,7 @@ export const Navigation: React.FC<{ className?: string }> = ({ className }) => { key={session.id} session={session} active={session.id === activeSessionId} + isLiveVoiceActive={session.id === activeLiveVoiceSessionId} status={sessionStatuses.get(session.id)} onClick={() => { clearUnread(session.id); @@ -386,6 +408,7 @@ export const Navigation: React.FC<{ className?: string }> = ({ className }) => { key={session.id} session={session} active={session.id === activeSessionId} + isLiveVoiceActive={session.id === activeLiveVoiceSessionId} status={sessionStatuses.get(session.id)} onClick={() => { clearUnread(session.id); diff --git a/ui/desktop/src/components/LiveVoiceButton.test.tsx b/ui/desktop/src/components/LiveVoiceButton.test.tsx new file mode 100644 index 0000000000..fb1fc7ae4b --- /dev/null +++ b/ui/desktop/src/components/LiveVoiceButton.test.tsx @@ -0,0 +1,113 @@ +import { render, screen } from '@testing-library/react'; +import type { ComponentProps } from 'react'; +import { describe, expect, it, vi } from 'vitest'; +import { IntlTestWrapper } from '../i18n/test-utils'; +import { LiveVoiceButton } from './LiveVoiceButton'; + +const baseProps: ComponentProps = { + availability: { status: 'ready', message: 'Start Live voice' }, + composerEmpty: true, + phase: 'idle', + muted: false, + activeInAnotherSession: false, + onStart: vi.fn(), + onStop: vi.fn(), + onToggleMute: vi.fn(), +}; + +function renderButton(props: Partial = {}) { + return render(, { wrapper: IntlTestWrapper }); +} + +describe('LiveVoiceButton', () => { + it('shows an eligible state for an empty composer', () => { + renderButton(); + + const button = screen.getByTestId('live-voice-button'); + expect(button).toBeEnabled(); + expect(button).toHaveAttribute('data-eligible', 'true'); + expect(button).toHaveAccessibleName('Start Live voice'); + }); + + it.each(['idle', 'error'] as const)('shows the unavailable reason while %s', (phase) => { + renderButton({ + availability: { + status: 'unavailable', + message: 'Live voice requires Autonomous mode', + }, + phase, + }); + + const button = screen.getByTestId('live-voice-button'); + expect(button).toBeDisabled(); + expect(button).toHaveAttribute('data-eligible', 'false'); + expect(button).toHaveAccessibleName('Live voice requires Autonomous mode'); + }); + + it.each(['idle', 'error'] as const)('applies empty-composer gate while %s', (phase) => { + renderButton({ composerEmpty: false, phase }); + + const button = screen.getByTestId('live-voice-button'); + expect(button).toBeDisabled(); + expect(button).toHaveAttribute('data-eligible', 'false'); + expect(button).toHaveAccessibleName('Clear the message and attachments to use Live voice'); + }); + + it('does not render without eligibility for the displayed session', () => { + renderButton({ availability: null }); + expect(screen.queryByTestId('live-voice-button')).not.toBeInTheDocument(); + }); + + it('returns to an interaction active in another session', () => { + const onStart = vi.fn(); + renderButton({ + availability: null, + composerEmpty: false, + activeInAnotherSession: true, + onStart, + }); + + const button = screen.getByTestId('live-voice-button'); + expect(button).toBeEnabled(); + expect(button).toHaveAccessibleName('Return to active Live voice'); + button.click(); + expect(onStart).toHaveBeenCalledOnce(); + }); + + it('starts when idle and keeps stop controls available while availability refreshes', () => { + const onStart = vi.fn(); + const onStop = vi.fn(); + const activeProps = { ...baseProps, availability: null, onStart, onStop }; + const { container, rerender } = renderButton({ onStart, onStop }); + + screen.getByTestId('live-voice-button').click(); + expect(onStart).toHaveBeenCalledOnce(); + + rerender(); + expect(container.querySelector('.animate-spin')).toBeInTheDocument(); + screen.getByTestId('live-voice-button').click(); + expect(onStop).toHaveBeenCalledOnce(); + + rerender(); + screen.getByTestId('live-voice-button').click(); + expect(onStop).toHaveBeenCalledTimes(2); + }); + + it('disables the stopping state', () => { + renderButton({ availability: null, phase: 'stopping' }); + expect(screen.getByTestId('live-voice-button')).toBeDisabled(); + }); + + it('toggles microphone mute while live', () => { + const onToggleMute = vi.fn(); + const liveProps = { availability: null, phase: 'live', onToggleMute } as const; + const { rerender } = renderButton(liveProps); + + screen.getByTestId('live-voice-mute-button').click(); + expect(onToggleMute).toHaveBeenCalledOnce(); + expect(screen.getByTestId('live-voice-mute-button')).toHaveAccessibleName('Mute microphone'); + + rerender(); + expect(screen.getByTestId('live-voice-mute-button')).toHaveAccessibleName('Unmute microphone'); + }); +}); diff --git a/ui/desktop/src/components/LiveVoiceButton.tsx b/ui/desktop/src/components/LiveVoiceButton.tsx new file mode 100644 index 0000000000..c57c38a483 --- /dev/null +++ b/ui/desktop/src/components/LiveVoiceButton.tsx @@ -0,0 +1,145 @@ +import type { LiveVoiceAvailabilityResponse_unstable } from '@aaif/goose-acp-client'; +import { AudioLines, LoaderCircle, Mic, MicOff, Square } from 'lucide-react'; +import { defineMessages, useIntl } from '../i18n'; +import { cn } from '../utils'; +import { Button } from './ui/button'; +import { Tooltip, TooltipContent, TooltipTrigger } from './ui/Tooltip'; +import { isLiveVoiceActive, type LiveVoicePhase } from '../liveVoice/useLiveVoice'; + +const i18n = defineMessages({ + emptyComposerRequired: { + id: 'liveVoice.emptyComposerRequired', + defaultMessage: 'Clear the message and attachments to use Live voice', + }, + connecting: { + id: 'liveVoice.connecting', + defaultMessage: 'Connecting Live voice', + }, + live: { + id: 'liveVoice.live', + defaultMessage: 'Stop Live voice', + }, + stopping: { + id: 'liveVoice.stopping', + defaultMessage: 'Stopping Live voice', + }, + error: { + id: 'liveVoice.error', + defaultMessage: 'Live voice failed. Try again', + }, + mute: { + id: 'liveVoice.mute', + defaultMessage: 'Mute microphone', + }, + unmute: { + id: 'liveVoice.unmute', + defaultMessage: 'Unmute microphone', + }, + returnToActive: { + id: 'liveVoice.returnToActive', + defaultMessage: 'Return to active Live voice', + }, +}); + +interface LiveVoiceButtonProps { + availability: LiveVoiceAvailabilityResponse_unstable | null; + composerEmpty: boolean; + phase: LiveVoicePhase; + muted: boolean; + activeInAnotherSession: boolean; + onStart: () => void; + onStop: () => void; + onToggleMute: () => void; +} + +export function LiveVoiceButton({ + availability, + composerEmpty, + phase, + muted, + activeInAnotherSession, + onStart, + onStop, + onToggleMute, +}: LiveVoiceButtonProps) { + const intl = useIntl(); + if (availability === null && !isLiveVoiceActive(phase) && !activeInAnotherSession) return null; + + const eligible = activeInAnotherSession || (availability?.status === 'ready' && composerEmpty); + const stopping = phase === 'stopping'; + const canStart = phase === 'idle' || phase === 'error'; + const canStop = phase === 'connecting' || phase === 'live'; + let label = availability?.message; + if (activeInAnotherSession) { + label = intl.formatMessage(i18n.returnToActive); + } else if (phase === 'connecting') { + label = intl.formatMessage(i18n.connecting); + } else if (phase === 'live') { + label = intl.formatMessage(i18n.live); + } else if (phase === 'stopping') { + label = intl.formatMessage(i18n.stopping); + } else if (!composerEmpty) { + label = intl.formatMessage(i18n.emptyComposerRequired); + } else if (availability?.status === 'ready' && phase === 'error') { + label = intl.formatMessage(i18n.error); + } + const disabled = activeInAnotherSession ? false : canStart ? !eligible : stopping; + + return ( + <> + {phase === 'live' && ( + + + + + {intl.formatMessage(muted ? i18n.unmute : i18n.mute)} + + )} + + + + + + + {label} + + + ); +} diff --git a/ui/desktop/src/components/settings/chat/ChatSettingsSection.tsx b/ui/desktop/src/components/settings/chat/ChatSettingsSection.tsx index ebbf7acc2c..13f690027c 100644 --- a/ui/desktop/src/components/settings/chat/ChatSettingsSection.tsx +++ b/ui/desktop/src/components/settings/chat/ChatSettingsSection.tsx @@ -3,6 +3,7 @@ import { DictationSettings } from '../dictation/DictationSettings'; import { SecurityToggle } from '../security/SecurityToggle'; import { ResponseStylesSection } from '../response_styles/ResponseStylesSection'; import { GoosehintsSection } from './GoosehintsSection'; +import { LiveVoiceSettings } from './LiveVoiceSettings'; import { SpellcheckToggle } from './SpellcheckToggle'; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '../../ui/card'; import { defineMessages, useIntl } from '../../../i18n'; @@ -50,6 +51,7 @@ export default function ChatSettingsSection() { + diff --git a/ui/desktop/src/components/settings/chat/LiveVoiceSettings.test.tsx b/ui/desktop/src/components/settings/chat/LiveVoiceSettings.test.tsx new file mode 100644 index 0000000000..d488d74210 --- /dev/null +++ b/ui/desktop/src/components/settings/chat/LiveVoiceSettings.test.tsx @@ -0,0 +1,35 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { IntlTestWrapper } from '../../../i18n/test-utils'; +import { useConfig } from '../../ConfigContext'; +import { LiveVoiceSettings } from './LiveVoiceSettings'; + +vi.mock('../../ConfigContext', () => ({ useConfig: vi.fn() })); + +describe('LiveVoiceSettings', () => { + const read = vi.fn(); + const upsert = vi.fn(); + + beforeEach(() => { + vi.clearAllMocks(); + read.mockResolvedValue(true); + vi.mocked(useConfig).mockReturnValue({ read, upsert } as unknown as ReturnType< + typeof useConfig + >); + }); + + it('loads and persists the Live voice toggle', async () => { + render(, { wrapper: IntlTestWrapper }); + + const toggle = screen.getByRole('switch', { name: 'Live voice' }); + await waitFor(() => expect(toggle).toBeChecked()); + expect(read).toHaveBeenCalledWith('GOOSE_LIVE_VOICE_ENABLED', false); + + fireEvent.click(toggle); + + await waitFor(() => + expect(upsert).toHaveBeenCalledWith('GOOSE_LIVE_VOICE_ENABLED', false, false) + ); + await waitFor(() => expect(toggle).not.toBeChecked()); + }); +}); diff --git a/ui/desktop/src/components/settings/chat/LiveVoiceSettings.tsx b/ui/desktop/src/components/settings/chat/LiveVoiceSettings.tsx new file mode 100644 index 0000000000..5d5a163971 --- /dev/null +++ b/ui/desktop/src/components/settings/chat/LiveVoiceSettings.tsx @@ -0,0 +1,52 @@ +import { useEffect, useState } from 'react'; +import { useConfig } from '../../ConfigContext'; +import { Switch } from '../../ui/switch'; +import { defineMessages, useIntl } from '../../../i18n'; + +const LIVE_VOICE_ENABLED_CONFIG_KEY = 'GOOSE_LIVE_VOICE_ENABLED'; + +const i18n = defineMessages({ + title: { + id: 'liveVoiceSettings.title', + defaultMessage: 'Live voice', + }, + description: { + id: 'liveVoiceSettings.description', + defaultMessage: 'Have a real-time voice conversation with Goose', + }, +}); + +export const LiveVoiceSettings = () => { + const intl = useIntl(); + const { read, upsert } = useConfig(); + const [enabled, setEnabled] = useState(null); + + useEffect(() => { + void read(LIVE_VOICE_ENABLED_CONFIG_KEY, false).then((value) => { + setEnabled(value === true || value === 1 || value === '1' || value === 'true'); + }); + }, [read]); + + const handleToggle = async (checked: boolean) => { + await upsert(LIVE_VOICE_ENABLED_CONFIG_KEY, checked, false); + setEnabled(checked); + }; + + return ( +
+
+

{intl.formatMessage(i18n.title)}

+

+ {intl.formatMessage(i18n.description)} +

+
+ +
+ ); +}; diff --git a/ui/desktop/src/hooks/useChatSession.ts b/ui/desktop/src/hooks/useChatSession.ts index 3659dbd70b..30dc9ab9d8 100644 --- a/ui/desktop/src/hooks/useChatSession.ts +++ b/ui/desktop/src/hooks/useChatSession.ts @@ -68,6 +68,7 @@ export function useChatSession({ const sessionLoadError = acpSnapshot?.sessionLoadError; const tokenState = acpSnapshot?.tokenState ?? initialTokenState; const queueProcessingBlocked = acpSnapshot?.pendingCancelPromptAttemptId != null; + const hasActiveRun = acpSnapshot?.activeRunId != null; const snapshotRef = useRef(acpSnapshot); snapshotRef.current = acpSnapshot; @@ -354,6 +355,7 @@ export function useChatSession({ notifications: notificationsMap, pauseQueueOnStop: false, queueProcessingBlocked, + hasActiveRun, onMessageUpdate, }; } diff --git a/ui/desktop/src/hooks/useChatSessionTypes.ts b/ui/desktop/src/hooks/useChatSessionTypes.ts index ccd5fbad93..5f3b1f5622 100644 --- a/ui/desktop/src/hooks/useChatSessionTypes.ts +++ b/ui/desktop/src/hooks/useChatSessionTypes.ts @@ -28,6 +28,7 @@ export interface UseChatSessionResult { notifications: Map; pauseQueueOnStop: boolean; queueProcessingBlocked: boolean; + hasActiveRun: boolean; onMessageUpdate: ( messageId: string, newContent: string, diff --git a/ui/desktop/src/i18n/messages/de.json b/ui/desktop/src/i18n/messages/de.json index f0460ff262..349c874753 100644 --- a/ui/desktop/src/i18n/messages/de.json +++ b/ui/desktop/src/i18n/messages/de.json @@ -1733,6 +1733,36 @@ "launcher.placeholder": { "defaultMessage": "Frag goose irgendetwas..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Führe ein Echtzeit-Sprachgespräch mit Goose" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Live-Sprachmodus" + }, "loadingGoose.compacting": { "defaultMessage": "goose komprimiert die Unterhaltung..." }, diff --git a/ui/desktop/src/i18n/messages/en.json b/ui/desktop/src/i18n/messages/en.json index 3c6ec8d996..66829c7036 100644 --- a/ui/desktop/src/i18n/messages/en.json +++ b/ui/desktop/src/i18n/messages/en.json @@ -1754,6 +1754,36 @@ "launcher.placeholder": { "defaultMessage": "Ask goose anything..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Have a real-time voice conversation with Goose" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Live voice" + }, "loadingGoose.compacting": { "defaultMessage": "goose is compacting the conversation..." }, diff --git a/ui/desktop/src/i18n/messages/es.json b/ui/desktop/src/i18n/messages/es.json index 2738dc9b28..08008e94c5 100644 --- a/ui/desktop/src/i18n/messages/es.json +++ b/ui/desktop/src/i18n/messages/es.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "No se pudo actualizar el directorio de trabajo" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "Pregúntale lo que sea a goose..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Mantén una conversación de voz en tiempo real con Goose" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Voz en directo" + }, "loadingGoose.compacting": { "defaultMessage": "goose está compactando la conversación..." }, diff --git a/ui/desktop/src/i18n/messages/fr.json b/ui/desktop/src/i18n/messages/fr.json index 20d41c35bc..6160cdfc49 100644 --- a/ui/desktop/src/i18n/messages/fr.json +++ b/ui/desktop/src/i18n/messages/fr.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "Échec de la mise à jour du répertoire de travail" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "Demandez n'importe quoi à goose..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Discutez avec Goose en temps réel par la voix" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Voix en direct" + }, "loadingGoose.compacting": { "defaultMessage": "goose compacte la conversation..." }, diff --git a/ui/desktop/src/i18n/messages/hi.json b/ui/desktop/src/i18n/messages/hi.json index 62045c13f0..bb0d1c46d2 100644 --- a/ui/desktop/src/i18n/messages/hi.json +++ b/ui/desktop/src/i18n/messages/hi.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "कार्यशील निर्देशिका अद्यतन करने में विफल" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "goose से कुछ भी पूछें..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Goose के साथ रीयल-टाइम वॉइस बातचीत करें" + }, + "liveVoiceSettings.title": { + "defaultMessage": "लाइव वॉइस" + }, "loadingGoose.compacting": { "defaultMessage": "goose बातचीत को संक्षिप्त कर रहा है..." }, diff --git a/ui/desktop/src/i18n/messages/id.json b/ui/desktop/src/i18n/messages/id.json index a84d32c09b..28602f2a9e 100644 --- a/ui/desktop/src/i18n/messages/id.json +++ b/ui/desktop/src/i18n/messages/id.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "Gagal memperbarui direktori kerja" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "Tanyakan apa saja kepada goose..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Lakukan percakapan suara real-time dengan Goose" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Suara langsung" + }, "loadingGoose.compacting": { "defaultMessage": "goose sedang meringkas percakapan..." }, diff --git a/ui/desktop/src/i18n/messages/it.json b/ui/desktop/src/i18n/messages/it.json index e32340e06a..2a4d0fc17e 100644 --- a/ui/desktop/src/i18n/messages/it.json +++ b/ui/desktop/src/i18n/messages/it.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "Impossibile aggiornare la cartella di lavoro" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "Chiedi qualsiasi cosa a goose..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Parla con Goose in tempo reale" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Voce in tempo reale" + }, "loadingGoose.compacting": { "defaultMessage": "goose sta compattando la conversazione..." }, diff --git a/ui/desktop/src/i18n/messages/ja.json b/ui/desktop/src/i18n/messages/ja.json index c84e48df92..74b67bc740 100644 --- a/ui/desktop/src/i18n/messages/ja.json +++ b/ui/desktop/src/i18n/messages/ja.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "作業ディレクトリの更新に失敗しました" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "gooseに何でも聞いて..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Goose とリアルタイムで音声会話します" + }, + "liveVoiceSettings.title": { + "defaultMessage": "ライブ音声" + }, "loadingGoose.compacting": { "defaultMessage": "gooseが会話を圧縮しています..." }, diff --git a/ui/desktop/src/i18n/messages/ko.json b/ui/desktop/src/i18n/messages/ko.json index d4c302eb64..b6d7bd263f 100644 --- a/ui/desktop/src/i18n/messages/ko.json +++ b/ui/desktop/src/i18n/messages/ko.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "작업 디렉터리를 업데이트하지 못했습니다." }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "goose에게 무엇이든 물어보세요..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Goose와 실시간으로 음성 대화" + }, + "liveVoiceSettings.title": { + "defaultMessage": "실시간 음성" + }, "loadingGoose.compacting": { "defaultMessage": "goose가 대화를 압축하고 있어요..." }, diff --git a/ui/desktop/src/i18n/messages/ms.json b/ui/desktop/src/i18n/messages/ms.json index bc93bc147d..9cedd208ef 100644 --- a/ui/desktop/src/i18n/messages/ms.json +++ b/ui/desktop/src/i18n/messages/ms.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "Gagal mengemas kini direktori kerja" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "Tanya goose apa sahaja..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Adakan perbualan suara masa nyata dengan Goose" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Suara langsung" + }, "loadingGoose.compacting": { "defaultMessage": "goose sedang memadatkan perbualan..." }, diff --git a/ui/desktop/src/i18n/messages/pt.json b/ui/desktop/src/i18n/messages/pt.json index 00ce6623c8..3690e3d29e 100644 --- a/ui/desktop/src/i18n/messages/pt.json +++ b/ui/desktop/src/i18n/messages/pt.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "Falha ao atualizar o diretório de trabalho" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "Pergunte qualquer coisa ao goose..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Tenha uma conversa por voz em tempo real com o Goose" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Voz em tempo real" + }, "loadingGoose.compacting": { "defaultMessage": "o goose está compactando a conversa..." }, diff --git a/ui/desktop/src/i18n/messages/ru.json b/ui/desktop/src/i18n/messages/ru.json index 8bd21248f0..b5d4851c81 100644 --- a/ui/desktop/src/i18n/messages/ru.json +++ b/ui/desktop/src/i18n/messages/ru.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "Не удалось обновить рабочий каталог" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "Спросите goose о чем угодно..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Общайтесь с Goose голосом в реальном времени" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Голосовой режим" + }, "loadingGoose.compacting": { "defaultMessage": "goose сжимает разговор..." }, diff --git a/ui/desktop/src/i18n/messages/tr.json b/ui/desktop/src/i18n/messages/tr.json index 96ca06a5bb..bf4b564e3c 100644 --- a/ui/desktop/src/i18n/messages/tr.json +++ b/ui/desktop/src/i18n/messages/tr.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "Çalışma dizini güncellenemedi" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "goose'a herhangi bir şey sorun..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Goose ile gerçek zamanlı sesli sohbet edin" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Canlı ses" + }, "loadingGoose.compacting": { "defaultMessage": "goose konuşmayı sıkıştırıyor..." }, diff --git a/ui/desktop/src/i18n/messages/vi.json b/ui/desktop/src/i18n/messages/vi.json index 946dce0745..f17ac5abd2 100644 --- a/ui/desktop/src/i18n/messages/vi.json +++ b/ui/desktop/src/i18n/messages/vi.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "Không thể cập nhật thư mục làm việc" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "Hỏi goose bất cứ điều gì..." }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "Trò chuyện bằng giọng nói theo thời gian thực với Goose" + }, + "liveVoiceSettings.title": { + "defaultMessage": "Giọng nói trực tiếp" + }, "loadingGoose.compacting": { "defaultMessage": "goose đang nén cuộc trò chuyện..." }, diff --git a/ui/desktop/src/i18n/messages/zh-CN.json b/ui/desktop/src/i18n/messages/zh-CN.json index 2a468db543..ca5a007bc0 100644 --- a/ui/desktop/src/i18n/messages/zh-CN.json +++ b/ui/desktop/src/i18n/messages/zh-CN.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "更新工作目录失败" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "问 goose 任何问题…" }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "与 Goose 进行实时语音对话" + }, + "liveVoiceSettings.title": { + "defaultMessage": "实时语音" + }, "loadingGoose.compacting": { "defaultMessage": "goose 正在压缩对话…" }, diff --git a/ui/desktop/src/i18n/messages/zh-TW.json b/ui/desktop/src/i18n/messages/zh-TW.json index e365a5c7a5..a5c6437970 100644 --- a/ui/desktop/src/i18n/messages/zh-TW.json +++ b/ui/desktop/src/i18n/messages/zh-TW.json @@ -929,6 +929,10 @@ "dirSwitcher.enterPathInvalid": { "defaultMessage": "Working directory must be an absolute path" }, + + + + "dirSwitcher.failedToUpdateWorkingDir": { "defaultMessage": "無法更新工作目錄" }, @@ -1733,6 +1737,36 @@ "launcher.placeholder": { "defaultMessage": "向 goose 詢問任何問題…" }, + "liveVoice.connecting": { + "defaultMessage": "Connecting Live voice" + }, + "liveVoice.emptyComposerRequired": { + "defaultMessage": "Clear the message and attachments to use Live voice" + }, + "liveVoice.error": { + "defaultMessage": "Live voice failed. Try again" + }, + "liveVoice.live": { + "defaultMessage": "Stop Live voice" + }, + "liveVoice.mute": { + "defaultMessage": "Mute microphone" + }, + "liveVoice.returnToActive": { + "defaultMessage": "Return to active Live voice" + }, + "liveVoice.stopping": { + "defaultMessage": "Stopping Live voice" + }, + "liveVoice.unmute": { + "defaultMessage": "Unmute microphone" + }, + "liveVoiceSettings.description": { + "defaultMessage": "與 Goose 進行即時語音對話" + }, + "liveVoiceSettings.title": { + "defaultMessage": "即時語音" + }, "loadingGoose.compacting": { "defaultMessage": "goose 正在壓縮對話…" }, diff --git a/ui/desktop/src/liveVoice/LiveVoiceMediaSession.test.ts b/ui/desktop/src/liveVoice/LiveVoiceMediaSession.test.ts new file mode 100644 index 0000000000..d14dcd0aed --- /dev/null +++ b/ui/desktop/src/liveVoice/LiveVoiceMediaSession.test.ts @@ -0,0 +1,191 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { LiveVoiceMediaSession } from './LiveVoiceMediaSession'; + +const createElement = document.createElement.bind(document); + +class FakeTrack extends EventTarget { + enabled = true; + stop = vi.fn(); +} + +class FakeStream { + constructor(private readonly tracks: FakeTrack[]) {} + getTracks = () => this.tracks; + getAudioTracks = () => this.tracks; +} + +class FakeDataChannel extends EventTarget { + readyState: RTCDataChannelState = 'open'; + close = vi.fn(); +} + +class FakePeerConnection extends EventTarget { + iceGatheringState: RTCIceGatheringState = 'complete'; + connectionState: RTCPeerConnectionState = 'connected'; + localDescription: RTCSessionDescriptionInit | null = null; + ontrack: ((event: RTCTrackEvent) => void) | null = null; + dataChannel = new FakeDataChannel(); + sessionEvent = { type: 'session.started' }; + close = vi.fn(); + addTrack = vi.fn(); + createDataChannel = vi.fn(() => this.dataChannel as unknown as RTCDataChannel); + createOffer = vi.fn(async () => ({ type: 'offer' as const, sdp: 'bounded-offer' })); + setLocalDescription = vi.fn(async (description: RTCSessionDescriptionInit) => { + this.localDescription = description; + }); + setRemoteDescription = vi.fn(async () => { + this.ontrack?.({ + streams: [new FakeStream([new FakeTrack()])], + track: new FakeTrack(), + } as unknown as RTCTrackEvent); + this.dataChannel.dispatchEvent( + new MessageEvent('message', { data: JSON.stringify(this.sessionEvent) }) + ); + }); +} + +describe('LiveVoiceMediaSession', () => { + let localTrack: FakeTrack; + let peerConnection: FakePeerConnection; + let play: ReturnType Promise>>; + + beforeEach(() => { + localTrack = new FakeTrack(); + peerConnection = new FakePeerConnection(); + play = vi.fn<() => Promise>().mockResolvedValue(undefined); + vi.stubGlobal('MediaStream', FakeStream); + vi.stubGlobal( + 'RTCPeerConnection', + vi.fn(function RTCPeerConnectionMock() { + return peerConnection; + }) + ); + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { + getUserMedia: vi.fn(async () => new FakeStream([localTrack])), + }, + }); + vi.spyOn(document, 'createElement').mockImplementation(((tagName: string) => { + const element = createElement(tagName); + if (tagName === 'audio') { + vi.spyOn(element as HTMLAudioElement, 'play').mockImplementation(play); + vi.spyOn(element as HTMLAudioElement, 'pause').mockImplementation(() => undefined); + } + return element; + }) as typeof document.createElement); + }); + + it('keeps the microphone disabled until media is connected, then tears down once', async () => { + const media = new LiveVoiceMediaSession(vi.fn()); + + await expect(media.createOffer()).resolves.toBe('bounded-offer'); + expect(localTrack.enabled).toBe(false); + expect(peerConnection.createDataChannel).toHaveBeenCalledWith('oai-events'); + + await media.applyAnswer('bounded-answer'); + media.setMuted(false); + expect(localTrack.enabled).toBe(true); + + media.setMuted(true); + media.setMuted(false); + media.setMuted(true); + expect(localTrack.enabled).toBe(false); + + media.teardown(); + media.teardown(); + expect(localTrack.stop).toHaveBeenCalledOnce(); + expect(peerConnection.dataChannel.close).toHaveBeenCalledOnce(); + expect(peerConnection.close).toHaveBeenCalledOnce(); + }); + + it('returns a product-safe error when microphone acquisition fails', async () => { + vi.mocked(navigator.mediaDevices.getUserMedia).mockRejectedValueOnce( + new Error('native device identifier') + ); + + await expect(new LiveVoiceMediaSession(vi.fn()).createOffer()).rejects.toThrow( + 'Microphone or media setup failed' + ); + }); + + it('rejects setup when the data channel reports a startup error', async () => { + const media = new LiveVoiceMediaSession(vi.fn()); + peerConnection.sessionEvent = { type: 'error' }; + + await media.createOffer(); + + await expect(media.applyAnswer('bounded-answer')).rejects.toThrow( + 'Live voice could not connect media' + ); + expect(localTrack.stop).toHaveBeenCalledOnce(); + }); + + it('rejects setup when audio playback cannot start', async () => { + const media = new LiveVoiceMediaSession(vi.fn()); + play.mockRejectedValueOnce(new Error('autoplay blocked')); + + await media.createOffer(); + + await expect(media.applyAnswer('bounded-answer')).rejects.toThrow( + 'Live voice could not connect media' + ); + expect(localTrack.stop).toHaveBeenCalledOnce(); + }); + + it('stops a microphone stream acquired after teardown', async () => { + let provideStream!: (stream: MediaStream) => void; + vi.mocked(navigator.mediaDevices.getUserMedia).mockReturnValueOnce( + new Promise((resolve) => { + provideStream = resolve; + }) + ); + const lateTrack = new FakeTrack(); + const media = new LiveVoiceMediaSession(vi.fn()); + + const offer = media.createOffer(); + media.teardown(); + provideStream(new FakeStream([lateTrack]) as unknown as MediaStream); + + await expect(offer).rejects.toThrow('Microphone or media setup failed'); + expect(lateTrack.stop).toHaveBeenCalledOnce(); + expect(peerConnection.createOffer).not.toHaveBeenCalled(); + }); + + it('reports terminal microphone failure after media connects', async () => { + const onMediaFailure = vi.fn(); + const media = new LiveVoiceMediaSession(onMediaFailure); + await media.createOffer(); + await media.applyAnswer('bounded-answer'); + + localTrack.dispatchEvent(new Event('ended')); + + expect(onMediaFailure).toHaveBeenCalledOnce(); + }); + + it('reports terminal peer failure after media connects', async () => { + const onMediaFailure = vi.fn(); + const media = new LiveVoiceMediaSession(onMediaFailure); + await media.createOffer(); + await media.applyAnswer('bounded-answer'); + + peerConnection.connectionState = 'failed'; + peerConnection.dispatchEvent(new Event('connectionstatechange')); + + expect(onMediaFailure).toHaveBeenCalledOnce(); + }); + + it('does not report its own teardown as a media failure', async () => { + const onMediaFailure = vi.fn(); + const media = new LiveVoiceMediaSession(onMediaFailure); + await media.createOffer(); + await media.applyAnswer('bounded-answer'); + + media.teardown(); + localTrack.dispatchEvent(new Event('ended')); + peerConnection.connectionState = 'failed'; + peerConnection.dispatchEvent(new Event('connectionstatechange')); + + expect(onMediaFailure).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/desktop/src/liveVoice/LiveVoiceMediaSession.ts b/ui/desktop/src/liveVoice/LiveVoiceMediaSession.ts new file mode 100644 index 0000000000..07cc1a24d4 --- /dev/null +++ b/ui/desktop/src/liveVoice/LiveVoiceMediaSession.ts @@ -0,0 +1,275 @@ +const MEDIA_SETUP_TIMEOUT_MS = 15_000; + +export class LiveVoiceMediaSession { + private peerConnection: RTCPeerConnection | null = null; + private localStream: MediaStream | null = null; + private remoteStream: MediaStream | null = null; + private dataChannel: RTCDataChannel | null = null; + private audioElement: HTMLAudioElement | null = null; + private playbackStarted: Promise | null = null; + private tornDown = false; + + constructor(private readonly onMediaFailure: () => void) {} + + async createOffer(): Promise { + try { + const localStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (this.tornDown) { + for (const track of localStream.getTracks()) { + track.enabled = false; + track.stop(); + } + throw new Error('Live voice media is unavailable'); + } + + this.localStream = localStream; + for (const track of localStream.getAudioTracks()) { + track.enabled = false; + } + + const peerConnection = new RTCPeerConnection(); + this.peerConnection = peerConnection; + this.audioElement = document.createElement('audio'); + this.audioElement.autoplay = true; + this.dataChannel = peerConnection.createDataChannel('oai-events'); + peerConnection.ontrack = (event) => { + const stream = event.streams[0] ?? new MediaStream([event.track]); + this.remoteStream = stream; + if (this.audioElement) { + this.audioElement.srcObject = stream; + this.playbackStarted = this.audioElement.play(); + void this.playbackStarted.catch(() => undefined); + } + }; + for (const track of localStream.getTracks()) { + peerConnection.addTrack(track, localStream); + } + + const offer = await peerConnection.createOffer(); + await peerConnection.setLocalDescription(offer); + await waitForIceGathering(peerConnection); + const offerSdp = peerConnection.localDescription?.sdp; + if (!offerSdp) { + throw new Error('Live voice could not create a media offer'); + } + return offerSdp; + } catch { + this.teardown(); + throw new Error('Microphone or media setup failed'); + } + } + + async applyAnswer(answerSdp: string): Promise { + const peerConnection = this.peerConnection; + const dataChannel = this.dataChannel; + if (!peerConnection || !dataChannel || this.tornDown) { + throw new Error('Live voice media is unavailable'); + } + + try { + const sessionStarted = waitForSessionStarted(dataChannel); + try { + await peerConnection.setRemoteDescription({ type: 'answer', sdp: answerSdp }); + await Promise.all([ + waitForPeerConnection(peerConnection), + waitForDataChannel(dataChannel), + waitForRemoteTrack(this), + sessionStarted.promise, + ]); + await this.playbackStarted; + this.observeMediaFailures(peerConnection); + } finally { + sessionStarted.cancel(); + } + } catch { + this.teardown(); + throw new Error('Live voice could not connect media'); + } + } + + setMuted(muted: boolean): void { + if (this.tornDown) return; + for (const track of this.localStream?.getAudioTracks() ?? []) { + track.enabled = !muted; + } + } + + sendCommentary(text: string): void { + const content = text.trim(); + if (this.tornDown || !content || this.dataChannel?.readyState !== 'open') return; + + try { + this.dataChannel.send( + JSON.stringify({ + type: 'session.commentary.append', + event_id: `event_${globalThis.crypto.randomUUID()}`, + delegation_id: null, + content, + }) + ); + } catch { + // Greeting failure must not end an otherwise healthy Live voice interaction. + } + } + + hasRemoteTrack(): boolean { + return this.remoteStream !== null; + } + + teardown(): void { + if (this.tornDown) return; + this.tornDown = true; + for (const track of this.localStream?.getTracks() ?? []) { + track.enabled = false; + track.stop(); + } + for (const track of this.remoteStream?.getTracks() ?? []) { + track.stop(); + } + this.dataChannel?.close(); + this.peerConnection?.close(); + if (this.audioElement) { + this.audioElement.pause(); + this.audioElement.srcObject = null; + } + this.localStream = null; + this.remoteStream = null; + this.dataChannel = null; + this.peerConnection = null; + this.audioElement = null; + this.playbackStarted = null; + } + + private observeMediaFailures(peerConnection: RTCPeerConnection): void { + for (const track of this.localStream?.getAudioTracks() ?? []) { + track.addEventListener('ended', this.reportMediaFailure, { once: true }); + } + peerConnection.addEventListener('connectionstatechange', () => { + if (peerConnection.connectionState === 'failed') { + this.reportMediaFailure(); + } + }); + } + + private reportMediaFailure = (): void => { + if (this.tornDown) return; + this.onMediaFailure(); + }; +} + +function waitForIceGathering(peerConnection: RTCPeerConnection): Promise { + if (peerConnection.iceGatheringState === 'complete') return Promise.resolve(); + return waitForEvent( + peerConnection, + 'icegatheringstatechange', + () => peerConnection.iceGatheringState === 'complete' + ); +} + +function waitForPeerConnection(peerConnection: RTCPeerConnection): Promise { + if (peerConnection.connectionState === 'connected') return Promise.resolve(); + return waitForEvent(peerConnection, 'connectionstatechange', () => { + if ( + peerConnection.connectionState === 'failed' || + peerConnection.connectionState === 'closed' + ) { + throw new Error('Peer connection failed'); + } + return peerConnection.connectionState === 'connected'; + }); +} + +function waitForDataChannel(dataChannel: RTCDataChannel): Promise { + if (dataChannel.readyState === 'open') return Promise.resolve(); + return waitForEvent(dataChannel, 'open', () => true); +} + +function waitForSessionStarted(dataChannel: RTCDataChannel): { + promise: Promise; + cancel: () => void; +} { + let cancel: () => void = () => undefined; + const promise = new Promise((resolve, reject) => { + const timeoutId = setTimeout( + () => finish(new Error('Live session startup timed out')), + MEDIA_SETUP_TIMEOUT_MS + ); + const listener = (event: MessageEvent) => { + if (typeof event.data !== 'string') return; + let message: { type?: unknown }; + try { + message = JSON.parse(event.data) as { type?: unknown }; + } catch { + return; + } + if (message.type === 'session.started') { + finish(); + } else if (message.type === 'error') { + finish(new Error('Live session failed to start')); + } + }; + const finish = (error?: Error) => { + clearTimeout(timeoutId); + dataChannel.removeEventListener('message', listener); + if (error) { + reject(error); + } else { + resolve(); + } + }; + cancel = finish; + dataChannel.addEventListener('message', listener); + }); + return { promise, cancel }; +} + +function waitForRemoteTrack(media: LiveVoiceMediaSession): Promise { + if (media.hasRemoteTrack()) return Promise.resolve(); + return pollUntil(() => media.hasRemoteTrack()); +} + +function waitForEvent( + target: EventTarget, + eventName: string, + isReady: () => boolean +): Promise { + return new Promise((resolve, reject) => { + const timeoutId = setTimeout( + () => finish(new Error('Media setup timed out')), + MEDIA_SETUP_TIMEOUT_MS + ); + const listener = () => { + try { + if (isReady()) finish(); + } catch (error) { + finish(error instanceof Error ? error : new Error('Media setup failed')); + } + }; + const finish = (error?: Error) => { + clearTimeout(timeoutId); + target.removeEventListener(eventName, listener); + if (error) { + reject(error); + } else { + resolve(); + } + }; + target.addEventListener(eventName, listener); + }); +} + +function pollUntil(isReady: () => boolean): Promise { + return new Promise((resolve, reject) => { + const startedAt = Date.now(); + const poll = () => { + if (isReady()) { + resolve(); + } else if (Date.now() - startedAt >= MEDIA_SETUP_TIMEOUT_MS) { + reject(new Error('Media setup timed out')); + } else { + setTimeout(poll, 25); + } + }; + poll(); + }); +} diff --git a/ui/desktop/src/liveVoice/useLiveVoice.test.ts b/ui/desktop/src/liveVoice/useLiveVoice.test.ts new file mode 100644 index 0000000000..4fdd0d21a2 --- /dev/null +++ b/ui/desktop/src/liveVoice/useLiveVoice.test.ts @@ -0,0 +1,317 @@ +import { act, renderHook, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { isAcpRecovering, subscribeToAcpRecovery } from '../acp/acpConnection'; +import { acpStartLiveVoice, acpStopLiveVoice } from '../acp/liveVoice'; +import { publishLiveVoiceInteractionEnded } from '../acp/liveVoiceNotifications'; +import { LiveVoiceMediaSession } from './LiveVoiceMediaSession'; +import { useLiveVoice } from './useLiveVoice'; + +vi.mock('../acp/liveVoice', () => ({ + acpStartLiveVoice: vi.fn(), + acpStopLiveVoice: vi.fn(), +})); + +vi.mock('../acp/acpConnection', () => ({ + isAcpRecovering: vi.fn(() => false), + subscribeToAcpRecovery: vi.fn(), +})); + +vi.mock('./LiveVoiceMediaSession', () => ({ + LiveVoiceMediaSession: vi.fn(), +})); + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +describe('useLiveVoice', () => { + let mediaFailure: () => void; + let recoveryChanged: (recovering: boolean) => void; + const media = { + createOffer: vi.fn(), + applyAnswer: vi.fn(), + setMuted: vi.fn(), + teardown: vi.fn(), + }; + beforeEach(() => { + vi.clearAllMocks(); + media.createOffer.mockResolvedValue('offer'); + media.applyAnswer.mockResolvedValue(undefined); + vi.mocked(isAcpRecovering).mockReturnValue(false); + vi.mocked(subscribeToAcpRecovery).mockImplementation((listener) => { + recoveryChanged = listener; + return () => undefined; + }); + vi.mocked(LiveVoiceMediaSession).mockImplementation( + function LiveVoiceMediaSessionMock(onFailure) { + mediaFailure = onFailure; + return media as unknown as LiveVoiceMediaSession; + } + ); + vi.mocked(acpStartLiveVoice).mockResolvedValue({ + interactionId: 'live-opaque', + answerSdp: 'answer', + }); + vi.mocked(acpStopLiveVoice).mockResolvedValue(undefined); + }); + + it('starts media from the start response and stops the same connection', async () => { + const { result } = renderHook(() => useLiveVoice()); + + await act(async () => result.current.start('main-session')); + expect(media.applyAnswer).toHaveBeenCalledWith('answer'); + expect(media.setMuted).toHaveBeenCalledWith(false); + expect(result.current.phase).toBe('live'); + expect(result.current.activeSessionId).toBe('main-session'); + + await act(async () => result.current.stop()); + expect(media.teardown).toHaveBeenCalledOnce(); + expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'live-opaque'); + expect(result.current.phase).toBe('idle'); + expect(result.current.activeSessionId).toBeNull(); + }); + + it('blocks retry until the server interaction stops after media setup fails', async () => { + const cleanup = deferred(); + media.applyAnswer.mockRejectedValueOnce(new Error('media failed')); + vi.mocked(acpStopLiveVoice).mockReturnValueOnce(cleanup.promise); + const { result } = renderHook(() => useLiveVoice()); + + let startPromise!: Promise; + act(() => { + startPromise = result.current.start('main-session'); + }); + await waitFor(() => expect(acpStopLiveVoice).toHaveBeenCalledOnce()); + + expect(media.teardown).toHaveBeenCalledOnce(); + expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'live-opaque'); + expect(result.current.phase).toBe('stopping'); + + await act(async () => result.current.start('main-session')); + expect(acpStartLiveVoice).toHaveBeenCalledOnce(); + + await act(async () => { + cleanup.resolve(); + await startPromise; + }); + expect(result.current.phase).toBe('error'); + }); + + it('tears down and stops an active interaction when unmounted', async () => { + const { result, unmount } = renderHook(() => useLiveVoice()); + await act(async () => result.current.start('main-session')); + + unmount(); + + expect(media.teardown).toHaveBeenCalledOnce(); + expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'live-opaque'); + }); + + it('keeps one interaction when another session tries to start', async () => { + const { result } = renderHook(() => useLiveVoice()); + await act(async () => result.current.start('main-session')); + + await act(async () => result.current.start('other-session')); + + expect(result.current.activeSessionId).toBe('main-session'); + expect(media.teardown).not.toHaveBeenCalled(); + expect(acpStopLiveVoice).not.toHaveBeenCalled(); + expect(LiveVoiceMediaSession).toHaveBeenCalledOnce(); + expect(acpStartLiveVoice).toHaveBeenCalledOnce(); + }); + + it.each(['unmount', 'stop'] as const)( + 'closes an interaction that finishes after %s while connecting', + async (action) => { + const pending = deferred<{ interactionId: string; answerSdp: string }>(); + vi.mocked(acpStartLiveVoice).mockReturnValueOnce(pending.promise); + const { result, unmount } = renderHook(() => useLiveVoice()); + act(() => { + void result.current.start('main-session'); + }); + await waitFor(() => expect(acpStartLiveVoice).toHaveBeenCalledOnce()); + + if (action === 'unmount') { + unmount(); + } else { + await act(async () => result.current.stop()); + expect(result.current.phase).toBe('stopping'); + } + await act(async () => { + pending.resolve({ interactionId: 'late-interaction', answerSdp: 'late-answer' }); + }); + + expect(media.teardown).toHaveBeenCalledOnce(); + expect(media.applyAnswer).not.toHaveBeenCalled(); + expect(media.setMuted).not.toHaveBeenCalled(); + expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'late-interaction'); + if (action === 'stop') expect(result.current.phase).toBe('idle'); + } + ); + + it('does not become live after stopping during answer setup', async () => { + const pending = deferred(); + media.applyAnswer.mockReturnValueOnce(pending.promise); + const { result } = renderHook(() => useLiveVoice()); + act(() => { + void result.current.start('main-session'); + }); + await waitFor(() => expect(media.applyAnswer).toHaveBeenCalledOnce()); + + await act(async () => result.current.stop()); + await act(async () => pending.resolve(undefined)); + + expect(media.setMuted).not.toHaveBeenCalled(); + expect(result.current.phase).toBe('idle'); + }); + + it('applies rapid mute changes to the current media session', async () => { + const { result } = renderHook(() => useLiveVoice()); + await act(async () => result.current.start('main-session')); + + act(() => { + result.current.toggleMute(); + result.current.toggleMute(); + result.current.toggleMute(); + }); + + expect(media.setMuted.mock.calls).toEqual([[false], [true], [false], [true]]); + expect(result.current.muted).toBe(true); + }); + + it('cleans up when the backend reports that the current interaction failed', async () => { + const { result } = renderHook(() => useLiveVoice()); + await act(async () => result.current.start('main-session')); + + act(() => { + publishLiveVoiceInteractionEnded({ + sessionId: 'main-session', + update: { + sessionUpdate: 'live_voice_interaction_ended', + interactionId: 'live-opaque', + outcome: 'failed', + }, + }); + publishLiveVoiceInteractionEnded({ + sessionId: 'main-session', + update: { + sessionUpdate: 'live_voice_interaction_ended', + interactionId: 'live-opaque', + outcome: 'failed', + }, + }); + }); + + expect(media.teardown).toHaveBeenCalledOnce(); + expect(acpStopLiveVoice).not.toHaveBeenCalled(); + expect(result.current.phase).toBe('error'); + }); + + it('ignores stale session and interaction endings', async () => { + const { result } = renderHook(() => useLiveVoice()); + await act(async () => result.current.start('main-session')); + + act(() => { + publishLiveVoiceInteractionEnded({ + sessionId: 'other-session', + update: { + sessionUpdate: 'live_voice_interaction_ended', + interactionId: 'live-opaque', + outcome: 'failed', + }, + }); + publishLiveVoiceInteractionEnded({ + sessionId: 'main-session', + update: { + sessionUpdate: 'live_voice_interaction_ended', + interactionId: 'old-interaction', + outcome: 'failed', + }, + }); + }); + + expect(media.teardown).not.toHaveBeenCalled(); + expect(result.current.phase).toBe('live'); + }); + + it('reconciles a terminal update that arrives before the start response', async () => { + const pending = deferred<{ interactionId: string; answerSdp: string }>(); + vi.mocked(acpStartLiveVoice).mockReturnValueOnce(pending.promise); + const { result } = renderHook(() => useLiveVoice()); + act(() => { + void result.current.start('main-session'); + }); + await waitFor(() => expect(acpStartLiveVoice).toHaveBeenCalledOnce()); + + act(() => { + publishLiveVoiceInteractionEnded({ + sessionId: 'main-session', + update: { + sessionUpdate: 'live_voice_interaction_ended', + interactionId: 'early-interaction', + outcome: 'failed', + }, + }); + }); + await act(async () => + pending.resolve({ interactionId: 'early-interaction', answerSdp: 'answer' }) + ); + + expect(media.applyAnswer).not.toHaveBeenCalled(); + expect(media.teardown).toHaveBeenCalledOnce(); + expect(result.current.phase).toBe('error'); + }); + + it('cleans up locally without stopping through a recovering ACP connection', async () => { + const { result } = renderHook(() => useLiveVoice()); + await act(async () => result.current.start('main-session')); + + act(() => recoveryChanged(true)); + + expect(media.teardown).toHaveBeenCalledOnce(); + expect(acpStopLiveVoice).not.toHaveBeenCalled(); + expect(result.current.phase).toBe('idle'); + }); + + it('does not stop a late interaction response through a recovered ACP connection', async () => { + const pending = deferred<{ interactionId: string; answerSdp: string }>(); + vi.mocked(acpStartLiveVoice).mockReturnValueOnce(pending.promise); + const { result } = renderHook(() => useLiveVoice()); + act(() => { + void result.current.start('main-session'); + }); + await waitFor(() => expect(acpStartLiveVoice).toHaveBeenCalledOnce()); + + act(() => recoveryChanged(true)); + await act(async () => + pending.resolve({ interactionId: 'old-interaction', answerSdp: 'answer' }) + ); + + expect(media.teardown).toHaveBeenCalledOnce(); + expect(acpStopLiveVoice).not.toHaveBeenCalled(); + expect(result.current.phase).toBe('idle'); + }); + + it('blocks retry until the server interaction stops after active media fails', async () => { + const cleanup = deferred(); + const { result } = renderHook(() => useLiveVoice()); + await act(async () => result.current.start('main-session')); + vi.mocked(acpStopLiveVoice).mockReturnValueOnce(cleanup.promise); + + act(() => mediaFailure()); + + expect(media.teardown).toHaveBeenCalledOnce(); + expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'live-opaque'); + expect(result.current.phase).toBe('stopping'); + + await act(async () => result.current.start('main-session')); + expect(acpStartLiveVoice).toHaveBeenCalledOnce(); + + await act(async () => cleanup.resolve()); + expect(result.current.phase).toBe('error'); + }); +}); diff --git a/ui/desktop/src/liveVoice/useLiveVoice.ts b/ui/desktop/src/liveVoice/useLiveVoice.ts new file mode 100644 index 0000000000..bfdabfd19f --- /dev/null +++ b/ui/desktop/src/liveVoice/useLiveVoice.ts @@ -0,0 +1,242 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { isAcpRecovering, subscribeToAcpRecovery } from '../acp/acpConnection'; +import { acpStartLiveVoice, acpStopLiveVoice } from '../acp/liveVoice'; +import { + subscribeToLiveVoiceInteractionEnded, + type LiveVoiceInteractionEndedNotification, +} from '../acp/liveVoiceNotifications'; +import { LiveVoiceMediaSession } from './LiveVoiceMediaSession'; + +export type LiveVoicePhase = 'idle' | 'connecting' | 'live' | 'stopping' | 'error'; + +export function isLiveVoiceActive(phase: LiveVoicePhase): boolean { + return phase === 'connecting' || phase === 'live' || phase === 'stopping'; +} + +export interface LiveVoiceController { + activeSessionId: string | null; + liveVoiceSessionId: string | null; + phase: LiveVoicePhase; + muted: boolean; + start: (sessionId: string, initialCommentary?: string) => Promise; + stop: () => Promise; + toggleMute: () => void; +} + +interface LiveVoiceInteraction { + sessionId: string; + interactionId?: string; + remoteStartPending: boolean; + media: LiveVoiceMediaSession; + mediaReady: boolean; + invalidated: boolean; + acpConnectionLost: boolean; + pendingOutcomesByInteractionId: Map< + string, + LiveVoiceInteractionEndedNotification['update']['outcome'] + >; +} + +async function stopRemoteInteraction( + interaction: LiveVoiceInteraction +): Promise<'stopped' | 'failed'> { + if (!interaction.interactionId || interaction.acpConnectionLost) return 'stopped'; + try { + await acpStopLiveVoice(interaction.sessionId, interaction.interactionId); + return 'stopped'; + } catch { + return 'failed'; + } +} + +export function useLiveVoice(): LiveVoiceController { + const [liveVoiceSessionId, setLiveVoiceSessionId] = useState(null); + const [phase, setPhase] = useState('idle'); + const [muted, setMuted] = useState(false); + const mutedRef = useRef(false); + const interactionRef = useRef(null); + + const invalidateInteractionAndReleaseMedia = useCallback((interaction: LiveVoiceInteraction) => { + if (interaction.invalidated) return; + interaction.invalidated = true; + interaction.media.teardown(); + mutedRef.current = false; + }, []); + + const finishCurrentInteraction = useCallback( + ( + interaction: LiveVoiceInteraction, + outcome: LiveVoiceInteractionEndedNotification['update']['outcome'] + ) => { + if (interactionRef.current !== interaction) return false; + + interactionRef.current = null; + invalidateInteractionAndReleaseMedia(interaction); + if (outcome !== 'failed') { + setLiveVoiceSessionId(null); + } + setMuted(false); + setPhase(outcome === 'failed' ? 'error' : 'idle'); + return true; + }, + [invalidateInteractionAndReleaseMedia] + ); + + const failCurrentInteraction = useCallback( + async (interaction: LiveVoiceInteraction) => { + if (interactionRef.current !== interaction || interaction.invalidated) return; + + invalidateInteractionAndReleaseMedia(interaction); + setMuted(false); + if (interaction.interactionId) { + setPhase('stopping'); + await stopRemoteInteraction(interaction); + } + finishCurrentInteraction(interaction, 'failed'); + }, + [finishCurrentInteraction, invalidateInteractionAndReleaseMedia] + ); + + useEffect(() => { + return () => { + const interaction = interactionRef.current; + if (!interaction) return; + + interactionRef.current = null; + invalidateInteractionAndReleaseMedia(interaction); + void stopRemoteInteraction(interaction); + }; + }, [invalidateInteractionAndReleaseMedia]); + + useEffect(() => { + return subscribeToLiveVoiceInteractionEnded((notification) => { + const interaction = interactionRef.current; + if (!interaction || interaction.sessionId !== notification.sessionId) return; + + if (!interaction.interactionId) { + interaction.pendingOutcomesByInteractionId.set( + notification.update.interactionId, + notification.update.outcome + ); + return; + } + if ( + !interaction.invalidated && + interaction.interactionId === notification.update.interactionId + ) { + finishCurrentInteraction(interaction, notification.update.outcome); + } + }); + }, [finishCurrentInteraction]); + + useEffect(() => { + return subscribeToAcpRecovery((recovering) => { + if (!recovering) return; + + const interaction = interactionRef.current; + if (interaction) { + interaction.acpConnectionLost = true; + finishCurrentInteraction(interaction, 'stopped'); + } + }); + }, [finishCurrentInteraction]); + + const start = useCallback( + async (sessionId: string, initialCommentary?: string) => { + if (interactionRef.current || isAcpRecovering()) return; + + setLiveVoiceSessionId(sessionId); + mutedRef.current = false; + setMuted(false); + setPhase('connecting'); + let interaction: LiveVoiceInteraction; + const media = new LiveVoiceMediaSession(() => { + void failCurrentInteraction(interaction); + }); + interaction = { + sessionId, + remoteStartPending: false, + media, + mediaReady: false, + invalidated: false, + acpConnectionLost: false, + pendingOutcomesByInteractionId: new Map(), + }; + interactionRef.current = interaction; + const isCurrent = () => interactionRef.current === interaction && !interaction.invalidated; + + try { + const offerSdp = await interaction.media.createOffer(); + if (!isCurrent()) return; + + interaction.remoteStartPending = true; + const response = await acpStartLiveVoice(sessionId, offerSdp); + interaction.remoteStartPending = false; + interaction.interactionId = response.interactionId; + const pendingOutcome = interaction.pendingOutcomesByInteractionId.get( + interaction.interactionId + ); + interaction.pendingOutcomesByInteractionId.clear(); + if (pendingOutcome) { + finishCurrentInteraction(interaction, pendingOutcome); + return; + } + if (!isCurrent()) { + finishCurrentInteraction(interaction, await stopRemoteInteraction(interaction)); + return; + } + + await interaction.media.applyAnswer(response.answerSdp); + if (!isCurrent()) return; + + interaction.mediaReady = true; + interaction.media.setMuted(mutedRef.current); + setPhase('live'); + if (initialCommentary) { + interaction.media.sendCommentary(initialCommentary); + } + } catch { + interaction.remoteStartPending = false; + if (interaction.invalidated) { + if (!interaction.interactionId) finishCurrentInteraction(interaction, 'stopped'); + return; + } + await failCurrentInteraction(interaction); + } + }, + [failCurrentInteraction, finishCurrentInteraction] + ); + + const toggleMute = useCallback(() => { + const interaction = interactionRef.current; + if (!interaction || interaction.invalidated || !interaction.mediaReady) return; + + mutedRef.current = !mutedRef.current; + interaction.media.setMuted(mutedRef.current); + setMuted(mutedRef.current); + }, []); + + const stop = useCallback(async () => { + const interaction = interactionRef.current; + if (!interaction || interaction.invalidated) return; + + invalidateInteractionAndReleaseMedia(interaction); + setMuted(false); + if (!interaction.interactionId) { + if (interaction.remoteStartPending) { + setPhase('stopping'); + return; + } + interactionRef.current = null; + setLiveVoiceSessionId(null); + setPhase('idle'); + return; + } + + setPhase('stopping'); + finishCurrentInteraction(interaction, await stopRemoteInteraction(interaction)); + }, [finishCurrentInteraction, invalidateInteractionAndReleaseMedia]); + + const activeSessionId = isLiveVoiceActive(phase) ? liveVoiceSessionId : null; + return { activeSessionId, liveVoiceSessionId, phase, muted, start, stop, toggleMute }; +} diff --git a/ui/desktop/src/utils/navigationUtils.ts b/ui/desktop/src/utils/navigationUtils.ts index 85a8662096..72ec69cb08 100644 --- a/ui/desktop/src/utils/navigationUtils.ts +++ b/ui/desktop/src/utils/navigationUtils.ts @@ -29,6 +29,7 @@ export type ViewOptions = { disableAnimation?: boolean; initialMessage?: UserInput; resumeSessionId?: string; + startLiveVoice?: boolean; pendingScheduleDeepLink?: string; }; diff --git a/ui/goose-acp-client/src/generated/client.gen.ts b/ui/goose-acp-client/src/generated/client.gen.ts index 131a835e84..4635477ff8 100644 --- a/ui/goose-acp-client/src/generated/client.gen.ts +++ b/ui/goose-acp-client/src/generated/client.gen.ts @@ -97,6 +97,11 @@ import type { ListSlashCommandsResponse_unstable, ListSourcesRequest_unstable, ListSourcesResponse_unstable, + LiveVoiceAvailabilityRequest_unstable, + LiveVoiceAvailabilityResponse_unstable, + LiveVoiceStartRequest_unstable, + LiveVoiceStartResponse_unstable, + LiveVoiceStopRequest_unstable, LocalInferenceBuiltinChatTemplatesListRequest_unstable, LocalInferenceBuiltinChatTemplatesListResponse_unstable, LocalInferenceHuggingFaceRepoVariantsRequest_unstable, @@ -227,6 +232,8 @@ import { zListSchedulesResponse_unstable, zListSlashCommandsResponse_unstable, zListSourcesResponse_unstable, + zLiveVoiceAvailabilityResponse_unstable, + zLiveVoiceStartResponse_unstable, zLocalInferenceBuiltinChatTemplatesListResponse_unstable, zLocalInferenceHuggingFaceRepoVariantsResponse_unstable, zLocalInferenceHuggingFaceSearchResponse_unstable, @@ -384,6 +391,36 @@ export class GooseExtClient { ) as SteerSessionResponse_unstable; } + async sessionLiveVoiceAvailability_unstable( + params: LiveVoiceAvailabilityRequest_unstable, + ): Promise { + const raw = await this.conn.request( + "_goose/unstable/session/live-voice/availability", + params, + ); + return zLiveVoiceAvailabilityResponse_unstable.parse( + raw, + ) as LiveVoiceAvailabilityResponse_unstable; + } + + async sessionLiveVoiceStart_unstable( + params: LiveVoiceStartRequest_unstable, + ): Promise { + const raw = await this.conn.request( + "_goose/unstable/session/live-voice/start", + params, + ); + return zLiveVoiceStartResponse_unstable.parse( + raw, + ) as LiveVoiceStartResponse_unstable; + } + + async sessionLiveVoiceStop_unstable( + params: LiveVoiceStopRequest_unstable, + ): Promise { + await this.conn.request("_goose/unstable/session/live-voice/stop", params); + } + async diagnosticsGet_unstable( params: DiagnosticsGetRequest_unstable, ): Promise { diff --git a/ui/goose-acp-client/src/generated/index.ts b/ui/goose-acp-client/src/generated/index.ts index dcf8b2e484..60065c22e1 100644 --- a/ui/goose-acp-client/src/generated/index.ts +++ b/ui/goose-acp-client/src/generated/index.ts @@ -1,6 +1,6 @@ // This file is auto-generated by @hey-api/openapi-ts -export type { AddConfigExtensionRequest_unstable, AddSessionExtensionRequest_unstable, AgentMention, Annotations, AppsDeleteRequest_unstable, AppsDeleteResponse_unstable, AppsExportRequest_unstable, AppsExportResponse_unstable, AppsImportRequest_unstable, AppsImportResponse_unstable, AppsListRequest_unstable, AppsListResponse_unstable, ArchiveSessionRequest_unstable, AudioContent, AvailableCommand, AvailableCommandInput, BlobResourceContents, CanonicalModelInfoDto, CanonicalModelInfoRequest_unstable, CanonicalModelInfoResponse_unstable, ConfigReadAllRequest_unstable, ConfigReadAllResponse_unstable, ConfigReadRequest_unstable, ConfigReadResponse_unstable, ConfigRemoveRequest_unstable, ConfigUpsertRequest_unstable, ContentBlock, CostSourceData, CreateScheduleRequest_unstable, CreateScheduleResponse_unstable, CreateSourceRequest_unstable, CreateSourceResponse_unstable, CustomProviderConfigDto, CustomProviderCreateRequest_unstable, CustomProviderCreateResponse_unstable, CustomProviderDeleteRequest_unstable, CustomProviderDeleteResponse_unstable, CustomProviderReadRequest_unstable, CustomProviderReadResponse_unstable, CustomProviderUpdateRequest_unstable, CustomProviderUpdateResponse_unstable, DecodeRecipeRequest_unstable, DecodeRecipeResponse_unstable, DefaultsClearRequest_unstable, DefaultsReadRequest_unstable, DefaultsReadResponse_unstable, DefaultsSaveRequest_unstable, DeleteRecipeRequest_unstable, DeleteScheduleRequest_unstable, DeleteSourceRequest_unstable, DiagnosticsGetRequest_unstable, DiagnosticsGetResponse_unstable, DiagnosticsReportLevel, DictationConfigRequest_unstable, DictationConfigResponse_unstable, DictationDownloadProgress, DictationLocalModelStatus, DictationModelCancelRequest_unstable, DictationModelDeleteRequest_unstable, DictationModelDownloadProgressRequest_unstable, DictationModelDownloadProgressResponse_unstable, DictationModelDownloadRequest_unstable, DictationModelOption, DictationModelsListRequest_unstable, DictationModelsListResponse_unstable, DictationProviderStatusEntry, DictationTranscribeRequest_unstable, DictationTranscribeResponse_unstable, EmbeddedResource, EmbeddedResourceResource, EmptyResponse, EncodeRecipeRequest_unstable, EncodeRecipeResponse_unstable, EnvVariable, ExportSessionRequest_unstable, ExportSessionResponse_unstable, ExportSourceRequest_unstable, ExportSourceResponse_unstable, ExtAgentRequest, ExtAgentResponse, ExtNotification, ExtRequest, ExtResponse, GetConfigExtensionsRequest_unstable, GetConfigExtensionsResponse_unstable, GetPromptRequest_unstable, GetPromptResponse_unstable, GetSessionExtensionsRequest_unstable, GetSessionExtensionsResponse_unstable, GetSessionInfoRequest_unstable, GetSessionInfoResponse_unstable, GetToolsRequest_unstable, GetToolsResponse_unstable, GooseExtension, GooseExtensionEntry, GooseSessionNotification_unstable, GooseSessionUpdate, GooseToolCallRequest_unstable, GooseToolCallResponse_unstable, HttpHeader, ImageContent, ImportSessionRequest_unstable, ImportSessionResponse_unstable, ImportSourcesRequest_unstable, ImportSourcesResponse_unstable, InspectRunningJobRequest_unstable, InspectRunningJobResponse_unstable, KillRunningJobRequest_unstable, KillRunningJobResponse_unstable, ListAgentMentionsRequest_unstable, ListAgentMentionsResponse_unstable, ListPromptsRequest_unstable, ListPromptsResponse_unstable, ListProvidersRequest_unstable, ListProvidersResponse_unstable, ListRecipesRequest_unstable, ListRecipesResponse_unstable, ListScheduleSessionsRequest_unstable, ListScheduleSessionsResponse_unstable, ListSchedulesRequest_unstable, ListSchedulesResponse_unstable, ListSlashCommandsRequest_unstable, ListSlashCommandsResponse_unstable, ListSourcesRequest_unstable, ListSourcesResponse_unstable, LocalInferenceBuiltinChatTemplatesListRequest_unstable, LocalInferenceBuiltinChatTemplatesListResponse_unstable, LocalInferenceChatTemplate, LocalInferenceDownloadProgressDto, LocalInferenceDownloadState, LocalInferenceHfGgufFileDto, LocalInferenceHfModelInfoDto, LocalInferenceHfModelVariantDto, LocalInferenceHuggingFaceRepoVariantsRequest_unstable, LocalInferenceHuggingFaceRepoVariantsResponse_unstable, LocalInferenceHuggingFaceSearchRequest_unstable, LocalInferenceHuggingFaceSearchResponse_unstable, LocalInferenceModelDeleteRequest_unstable, LocalInferenceModelDownloadCancelRequest_unstable, LocalInferenceModelDownloadProgressRequest_unstable, LocalInferenceModelDownloadProgressResponse_unstable, LocalInferenceModelDownloadRequest_unstable, LocalInferenceModelDownloadResponse_unstable, LocalInferenceModelDownloadStatusDto, LocalInferenceModelDto, LocalInferenceModelEvictRequest_unstable, LocalInferenceModelSettingsDto, LocalInferenceModelSettingsReadRequest_unstable, LocalInferenceModelSettingsReadResponse_unstable, LocalInferenceModelSettingsUpdateRequest_unstable, LocalInferenceModelSettingsUpdateResponse_unstable, LocalInferenceModelsListRequest_unstable, LocalInferenceModelsListResponse_unstable, LocalInferenceSamplingConfig, LocalInferenceToolCallingMode, McpServer, McpServerHttp, McpServerSse, McpServerStdio, MessageUsageData, MessageUsageUpdate, OnboardingImportApplyRequest_unstable, OnboardingImportApplyResponse_unstable, OnboardingImportCandidate, OnboardingImportCounts, OnboardingImportScanRequest_unstable, OnboardingImportScanResponse_unstable, OnboardingImportSourceKind, ParseRecipeRequest_unstable, ParseRecipeResponse_unstable, PauseScheduleRequest_unstable, PreferenceKey, PreferencesReadRequest_unstable, PreferencesReadResponse_unstable, PreferencesSaveRequest_unstable, PreferenceValue, PromptOperationResponse_unstable, PromptTemplateEntry, ProviderCatalogListRequest_unstable, ProviderCatalogListResponse_unstable, ProviderCatalogTemplateRequest_unstable, ProviderCatalogTemplateResponse_unstable, ProviderConfigAuthenticateRequest_unstable, ProviderConfigChangeResponse_unstable, ProviderConfigDeleteRequest_unstable, ProviderConfigFieldUpdate, ProviderConfigFieldValueDto, ProviderConfigKey, ProviderConfigReadRequest_unstable, ProviderConfigReadResponse_unstable, ProviderConfigSaveRequest_unstable, ProviderConfigStatusDto, ProviderConfigStatusRequest_unstable, ProviderConfigStatusResponse_unstable, ProviderDeviceCodeNotification_unstable, ProviderInventoryEntryDto, ProviderInventoryModelDto, ProviderReadinessCheckRequest_unstable, ProviderReadinessCheckResponse_unstable, ProviderSecretDeleteRequest_unstable, ProviderSecretDto, ProviderSecretsListRequest_unstable, ProviderSecretsListResponse_unstable, ProviderSecretStatusDto, ProviderSecretStorageDto, ProviderSetupCatalogEntryDto, ProviderSetupCatalogListRequest_unstable, ProviderSetupCatalogListResponse_unstable, ProviderSetupCategoryDto, ProviderSetupFieldDto, ProviderSetupGroupDto, ProviderSetupMethodDto, ProviderSupportedModelsListRequest_unstable, ProviderSupportedModelsListResponse_unstable, ProviderTemplateCapabilitiesDto, ProviderTemplateCatalogEntryDto, ProviderTemplateDto, ProviderTemplateModelDto, ReadResourceRequest_unstable, ReadResourceResponse_unstable, RecipeAuthorDto, RecipeDto, RecipeExtensionDto, RecipeListEntryDto, RecipeParameterDto, RecipeParameterInputTypeDto, RecipeParameterRequirementDto, RecipeParamsAction, RecipeParamsResponse_unstable, RecipeResponseDto, RecipeRetryConfigDto, RecipeSettingsDto, RecipeSuccessCheckDto, RecipeToYamlRequest_unstable, RecipeToYamlResponse_unstable, RefreshProviderInventoryRequest_unstable, RefreshProviderInventoryResponse_unstable, RefreshProviderInventorySkipDto, RefreshProviderInventorySkipReasonDto, RemoveConfigExtensionRequest_unstable, RemoveSessionExtensionRequest_unstable, RenameSessionRequest_unstable, RequestRecipeParams_unstable, ResetPromptRequest_unstable, ResourceLink, Role, RunScheduleNowRequest_unstable, RunScheduleNowResponse_unstable, RunScheduleNowStatus, SavePromptRequest_unstable, SaveRecipeRequest_unstable, SaveRecipeResponse_unstable, ScanRecipeRequest_unstable, ScanRecipeResponse_unstable, ScheduledJobDto, ScheduleRecipeRequest_unstable, SessionExportFormat, SessionExtensionEntry, SessionId, SessionImportSource, SessionInfo, SessionSystemPromptMode, SessionUsageUpdate, SetConfigExtensionEnabledRequest_unstable, SetRecipeSlashCommandRequest_unstable, SetSessionSystemPromptRequest_unstable, SetToolPermissionsRequest_unstable, SetToolPermissionsResponse_unstable, ShareSessionNostrRequest_unstable, ShareSessionNostrResponse_unstable, SourceEntry, SourceScope, SourceType, StatusMessage, StatusMessageUpdate, SteerSessionRequest_unstable, SteerSessionResponse_unstable, SubRecipeDto, TextContent, TextResourceContents, ToolListItem, ToolPermissionEntry, ToolPermissionLevel, TruncateSessionConversationRequest_unstable, UnarchiveSessionRequest_unstable, UnpauseScheduleRequest_unstable, UnstructuredCommandInput, UpdateScheduleRequest_unstable, UpdateScheduleResponse_unstable, UpdateSessionProjectRequest_unstable, UpdateSourceRequest_unstable, UpdateSourceResponse_unstable, UpdateWorkingDirRequest_unstable } from './types.gen.js'; +export type { AddConfigExtensionRequest_unstable, AddSessionExtensionRequest_unstable, AgentMention, Annotations, AppsDeleteRequest_unstable, AppsDeleteResponse_unstable, AppsExportRequest_unstable, AppsExportResponse_unstable, AppsImportRequest_unstable, AppsImportResponse_unstable, AppsListRequest_unstable, AppsListResponse_unstable, ArchiveSessionRequest_unstable, AudioContent, AvailableCommand, AvailableCommandInput, BlobResourceContents, CanonicalModelInfoDto, CanonicalModelInfoRequest_unstable, CanonicalModelInfoResponse_unstable, ConfigReadAllRequest_unstable, ConfigReadAllResponse_unstable, ConfigReadRequest_unstable, ConfigReadResponse_unstable, ConfigRemoveRequest_unstable, ConfigUpsertRequest_unstable, ContentBlock, CostSourceData, CreateScheduleRequest_unstable, CreateScheduleResponse_unstable, CreateSourceRequest_unstable, CreateSourceResponse_unstable, CustomProviderConfigDto, CustomProviderCreateRequest_unstable, CustomProviderCreateResponse_unstable, CustomProviderDeleteRequest_unstable, CustomProviderDeleteResponse_unstable, CustomProviderReadRequest_unstable, CustomProviderReadResponse_unstable, CustomProviderUpdateRequest_unstable, CustomProviderUpdateResponse_unstable, DecodeRecipeRequest_unstable, DecodeRecipeResponse_unstable, DefaultsClearRequest_unstable, DefaultsReadRequest_unstable, DefaultsReadResponse_unstable, DefaultsSaveRequest_unstable, DeleteRecipeRequest_unstable, DeleteScheduleRequest_unstable, DeleteSourceRequest_unstable, DiagnosticsGetRequest_unstable, DiagnosticsGetResponse_unstable, DiagnosticsReportLevel, DictationConfigRequest_unstable, DictationConfigResponse_unstable, DictationDownloadProgress, DictationLocalModelStatus, DictationModelCancelRequest_unstable, DictationModelDeleteRequest_unstable, DictationModelDownloadProgressRequest_unstable, DictationModelDownloadProgressResponse_unstable, DictationModelDownloadRequest_unstable, DictationModelOption, DictationModelsListRequest_unstable, DictationModelsListResponse_unstable, DictationProviderStatusEntry, DictationTranscribeRequest_unstable, DictationTranscribeResponse_unstable, EmbeddedResource, EmbeddedResourceResource, EmptyResponse, EncodeRecipeRequest_unstable, EncodeRecipeResponse_unstable, EnvVariable, ExportSessionRequest_unstable, ExportSessionResponse_unstable, ExportSourceRequest_unstable, ExportSourceResponse_unstable, ExtAgentRequest, ExtAgentResponse, ExtNotification, ExtRequest, ExtResponse, GetConfigExtensionsRequest_unstable, GetConfigExtensionsResponse_unstable, GetPromptRequest_unstable, GetPromptResponse_unstable, GetSessionExtensionsRequest_unstable, GetSessionExtensionsResponse_unstable, GetSessionInfoRequest_unstable, GetSessionInfoResponse_unstable, GetToolsRequest_unstable, GetToolsResponse_unstable, GooseExtension, GooseExtensionEntry, GooseSessionNotification_unstable, GooseSessionUpdate, GooseToolCallRequest_unstable, GooseToolCallResponse_unstable, HttpHeader, ImageContent, ImportSessionRequest_unstable, ImportSessionResponse_unstable, ImportSourcesRequest_unstable, ImportSourcesResponse_unstable, InspectRunningJobRequest_unstable, InspectRunningJobResponse_unstable, KillRunningJobRequest_unstable, KillRunningJobResponse_unstable, ListAgentMentionsRequest_unstable, ListAgentMentionsResponse_unstable, ListPromptsRequest_unstable, ListPromptsResponse_unstable, ListProvidersRequest_unstable, ListProvidersResponse_unstable, ListRecipesRequest_unstable, ListRecipesResponse_unstable, ListScheduleSessionsRequest_unstable, ListScheduleSessionsResponse_unstable, ListSchedulesRequest_unstable, ListSchedulesResponse_unstable, ListSlashCommandsRequest_unstable, ListSlashCommandsResponse_unstable, ListSourcesRequest_unstable, ListSourcesResponse_unstable, LiveVoiceAvailabilityRequest_unstable, LiveVoiceAvailabilityResponse_unstable, LiveVoiceInteractionEndedUpdate, LiveVoiceInteractionOutcome, LiveVoiceStartRequest_unstable, LiveVoiceStartResponse_unstable, LiveVoiceStatus, LiveVoiceStopRequest_unstable, LocalInferenceBuiltinChatTemplatesListRequest_unstable, LocalInferenceBuiltinChatTemplatesListResponse_unstable, LocalInferenceChatTemplate, LocalInferenceDownloadProgressDto, LocalInferenceDownloadState, LocalInferenceHfGgufFileDto, LocalInferenceHfModelInfoDto, LocalInferenceHfModelVariantDto, LocalInferenceHuggingFaceRepoVariantsRequest_unstable, LocalInferenceHuggingFaceRepoVariantsResponse_unstable, LocalInferenceHuggingFaceSearchRequest_unstable, LocalInferenceHuggingFaceSearchResponse_unstable, LocalInferenceModelDeleteRequest_unstable, LocalInferenceModelDownloadCancelRequest_unstable, LocalInferenceModelDownloadProgressRequest_unstable, LocalInferenceModelDownloadProgressResponse_unstable, LocalInferenceModelDownloadRequest_unstable, LocalInferenceModelDownloadResponse_unstable, LocalInferenceModelDownloadStatusDto, LocalInferenceModelDto, LocalInferenceModelEvictRequest_unstable, LocalInferenceModelSettingsDto, LocalInferenceModelSettingsReadRequest_unstable, LocalInferenceModelSettingsReadResponse_unstable, LocalInferenceModelSettingsUpdateRequest_unstable, LocalInferenceModelSettingsUpdateResponse_unstable, LocalInferenceModelsListRequest_unstable, LocalInferenceModelsListResponse_unstable, LocalInferenceSamplingConfig, LocalInferenceToolCallingMode, McpServer, McpServerHttp, McpServerSse, McpServerStdio, MessageUsageData, MessageUsageUpdate, OnboardingImportApplyRequest_unstable, OnboardingImportApplyResponse_unstable, OnboardingImportCandidate, OnboardingImportCounts, OnboardingImportScanRequest_unstable, OnboardingImportScanResponse_unstable, OnboardingImportSourceKind, ParseRecipeRequest_unstable, ParseRecipeResponse_unstable, PauseScheduleRequest_unstable, PreferenceKey, PreferencesReadRequest_unstable, PreferencesReadResponse_unstable, PreferencesSaveRequest_unstable, PreferenceValue, PromptOperationResponse_unstable, PromptTemplateEntry, ProviderCatalogListRequest_unstable, ProviderCatalogListResponse_unstable, ProviderCatalogTemplateRequest_unstable, ProviderCatalogTemplateResponse_unstable, ProviderConfigAuthenticateRequest_unstable, ProviderConfigChangeResponse_unstable, ProviderConfigDeleteRequest_unstable, ProviderConfigFieldUpdate, ProviderConfigFieldValueDto, ProviderConfigKey, ProviderConfigReadRequest_unstable, ProviderConfigReadResponse_unstable, ProviderConfigSaveRequest_unstable, ProviderConfigStatusDto, ProviderConfigStatusRequest_unstable, ProviderConfigStatusResponse_unstable, ProviderDeviceCodeNotification_unstable, ProviderInventoryEntryDto, ProviderInventoryModelDto, ProviderReadinessCheckRequest_unstable, ProviderReadinessCheckResponse_unstable, ProviderSecretDeleteRequest_unstable, ProviderSecretDto, ProviderSecretsListRequest_unstable, ProviderSecretsListResponse_unstable, ProviderSecretStatusDto, ProviderSecretStorageDto, ProviderSetupCatalogEntryDto, ProviderSetupCatalogListRequest_unstable, ProviderSetupCatalogListResponse_unstable, ProviderSetupCategoryDto, ProviderSetupFieldDto, ProviderSetupGroupDto, ProviderSetupMethodDto, ProviderSupportedModelsListRequest_unstable, ProviderSupportedModelsListResponse_unstable, ProviderTemplateCapabilitiesDto, ProviderTemplateCatalogEntryDto, ProviderTemplateDto, ProviderTemplateModelDto, ReadResourceRequest_unstable, ReadResourceResponse_unstable, RecipeAuthorDto, RecipeDto, RecipeExtensionDto, RecipeListEntryDto, RecipeParameterDto, RecipeParameterInputTypeDto, RecipeParameterRequirementDto, RecipeParamsAction, RecipeParamsResponse_unstable, RecipeResponseDto, RecipeRetryConfigDto, RecipeSettingsDto, RecipeSuccessCheckDto, RecipeToYamlRequest_unstable, RecipeToYamlResponse_unstable, RefreshProviderInventoryRequest_unstable, RefreshProviderInventoryResponse_unstable, RefreshProviderInventorySkipDto, RefreshProviderInventorySkipReasonDto, RemoveConfigExtensionRequest_unstable, RemoveSessionExtensionRequest_unstable, RenameSessionRequest_unstable, RequestRecipeParams_unstable, ResetPromptRequest_unstable, ResourceLink, Role, RunScheduleNowRequest_unstable, RunScheduleNowResponse_unstable, RunScheduleNowStatus, SavePromptRequest_unstable, SaveRecipeRequest_unstable, SaveRecipeResponse_unstable, ScanRecipeRequest_unstable, ScanRecipeResponse_unstable, ScheduledJobDto, ScheduleRecipeRequest_unstable, SessionExportFormat, SessionExtensionEntry, SessionId, SessionImportSource, SessionInfo, SessionSystemPromptMode, SessionUsageUpdate, SetConfigExtensionEnabledRequest_unstable, SetRecipeSlashCommandRequest_unstable, SetSessionSystemPromptRequest_unstable, SetToolPermissionsRequest_unstable, SetToolPermissionsResponse_unstable, ShareSessionNostrRequest_unstable, ShareSessionNostrResponse_unstable, SourceEntry, SourceScope, SourceType, StatusMessage, StatusMessageUpdate, SteerSessionRequest_unstable, SteerSessionResponse_unstable, SubRecipeDto, TextContent, TextResourceContents, ToolListItem, ToolPermissionEntry, ToolPermissionLevel, TruncateSessionConversationRequest_unstable, UnarchiveSessionRequest_unstable, UnpauseScheduleRequest_unstable, UnstructuredCommandInput, UpdateScheduleRequest_unstable, UpdateScheduleResponse_unstable, UpdateSessionProjectRequest_unstable, UpdateSourceRequest_unstable, UpdateSourceResponse_unstable, UpdateWorkingDirRequest_unstable } from './types.gen.js'; export const GOOSE_EXT_METHODS = [ { @@ -68,6 +68,21 @@ export const GOOSE_EXT_METHODS = [ requestType: "SteerSessionRequest_unstable", responseType: "SteerSessionResponse_unstable", }, + { + method: "_goose/unstable/session/live-voice/availability", + requestType: "LiveVoiceAvailabilityRequest_unstable", + responseType: "LiveVoiceAvailabilityResponse_unstable", + }, + { + method: "_goose/unstable/session/live-voice/start", + requestType: "LiveVoiceStartRequest_unstable", + responseType: "LiveVoiceStartResponse_unstable", + }, + { + method: "_goose/unstable/session/live-voice/stop", + requestType: "LiveVoiceStopRequest_unstable", + responseType: "EmptyResponse", + }, { method: "_goose/unstable/diagnostics/get", requestType: "DiagnosticsGetRequest_unstable", diff --git a/ui/goose-acp-client/src/generated/types.gen.ts b/ui/goose-acp-client/src/generated/types.gen.ts index ece6407c77..fe59c54cea 100644 --- a/ui/goose-acp-client/src/generated/types.gen.ts +++ b/ui/goose-acp-client/src/generated/types.gen.ts @@ -678,9 +678,44 @@ export type SteerSessionResponse_unstable = { messageId: string; }; +/** + * Ask whether a new or existing chat can enter direct Live voice. + */ +export type LiveVoiceAvailabilityRequest_unstable = { + sessionId?: string | null; + _meta?: { + [key: string]: unknown; + } | null; +}; + +export type LiveVoiceAvailabilityResponse_unstable = { + status: LiveVoiceStatus; + message: string; +}; + +export type LiveVoiceStatus = 'ready' | 'unavailable'; + /** * Get a diagnostic report for a session. */ +export type LiveVoiceStartRequest_unstable = { + sessionId: string; + offerSdp: string; + _meta?: { + [key: string]: unknown; + } | null; +}; + +export type LiveVoiceStartResponse_unstable = { + interactionId: string; + answerSdp: string; +}; + +export type LiveVoiceStopRequest_unstable = { + sessionId: string; + interactionId: string; +}; + export type DiagnosticsGetRequest_unstable = { sessionId: string; level?: DiagnosticsReportLevel; @@ -2646,7 +2681,9 @@ export type GooseSessionUpdate = ({ sessionUpdate: 'status_message'; } & StatusMessageUpdate) | ({ sessionUpdate: 'message_usage'; -} & MessageUsageUpdate); +} & MessageUsageUpdate) | ({ + sessionUpdate: 'live_voice_interaction_ended'; +} & LiveVoiceInteractionEndedUpdate); /** * Streaming context-window usage update for a session. @@ -2712,6 +2749,13 @@ export type MessageUsageUpdate = { usage: MessageUsageData; }; +export type LiveVoiceInteractionOutcome = 'stopped' | 'failed'; + +export type LiveVoiceInteractionEndedUpdate = { + interactionId: string; + outcome: LiveVoiceInteractionOutcome; +}; + /** * Dedicated provider notification for OAuth device-code flow. * Sent during provider authentication when the ACP client supports @@ -2745,14 +2789,14 @@ export type RecipeParamsAction = 'submit' | 'cancel'; export type ExtRequest = { id: string; method: string; - params?: AddSessionExtensionRequest_unstable | RemoveSessionExtensionRequest_unstable | GetToolsRequest_unstable | SetToolPermissionsRequest_unstable | GooseToolCallRequest_unstable | ReadResourceRequest_unstable | AppsListRequest_unstable | AppsExportRequest_unstable | AppsImportRequest_unstable | AppsDeleteRequest_unstable | UpdateWorkingDirRequest_unstable | SetSessionSystemPromptRequest_unstable | SteerSessionRequest_unstable | DiagnosticsGetRequest_unstable | ListPromptsRequest_unstable | GetPromptRequest_unstable | SavePromptRequest_unstable | ResetPromptRequest_unstable | GetConfigExtensionsRequest_unstable | AddConfigExtensionRequest_unstable | RemoveConfigExtensionRequest_unstable | SetConfigExtensionEnabledRequest_unstable | GetSessionExtensionsRequest_unstable | ListProvidersRequest_unstable | ProviderSupportedModelsListRequest_unstable | ProviderCatalogListRequest_unstable | ProviderSetupCatalogListRequest_unstable | ProviderCatalogTemplateRequest_unstable | CustomProviderCreateRequest_unstable | CustomProviderReadRequest_unstable | CustomProviderUpdateRequest_unstable | CustomProviderDeleteRequest_unstable | RefreshProviderInventoryRequest_unstable | ProviderReadinessCheckRequest_unstable | ProviderConfigReadRequest_unstable | ProviderConfigStatusRequest_unstable | ProviderConfigSaveRequest_unstable | ProviderConfigDeleteRequest_unstable | ProviderConfigAuthenticateRequest_unstable | ProviderSecretsListRequest_unstable | ProviderSecretDeleteRequest_unstable | CanonicalModelInfoRequest_unstable | PreferencesReadRequest_unstable | PreferencesSaveRequest_unstable | ConfigReadRequest_unstable | ConfigUpsertRequest_unstable | ConfigRemoveRequest_unstable | ConfigReadAllRequest_unstable | DefaultsReadRequest_unstable | DefaultsSaveRequest_unstable | DefaultsClearRequest_unstable | OnboardingImportScanRequest_unstable | OnboardingImportApplyRequest_unstable | ExportSessionRequest_unstable | ImportSessionRequest_unstable | ShareSessionNostrRequest_unstable | EncodeRecipeRequest_unstable | DecodeRecipeRequest_unstable | ScanRecipeRequest_unstable | ListRecipesRequest_unstable | DeleteRecipeRequest_unstable | ScheduleRecipeRequest_unstable | SetRecipeSlashCommandRequest_unstable | SaveRecipeRequest_unstable | ParseRecipeRequest_unstable | RecipeToYamlRequest_unstable | ListSchedulesRequest_unstable | ListScheduleSessionsRequest_unstable | CreateScheduleRequest_unstable | DeleteScheduleRequest_unstable | PauseScheduleRequest_unstable | UnpauseScheduleRequest_unstable | UpdateScheduleRequest_unstable | RunScheduleNowRequest_unstable | KillRunningJobRequest_unstable | InspectRunningJobRequest_unstable | GetSessionInfoRequest_unstable | TruncateSessionConversationRequest_unstable | UpdateSessionProjectRequest_unstable | RenameSessionRequest_unstable | ArchiveSessionRequest_unstable | UnarchiveSessionRequest_unstable | CreateSourceRequest_unstable | ListSourcesRequest_unstable | ListAgentMentionsRequest_unstable | ListSlashCommandsRequest_unstable | UpdateSourceRequest_unstable | DeleteSourceRequest_unstable | ExportSourceRequest_unstable | ImportSourcesRequest_unstable | DictationTranscribeRequest_unstable | DictationConfigRequest_unstable | DictationModelsListRequest_unstable | DictationModelDownloadRequest_unstable | DictationModelDownloadProgressRequest_unstable | DictationModelCancelRequest_unstable | DictationModelDeleteRequest_unstable | LocalInferenceModelsListRequest_unstable | LocalInferenceModelDownloadRequest_unstable | LocalInferenceModelDownloadProgressRequest_unstable | LocalInferenceModelDownloadCancelRequest_unstable | LocalInferenceModelDeleteRequest_unstable | LocalInferenceModelEvictRequest_unstable | LocalInferenceModelSettingsReadRequest_unstable | LocalInferenceModelSettingsUpdateRequest_unstable | LocalInferenceHuggingFaceSearchRequest_unstable | LocalInferenceHuggingFaceRepoVariantsRequest_unstable | LocalInferenceBuiltinChatTemplatesListRequest_unstable | { + params?: AddSessionExtensionRequest_unstable | RemoveSessionExtensionRequest_unstable | GetToolsRequest_unstable | SetToolPermissionsRequest_unstable | GooseToolCallRequest_unstable | ReadResourceRequest_unstable | AppsListRequest_unstable | AppsExportRequest_unstable | AppsImportRequest_unstable | AppsDeleteRequest_unstable | UpdateWorkingDirRequest_unstable | SetSessionSystemPromptRequest_unstable | SteerSessionRequest_unstable | LiveVoiceAvailabilityRequest_unstable | LiveVoiceStartRequest_unstable | LiveVoiceStopRequest_unstable | DiagnosticsGetRequest_unstable | ListPromptsRequest_unstable | GetPromptRequest_unstable | SavePromptRequest_unstable | ResetPromptRequest_unstable | GetConfigExtensionsRequest_unstable | AddConfigExtensionRequest_unstable | RemoveConfigExtensionRequest_unstable | SetConfigExtensionEnabledRequest_unstable | GetSessionExtensionsRequest_unstable | ListProvidersRequest_unstable | ProviderSupportedModelsListRequest_unstable | ProviderCatalogListRequest_unstable | ProviderSetupCatalogListRequest_unstable | ProviderCatalogTemplateRequest_unstable | CustomProviderCreateRequest_unstable | CustomProviderReadRequest_unstable | CustomProviderUpdateRequest_unstable | CustomProviderDeleteRequest_unstable | RefreshProviderInventoryRequest_unstable | ProviderReadinessCheckRequest_unstable | ProviderConfigReadRequest_unstable | ProviderConfigStatusRequest_unstable | ProviderConfigSaveRequest_unstable | ProviderConfigDeleteRequest_unstable | ProviderConfigAuthenticateRequest_unstable | ProviderSecretsListRequest_unstable | ProviderSecretDeleteRequest_unstable | CanonicalModelInfoRequest_unstable | PreferencesReadRequest_unstable | PreferencesSaveRequest_unstable | ConfigReadRequest_unstable | ConfigUpsertRequest_unstable | ConfigRemoveRequest_unstable | ConfigReadAllRequest_unstable | DefaultsReadRequest_unstable | DefaultsSaveRequest_unstable | DefaultsClearRequest_unstable | OnboardingImportScanRequest_unstable | OnboardingImportApplyRequest_unstable | ExportSessionRequest_unstable | ImportSessionRequest_unstable | ShareSessionNostrRequest_unstable | EncodeRecipeRequest_unstable | DecodeRecipeRequest_unstable | ScanRecipeRequest_unstable | ListRecipesRequest_unstable | DeleteRecipeRequest_unstable | ScheduleRecipeRequest_unstable | SetRecipeSlashCommandRequest_unstable | SaveRecipeRequest_unstable | ParseRecipeRequest_unstable | RecipeToYamlRequest_unstable | ListSchedulesRequest_unstable | ListScheduleSessionsRequest_unstable | CreateScheduleRequest_unstable | DeleteScheduleRequest_unstable | PauseScheduleRequest_unstable | UnpauseScheduleRequest_unstable | UpdateScheduleRequest_unstable | RunScheduleNowRequest_unstable | KillRunningJobRequest_unstable | InspectRunningJobRequest_unstable | GetSessionInfoRequest_unstable | TruncateSessionConversationRequest_unstable | UpdateSessionProjectRequest_unstable | RenameSessionRequest_unstable | ArchiveSessionRequest_unstable | UnarchiveSessionRequest_unstable | CreateSourceRequest_unstable | ListSourcesRequest_unstable | ListAgentMentionsRequest_unstable | ListSlashCommandsRequest_unstable | UpdateSourceRequest_unstable | DeleteSourceRequest_unstable | ExportSourceRequest_unstable | ImportSourcesRequest_unstable | DictationTranscribeRequest_unstable | DictationConfigRequest_unstable | DictationModelsListRequest_unstable | DictationModelDownloadRequest_unstable | DictationModelDownloadProgressRequest_unstable | DictationModelCancelRequest_unstable | DictationModelDeleteRequest_unstable | LocalInferenceModelsListRequest_unstable | LocalInferenceModelDownloadRequest_unstable | LocalInferenceModelDownloadProgressRequest_unstable | LocalInferenceModelDownloadCancelRequest_unstable | LocalInferenceModelDeleteRequest_unstable | LocalInferenceModelEvictRequest_unstable | LocalInferenceModelSettingsReadRequest_unstable | LocalInferenceModelSettingsUpdateRequest_unstable | LocalInferenceHuggingFaceSearchRequest_unstable | LocalInferenceHuggingFaceRepoVariantsRequest_unstable | LocalInferenceBuiltinChatTemplatesListRequest_unstable | { [key: string]: unknown; } | null; }; export type ExtResponse = { id: string; - result?: EmptyResponse | GetToolsResponse_unstable | SetToolPermissionsResponse_unstable | GooseToolCallResponse_unstable | ReadResourceResponse_unstable | AppsListResponse_unstable | AppsExportResponse_unstable | AppsImportResponse_unstable | AppsDeleteResponse_unstable | SteerSessionResponse_unstable | DiagnosticsGetResponse_unstable | ListPromptsResponse_unstable | GetPromptResponse_unstable | PromptOperationResponse_unstable | GetConfigExtensionsResponse_unstable | GetSessionExtensionsResponse_unstable | ListProvidersResponse_unstable | ProviderSupportedModelsListResponse_unstable | ProviderCatalogListResponse_unstable | ProviderSetupCatalogListResponse_unstable | ProviderCatalogTemplateResponse_unstable | CustomProviderCreateResponse_unstable | CustomProviderReadResponse_unstable | CustomProviderUpdateResponse_unstable | CustomProviderDeleteResponse_unstable | RefreshProviderInventoryResponse_unstable | ProviderReadinessCheckResponse_unstable | ProviderConfigReadResponse_unstable | ProviderConfigStatusResponse_unstable | ProviderConfigChangeResponse_unstable | ProviderSecretsListResponse_unstable | CanonicalModelInfoResponse_unstable | PreferencesReadResponse_unstable | ConfigReadResponse_unstable | ConfigReadAllResponse_unstable | DefaultsReadResponse_unstable | OnboardingImportScanResponse_unstable | OnboardingImportApplyResponse_unstable | ExportSessionResponse_unstable | ImportSessionResponse_unstable | ShareSessionNostrResponse_unstable | EncodeRecipeResponse_unstable | DecodeRecipeResponse_unstable | ScanRecipeResponse_unstable | ListRecipesResponse_unstable | SaveRecipeResponse_unstable | ParseRecipeResponse_unstable | RecipeToYamlResponse_unstable | ListSchedulesResponse_unstable | ListScheduleSessionsResponse_unstable | CreateScheduleResponse_unstable | UpdateScheduleResponse_unstable | RunScheduleNowResponse_unstable | KillRunningJobResponse_unstable | InspectRunningJobResponse_unstable | GetSessionInfoResponse_unstable | CreateSourceResponse_unstable | ListSourcesResponse_unstable | ListAgentMentionsResponse_unstable | ListSlashCommandsResponse_unstable | UpdateSourceResponse_unstable | ExportSourceResponse_unstable | ImportSourcesResponse_unstable | DictationTranscribeResponse_unstable | DictationConfigResponse_unstable | DictationModelsListResponse_unstable | DictationModelDownloadProgressResponse_unstable | LocalInferenceModelsListResponse_unstable | LocalInferenceModelDownloadResponse_unstable | LocalInferenceModelDownloadProgressResponse_unstable | LocalInferenceModelSettingsReadResponse_unstable | LocalInferenceModelSettingsUpdateResponse_unstable | LocalInferenceHuggingFaceSearchResponse_unstable | LocalInferenceHuggingFaceRepoVariantsResponse_unstable | LocalInferenceBuiltinChatTemplatesListResponse_unstable | unknown; + result?: EmptyResponse | GetToolsResponse_unstable | SetToolPermissionsResponse_unstable | GooseToolCallResponse_unstable | ReadResourceResponse_unstable | AppsListResponse_unstable | AppsExportResponse_unstable | AppsImportResponse_unstable | AppsDeleteResponse_unstable | SteerSessionResponse_unstable | LiveVoiceAvailabilityResponse_unstable | LiveVoiceStartResponse_unstable | DiagnosticsGetResponse_unstable | ListPromptsResponse_unstable | GetPromptResponse_unstable | PromptOperationResponse_unstable | GetConfigExtensionsResponse_unstable | GetSessionExtensionsResponse_unstable | ListProvidersResponse_unstable | ProviderSupportedModelsListResponse_unstable | ProviderCatalogListResponse_unstable | ProviderSetupCatalogListResponse_unstable | ProviderCatalogTemplateResponse_unstable | CustomProviderCreateResponse_unstable | CustomProviderReadResponse_unstable | CustomProviderUpdateResponse_unstable | CustomProviderDeleteResponse_unstable | RefreshProviderInventoryResponse_unstable | ProviderReadinessCheckResponse_unstable | ProviderConfigReadResponse_unstable | ProviderConfigStatusResponse_unstable | ProviderConfigChangeResponse_unstable | ProviderSecretsListResponse_unstable | CanonicalModelInfoResponse_unstable | PreferencesReadResponse_unstable | ConfigReadResponse_unstable | ConfigReadAllResponse_unstable | DefaultsReadResponse_unstable | OnboardingImportScanResponse_unstable | OnboardingImportApplyResponse_unstable | ExportSessionResponse_unstable | ImportSessionResponse_unstable | ShareSessionNostrResponse_unstable | EncodeRecipeResponse_unstable | DecodeRecipeResponse_unstable | ScanRecipeResponse_unstable | ListRecipesResponse_unstable | SaveRecipeResponse_unstable | ParseRecipeResponse_unstable | RecipeToYamlResponse_unstable | ListSchedulesResponse_unstable | ListScheduleSessionsResponse_unstable | CreateScheduleResponse_unstable | UpdateScheduleResponse_unstable | RunScheduleNowResponse_unstable | KillRunningJobResponse_unstable | InspectRunningJobResponse_unstable | GetSessionInfoResponse_unstable | CreateSourceResponse_unstable | ListSourcesResponse_unstable | ListAgentMentionsResponse_unstable | ListSlashCommandsResponse_unstable | UpdateSourceResponse_unstable | ExportSourceResponse_unstable | ImportSourcesResponse_unstable | DictationTranscribeResponse_unstable | DictationConfigResponse_unstable | DictationModelsListResponse_unstable | DictationModelDownloadProgressResponse_unstable | LocalInferenceModelsListResponse_unstable | LocalInferenceModelDownloadResponse_unstable | LocalInferenceModelDownloadProgressResponse_unstable | LocalInferenceModelSettingsReadResponse_unstable | LocalInferenceModelSettingsUpdateResponse_unstable | LocalInferenceHuggingFaceSearchResponse_unstable | LocalInferenceHuggingFaceRepoVariantsResponse_unstable | LocalInferenceBuiltinChatTemplatesListResponse_unstable | unknown; } | { error: { code: number; diff --git a/ui/goose-acp-client/src/generated/zod.gen.ts b/ui/goose-acp-client/src/generated/zod.gen.ts index d72023fa9b..9c6ac5d708 100644 --- a/ui/goose-acp-client/src/generated/zod.gen.ts +++ b/ui/goose-acp-client/src/generated/zod.gen.ts @@ -423,11 +423,42 @@ export const zSteerSessionResponse_unstable = z.object({ messageId: z.string() }); -export const zDiagnosticsReportLevel = z.enum(['summary', 'full']); +/** + * Ask whether a new or existing chat can enter direct Live voice. + */ +export const zLiveVoiceAvailabilityRequest_unstable = z.object({ + sessionId: z.string().nullish(), + _meta: z.record(z.string(), z.unknown()).nullish() +}); + +export const zLiveVoiceStatus = z.enum(['ready', 'unavailable']); + +export const zLiveVoiceAvailabilityResponse_unstable = z.object({ + status: zLiveVoiceStatus, + message: z.string() +}); /** * Get a diagnostic report for a session. */ +export const zLiveVoiceStartRequest_unstable = z.object({ + sessionId: z.string(), + offerSdp: z.string(), + _meta: z.record(z.string(), z.unknown()).nullish() +}); + +export const zLiveVoiceStartResponse_unstable = z.object({ + interactionId: z.string(), + answerSdp: z.string() +}); + +export const zLiveVoiceStopRequest_unstable = z.object({ + sessionId: z.string(), + interactionId: z.string() +}); + +export const zDiagnosticsReportLevel = z.enum(['summary', 'full']); + export const zDiagnosticsGetRequest_unstable = z.object({ sessionId: z.string(), level: zDiagnosticsReportLevel.optional().default('summary') @@ -2234,6 +2265,13 @@ export const zMessageUsageUpdate = z.object({ usage: zMessageUsageData }); +export const zLiveVoiceInteractionOutcome = z.enum(['stopped', 'failed']); + +export const zLiveVoiceInteractionEndedUpdate = z.object({ + interactionId: z.string(), + outcome: zLiveVoiceInteractionOutcome +}); + /** * Discriminated union of goose-specific session update payloads. * Variant tag matches ACP's convention (`sessionUpdate: ""`). @@ -2245,7 +2283,8 @@ export const zMessageUsageUpdate = z.object({ export const zGooseSessionUpdate = z.discriminatedUnion('sessionUpdate', [ zSessionUsageUpdate.extend({ sessionUpdate: z.literal('usage_update') }), zStatusMessageUpdate.extend({ sessionUpdate: z.literal('status_message') }), - zMessageUsageUpdate.extend({ sessionUpdate: z.literal('message_usage') }) + zMessageUsageUpdate.extend({ sessionUpdate: z.literal('message_usage') }), + zLiveVoiceInteractionEndedUpdate.extend({ sessionUpdate: z.literal('live_voice_interaction_ended') }) ]); /** @@ -2303,6 +2342,9 @@ export const zExtRequest = z.object({ zUpdateWorkingDirRequest_unstable, zSetSessionSystemPromptRequest_unstable, zSteerSessionRequest_unstable, + zLiveVoiceAvailabilityRequest_unstable, + zLiveVoiceStartRequest_unstable, + zLiveVoiceStopRequest_unstable, zDiagnosticsGetRequest_unstable, zListPromptsRequest_unstable, zGetPromptRequest_unstable, @@ -2418,6 +2460,8 @@ export const zExtResponse = z.union([ zAppsImportResponse_unstable, zAppsDeleteResponse_unstable, zSteerSessionResponse_unstable, + zLiveVoiceAvailabilityResponse_unstable, + zLiveVoiceStartResponse_unstable, zDiagnosticsGetResponse_unstable, zListPromptsResponse_unstable, zGetPromptResponse_unstable,