feat: add Live voice conversations to the desktop app (#12093)
Some checks failed
Canary / Prepare Version (push) Waiting to run
Canary / build-cli-linux (push) Blocked by required conditions
Canary / Upload Install Script (push) Waiting to run
Canary / bundle-macos-arm64 (push) Blocked by required conditions
Canary / bundle-macos-x64 (push) Blocked by required conditions
Canary / bundle-windows (push) Blocked by required conditions
Canary / bundle-windows-cuda (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / Test Desktop Updater Install (ubuntu-latest) (push) Blocked by required conditions
CI / Test Desktop Updater Install (windows-latest) (push) Blocked by required conditions
CI / Build Rust Project on Windows (push) Waiting to run
CI / Build and Test TLS Backend (rustls-tls) (push) Blocked by required conditions
CI / changes (push) Waiting to run
CI / Build and Test Roaming Feature (push) Blocked by required conditions
CI / goose ACP npm packages (push) Blocked by required conditions
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Check goose-sdk UniFFI (push) Blocked by required conditions
CI / Test Kotlin GDK package (push) Blocked by required conditions
CI / Build and Test TLS Backend (native-tls) (push) Blocked by required conditions
CI / Check MSRV (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check Generated Schemas are Up-to-Date (push) Blocked by required conditions
CI / Check GDK Release Configuration (push) Waiting to run
CI / Check GDK API Reference is Up-to-Date (push) Waiting to run
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
Deploy Documentation / deploy (push) Waiting to run
Prepare GDK release / Create or update GDK release PR (push) Waiting to run
MCP Conformance / Build Conformance Binaries (push) Waiting to run
MCP Conformance / Conformance 2025-11-25 / 0.1.16 (push) Blocked by required conditions
MCP Conformance / Conformance 2025-11-25 / 0.2.0-alpha.10 (push) Blocked by required conditions
MCP Conformance / Conformance 2026-07-28 / 0.2.0-alpha.10 (push) Blocked by required conditions
Create Minor Release PR / check-version-bump-pr (push) Waiting to run
Create Minor Release PR / release (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Publish Ask AI Bot Docker Image / docker (push) Waiting to run
Publish Docker Image / docker (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Cargo Deny / deny (push) Has been cancelled

This commit is contained in:
Lifei Zhou 2026-09-17 03:25:45 +00:00 • committed by GitHub
parent 3952f64e71
commit d213a3b135
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
84 changed files with 6980 additions and 451 deletions

View file

@ -177,10 +177,10 @@ jobs:
run: |
gnome-keyring-daemon --components=secrets --daemonize --unlock <<< 'foobar'
export CARGO_INCREMENTAL=0
cargo test -p goose --no-default-features --features ${{ matrix.tls-feature }},code-mode
cargo test -p goose-providers --no-default-features --features ${{ matrix.tls-feature }}
cargo test -p goose-cli --no-default-features --features ${{ matrix.tls-feature }},code-mode -- --skip scenario_tests::scenarios::tests
cargo test -p goose-cli --no-default-features --features ${{ matrix.tls-feature }},code-mode --jobs 1 scenario_tests::scenarios::tests
cargo test -p goose --no-default-features --features ${{ matrix.tls-feature }},code-mode,live-voice
cargo test -p goose-providers --no-default-features --features ${{ matrix.tls-feature }},live-websocket
cargo test -p goose-cli --no-default-features --features ${{ matrix.tls-feature }},code-mode,live-voice -- --skip scenario_tests::scenarios::tests
cargo test -p goose-cli --no-default-features --features ${{ matrix.tls-feature }},code-mode,live-voice --jobs 1 scenario_tests::scenarios::tests
env:
RUST_MIN_STACK: 8388608

2
.gitignore vendored
View file

@ -97,3 +97,5 @@ gooseselftest/
/crates/goose-sdk/examples/uniffi/kotlin/.gradle/
/crates/goose-sdk/examples/uniffi/kotlin/.kotlin/
/crates/goose-sdk/examples/uniffi/kotlin/build/
live-in-desktop-design

View file

@ -25,7 +25,8 @@ use syn::{
///
/// # Handler signatures
///
/// Handlers may take zero or one parameter (beyond `&self`):
/// Handlers may take zero or one request parameter (beyond `&self`). A handler
/// that needs the active ACP connection may take it before the request:
///
/// ```ignore
/// // No params — called for requests with no/empty params
@ -35,6 +36,14 @@ use syn::{
/// // Typed params — JSON params auto-deserialized
/// #[custom_method(GetSessionRequest)]
/// async fn on_get_session(&self, req: GetSessionRequest) -> Result<GetSessionResponse, agent_client_protocol::Error> { .. }
///
/// // Connection and typed request
/// #[custom_method(StartCallRequest)]
/// async fn on_start_call(
/// &self,
/// cx: &agent_client_protocol::ConnectionTo<agent_client_protocol::Client>,
/// req: StartCallRequest,
/// ) -> Result<StartCallResponse, agent_client_protocol::Error> { .. }
/// ```
///
/// The return type must be `Result<T, agent_client_protocol::Error>` where `T: Serialize`.
@ -64,7 +73,13 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream {
if let Some(req_type) = request_type {
let fn_ident = method.sig.ident.clone();
let param_type = extract_param_type(&method.sig);
let parameter_types = extract_parameter_types(&method.sig);
let (uses_connection, param_type) = match parameter_types.as_slice() {
[] => (false, None),
[request] => (false, Some(request.clone())),
[_connection, request] => (true, Some(request.clone())),
_ => panic!("custom method handlers accept at most a connection and request"),
};
let return_type = extract_return_type(&method.sig);
let ok_type = extract_result_ok_type(&method.sig);
@ -72,6 +87,7 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream {
request_type: req_type,
fn_ident,
param_type,
uses_connection,
return_type,
ok_type,
});
@ -85,6 +101,11 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream {
.map(|route| {
let req_type = &route.request_type;
let fn_ident = &route.fn_ident;
let call = if route.uses_connection {
quote! { self.#fn_ident(cx, req).await? }
} else {
quote! { self.#fn_ident(req).await? }
};
match &route.param_type {
Some(_) => {
@ -92,7 +113,7 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream {
if <#req_type as agent_client_protocol::JsonRpcMessage>::matches_method(method) {
let req = serde_json::from_value(params)
.map_err(|e| agent_client_protocol::Error::invalid_params().data(e.to_string()))?;
let result = self.#fn_ident(req).await?;
let result = #call;
return serde_json::to_value(&result)
.map_err(|e| agent_client_protocol::Error::internal_error().data(e.to_string()));
}
@ -183,7 +204,8 @@ pub fn custom_methods(_attr: TokenStream, item: TokenStream) -> TokenStream {
// Generate the handle_custom_request method.
let dispatcher = quote! {
async fn handle_custom_request(
&self,
self: &std::sync::Arc<Self>,
cx: &agent_client_protocol::ConnectionTo<agent_client_protocol::Client>,
method: &str,
params: serde_json::Value,
) -> Result<serde_json::Value, agent_client_protocol::Error> {
@ -216,24 +238,26 @@ struct Route {
request_type: Type,
fn_ident: syn::Ident,
param_type: Option<Type>,
uses_connection: bool,
#[allow(dead_code)]
return_type: Option<Type>,
ok_type: Option<Type>,
}
/// Extract the type of the first non-self parameter, if any.
fn extract_param_type(sig: &syn::Signature) -> Option<Type> {
for input in &sig.inputs {
if let FnArg::Typed(pat_type) = input {
if let Pat::Ident(pat_ident) = &*pat_type.pat {
if pat_ident.ident == "self" {
continue;
fn extract_parameter_types(sig: &syn::Signature) -> Vec<Type> {
sig.inputs
.iter()
.filter_map(|input| match input {
FnArg::Typed(pat_type) => {
if matches!(&*pat_type.pat, Pat::Ident(pat) if pat.ident == "self") {
None
} else {
Some((*pat_type.ty).clone())
}
}
return Some((*pat_type.ty).clone());
}
}
None
FnArg::Receiver(_) => None,
})
.collect()
}
/// Extract the full return type (e.g. `Result<T, E>`).

View file

@ -100,11 +100,13 @@ default = [
"nostr",
"otel",
"rustls-tls",
"live-voice",
"system-keyring",
"update",
]
roaming = ["dep:goose-roaming", "dep:qrcode", "dep:fs2", "dep:agent-client-protocol"]
code-mode = ["goose/code-mode"]
live-voice = ["goose/live-voice"]
local-inference = ["goose/local-inference"]
aws-providers = ["goose/aws-providers"]
cuda = ["goose/cuda", "local-inference"]
@ -115,7 +117,7 @@ nostr = ["goose/nostr"]
otel = ["goose/otel"]
system-keyring = ["goose/system-keyring"]
update = ["dep:sigstore-verify", "dep:snap"]
portable-default = ["rustls-tls", "aws-providers", "telemetry", "otel"]
portable-default = ["rustls-tls", "live-voice", "aws-providers", "telemetry", "otel"]
# disables the update command
disable-update = []
rustls-tls = [

View file

@ -13,7 +13,6 @@ workspace = true
[features]
default = []
test-utils = []
local-inference = ["dep:goose-local-inference"]
local-inference-hf-hub = ["local-inference", "goose-local-inference/hf-hub"]
cuda = ["local-inference", "goose-local-inference/cuda"]

View file

@ -38,38 +38,51 @@ fn valid_sdp(sdp: &str) -> bool {
#[async_trait]
pub trait ProviderConnection: Send {
async fn next_event(&mut self) -> ProviderConnectionEvent;
async fn send_delegation_update(
&mut self,
update: DelegationUpdate,
) -> Result<DelegationUpdateDelivery>;
async fn stop(&mut self) -> Result<()>;
}
pub enum DelegationUpdateDelivery {
Delivered,
Undelivered,
}
#[derive(Debug, PartialEq)]
pub enum ProviderConnectionEvent {
TranscriptDelta {
event_id: String,
role: rmcp::model::Role,
text: String,
start_ms: u64,
end_ms: u64,
},
DelegationRequested {
event_id: String,
delegation_id: String,
offset_ms: u64,
},
ReceiverLagged,
Closed,
Failed,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DelegationUpdate {
pub provider_delegation_id: String,
pub text: String,
}
#[derive(Clone, Debug, PartialEq)]
pub struct LiveVoiceInputMessage {
pub role: rmcp::model::Role,
pub text: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LiveVoiceProviderAvailability {
Ready,
Disabled,
Unavailable,
}
#[async_trait]
pub trait LiveVoiceProvider: Send + Sync {
fn availability(&self) -> LiveVoiceProviderAvailability;
async fn start(
&self,
offer: WebRtcOffer,
@ -89,137 +102,3 @@ mod tests {
assert!(WebRtcAnswer::new("answer".into()).is_some());
}
}
#[cfg(any(test, feature = "test-utils"))]
pub mod fake {
use super::*;
use tokio::sync::{mpsc, oneshot};
pub fn provider_channel() -> (
std::sync::Arc<FakeLiveVoiceProvider>,
mpsc::UnboundedReceiver<FakeStartRequest>,
) {
provider_channel_with_availability(LiveVoiceProviderAvailability::Ready)
}
pub fn provider_channel_with_availability(
availability: LiveVoiceProviderAvailability,
) -> (
std::sync::Arc<FakeLiveVoiceProvider>,
mpsc::UnboundedReceiver<FakeStartRequest>,
) {
let (start_tx, start_rx) = mpsc::unbounded_channel();
(
std::sync::Arc::new(FakeLiveVoiceProvider {
availability,
start_tx,
}),
start_rx,
)
}
pub struct FakeLiveVoiceProvider {
availability: LiveVoiceProviderAvailability,
start_tx: mpsc::UnboundedSender<FakeStartRequest>,
}
#[async_trait]
impl LiveVoiceProvider for FakeLiveVoiceProvider {
fn availability(&self) -> LiveVoiceProviderAvailability {
self.availability
}
async fn start(
&self,
offer: WebRtcOffer,
input_messages: Vec<LiveVoiceInputMessage>,
) -> Result<(WebRtcAnswer, Box<dyn ProviderConnection>)> {
let (response_tx, response_rx) = oneshot::channel();
self.start_tx
.send(FakeStartRequest {
offer,
input_messages,
response_tx,
})
.map_err(|_| anyhow::anyhow!("fake provider driver dropped"))?;
response_rx
.await
.map_err(|_| anyhow::anyhow!("fake provider start response dropped"))?
}
}
pub struct FakeStartRequest {
pub offer: WebRtcOffer,
pub input_messages: Vec<LiveVoiceInputMessage>,
response_tx: oneshot::Sender<Result<(WebRtcAnswer, Box<dyn ProviderConnection>)>>,
}
impl FakeStartRequest {
pub fn accept(self, answer: WebRtcAnswer) -> Result<FakeConnectionDriver> {
let (stop_request_tx, stop_request_rx) = mpsc::unbounded_channel();
let (event_tx, event_rx) = mpsc::unbounded_channel();
self.response_tx
.send(Ok((
answer,
Box::new(FakeProviderConnection {
stop_request_tx,
event_rx,
}),
)))
.map_err(|_| anyhow::anyhow!("fake provider start caller dropped"))?;
Ok(FakeConnectionDriver {
stop_request_rx,
event_tx,
})
}
pub fn reject(self, message: impl Into<String>) -> Result<()> {
self.response_tx
.send(Err(anyhow::anyhow!(message.into())))
.map_err(|_| anyhow::anyhow!("fake provider start caller dropped"))
}
}
pub struct FakeConnectionDriver {
stop_request_rx: mpsc::UnboundedReceiver<oneshot::Sender<Result<(), String>>>,
event_tx: mpsc::UnboundedSender<ProviderConnectionEvent>,
}
impl FakeConnectionDriver {
pub async fn next_stop_request(&mut self) -> Option<oneshot::Sender<Result<(), String>>> {
self.stop_request_rx.recv().await
}
pub fn send_event(&self, event: ProviderConnectionEvent) -> Result<()> {
self.event_tx
.send(event)
.map_err(|_| anyhow::anyhow!("fake provider event receiver dropped"))
}
}
struct FakeProviderConnection {
stop_request_tx: mpsc::UnboundedSender<oneshot::Sender<Result<(), String>>>,
event_rx: mpsc::UnboundedReceiver<ProviderConnectionEvent>,
}
#[async_trait]
impl ProviderConnection for FakeProviderConnection {
async fn next_event(&mut self) -> ProviderConnectionEvent {
self.event_rx
.recv()
.await
.unwrap_or(ProviderConnectionEvent::Failed)
}
async fn stop(&mut self) -> Result<()> {
let (response_tx, response_rx) = oneshot::channel();
self.stop_request_tx
.send(response_tx)
.map_err(|_| anyhow::anyhow!("fake provider driver dropped"))?;
response_rx
.await
.map_err(|_| anyhow::anyhow!("fake provider stop response dropped"))?
.map_err(anyhow::Error::msg)
}
}
}

View file

@ -3,108 +3,69 @@
use crate::{
live::{LiveSessionEndReason, LiveSessionEvent},
live_voice_provider::{
LiveVoiceInputMessage, LiveVoiceProvider, LiveVoiceProviderAvailability,
DelegationUpdate, DelegationUpdateDelivery, LiveVoiceInputMessage, LiveVoiceProvider,
ProviderConnection, ProviderConnectionEvent, WebRtcAnswer, WebRtcOffer,
},
openai_live::{
ConnectedOpenAiLiveSession, OpenAiLiveClient, OpenAiLiveEvent, OpenAiLiveEventKind,
OpenAiLiveMessage, OpenAiLiveMessageRole, OpenAiLiveSessionConfig, OpenAiLiveSessionId,
ConnectedOpenAiLiveSession, OpenAiLiveClient, OpenAiLiveContext, OpenAiLiveContextChannel,
OpenAiLiveDelegationId, OpenAiLiveEvent, OpenAiLiveEventKind, OpenAiLiveMessage,
OpenAiLiveMessageRole, OpenAiLiveSessionConfig, OpenAiLiveSessionId,
},
};
use anyhow::{bail, Result};
use async_trait::async_trait;
use std::time::Duration;
use std::{collections::VecDeque, time::Duration};
use tokio::{
sync::broadcast::error::RecvError,
time::{sleep_until, timeout, timeout_at, Instant},
};
pub const OPENAI_LIVE_VOICE_GATE_ENV: &str = "GOOSE_LIVE_VOICE_ENABLED";
pub const OPENAI_LIVE_MODEL_ENV: &str = "GOOSE_LIVE_VOICE_MODEL";
pub const OPENAI_LIVE_VOICE_ENV: &str = "GOOSE_LIVE_VOICE";
pub const OPENAI_LIVE_API_KEY_ENV: &str = "OPENAI_API_KEY";
pub const DEFAULT_OPENAI_LIVE_MODEL: &str = "gpt-live-1";
pub const DEFAULT_OPENAI_LIVE_VOICE: &str = "marin";
const OPENAI_LIVE_MODEL: &str = "gpt-live-1";
const HTTP_SETUP_TIMEOUT: Duration = Duration::from_secs(15);
const SIDEBAND_ATTACH_TIMEOUT: Duration = Duration::from_secs(10);
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OpenAiLiveVoiceConfig {
pub enabled: bool,
pub model: String,
pub voice: String,
}
impl OpenAiLiveVoiceConfig {
pub fn from_env() -> Self {
Self {
enabled: std::env::var(OPENAI_LIVE_VOICE_GATE_ENV)
.is_ok_and(|value| value.eq_ignore_ascii_case("true") || value == "1"),
model: std::env::var(OPENAI_LIVE_MODEL_ENV)
.unwrap_or_else(|_| DEFAULT_OPENAI_LIVE_MODEL.into()),
voice: std::env::var(OPENAI_LIVE_VOICE_ENV)
.unwrap_or_else(|_| DEFAULT_OPENAI_LIVE_VOICE.into()),
}
}
}
const DELEGATION_UPDATE_ACK_TIMEOUT: Duration = Duration::from_secs(10);
pub struct OpenAiLiveVoiceProvider {
client: Option<OpenAiLiveClient>,
config: OpenAiLiveVoiceConfig,
client: OpenAiLiveClient,
voice: String,
instructions: String,
delivery_failure_instructions: String,
}
impl OpenAiLiveVoiceProvider {
pub fn from_env() -> Self {
let config = OpenAiLiveVoiceConfig::from_env();
let client = std::env::var(OPENAI_LIVE_API_KEY_ENV)
.ok()
.filter(|key| !key.trim().is_empty())
.map(OpenAiLiveClient::new);
Self { client, config }
}
pub fn new(api_key: impl Into<String>, config: OpenAiLiveVoiceConfig) -> Result<Self> {
pub fn new(
api_key: impl Into<String>,
voice: String,
instructions: String,
delivery_failure_instructions: String,
) -> Result<Self> {
let api_key = api_key.into();
if api_key.trim().is_empty() {
bail!("OpenAI API key is empty");
}
if voice.trim().is_empty() {
bail!("OpenAI Live voice is empty");
}
Ok(Self {
client: Some(OpenAiLiveClient::new(api_key)),
config,
client: OpenAiLiveClient::new(api_key),
voice,
instructions,
delivery_failure_instructions,
})
}
}
#[async_trait]
impl LiveVoiceProvider for OpenAiLiveVoiceProvider {
fn availability(&self) -> LiveVoiceProviderAvailability {
if !self.config.enabled {
LiveVoiceProviderAvailability::Disabled
} else if self.client.is_none() {
LiveVoiceProviderAvailability::Unavailable
} else {
LiveVoiceProviderAvailability::Ready
}
}
async fn start(
&self,
offer: WebRtcOffer,
input_messages: Vec<LiveVoiceInputMessage>,
) -> Result<(WebRtcAnswer, Box<dyn ProviderConnection>)> {
if !self.config.enabled {
bail!("OpenAI Live voice is disabled");
}
let client = self
.client
.as_ref()
.ok_or_else(|| anyhow::anyhow!("OpenAI Live credentials are unavailable"))?
.clone();
let config = OpenAiLiveSessionConfig {
model: self.config.model.clone(),
instructions: String::new(),
voice: Some(self.config.voice.clone()),
model: OPENAI_LIVE_MODEL.into(),
instructions: self.instructions.clone(),
voice: Some(self.voice.clone()),
input_messages: input_messages
.into_iter()
.map(|message| OpenAiLiveMessage {
@ -119,25 +80,44 @@ impl LiveVoiceProvider for OpenAiLiveVoiceProvider {
};
let negotiation = timeout(
HTTP_SETUP_TIMEOUT,
client.webrtc(config).negotiate(offer.into_sdp()),
self.client.webrtc(config).negotiate(offer.into_sdp()),
)
.await
.map_err(|_| anyhow::anyhow!("OpenAI Live HTTP setup timed out"))??;
let session_id = negotiation.session_id;
let answer = WebRtcAnswer::new(negotiation.answer_sdp)
.ok_or_else(|| anyhow::anyhow!("OpenAI Live returned an invalid WebRTC answer"))?;
let sideband = connect_sideband(&client, session_id).await?;
Ok((answer, Box::new(OpenAiProviderConnection { sideband })))
let sideband = connect_sideband(&self.client, session_id).await?;
Ok((
answer,
Box::new(OpenAiProviderConnection {
sideband,
pending_events: VecDeque::new(),
delivery_failure_instructions: self.delivery_failure_instructions.clone(),
}),
))
}
}
struct OpenAiProviderConnection {
sideband: ConnectedOpenAiLiveSession,
pending_events: VecDeque<ProviderConnectionEvent>,
delivery_failure_instructions: String,
}
#[derive(Debug, PartialEq)]
enum AppendContextOutcome {
Accepted,
Rejected,
TimedOut,
}
#[async_trait]
impl ProviderConnection for OpenAiProviderConnection {
async fn next_event(&mut self) -> ProviderConnectionEvent {
if let Some(event) = self.pending_events.pop_front() {
return event;
}
loop {
if let Some(event) = provider_connection_event(self.sideband.recv().await) {
return event;
@ -145,11 +125,103 @@ impl ProviderConnection for OpenAiProviderConnection {
}
}
async fn send_delegation_update(
&mut self,
update: DelegationUpdate,
) -> Result<DelegationUpdateDelivery> {
match self
.append_context(
Some(OpenAiLiveDelegationId(update.provider_delegation_id)),
update.text,
)
.await?
{
AppendContextOutcome::Accepted => Ok(DelegationUpdateDelivery::Delivered),
AppendContextOutcome::Rejected => {
let _ = self
.append_context(None, self.delivery_failure_instructions.clone())
.await?;
Ok(DelegationUpdateDelivery::Undelivered)
}
AppendContextOutcome::TimedOut => Ok(DelegationUpdateDelivery::Undelivered),
}
}
async fn stop(&mut self) -> Result<()> {
self.sideband.close().await
}
}
impl OpenAiProviderConnection {
async fn append_context(
&mut self,
delegation_id: Option<OpenAiLiveDelegationId>,
text: String,
) -> Result<AppendContextOutcome> {
let event_id = format!("event_{}", uuid::Uuid::new_v4());
self.sideband
.send(crate::openai_live::OpenAiLiveCommand::AppendContext {
event_id: event_id.clone(),
delegation_id,
context: OpenAiLiveContext {
text,
channel: OpenAiLiveContextChannel::Commentary,
},
})
.await?;
match timeout(DELEGATION_UPDATE_ACK_TIMEOUT, async {
loop {
let event = self.sideband.recv().await;
match append_context_response(&event, &event_id) {
Some(outcome) => return Ok(outcome),
None => match provider_connection_event(event) {
Some(
ProviderConnectionEvent::Closed
| ProviderConnectionEvent::Failed
| ProviderConnectionEvent::ReceiverLagged,
) => {
bail!("OpenAI Live session ended while delivering a delegation update")
}
Some(event) => self.pending_events.push_back(event),
None => {}
},
}
}
})
.await
{
Ok(outcome) => outcome,
Err(_) => Ok(AppendContextOutcome::TimedOut),
}
}
}
fn append_context_response(
event: &std::result::Result<LiveSessionEvent<OpenAiLiveEvent>, RecvError>,
event_id: &str,
) -> Option<AppendContextOutcome> {
match event {
Ok(LiveSessionEvent::Message(OpenAiLiveEvent {
kind:
OpenAiLiveEventKind::ContextAppended {
client_event_id: Some(client_event_id),
..
},
..
})) if client_event_id == event_id => Some(AppendContextOutcome::Accepted),
Ok(LiveSessionEvent::Message(OpenAiLiveEvent {
kind:
OpenAiLiveEventKind::Error {
client_event_id: Some(client_event_id),
..
},
..
})) if client_event_id == event_id => Some(AppendContextOutcome::Rejected),
_ => None,
}
}
fn provider_connection_event(
event: std::result::Result<LiveSessionEvent<OpenAiLiveEvent>, RecvError>,
) -> Option<ProviderConnectionEvent> {
@ -159,12 +231,31 @@ fn provider_connection_event(
event_id,
role,
delta,
start_ms,
end_ms,
..
} => Some(ProviderConnectionEvent::TranscriptDelta {
event_id,
role,
text: delta,
start_ms,
end_ms,
}),
OpenAiLiveEventKind::DelegationCreated {
event_id,
delegation,
..
} if matches!(
&delegation.target,
crate::openai_live::OpenAiLiveDelegationTarget::Client
) =>
{
Some(ProviderConnectionEvent::DelegationRequested {
event_id,
delegation_id: delegation.id.0,
offset_ms: delegation.offset_ms,
})
}
OpenAiLiveEventKind::SessionClosed { .. } => Some(ProviderConnectionEvent::Closed),
_ => None,
},
@ -209,20 +300,9 @@ mod tests {
use super::*;
#[test]
fn configuration_is_gated() {
let provider = OpenAiLiveVoiceProvider::new(
"key",
OpenAiLiveVoiceConfig {
enabled: false,
model: DEFAULT_OPENAI_LIVE_MODEL.into(),
voice: DEFAULT_OPENAI_LIVE_VOICE.into(),
},
)
.unwrap();
assert_eq!(
provider.availability(),
LiveVoiceProviderAvailability::Disabled
);
fn configuration_must_be_valid() {
assert!(OpenAiLiveVoiceProvider::new("", "voice".into(), "".into(), "".into()).is_err());
assert!(OpenAiLiveVoiceProvider::new("key", String::new(), "".into(), "".into()).is_err());
}
#[test]
@ -247,6 +327,8 @@ mod tests {
event_id: "event_transcript_1".into(),
role: rmcp::model::Role::Assistant,
text: "hello".into(),
start_ms: 10,
end_ms: 20,
}),
),
(
@ -305,4 +387,39 @@ mod tests {
assert_eq!(provider_connection_event(event), expected);
}
}
#[test]
fn correlates_delegation_update_responses() {
let message = |kind| {
Ok(LiveSessionEvent::Message(OpenAiLiveEvent {
kind,
raw: None,
}))
};
let accepted = message(OpenAiLiveEventKind::ContextAppended {
event_id: "event_accepted".into(),
channel: OpenAiLiveContextChannel::Commentary,
client_event_id: Some("client_event".into()),
start_ms: 10,
end_ms: 20,
});
let rejected = message(OpenAiLiveEventKind::Error {
event_id: "event_rejected".into(),
error_type: "invalid_request_error".into(),
code: "invalid_delegation".into(),
message: "delegation is stale".into(),
parameter: Some("delegation_id".into()),
client_event_id: Some("client_event".into()),
});
assert_eq!(
append_context_response(&accepted, "client_event"),
Some(AppendContextOutcome::Accepted)
);
assert_eq!(
append_context_response(&rejected, "client_event"),
Some(AppendContextOutcome::Rejected)
);
assert_eq!(append_context_response(&accepted, "other_event"), None);
}
}

View file

@ -26,13 +26,29 @@ pub struct GooseSessionNotification {
"mapping": {
"usage_update": "#/$defs/SessionUsageUpdate",
"status_message": "#/$defs/StatusMessageUpdate",
"message_usage": "#/$defs/MessageUsageUpdate"
"message_usage": "#/$defs/MessageUsageUpdate",
"live_voice_interaction_ended": "#/$defs/LiveVoiceInteractionEndedUpdate"
}
}))]
pub enum GooseSessionUpdate {
UsageUpdate(SessionUsageUpdate),
StatusMessage(StatusMessageUpdate),
MessageUsage(MessageUsageUpdate),
LiveVoiceInteractionEnded(LiveVoiceInteractionEndedUpdate),
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct LiveVoiceInteractionEndedUpdate {
pub interaction_id: String,
pub outcome: LiveVoiceInteractionOutcome,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum LiveVoiceInteractionOutcome {
Stopped,
Failed,
}
/// Dedicated provider notification for OAuth device-code flow.
@ -194,6 +210,33 @@ mod tests {
);
}
#[test]
fn live_voice_interaction_ended_serializes_to_expected_wire_shape() {
let notification = GooseSessionNotification {
session_id: "s1".to_string(),
update: GooseSessionUpdate::LiveVoiceInteractionEnded(
LiveVoiceInteractionEndedUpdate {
interaction_id: "live_opaque".to_string(),
outcome: LiveVoiceInteractionOutcome::Failed,
},
),
};
let value = serde_json::to_value(notification).unwrap();
assert_eq!(
value,
json!({
"sessionId": "s1",
"update": {
"sessionUpdate": "live_voice_interaction_ended",
"interactionId": "live_opaque",
"outcome": "failed"
}
})
);
}
#[test]
fn message_usage_serializes_to_expected_wire_shape() {
let notification = GooseSessionNotification {

View file

@ -1,4 +1,6 @@
use agent_client_protocol::schema::v1::{AvailableCommand, ContentBlock, McpServer, SessionInfo};
use agent_client_protocol::schema::v1::{
AvailableCommand, ContentBlock, McpServer, Meta, SessionInfo,
};
use agent_client_protocol::{JsonRpcRequest, JsonRpcResponse};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -239,7 +241,65 @@ pub struct SteerSessionResponse {
pub message_id: String,
}
/// Ask whether a new or existing chat can enter direct Live voice.
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
#[request(
method = "_goose/unstable/session/live-voice/availability",
response = LiveVoiceAvailabilityResponse
)]
#[serde(rename_all = "camelCase")]
pub struct LiveVoiceAvailabilityRequest {
pub session_id: Option<String>,
#[serde(default, rename = "_meta", skip_serializing_if = "Option::is_none")]
pub meta: Option<Meta>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum LiveVoiceStatus {
Ready,
Unavailable,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
#[serde(rename_all = "camelCase")]
pub struct LiveVoiceAvailabilityResponse {
pub status: LiveVoiceStatus,
pub message: String,
}
/// Get a diagnostic report for a session.
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
#[request(
method = "_goose/unstable/session/live-voice/start",
response = LiveVoiceStartResponse
)]
#[serde(rename_all = "camelCase")]
pub struct LiveVoiceStartRequest {
pub session_id: String,
pub offer_sdp: String,
#[serde(default, rename = "_meta", skip_serializing_if = "Option::is_none")]
pub meta: Option<Meta>,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
#[serde(rename_all = "camelCase")]
pub struct LiveVoiceStartResponse {
pub interaction_id: String,
pub answer_sdp: String,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
#[request(
method = "_goose/unstable/session/live-voice/stop",
response = EmptyResponse
)]
#[serde(rename_all = "camelCase")]
pub struct LiveVoiceStopRequest {
pub session_id: String,
pub interaction_id: String,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
#[request(
method = "_goose/unstable/diagnostics/get",

View file

@ -20,6 +20,7 @@ otel = [
"dep:opentelemetry-stdout",
]
code-mode = ["dep:pctx_code_mode"]
live-voice = ["goose-providers/live-websocket"]
local-inference = [
"dep:candle-core",
"dep:candle-nn",
@ -77,7 +78,7 @@ native-tls = [
"goose-providers/native-tls",
]
system-keyring = ["dep:keyring"]
portable-default = ["rustls-tls", "aws-providers", "telemetry", "otel"]
portable-default = ["rustls-tls", "live-voice", "aws-providers", "telemetry", "otel"]
nostr = ["dep:nostr", "dep:nostr-sdk"]
[lints]
@ -256,7 +257,7 @@ libc = { version = "0.2.182", default-features = false, features = ["std"] }
serial_test = { workspace = true }
mockall = { version = "0.15", default-features = false }
wiremock = { workspace = true }
tokio = { workspace = true }
tokio = { workspace = true, features = ["test-util"] }
tokio-util = { workspace = true, features = ["compat"] }
dotenvy = { workspace = true }
ctor = { version = "1.0", default-features = false, features = ["proc_macro"] }

View file

@ -65,6 +65,21 @@
"requestType": "SteerSessionRequest_unstable",
"responseType": "SteerSessionResponse_unstable"
},
{
"method": "_goose/unstable/session/live-voice/availability",
"requestType": "LiveVoiceAvailabilityRequest_unstable",
"responseType": "LiveVoiceAvailabilityResponse_unstable"
},
{
"method": "_goose/unstable/session/live-voice/start",
"requestType": "LiveVoiceStartRequest_unstable",
"responseType": "LiveVoiceStartResponse_unstable"
},
{
"method": "_goose/unstable/session/live-voice/stop",
"requestType": "LiveVoiceStopRequest_unstable",
"responseType": "EmptyResponse"
},
{
"method": "_goose/unstable/diagnostics/get",
"requestType": "DiagnosticsGetRequest_unstable",

View file

@ -1299,6 +1299,110 @@
"x-side": "agent",
"x-method": "_goose/unstable/session/steer"
},
"LiveVoiceAvailabilityRequest_unstable": {
"type": "object",
"properties": {
"sessionId": {
"type": [
"string",
"null"
]
},
"_meta": {
"type": [
"object",
"null"
],
"additionalProperties": {}
}
},
"description": "Ask whether a new or existing chat can enter direct Live voice.",
"x-side": "agent",
"x-method": "_goose/unstable/session/live-voice/availability"
},
"LiveVoiceAvailabilityResponse_unstable": {
"type": "object",
"properties": {
"status": {
"$ref": "#/$defs/LiveVoiceStatus"
},
"message": {
"type": "string"
}
},
"required": [
"status",
"message"
],
"x-side": "agent",
"x-method": "_goose/unstable/session/live-voice/availability"
},
"LiveVoiceStatus": {
"type": "string",
"enum": [
"ready",
"unavailable"
]
},
"LiveVoiceStartRequest_unstable": {
"type": "object",
"properties": {
"sessionId": {
"type": "string"
},
"offerSdp": {
"type": "string"
},
"_meta": {
"type": [
"object",
"null"
],
"additionalProperties": {}
}
},
"required": [
"sessionId",
"offerSdp"
],
"description": "Get a diagnostic report for a session.",
"x-side": "agent",
"x-method": "_goose/unstable/session/live-voice/start"
},
"LiveVoiceStartResponse_unstable": {
"type": "object",
"properties": {
"interactionId": {
"type": "string"
},
"answerSdp": {
"type": "string"
}
},
"required": [
"interactionId",
"answerSdp"
],
"x-side": "agent",
"x-method": "_goose/unstable/session/live-voice/start"
},
"LiveVoiceStopRequest_unstable": {
"type": "object",
"properties": {
"sessionId": {
"type": "string"
},
"interactionId": {
"type": "string"
}
},
"required": [
"sessionId",
"interactionId"
],
"x-side": "agent",
"x-method": "_goose/unstable/session/live-voice/stop"
},
"DiagnosticsGetRequest_unstable": {
"type": "object",
"properties": {
@ -1313,7 +1417,6 @@
"required": [
"sessionId"
],
"description": "Get a diagnostic report for a session.",
"x-side": "agent",
"x-method": "_goose/unstable/diagnostics/get"
},
@ -6689,6 +6792,19 @@
"required": [
"sessionUpdate"
]
},
{
"$ref": "#/$defs/LiveVoiceInteractionEndedUpdate",
"type": "object",
"properties": {
"sessionUpdate": {
"type": "string",
"const": "live_voice_interaction_ended"
}
},
"required": [
"sessionUpdate"
]
}
],
"description": "Discriminated union of goose-specific session update payloads.\nVariant tag matches ACP's convention (`sessionUpdate: \"<snake_case>\"`).\n\n`discriminator.mapping` is what makes TS codegen (`@hey-api/openapi-ts`)\nemit the correct snake_case tag value even when this enum has a single\nvariant. Add a mapping entry per variant.",
@ -6697,7 +6813,8 @@
"mapping": {
"usage_update": "#/$defs/SessionUsageUpdate",
"status_message": "#/$defs/StatusMessageUpdate",
"message_usage": "#/$defs/MessageUsageUpdate"
"message_usage": "#/$defs/MessageUsageUpdate",
"live_voice_interaction_ended": "#/$defs/LiveVoiceInteractionEndedUpdate"
}
}
},
@ -6890,6 +7007,28 @@
],
"description": "Per-message token usage/cost/timing, keyed by the message id used for\nchunk matching. Sent live after a turn's messages and on replay."
},
"LiveVoiceInteractionOutcome": {
"type": "string",
"enum": [
"stopped",
"failed"
]
},
"LiveVoiceInteractionEndedUpdate": {
"type": "object",
"properties": {
"interactionId": {
"type": "string"
},
"outcome": {
"$ref": "#/$defs/LiveVoiceInteractionOutcome"
}
},
"required": [
"interactionId",
"outcome"
]
},
"ProviderDeviceCodeNotification_unstable": {
"type": "object",
"properties": {
@ -7098,6 +7237,33 @@
"description": "Params for _goose/unstable/session/steer",
"title": "SteerSessionRequest_unstable"
},
{
"allOf": [
{
"$ref": "#/$defs/LiveVoiceAvailabilityRequest_unstable"
}
],
"description": "Params for _goose/unstable/session/live-voice/availability",
"title": "LiveVoiceAvailabilityRequest_unstable"
},
{
"allOf": [
{
"$ref": "#/$defs/LiveVoiceStartRequest_unstable"
}
],
"description": "Params for _goose/unstable/session/live-voice/start",
"title": "LiveVoiceStartRequest_unstable"
},
{
"allOf": [
{
"$ref": "#/$defs/LiveVoiceStopRequest_unstable"
}
],
"description": "Params for _goose/unstable/session/live-voice/stop",
"title": "LiveVoiceStopRequest_unstable"
},
{
"allOf": [
{
@ -8063,6 +8229,22 @@
],
"title": "SteerSessionResponse_unstable"
},
{
"allOf": [
{
"$ref": "#/$defs/LiveVoiceAvailabilityResponse_unstable"
}
],
"title": "LiveVoiceAvailabilityResponse_unstable"
},
{
"allOf": [
{
"$ref": "#/$defs/LiveVoiceStartResponse_unstable"
}
],
"title": "LiveVoiceStartResponse_unstable"
},
{
"allOf": [
{

View file

@ -98,6 +98,8 @@ use self::tool_calls::enrichment::{spawn_chain_summary_enrichment, spawn_tool_ti
mod agent_requests;
pub use agent_requests::agent_request_schemas;
mod agent_mentions;
pub use crate::execution::ActiveRunRegistry;
use crate::execution::StartRunError;
mod apps;
mod config;
mod custom_dispatch;
@ -108,8 +110,10 @@ mod elicitation;
mod extensions;
mod fork_session;
mod list_sessions;
mod live_voice;
mod load_session;
mod local_inference;
pub use crate::live_voice::LiveVoiceService;
mod manage_sessions;
mod message_meta;
mod new_session;
@ -237,28 +241,11 @@ struct GooseAcpSession {
agent: Arc<Agent>,
}
pub struct ActivePromptRun {
run_id: String,
cancel_token: CancellationToken,
/// The agent actually running this prompt. Roaming gives each connection
/// its own agent, so a steer arriving on a second connection must be
/// routed here rather than to the caller's connection-local agent.
agent: Arc<Agent>,
}
struct AgentStreamOutcome {
was_cancelled: bool,
output_token_limit_reached: bool,
}
/// Per-session active-run registry, shared by every `GooseAcpAgent` created
/// from one `AcpServer`. Roaming spawns a fresh agent per connection, so two
/// paired clients loading the same session get distinct agents; sharing this
/// map across them is what makes the "session already has an active run" guard
/// fire between connections instead of letting two loops interleave writes on
/// one session.
pub type ActiveRunRegistry = Arc<Mutex<HashMap<String, ActivePromptRun>>>;
/// Releases a registry entry if the task consuming an agent stream is dropped
/// without reaching its explicit `clear_active_run` — e.g. a roaming
/// connection is revoked or lost mid-turn. Without this, the shared registry
@ -268,7 +255,7 @@ pub type ActiveRunRegistry = Arc<Mutex<HashMap<String, ActivePromptRun>>>;
/// The explicit clear still runs on normal paths; this drop is then a no-op
/// because the entry (matched by run id) is already gone.
struct ActiveRunDropGuard {
registry: ActiveRunRegistry,
registry: Arc<ActiveRunRegistry>,
session_id: String,
run_id: String,
cancel_token: CancellationToken,
@ -277,24 +264,15 @@ struct ActiveRunDropGuard {
impl Drop for ActiveRunDropGuard {
fn drop(&mut self) {
self.cancel_token.cancel();
let registry = self.registry.clone();
let session_id = std::mem::take(&mut self.session_id);
let run_id = std::mem::take(&mut self.run_id);
let agent = self.registry.remove_agent_run(&session_id, &run_id);
if let Ok(handle) = tokio::runtime::Handle::try_current() {
handle.spawn(async move {
let agent = {
let mut runs = registry.lock().await;
match runs.get(&session_id) {
Some(run) if run.run_id == run_id => {
runs.remove(&session_id).map(|run| run.agent)
}
_ => None,
}
};
if let Some(agent) = agent {
if let Some(agent) = agent {
handle.spawn(async move {
agent.discard_pending_steers(&session_id).await;
}
});
});
}
}
}
}
@ -333,15 +311,16 @@ pub struct GooseAcpAgentOptions {
/// When set, new sessions use this host-controlled working directory instead
/// of the `cwd` the connecting client sends (see `AcpServerFactoryConfig`).
pub session_cwd: Option<std::path::PathBuf>,
/// Active-run registry shared across all agents from one `AcpServer`, so the
/// active-run guard holds across roaming connections that each get a fresh
/// agent for the same session.
pub active_prompt_runs: ActiveRunRegistry,
/// Shared across roaming connections to coordinate prompt, Live, and
/// delegated agent runs for each session.
pub active_runs: Arc<ActiveRunRegistry>,
pub live_voice: Arc<LiveVoiceService>,
}
pub struct GooseAcpAgent {
sessions: Arc<Mutex<HashMap<String, GooseAcpSession>>>,
active_prompt_runs: Arc<Mutex<HashMap<String, ActivePromptRun>>>,
active_runs: Arc<ActiveRunRegistry>,
live_voice: Arc<LiveVoiceService>,
closed_session_ids: Arc<Mutex<HashSet<String>>>,
agent_manager: Arc<AgentManager>,
provider_factory: AcpProviderFactory,
@ -439,6 +418,13 @@ fn extract_timeout_from_meta(meta: &Option<Meta>) -> Option<u64> {
.and_then(|v| v.as_u64())
}
fn use_state_machine_from_meta(meta: Option<&Meta>) -> bool {
meta.and_then(|meta| meta.get("goose"))
.and_then(|goose| goose.get("unrolledAgentLoop"))
.and_then(|value| value.as_bool())
.unwrap_or_else(crate::agents::state_machine::enabled)
}
#[derive(Debug, Default, Deserialize)]
struct ClientCapabilitiesMeta {
#[serde(default)]
@ -836,8 +822,8 @@ pub(super) fn validate_absolute_cwd(cwd: &Path) -> Result<(), agent_client_proto
impl GooseAcpAgent {
#[cfg(test)]
pub(crate) fn active_run_registry(&self) -> &ActiveRunRegistry {
&self.active_prompt_runs
pub(crate) fn active_run_registry(&self) -> &Arc<ActiveRunRegistry> {
&self.active_runs
}
#[cfg(test)]
@ -854,7 +840,7 @@ impl GooseAcpAgent {
#[cfg(test)]
pub(crate) fn test_drop_active_run_guard(&self, session_id: &str, run_id: &str) {
drop(ActiveRunDropGuard {
registry: self.active_prompt_runs.clone(),
registry: self.active_runs.clone(),
session_id: session_id.to_string(),
run_id: run_id.to_string(),
cancel_token: CancellationToken::new(),
@ -983,7 +969,8 @@ impl GooseAcpAgent {
Ok(Self {
sessions: Arc::new(Mutex::new(HashMap::new())),
active_prompt_runs: options.active_prompt_runs,
active_runs: options.active_runs,
live_voice: options.live_voice,
closed_session_ids: Arc::new(Mutex::new(HashSet::new())),
agent_manager,
provider_factory: options.provider_factory,
@ -1917,40 +1904,24 @@ impl GooseAcpAgent {
.data(format!("Session not found: {}", session_id)));
}
let mut active_prompt_runs = self.active_prompt_runs.lock().await;
if let Some(active_run) = active_prompt_runs.get(session_id) {
return Err(agent_client_protocol::Error::invalid_params().data(format!(
"session already has active run `{}`; use _goose/unstable/session/steer",
active_run.run_id.as_str()
)));
}
active_prompt_runs.insert(
session_id.to_string(),
ActivePromptRun {
run_id,
cancel_token,
agent,
},
);
self.active_runs
.start_prompt_run(session_id, run_id, cancel_token, agent)
.map_err(|error| match error {
StartRunError::AgentRunExists { run_id } => {
let message = format!(
"session already has active run `{run_id}`; use _goose/unstable/session/steer"
);
agent_client_protocol::Error::invalid_params().data(message)
}
StartRunError::LiveVoiceInteractionExists => agent_client_protocol::Error::invalid_params()
.data("session already has an active Live run"),
StartRunError::LiveVoiceInteractionMissing => unreachable!("prompt runs do not require Live"),
})?;
Ok(())
}
async fn clear_active_run(&self, session_id: &str, run_id: &str) {
let agent = {
let mut active_prompt_runs = self.active_prompt_runs.lock().await;
let Some(active_run) = active_prompt_runs.get(session_id) else {
return;
};
if active_run.run_id != run_id {
return;
}
active_prompt_runs
.remove(session_id)
.map(|active_run| active_run.agent)
};
let agent = self.active_runs.remove_agent_run(session_id, run_id);
// Discard steers on the agent that owned the run; under roaming it may
// not be this connection's agent.
@ -1984,23 +1955,22 @@ impl GooseAcpAgent {
.data("expectedRunId must not be empty"));
}
let active_prompt_runs = self.active_prompt_runs.lock().await;
let active_run = active_prompt_runs.get(session_id).ok_or_else(|| {
let (active_run_id, agent) = self.active_runs.agent_run(session_id).ok_or_else(|| {
agent_client_protocol::Error::invalid_params().data("no active run to steer")
})?;
if active_run.run_id != expected_run_id {
if active_run_id != expected_run_id {
return Err(
agent_client_protocol::Error::invalid_params().data(serde_json::json!({
"message": format!(
"expected active run id `{expected_run_id}` but found `{}`",
active_run.run_id.as_str()
active_run_id.as_str()
),
"expectedRunId": expected_run_id,
"actualRunId": active_run.run_id.as_str(),
"actualRunId": active_run_id.as_str(),
})),
);
}
Ok((active_run.run_id.clone(), active_run.agent.clone()))
Ok((active_run_id, agent))
}
fn active_run_meta(active_run_id: Option<&str>) -> Meta {
@ -2317,7 +2287,7 @@ impl GooseAcpAgent {
// connection carrying it is revoked or lost); a normal completion's
// explicit clear wins and makes the guard's cleanup a no-op.
let _run_guard = ActiveRunDropGuard {
registry: self.active_prompt_runs.clone(),
registry: self.active_runs.clone(),
session_id: session_id.clone(),
run_id: run_id.clone(),
cancel_token: cancel_token.clone(),
@ -2344,13 +2314,7 @@ impl GooseAcpAgent {
}
let user_message = Self::convert_acp_prompt_to_message(&args.prompt);
let use_state_machine = args
.meta
.as_ref()
.and_then(|meta| meta.get("goose"))
.and_then(|goose| goose.get("unrolledAgentLoop"))
.and_then(|value| value.as_bool())
.unwrap_or_else(crate::agents::state_machine::enabled);
let use_state_machine = use_state_machine_from_meta(args.meta.as_ref());
let session_config = SessionConfig {
id: session_id.clone(),
schedule_id: None,
@ -2452,12 +2416,7 @@ impl GooseAcpAgent {
debug!(?args, "cancel request");
let session_id = args.session_id.0.to_string();
let token = {
let active_prompt_runs = self.active_prompt_runs.lock().await;
active_prompt_runs
.get(&session_id)
.map(|active_run| active_run.cancel_token.clone())
};
let token = self.active_runs.agent_cancel_token(&session_id);
if let Some(token) = token {
info!(session_id = %session_id, "prompt cancelled");
@ -2669,16 +2628,8 @@ impl GooseAcpAgent {
.await
.insert(session_id.to_string());
let active_run_token = {
let active_prompt_runs = self.active_prompt_runs.lock().await;
active_prompt_runs
.get(session_id)
.map(|active_run| active_run.cancel_token.clone())
};
if let Some(token) = active_run_token {
token.cancel();
}
self.active_runs.cancel_agent_run(session_id);
self.live_voice.stop_session_interaction(session_id).await;
let mut sessions = self.sessions.lock().await;
sessions.remove(session_id);
@ -3663,6 +3614,8 @@ print(\"hello, world\")
#[tokio::test]
async fn asynchronous_provider_effort_update_is_forwarded_to_client() {
let root = tempfile::tempdir().unwrap();
let active_runs = Arc::new(ActiveRunRegistry::default());
let live_voice = Arc::new(LiveVoiceService::from_config(active_runs.clone()));
let provider_factory: AcpProviderFactory = Arc::new(
|_provider_name, _extensions, _working_dir, _use_default_model| {
Box::pin(async { Err(anyhow::anyhow!("unused provider factory")) })
@ -3679,7 +3632,8 @@ print(\"hello, world\")
additional_source_roots: Vec::new(),
scheduler: None,
session_cwd: None,
active_prompt_runs: Default::default(),
active_runs,
live_voice,
})
.await
.unwrap(),

View file

@ -4,7 +4,8 @@ use goose_acp_macros::custom_methods;
#[custom_methods]
impl GooseAcpAgent {
pub async fn dispatch_custom_request(
&self,
self: &Arc<Self>,
cx: &ConnectionTo<Client>,
method: &str,
params: serde_json::Value,
) -> Result<serde_json::Value, agent_client_protocol::Error> {
@ -18,7 +19,7 @@ impl GooseAcpAgent {
});
}
self.handle_custom_request(method, params).await
self.handle_custom_request(cx, method, params).await
}
.await;
@ -133,6 +134,31 @@ impl GooseAcpAgent {
self.on_steer_session(req).await
}
#[custom_method(LiveVoiceAvailabilityRequest)]
async fn dispatch_live_voice_availability(
&self,
req: LiveVoiceAvailabilityRequest,
) -> Result<LiveVoiceAvailabilityResponse, agent_client_protocol::Error> {
self.on_live_voice_availability(req).await
}
#[custom_method(LiveVoiceStartRequest)]
async fn dispatch_live_voice_start(
self: &Arc<Self>,
cx: &ConnectionTo<Client>,
req: LiveVoiceStartRequest,
) -> Result<LiveVoiceStartResponse, agent_client_protocol::Error> {
self.on_live_voice_start(cx, req).await
}
#[custom_method(LiveVoiceStopRequest)]
async fn dispatch_live_voice_stop(
&self,
req: LiveVoiceStopRequest,
) -> Result<EmptyResponse, agent_client_protocol::Error> {
self.on_live_voice_stop(req).await
}
#[custom_method(DiagnosticsGetRequest)]
async fn dispatch_get_diagnostics(
&self,

View file

@ -475,8 +475,16 @@ impl HandleDispatchFrom<Client> for GooseAcpHandler {
|message: Dispatch| async move {
match message {
Dispatch::Request(req, responder) => {
let request_cx = cx.clone();
cx.spawn(async move {
match agent.dispatch_custom_request(&req.method, req.params).await {
match agent
.dispatch_custom_request(
&request_cx,
&req.method,
req.params,
)
.await
{
Ok(json) => responder.respond(json)?,
Err(e) => responder.respond_with_error(e)?,
}

View file

@ -0,0 +1,412 @@
use super::*;
use crate::live_voice::{
wait_for_completion, LiveMainAgent, LiveVoiceError, LiveVoiceInteractionCompletion,
LiveVoiceInteractionId, LiveVoiceTranscriptPublisher, StartLiveVoiceInteractionResult,
WebRtcOffer,
};
use futures::FutureExt;
const LIVE_DELEGATION_INSTRUCTION: &str = concat!(
"You support a live conversation. Treat the latest delegated input as an update to this session's earlier context. ",
"Voice transcripts may be incomplete. If required information is missing, ask a brief clarification instead of guessing. ",
"End every run with a concise, user-facing result below 400 tokens. State the outcome, what changed, validation performed, ",
"failures, and any required user action. Do not include raw tool output."
);
impl GooseAcpAgent {
async fn load_live_voice_session(
&self,
session_id: &str,
) -> Result<Session, agent_client_protocol::Error> {
self.session_manager
.get_session(session_id, false)
.await
.map_err(|_| {
agent_client_protocol::Error::resource_not_found(None)
.data("Session is not accessible")
})
}
pub(super) async fn on_live_voice_availability(
&self,
req: LiveVoiceAvailabilityRequest,
) -> Result<LiveVoiceAvailabilityResponse, agent_client_protocol::Error> {
if !use_state_machine_from_meta(req.meta.as_ref()) {
return Ok(LiveVoiceAvailabilityResponse {
status: LiveVoiceStatus::Unavailable,
message: "Live voice is unavailable while Use Legacy Agent Loop is enabled".into(),
});
}
let mode = match req.session_id.as_deref() {
Some(session_id) => self.load_live_voice_session(session_id).await?.goose_mode,
None => crate::config::Config::global()
.get_goose_mode()
.unwrap_or_default(),
};
Ok(
match self
.live_voice
.availability(req.session_id.as_deref(), mode)
{
Ok(()) => LiveVoiceAvailabilityResponse {
status: LiveVoiceStatus::Ready,
message: "Start Live voice".into(),
},
Err(message) => LiveVoiceAvailabilityResponse {
status: LiveVoiceStatus::Unavailable,
message: message.into(),
},
},
)
}
pub(super) async fn on_live_voice_start(
self: &Arc<Self>,
cx: &ConnectionTo<Client>,
req: LiveVoiceStartRequest,
) -> Result<LiveVoiceStartResponse, agent_client_protocol::Error> {
if !use_state_machine_from_meta(req.meta.as_ref()) {
return Err(map_live_voice_error(LiveVoiceError::Unavailable));
}
let offer = WebRtcOffer::new(req.offer_sdp)
.ok_or_else(agent_client_protocol::Error::invalid_params)?;
let session_id = req.session_id.clone();
let session = self.load_live_voice_session(&session_id).await?;
let reservation = self
.live_voice
.reserve_interaction(&session_id, session.goose_mode)
.map_err(map_live_voice_error)?;
let transcript_publisher = Self::live_transcript_publisher(cx, &req.session_id);
let agent = self
.prepare_live_agent(&req.session_id)
.await
.map_err(|error| agent_client_protocol::Error::internal_error().data(error))?;
let main_agent = self.live_main_agent(cx, agent);
let start = self.live_voice.start_interaction(
reservation,
offer,
self.session_manager.clone(),
transcript_publisher,
main_agent,
);
tokio::pin!(start);
let interaction = tokio::select! {
result = &mut start => result.map_err(map_live_voice_error)?,
_ = cx.incoming_closed() => {
return Err(agent_client_protocol::Error::internal_error()
.data("ACP connection closed while Live voice was starting"));
}
};
self.watch_live_interaction(cx, session_id, &interaction);
Ok(LiveVoiceStartResponse {
interaction_id: interaction.interaction_id.0,
answer_sdp: interaction.answer.into_sdp(),
})
}
fn live_transcript_publisher(
cx: &ConnectionTo<Client>,
session_id: &str,
) -> LiveVoiceTranscriptPublisher {
let connection = cx.clone();
let session_id = session_id.to_string();
Arc::new(move |message| {
let [MessageContent::Text(text)] = message.content.as_slice() else {
return;
};
let chunk = content_chunk_for_message(
&message,
ContentBlock::Text(TextContent::new(text.text.clone())),
);
let update = match message.role {
Role::User => SessionUpdate::UserMessageChunk(chunk),
Role::Assistant => SessionUpdate::AgentMessageChunk(chunk),
};
let _ = connection.send_notification(SessionNotification::new(
SessionId::new(session_id.clone()),
update,
));
})
}
fn live_main_agent(
self: &Arc<Self>,
cx: &ConnectionTo<Client>,
agent: Arc<Agent>,
) -> LiveMainAgent {
let start_owner = Arc::clone(self);
let start_connection = cx.clone();
let steer_owner = Arc::clone(self);
LiveMainAgent::new(
move |session_id, input| {
start_owner.clone().start_live_delegation(
session_id,
input,
start_connection.clone(),
agent.clone(),
)
},
move |session_id, input| {
let owner = steer_owner.clone();
async move { owner.steer_live_delegation(&session_id, input).await }.boxed()
},
)
}
fn watch_live_interaction(
&self,
cx: &ConnectionTo<Client>,
session_id: String,
interaction: &StartLiveVoiceInteractionResult,
) {
let interaction_id = interaction.interaction_id.clone();
let completion_rx = interaction.completion_rx.clone();
let live_voice = self.live_voice.clone();
let connection = cx.clone();
let notify_interaction_ended = self.supports_goose_custom_notifications();
tokio::spawn(async move {
tokio::select! {
biased;
_ = connection.incoming_closed() => {
let _ = live_voice.stop_interaction(&session_id, &interaction_id).await;
}
completion = wait_for_completion(completion_rx) => {
if notify_interaction_ended {
if let Ok(completion) = completion {
let outcome = match completion {
LiveVoiceInteractionCompletion::Stopped => LiveVoiceInteractionOutcome::Stopped,
LiveVoiceInteractionCompletion::Failed => LiveVoiceInteractionOutcome::Failed,
};
let _ = connection.send_notification(GooseSessionNotification {
session_id,
update: GooseSessionUpdate::LiveVoiceInteractionEnded(LiveVoiceInteractionEndedUpdate {
interaction_id: interaction_id.0,
outcome,
}),
});
}
}
}
}
});
}
pub(super) async fn on_live_voice_stop(
&self,
req: LiveVoiceStopRequest,
) -> Result<EmptyResponse, agent_client_protocol::Error> {
self.load_live_voice_session(&req.session_id).await?;
let interaction_id = LiveVoiceInteractionId(req.interaction_id);
self.live_voice
.stop_interaction(&req.session_id, &interaction_id)
.await
.map_err(map_live_voice_error)?;
Ok(EmptyResponse {})
}
fn start_live_delegation(
self: Arc<Self>,
session_id: String,
input: String,
cx: ConnectionTo<Client>,
agent: Arc<Agent>,
) -> Result<BoxFuture<'static, String>, String> {
let cancel_token = CancellationToken::new();
let run_id = format!("run_{}", Uuid::new_v4());
if self
.active_runs
.start_live_delegation(
&session_id,
run_id.clone(),
cancel_token.clone(),
agent.clone(),
)
.is_err()
{
return Err("Goose is already working on a task.".into());
}
let run_guard = ActiveRunDropGuard {
registry: self.active_runs.clone(),
session_id: session_id.clone(),
run_id: run_id.clone(),
cancel_token: cancel_token.clone(),
};
Ok(async move {
let _run_guard = run_guard;
self.run_live_delegation(session_id, input, cancel_token, cx, agent, run_id)
.await
}
.boxed())
}
async fn run_live_delegation(
&self,
session_id: String,
input: String,
cancel_token: CancellationToken,
cx: ConnectionTo<Client>,
agent: Arc<Agent>,
run_id: String,
) -> String {
let acp_session_id = SessionId::new(session_id.clone());
let _ = Self::send_active_run_update(&cx, &acp_session_id, Some(&run_id));
let session_config = SessionConfig {
id: session_id.clone(),
schedule_id: None,
max_turns: None,
retry_config: None,
};
let input_message =
Message::user().with_text(format!("{LIVE_DELEGATION_INSTRUCTION}\n\n{input}"));
let mut stream = match agent
.reply_live_delegation(input_message, session_config, cancel_token.clone())
.await
{
Ok(stream) => stream,
Err(_) => {
self.clear_active_run(&session_id, &run_id).await;
let _ = Self::send_active_run_update(&cx, &acp_session_id, None);
return "Goose couldn't start the task.".into();
}
};
let mut tool_requests = HashMap::new();
let mut outcome_message_id: Option<String> = None;
let mut outcome = String::new();
let target = SessionAgentTarget {
agent: agent.clone(),
session_id: session_id.clone(),
cancel_token: Some(cancel_token.clone()),
};
while let Some(event) = stream.next().await {
if cancel_token.is_cancelled() {
self.clear_active_run(&session_id, &run_id).await;
let _ = Self::send_active_run_update(&cx, &acp_session_id, None);
return "The task was cancelled.".into();
}
match event {
Ok(crate::agents::AgentEvent::Message(message)) => {
let is_user_visible = message.is_user_visible();
let is_visible_assistant = message.role == Role::Assistant && is_user_visible;
let projected_message = message.user_visible_content();
for content in &projected_message.content {
if let MessageContent::ToolRequest(tool_request) = content {
tool_requests.insert(tool_request.id.clone(), tool_request.clone());
}
let should_project = match content {
MessageContent::ToolRequest(_)
| MessageContent::ToolResponse(_)
| MessageContent::Thinking(_)
| MessageContent::ActionRequired(_) => true,
_ => is_user_visible,
};
if !should_project {
continue;
}
let _ = self
.handle_message_content(
content,
&projected_message,
&acp_session_id,
&target,
&tool_requests,
&cx,
)
.await;
}
if is_visible_assistant {
let text = projected_message
.content
.iter()
.filter_map(MessageContent::as_text)
.collect::<String>();
if !text.trim().is_empty() {
if outcome_message_id.is_some()
&& outcome_message_id == projected_message.id
{
outcome.push_str(&text);
} else {
outcome_message_id = projected_message.id;
outcome = text;
}
}
}
}
Ok(crate::agents::AgentEvent::McpNotification((request_id, notification))) => {
if let Some(update) =
tool_notifications::tool_notification_update(request_id, notification)
{
let _ = ToolCallNotifier::new(&cx, &acp_session_id).send_update(update);
}
}
Ok(crate::agents::AgentEvent::MessageUsage { message_id, usage }) => {
if self.supports_goose_custom_notifications() {
let _ = cx.send_notification(GooseSessionNotification {
session_id: session_id.clone(),
update: GooseSessionUpdate::MessageUsage(message_usage_update(
message_id, &usage,
)),
});
}
}
Ok(_) => {}
Err(_) => {
self.clear_active_run(&session_id, &run_id).await;
let _ = Self::send_active_run_update(&cx, &acp_session_id, None);
return "The task failed before it could complete.".into();
}
}
}
self.clear_active_run(&session_id, &run_id).await;
let _ = Self::send_active_run_update(&cx, &acp_session_id, None);
if cancel_token.is_cancelled() {
return "The task was cancelled.".into();
}
if outcome.is_empty() {
return "The task finished without a result to share.".into();
}
outcome
}
async fn steer_live_delegation(
&self,
session_id: &str,
input: String,
) -> Result<String, String> {
let Some((_, agent)) = self.active_runs.agent_run(session_id) else {
return Err("The task could not receive the latest instruction.".into());
};
agent
.steer(session_id, Message::user().with_text(input).agent_only())
.await;
Ok(
"The latest instruction was added to the work in progress. Wait for its updated result."
.into(),
)
}
async fn prepare_live_agent(&self, session_id: &str) -> Result<Arc<Agent>, String> {
self.get_session_agent(session_id)
.await
.map_err(|_| "Goose could not activate the coding agent.".to_string())
}
}
fn map_live_voice_error(error: LiveVoiceError) -> agent_client_protocol::Error {
match error {
LiveVoiceError::Unavailable => {
agent_client_protocol::Error::invalid_params().data("Live voice is unavailable")
}
LiveVoiceError::StartFailed => {
agent_client_protocol::Error::internal_error().data("Live voice could not start")
}
LiveVoiceError::StopFailed => {
agent_client_protocol::Error::internal_error().data("Live voice could not stop cleanly")
}
}
}

View file

@ -305,7 +305,7 @@ impl GooseAcpAgent {
let task_acp_session_id = acp_session_id.clone();
if let Err(error) = cx.spawn(async move {
let _run_guard = ActiveRunDropGuard {
registry: server.active_prompt_runs.clone(),
registry: server.active_runs.clone(),
session_id: task_session_id.clone(),
run_id: task_run_id.clone(),
cancel_token: task_cancel_token.clone(),

View file

@ -100,6 +100,8 @@ impl GooseAcpAgent {
req: DeleteSessionRequest,
) -> Result<DeleteSessionResponse, agent_client_protocol::Error> {
let session_id = req.session_id.0.to_string();
self.active_runs.cancel_agent_run(&session_id);
self.live_voice.stop_session_interaction(&session_id).await;
self.session_manager
.delete_session(&session_id)
.await

View file

@ -1,5 +1,6 @@
use crate::acp::server::{
AcpBuiltinSelection, AcpProviderFactory, ActiveRunRegistry, GooseAcpAgent, GooseAcpAgentOptions,
AcpBuiltinSelection, AcpProviderFactory, ActiveRunRegistry, GooseAcpAgent,
GooseAcpAgentOptions, LiveVoiceService,
};
use crate::agents::GoosePlatform;
use crate::scheduler_trait::SchedulerTrait;
@ -26,15 +27,19 @@ pub struct AcpServerFactoryConfig {
pub struct AcpServer {
config: AcpServerFactoryConfig,
scheduler: OnceCell<Arc<dyn SchedulerTrait>>,
active_prompt_runs: ActiveRunRegistry,
active_runs: Arc<ActiveRunRegistry>,
live_voice: Arc<crate::acp::server::LiveVoiceService>,
}
impl AcpServer {
pub fn new(config: AcpServerFactoryConfig) -> Self {
let active_runs = Arc::new(ActiveRunRegistry::default());
let live_voice = Arc::new(LiveVoiceService::from_config(active_runs.clone()));
Self {
config,
scheduler: OnceCell::new(),
active_prompt_runs: ActiveRunRegistry::default(),
active_runs,
live_voice,
}
}
@ -123,7 +128,8 @@ impl AcpServer {
additional_source_roots: self.config.additional_source_roots.clone(),
session_cwd,
scheduler,
active_prompt_runs: self.active_prompt_runs.clone(),
active_runs: self.active_runs.clone(),
live_voice: self.live_voice.clone(),
})
.await?;
info!("Created new ACP agent");
@ -175,8 +181,7 @@ mod tests {
assert!(
Arc::ptr_eq(a.active_run_registry(), b.active_run_registry()),
"each connection's agent must share one per-session run registry so \
the active-run guard holds across roaming connections"
"each connection's agent must share one per-session run registry"
);
}

View file

@ -2013,6 +2013,19 @@ impl Agent {
))
}
pub(crate) async fn reply_live_delegation(
&self,
user_message: Message,
session_config: SessionConfig,
cancel_token: CancellationToken,
) -> Result<BoxStream<'_, Result<AgentEvent>>> {
let user_message = user_message.agent_only();
let events = self
.reply_with_state_machine(user_message, session_config, Some(cancel_token))
.await?;
Ok(Box::pin(events.map_ok(ensure_message_event_id)))
}
#[instrument(
skip(self, user_message, session_config, use_state_machine, cancel_token),
fields(

View file

@ -7,7 +7,7 @@ use goose_agent::operation::{ConversationEffect, MachineEffect};
pub enum GooseEffect {
Conversation(ConversationEffect),
ReplaceConversation {
CompactConversation {
conversation: Conversation,
usage: Option<ProviderUsage>,
},
@ -20,7 +20,7 @@ impl MachineEffect for GooseEffect {
fn ensure_message_ids(&mut self) {
match self {
GooseEffect::Conversation(effect) => effect.ensure_message_ids(),
GooseEffect::ReplaceConversation { conversation, .. } => {
GooseEffect::CompactConversation { conversation, .. } => {
for message in conversation.messages_mut() {
if message.id.is_none() {
message.id = Some(format!("msg_{}", uuid::Uuid::new_v4()));

View file

@ -191,7 +191,7 @@ impl Operation<Session, GooseEffect> for CompactionOperation {
emit.message(command).await;
let response = emit.message(response).await;
yielded_with([
GooseEffect::ReplaceConversation {
GooseEffect::CompactConversation {
conversation: compacted,
usage: Some(usage),
},
@ -306,7 +306,7 @@ impl Operation<Session, GooseEffect> for CompactionOperation {
"Compaction complete",
))
.await;
applied([GooseEffect::ReplaceConversation {
applied([GooseEffect::CompactConversation {
conversation: compacted,
usage: Some(usage),
}])

View file

@ -19,7 +19,6 @@ fn contains_tool_confirmation_request(message: &Message) -> bool {
)
})
}
impl MachineSession for Session {
fn id(&self) -> &str {
&self.id
@ -63,14 +62,15 @@ impl EffectHandler<Session, GooseEffect> for SessionManager {
.apply()
.await?;
}
GooseEffect::ReplaceConversation {
GooseEffect::CompactConversation {
conversation,
usage: replacement_usage,
} => {
if let Some(provider_usage) = replacement_usage {
usage::record(self, session, provider_usage, true).await?;
}
self.replace_conversation(&session.id, conversation).await?;
self.save_compacted_conversation(&session.id, conversation)
.await?;
self.update(&session.id)
.usage(usage::estimate_context(conversation).await?)
.apply()
@ -137,7 +137,7 @@ impl EffectHandler<Session, GooseEffect> for SessionManager {
GooseEffect::Conversation(ConversationEffect::ReplaceConversation(
conversation,
))
| GooseEffect::ReplaceConversation { conversation, .. } => {
| GooseEffect::CompactConversation { conversation, .. } => {
emit.emit(AgentEvent::HistoryReplaced(conversation.clone()))
.await;
}
@ -158,7 +158,7 @@ impl EffectUsage<GooseEffect> for SessionManager {
) -> Option<goose_providers::conversation::token_usage::Usage> {
match effect {
GooseEffect::RecordUsage(usage)
| GooseEffect::ReplaceConversation {
| GooseEffect::CompactConversation {
usage: Some(usage), ..
} => Some(usage.usage),
_ => None,
@ -237,3 +237,76 @@ pub(crate) async fn run(
crate::agents::gen_ai_telemetry::record_usage(&tracing::Span::current(), &turn_usage);
Ok(session)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::GooseMode;
use crate::conversation::message::Message;
use crate::session::session_manager::SessionType;
#[tokio::test]
async fn live_replacement_preserves_concurrent_transcript_and_hidden_handoff() {
let manager = SessionManager::new(tempfile::tempdir().unwrap().keep());
let session = manager
.create_session(
"/tmp".into(),
"concurrent compaction".into(),
SessionType::User,
GooseMode::Auto,
)
.await
.unwrap();
let kickoff = Message::user()
.with_id("kickoff")
.with_text("delegated request")
.agent_only();
manager.add_message(&session.id, &kickoff).await.unwrap();
let snapshot = manager.get_session(&session.id, true).await.unwrap();
manager
.add_message(
&session.id,
&Message::user()
.with_id("concurrent-live")
.with_text("still speaking")
.user_only(),
)
.await
.unwrap();
let mut compacted = snapshot.conversation.clone().unwrap();
for message in compacted.messages_mut() {
message.metadata.agent_visible = false;
}
compacted.messages_mut().push(
Message::assistant()
.with_id("summary")
.with_text("summary")
.agent_only(),
);
manager
.save_compacted_conversation(&session.id, &compacted)
.await
.unwrap();
let reloaded = manager.get_session(&session.id, true).await.unwrap();
let messages = reloaded.conversation.unwrap();
let kickoff = messages
.messages()
.iter()
.find(|message| message.id.as_deref() == Some("kickoff"))
.unwrap();
assert!(!kickoff.is_user_visible());
assert!(!kickoff.is_agent_visible());
assert!(messages
.messages()
.iter()
.any(|message| message.id.as_deref() == Some("concurrent-live")));
assert!(messages
.messages()
.iter()
.any(|message| message.id.as_deref() == Some("summary")));
}
}

View file

@ -24,7 +24,7 @@ pub(super) fn enrich(session: &Session, effects: &mut [GooseEffect]) {
for index in 0..effects.len() {
let (usage, replaces_conversation) = match &effects[index] {
GooseEffect::RecordUsage(usage) => (usage.clone(), false),
GooseEffect::ReplaceConversation {
GooseEffect::CompactConversation {
usage: Some(usage), ..
} => (usage.clone(), true),
_ => continue,
@ -53,7 +53,7 @@ pub(super) fn enrich(session: &Session, effects: &mut [GooseEffect]) {
}
match &mut effects[index] {
GooseEffect::RecordUsage(usage) => *usage = enriched,
GooseEffect::ReplaceConversation { usage, .. } => *usage = Some(enriched),
GooseEffect::CompactConversation { usage, .. } => *usage = Some(enriched),
_ => {}
}
}

View file

@ -0,0 +1,244 @@
use crate::agents::Agent;
use std::{
collections::HashMap,
sync::{Arc, Mutex},
};
use tokio_util::sync::CancellationToken;
struct ActiveRun {
run_id: String,
cancel_token: CancellationToken,
/// Routes steering from another roaming connection to the run owner.
agent: Arc<Agent>,
}
struct SessionRunState {
agent_run: Option<ActiveRun>,
live_active: bool,
}
pub(crate) enum StartRunError {
AgentRunExists { run_id: String },
LiveVoiceInteractionExists,
LiveVoiceInteractionMissing,
}
#[derive(Default)]
pub struct ActiveRunRegistry {
runs_by_session: Mutex<HashMap<String, SessionRunState>>,
}
impl ActiveRunRegistry {
pub(crate) fn start_prompt_run(
&self,
session_id: &str,
run_id: String,
cancel_token: CancellationToken,
agent: Arc<Agent>,
) -> Result<(), StartRunError> {
let mut runs = self
.runs_by_session
.lock()
.expect("active run lock poisoned");
if let Some(state) = runs.get(session_id) {
if let Some(agent_run) = &state.agent_run {
return Err(StartRunError::AgentRunExists {
run_id: agent_run.run_id.clone(),
});
}
if state.live_active {
return Err(StartRunError::LiveVoiceInteractionExists);
}
}
runs.insert(
session_id.to_string(),
SessionRunState {
agent_run: Some(ActiveRun {
run_id,
cancel_token,
agent,
}),
live_active: false,
},
);
Ok(())
}
pub(crate) fn start_live_delegation(
&self,
session_id: &str,
run_id: String,
cancel_token: CancellationToken,
agent: Arc<Agent>,
) -> Result<(), StartRunError> {
let mut runs = self
.runs_by_session
.lock()
.expect("active run lock poisoned");
let Some(state) = runs.get_mut(session_id) else {
return Err(StartRunError::LiveVoiceInteractionMissing);
};
if let Some(agent_run) = &state.agent_run {
return Err(StartRunError::AgentRunExists {
run_id: agent_run.run_id.clone(),
});
}
state.agent_run = Some(ActiveRun {
run_id,
cancel_token,
agent,
});
Ok(())
}
pub(crate) fn agent_run(&self, session_id: &str) -> Option<(String, Arc<Agent>)> {
self.runs_by_session
.lock()
.expect("active run lock poisoned")
.get(session_id)
.and_then(|state| state.agent_run.as_ref())
.map(|run| (run.run_id.clone(), run.agent.clone()))
}
pub(crate) fn agent_cancel_token(&self, session_id: &str) -> Option<CancellationToken> {
self.runs_by_session
.lock()
.expect("active run lock poisoned")
.get(session_id)
.and_then(|state| state.agent_run.as_ref())
.map(|run| run.cancel_token.clone())
}
pub(crate) fn cancel_agent_run(&self, session_id: &str) {
if let Some(cancel_token) = self.agent_cancel_token(session_id) {
cancel_token.cancel();
}
}
pub(crate) fn remove_agent_run(&self, session_id: &str, run_id: &str) -> Option<Arc<Agent>> {
let mut runs = self
.runs_by_session
.lock()
.expect("active run lock poisoned");
let state = runs.get_mut(session_id)?;
if state.agent_run.as_ref()?.run_id != run_id {
return None;
}
let agent = state.agent_run.take()?.agent;
if !state.live_active {
runs.remove(session_id);
}
Some(agent)
}
pub(crate) fn start_live(&self, session_id: &str) -> bool {
let mut runs = self
.runs_by_session
.lock()
.expect("active run lock poisoned");
if runs.contains_key(session_id) {
return false;
}
runs.insert(
session_id.to_string(),
SessionRunState {
agent_run: None,
live_active: true,
},
);
true
}
pub(crate) fn finish_live(&self, session_id: &str) {
let mut runs = self
.runs_by_session
.lock()
.expect("active run lock poisoned");
if let Some(state) = runs.get_mut(session_id) {
state.live_active = false;
if state.agent_run.is_none() {
runs.remove(session_id);
}
}
}
pub(crate) fn is_active(&self, session_id: &str) -> bool {
self.runs_by_session
.lock()
.expect("active run lock poisoned")
.contains_key(session_id)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn live_runs_are_scoped_by_session() {
let registry = ActiveRunRegistry::default();
assert!(registry.start_live("one"));
assert!(!registry.start_live("one"));
assert!(registry.start_live("two"));
registry.finish_live("one");
assert!(registry.start_live("one"));
}
#[tokio::test]
async fn prompt_and_live_runs_conflict() {
let registry = ActiveRunRegistry::default();
assert!(registry
.start_prompt_run(
"session",
"run".into(),
CancellationToken::new(),
Arc::new(Agent::new()),
)
.is_ok());
assert!(!registry.start_live("session"));
registry.remove_agent_run("session", "run");
assert!(registry.start_live("session"));
assert!(matches!(
registry.start_prompt_run(
"session",
"run".into(),
CancellationToken::new(),
Arc::new(Agent::new()),
),
Err(StartRunError::LiveVoiceInteractionExists)
));
}
#[tokio::test]
async fn live_delegation_shares_the_live_session_without_admitting_another_prompt() {
let registry = ActiveRunRegistry::default();
assert!(registry.start_live("session"));
assert!(registry
.start_live_delegation(
"session",
"delegated".into(),
CancellationToken::new(),
Arc::new(Agent::new()),
)
.is_ok());
assert!(matches!(
registry.start_prompt_run(
"session",
"prompt".into(),
CancellationToken::new(),
Arc::new(Agent::new()),
),
Err(StartRunError::AgentRunExists { .. })
));
registry.finish_live("session");
assert_eq!(
registry.agent_run("session").map(|(run_id, _)| run_id),
Some("delegated".into())
);
}
}

View file

@ -3,8 +3,12 @@
//! This module provides centralized agent lifecycle management with session isolation,
//! enabling multiple concurrent sessions with independent agents, extensions, and providers.
mod active_run;
pub mod manager;
pub use active_run::ActiveRunRegistry;
pub(crate) use active_run::StartRunError;
use serde::{Deserialize, Serialize};
use std::fmt;

View file

@ -25,6 +25,7 @@ pub mod goose_apps;
pub mod hints;
pub mod hooks;
pub mod instance_id;
mod live_voice;
pub mod logging;
pub mod mcp_utils;
pub mod model_config;

View file

@ -0,0 +1,474 @@
use super::service::{
LiveVoiceInteractionCompletion, LiveVoiceInteractionGuard, LiveVoiceTranscriptPublisher,
};
use super::transcript::{DelegationContext, LiveTranscript};
use crate::{conversation::message::Message, session::SessionManager, token_counter::TokenCounter};
use futures::future::BoxFuture;
use goose_providers::live_voice_provider::{
DelegationUpdate, DelegationUpdateDelivery, ProviderConnection, ProviderConnectionEvent,
};
use rmcp::model::Role;
use std::{collections::HashSet, sync::Arc, time::Duration};
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
pub(super) const PROVIDER_CLEANUP_TIMEOUT: Duration = Duration::from_secs(20);
pub(super) const DELEGATION_INSTRUCTION: &str =
"Based on this conversation, identify and complete the user's request.";
const DELEGATION_UPDATE_TOKEN_LIMIT: usize = 500;
const UNDELIVERED_DELEGATION_UPDATE_NOTICE: &str =
"I couldn't confirm that the latest delegated update reached this voice conversation. Please ask me to share it again.";
const SAVED_RESULT_NOTICE: &str = "\n\nThe full result is saved in Goose.";
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct LiveVoiceInteractionId(pub(crate) String);
impl LiveVoiceInteractionId {
pub(super) fn new() -> Self {
Self(format!("live_{}", Uuid::now_v7()))
}
}
pub(super) struct LiveVoiceInteraction {
provider_connection: Box<dyn ProviderConnection>,
provider_event_ids: HashSet<String>,
provider_delegation_ids: HashSet<String>,
transcript: LiveTranscript,
delegated_main_agent_run: Option<DelegatedMainAgentRun>,
session_manager: Arc<SessionManager>,
transcript_publisher: LiveVoiceTranscriptPublisher,
main_agent: LiveMainAgent,
interaction_guard: LiveVoiceInteractionGuard,
}
struct DelegatedMainAgentRun {
// GPT-Live expects the final result on the delegation that started this run.
provider_delegation_id: String,
result_future: BoxFuture<'static, String>,
}
enum LiveVoiceInteractionEvent {
StopRequested,
MainAgentFinished(String),
Provider(ProviderConnectionEvent),
}
enum DelegationDecision {
Ignore,
Reject(String),
Accept(String),
}
impl LiveVoiceInteraction {
pub(super) fn new(
provider_connection: Box<dyn ProviderConnection>,
session_manager: Arc<SessionManager>,
transcript_publisher: LiveVoiceTranscriptPublisher,
main_agent: LiveMainAgent,
interaction_guard: LiveVoiceInteractionGuard,
) -> Self {
Self {
provider_connection,
provider_event_ids: HashSet::new(),
provider_delegation_ids: HashSet::new(),
transcript: LiveTranscript::default(),
delegated_main_agent_run: None,
session_manager,
transcript_publisher,
main_agent,
interaction_guard,
}
}
pub(super) async fn run(mut self) {
let stop_requested = self.interaction_guard.stop_requested().clone();
let mut stopping = false;
let completion = loop {
match self.next_event(&stop_requested, stopping).await {
LiveVoiceInteractionEvent::StopRequested => {
match timeout(PROVIDER_CLEANUP_TIMEOUT, self.cleanup_provider()).await {
Ok(Ok(())) => stopping = true,
_ => break LiveVoiceInteractionCompletion::Failed,
}
}
LiveVoiceInteractionEvent::MainAgentFinished(result) => {
if self.handle_main_agent_finished(result).await.is_err() {
self.stop_provider_after_error(stopping).await;
break LiveVoiceInteractionCompletion::Failed;
}
}
LiveVoiceInteractionEvent::Provider(ProviderConnectionEvent::TranscriptDelta {
event_id,
role,
text,
end_ms,
..
}) => {
if self
.receive_transcript(event_id, role, text, end_ms)
.await
.is_err()
{
self.stop_provider_after_error(stopping).await;
break LiveVoiceInteractionCompletion::Failed;
}
}
LiveVoiceInteractionEvent::Provider(
ProviderConnectionEvent::DelegationRequested {
event_id,
delegation_id,
offset_ms,
},
) => {
if stopping {
continue;
}
if self
.receive_delegation(event_id, delegation_id, offset_ms)
.await
.is_err()
{
self.stop_provider_after_error(stopping).await;
break LiveVoiceInteractionCompletion::Failed;
}
}
LiveVoiceInteractionEvent::Provider(ProviderConnectionEvent::Closed) => {
break if stopping {
LiveVoiceInteractionCompletion::Stopped
} else {
LiveVoiceInteractionCompletion::Failed
};
}
LiveVoiceInteractionEvent::Provider(
ProviderConnectionEvent::ReceiverLagged | ProviderConnectionEvent::Failed,
) => {
self.stop_provider_after_error(stopping).await;
break LiveVoiceInteractionCompletion::Failed;
}
}
};
self.finish_interaction(completion).await;
}
async fn next_event(
&mut self,
stop_requested: &CancellationToken,
stopping: bool,
) -> LiveVoiceInteractionEvent {
if stopping {
return LiveVoiceInteractionEvent::Provider(
self.provider_connection.next_event().await,
);
}
let delegated_run = &mut self.delegated_main_agent_run;
let provider_connection = &mut self.provider_connection;
tokio::select! {
biased;
_ = stop_requested.cancelled() => LiveVoiceInteractionEvent::StopRequested,
result = async {
delegated_run
.as_mut()
.expect("main agent is running")
.result_future
.as_mut()
.await
}, if delegated_run.is_some() => LiveVoiceInteractionEvent::MainAgentFinished(result),
event = provider_connection.next_event() => LiveVoiceInteractionEvent::Provider(event),
}
}
async fn receive_transcript(
&mut self,
event_id: String,
role: Role,
text: String,
end_ms: u64,
) -> anyhow::Result<()> {
let Some(transcript_delta_to_display) =
self.record_transcript(event_id, role, &text, end_ms)
else {
return Ok(());
};
(self.transcript_publisher)(transcript_delta_to_display);
save_raw_transcript_entries(
&self.session_manager,
self.interaction_guard.session_id(),
&mut self.transcript,
)
.await?;
Ok(())
}
fn record_transcript(
&mut self,
event_id: String,
role: Role,
delta_text: &str,
end_ms: u64,
) -> Option<Message> {
if !self.provider_event_ids.insert(event_id) {
return None;
}
self.transcript.append(role, delta_text, end_ms)
}
async fn receive_delegation(
&mut self,
event_id: String,
delegation_id: String,
offset_ms: u64,
) -> anyhow::Result<()> {
match self.handle_delegation_request(event_id, delegation_id.clone(), offset_ms) {
DelegationDecision::Ignore => {}
DelegationDecision::Reject(text) => {
self.send_delegation_update(delegation_id, bound_delegation_update(text).await)
.await?;
}
DelegationDecision::Accept(context) => {
let session_id = self.interaction_guard.session_id().to_string();
if self.delegated_main_agent_run.is_some() {
let input = format!("{context}\n{DELEGATION_INSTRUCTION}");
let response = match self.main_agent.steer(session_id, input).await {
Ok(response) => {
self.transcript
.mark_context_sent_to_main_agent_through(offset_ms);
response
}
Err(response) => response,
};
self.send_delegation_update(
delegation_id,
bound_delegation_update(response).await,
)
.await?;
} else {
self.transcript.finish_transcript_entry_being_built();
save_raw_transcript_entries(
&self.session_manager,
&session_id,
&mut self.transcript,
)
.await?;
let input = format!("{context}\n{DELEGATION_INSTRUCTION}");
match self.main_agent.start(session_id, input) {
Ok(completion) => {
self.delegated_main_agent_run = Some(DelegatedMainAgentRun {
provider_delegation_id: delegation_id,
result_future: completion,
});
self.transcript
.mark_context_sent_to_main_agent_through(offset_ms);
}
Err(response) => {
self.send_delegation_update(
delegation_id,
bound_delegation_update(response).await,
)
.await?;
}
}
}
}
}
Ok(())
}
fn handle_delegation_request(
&mut self,
event_id: String,
delegation_id: String,
offset_ms: u64,
) -> DelegationDecision {
if !self.provider_event_ids.insert(event_id)
|| !self.provider_delegation_ids.insert(delegation_id)
{
return DelegationDecision::Ignore;
}
match self.transcript.context_for_delegation(offset_ms) {
DelegationContext::Stale => {
DelegationDecision::Reject("The delegated conversation position is stale.".into())
}
DelegationContext::MissingUserInput => {
DelegationDecision::Reject("I couldn't identify a request to complete.".into())
}
DelegationContext::Available(context) => DelegationDecision::Accept(context),
}
}
async fn handle_main_agent_finished(&mut self, result: String) -> anyhow::Result<()> {
let run = self
.delegated_main_agent_run
.take()
.expect("main agent is running");
self.transcript.finish_transcript_entry_being_built();
save_raw_transcript_entries(
&self.session_manager,
self.interaction_guard.session_id(),
&mut self.transcript,
)
.await?;
self.send_delegation_update(
run.provider_delegation_id,
bound_delegation_update(result).await,
)
.await
}
async fn finish_interaction(mut self, mut completion: LiveVoiceInteractionCompletion) {
match self.delegated_main_agent_run.take() {
None => {
if save_transcript_and_context_waiting_for_main_agent(
&self.session_manager,
self.interaction_guard.session_id(),
&mut self.transcript,
)
.await
.is_err()
{
completion = LiveVoiceInteractionCompletion::Failed;
}
self.interaction_guard
.publish_completion_after_cleanup(completion);
}
Some(run) => {
self.interaction_guard.publish_completion(completion);
let _ = run.result_future.await;
let _ = save_transcript_and_context_waiting_for_main_agent(
&self.session_manager,
self.interaction_guard.session_id(),
&mut self.transcript,
)
.await;
}
}
}
async fn stop_provider_after_error(&mut self, stopping: bool) {
if !stopping {
let _ = timeout(PROVIDER_CLEANUP_TIMEOUT, self.cleanup_provider()).await;
}
}
async fn send_delegation_update(
&mut self,
provider_delegation_id: String,
text: String,
) -> anyhow::Result<()> {
let delivery = self
.provider_connection
.send_delegation_update(DelegationUpdate {
provider_delegation_id,
text,
})
.await?;
match delivery {
DelegationUpdateDelivery::Delivered => {}
DelegationUpdateDelivery::Undelivered => (self.transcript_publisher)(
Message::assistant()
.with_id(format!("msg_live_{}", Uuid::now_v7()))
.with_text(UNDELIVERED_DELEGATION_UPDATE_NOTICE)
.user_only(),
),
}
Ok(())
}
async fn cleanup_provider(&mut self) -> anyhow::Result<()> {
self.provider_connection.stop().await
}
}
pub(crate) struct LiveMainAgent {
start: Box<dyn Fn(String, String) -> Result<MainAgentRun, String> + Send + Sync>,
steer: Box<dyn Fn(String, String) -> MainAgentSteerResponse + Send + Sync>,
}
type MainAgentRun = BoxFuture<'static, String>;
type MainAgentSteerResponse = BoxFuture<'static, Result<String, String>>;
impl LiveMainAgent {
pub(crate) fn new(
start: impl Fn(String, String) -> Result<MainAgentRun, String> + Send + Sync + 'static,
steer: impl Fn(String, String) -> MainAgentSteerResponse + Send + Sync + 'static,
) -> Self {
Self {
start: Box::new(start),
steer: Box::new(steer),
}
}
fn start(&self, session_id: String, input: String) -> Result<MainAgentRun, String> {
(self.start)(session_id, input)
}
fn steer(&self, session_id: String, input: String) -> MainAgentSteerResponse {
(self.steer)(session_id, input)
}
}
async fn save_raw_transcript_entries(
session_manager: &SessionManager,
session_id: &str,
transcript: &mut LiveTranscript,
) -> anyhow::Result<()> {
for message in transcript.raw_transcript_entries_waiting_to_save() {
session_manager.add_message(session_id, message).await?;
}
transcript.mark_raw_transcript_entries_saved();
Ok(())
}
async fn save_transcript_and_context_waiting_for_main_agent(
session_manager: &SessionManager,
session_id: &str,
transcript: &mut LiveTranscript,
) -> anyhow::Result<()> {
transcript.finish_transcript_entry_being_built();
save_raw_transcript_entries(session_manager, session_id, transcript).await?;
if let Some(context) = transcript.agent_only_context_message_waiting_to_save() {
session_manager.add_message(session_id, &context).await?;
transcript.mark_context_saved_for_main_agent();
}
Ok(())
}
async fn bound_delegation_update(text: String) -> String {
let Ok(counter) = TokenCounter::new().await else {
return text;
};
if counter.count_tokens(&text) <= DELEGATION_UPDATE_TOKEN_LIMIT {
return text;
}
let allowed = DELEGATION_UPDATE_TOKEN_LIMIT - counter.count_tokens(SAVED_RESULT_NOTICE);
let boundaries = text
.char_indices()
.map(|(index, _)| index)
.collect::<Vec<_>>();
let mut low = 0;
let mut high = boundaries.len();
while low < high {
let middle = (low + high).div_ceil(2);
let end = boundaries.get(middle).copied().unwrap_or(text.len());
let candidate = text
.get(..end)
.expect("delegation update boundary comes from char_indices");
if counter.count_tokens(candidate) <= allowed {
low = middle;
} else {
high = middle - 1;
}
}
let end = boundaries.get(low).copied().unwrap_or(text.len());
let truncated = text
.get(..end)
.expect("delegation update boundary comes from char_indices");
format!("{}{}", truncated.trim_end(), SAVED_RESULT_NOTICE)
}
#[cfg(test)]
mod tests;

View file

@ -0,0 +1,249 @@
use super::*;
use async_trait::async_trait;
use tokio::sync::oneshot;
struct TestConnection {
stopped: Option<oneshot::Sender<()>>,
}
struct UndeliveredConnection;
fn test_interaction(
provider_connection: impl ProviderConnection + 'static,
) -> LiveVoiceInteraction {
test_interaction_with_publisher(provider_connection, Arc::new(|_| {}))
}
fn test_interaction_with_publisher(
provider_connection: impl ProviderConnection + 'static,
transcript_publisher: LiveVoiceTranscriptPublisher,
) -> LiveVoiceInteraction {
LiveVoiceInteraction::new(
Box::new(provider_connection),
Arc::new(SessionManager::new(tempfile::tempdir().unwrap().keep())),
transcript_publisher,
LiveMainAgent::new(
|_, _| Err("unused".into()),
|_, _| Box::pin(async { Err("unused".into()) }),
),
LiveVoiceInteractionGuard::for_test("test-session"),
)
}
#[async_trait]
impl ProviderConnection for TestConnection {
async fn next_event(&mut self) -> ProviderConnectionEvent {
std::future::pending().await
}
async fn send_delegation_update(
&mut self,
_update: DelegationUpdate,
) -> anyhow::Result<DelegationUpdateDelivery> {
Ok(DelegationUpdateDelivery::Delivered)
}
async fn stop(&mut self) -> anyhow::Result<()> {
if let Some(stopped) = self.stopped.take() {
let _ = stopped.send(());
}
Ok(())
}
}
#[async_trait]
impl ProviderConnection for UndeliveredConnection {
async fn next_event(&mut self) -> ProviderConnectionEvent {
std::future::pending().await
}
async fn send_delegation_update(
&mut self,
_update: DelegationUpdate,
) -> anyhow::Result<DelegationUpdateDelivery> {
Ok(DelegationUpdateDelivery::Undelivered)
}
async fn stop(&mut self) -> anyhow::Result<()> {
Ok(())
}
}
#[tokio::test]
async fn a_live_voice_interaction_owns_and_stops_its_provider_connection() {
let (stopped, did_stop) = oneshot::channel();
let mut interaction = test_interaction(TestConnection {
stopped: Some(stopped),
});
interaction.cleanup_provider().await.unwrap();
did_stop.await.unwrap();
}
#[tokio::test]
async fn an_undelivered_delegation_update_notifies_the_user() {
let (notice_tx, mut notice_rx) = tokio::sync::mpsc::unbounded_channel();
let transcript_publisher: LiveVoiceTranscriptPublisher = Arc::new(move |message| {
notice_tx.send(message).unwrap();
});
let mut interaction =
test_interaction_with_publisher(UndeliveredConnection, transcript_publisher);
interaction
.send_delegation_update("delegation-1".into(), "result".into())
.await
.unwrap();
let notice = notice_rx.recv().await.unwrap();
assert_eq!(notice.role, Role::Assistant);
assert_eq!(
notice.as_concat_text(),
UNDELIVERED_DELEGATION_UPDATE_NOTICE
);
}
#[tokio::test]
async fn delegation_preparation_respects_offset_and_suppresses_duplicates() {
let mut interaction = test_interaction(TestConnection { stopped: None });
interaction.record_transcript("1".into(), Role::Assistant, "ready", 5);
interaction.record_transcript("2".into(), Role::User, "do ", 10);
interaction.record_transcript("3".into(), Role::User, "this", 20);
interaction.record_transcript("4".into(), Role::Assistant, "crossing", 25);
interaction.record_transcript("5".into(), Role::Assistant, "later", 40);
let DelegationDecision::Accept(input) =
interaction.handle_delegation_request("event-1".into(), "delegation-1".into(), 20)
else {
panic!("delegation should be accepted");
};
assert_eq!(
input,
"Live conversation context:\nVoice assistant: ready\nUser: do this"
);
interaction
.transcript
.mark_context_sent_to_main_agent_through(20);
assert!(matches!(
interaction.handle_delegation_request("event-1".into(), "delegation-1".into(), 20),
DelegationDecision::Ignore
));
interaction
.record_transcript("6".into(), Role::User, " late detail", 20)
.unwrap();
assert_eq!(
interaction
.transcript
.raw_transcript_entries_waiting_to_save()
.last()
.unwrap()
.as_concat_text(),
"crossinglater"
);
let DelegationDecision::Accept(continuation) =
interaction.handle_delegation_request("event-2".into(), "delegation-2".into(), 20)
else {
panic!("continuation should be accepted");
};
assert_eq!(
continuation,
"Live conversation context:\nUser: late detail"
);
interaction
.transcript
.mark_context_sent_to_main_agent_through(20);
assert!(matches!(
interaction.handle_delegation_request("event-3".into(), "delegation-3".into(), 19),
DelegationDecision::Reject(_)
));
interaction.record_transcript("7".into(), Role::User, "again", 50);
let DelegationDecision::Accept(next) =
interaction.handle_delegation_request("event-4".into(), "delegation-4".into(), 50)
else {
panic!("later continuation should be accepted");
};
assert_eq!(
next,
"Live conversation context:\nVoice assistant: crossinglater\nUser: again"
);
interaction
.transcript
.mark_context_sent_to_main_agent_through(50);
let mut missing_user = test_interaction(TestConnection { stopped: None });
missing_user.record_transcript("1".into(), Role::Assistant, "hello", 10);
assert!(matches!(
missing_user.handle_delegation_request("event-1".into(), "delegation-1".into(), 10),
DelegationDecision::Reject(_)
));
}
#[tokio::test]
async fn context_waiting_for_main_agent_excludes_instruction_and_clears_after_handoff() {
let mut interaction = test_interaction(TestConnection { stopped: None });
interaction.record_transcript("1".into(), Role::Assistant, "Anything else?", 10);
interaction.record_transcript("2".into(), Role::User, "No thanks", 20);
let context = interaction
.transcript
.context_waiting_for_main_agent()
.unwrap();
assert_eq!(
context,
"Live conversation context:\nVoice assistant: Anything else?\nUser: No thanks"
);
assert_eq!(
interaction
.transcript
.context_waiting_for_main_agent()
.unwrap(),
context
);
interaction.record_transcript("3".into(), Role::User, "Already shared", 30);
interaction
.transcript
.mark_context_sent_to_main_agent_through(30);
assert!(interaction
.transcript
.context_waiting_for_main_agent()
.is_none());
}
#[tokio::test]
async fn transcript_grouping_projects_deltas_and_finalizes_messages() {
let mut interaction = test_interaction(TestConnection { stopped: None });
let first = interaction
.record_transcript("1".into(), Role::User, "hello", 10)
.unwrap();
assert!(first.is_user_visible());
assert!(!first.is_agent_visible());
let message_id = first.id.clone();
let second = interaction
.record_transcript("2".into(), Role::User, " world", 20)
.unwrap();
assert_eq!(second.id, message_id);
assert_eq!(second.as_concat_text(), " world");
assert!(interaction
.record_transcript("2".into(), Role::User, " world", 20)
.is_none());
let role_change = interaction
.record_transcript("3".into(), Role::Assistant, "hello", 30)
.unwrap();
assert_eq!(
interaction
.transcript
.raw_transcript_entries_waiting_to_save()
.last()
.unwrap()
.as_concat_text(),
"hello world"
);
assert_eq!(role_change.role, Role::Assistant);
assert!(role_change.is_user_visible());
assert!(!role_change.is_agent_visible());
}

View file

@ -0,0 +1,10 @@
mod interaction;
mod service;
mod transcript;
pub(crate) use interaction::{LiveMainAgent, LiveVoiceInteractionId};
pub use service::LiveVoiceService;
pub(crate) use service::{
wait_for_completion, LiveVoiceError, LiveVoiceInteractionCompletion,
LiveVoiceTranscriptPublisher, StartLiveVoiceInteractionResult, WebRtcOffer,
};

View file

@ -0,0 +1,463 @@
use super::interaction::{LiveMainAgent, LiveVoiceInteraction, LiveVoiceInteractionId};
use crate::config::GooseMode;
use crate::conversation::message::{Message, MessageContent};
use crate::conversation::Conversation;
use crate::execution::ActiveRunRegistry;
use crate::session::SessionManager;
use crate::token_counter::TokenCounter;
use goose_providers::live_voice_provider::{LiveVoiceInputMessage, LiveVoiceProvider};
pub(crate) use goose_providers::live_voice_provider::{WebRtcAnswer, WebRtcOffer};
#[cfg(feature = "live-voice")]
use goose_providers::openai_live_voice_provider::OpenAiLiveVoiceProvider;
use std::{
collections::HashMap,
sync::{Arc, Mutex},
};
use tokio::sync::watch;
use tokio_util::sync::CancellationToken;
const LIVE_VOICE_INPUT_MESSAGE_LIMIT: usize = 128;
const LIVE_VOICE_INPUT_TOKEN_LIMIT: usize = 8_192;
#[cfg(feature = "live-voice")]
const LIVE_VOICE_ENABLED_CONFIG_KEY: &str = "GOOSE_LIVE_VOICE_ENABLED";
#[cfg(feature = "live-voice")]
const LIVE_VOICE_CONFIG_KEY: &str = "GOOSE_LIVE_VOICE";
#[cfg(feature = "live-voice")]
const DEFAULT_OPENAI_LIVE_VOICE: &str = "marin";
#[cfg(feature = "live-voice")]
const LIVE_SESSION_INSTRUCTIONS: &str = concat!(
"You are Goose's live voice interface. Keep the conversation natural and concise.\n",
"Interruption policy: Stop speaking when the user interrupts and listen to what they say.\n",
"Delegation policy:\n",
"Backend tools:\n",
"- Goose can use backend reasoning and tools for longer tasks.\n",
"Delegate to Goose when:\n",
"- The user has finished stating a complete request that needs backend tools or reasoning.\n",
"- The user corrects or changes backend work already in progress.\n",
"Do not delegate to Goose when:\n",
"- The request is unfinished or is missing a required detail such as a location, object, ",
"command, or desired outcome. Ask one brief clarification and wait for the answer.\n",
"- The user is greeting you or making conversation that you can answer directly.\n",
"After delegating, briefly say the work is underway. Keep listening and accept corrections ",
"while Goose works. Do not guess the result. Present delegated results directly. Only say ",
"the task stopped or finished after Goose confirms it."
);
#[cfg(feature = "live-voice")]
const DELEGATION_DELIVERY_FAILURE_INSTRUCTIONS: &str = concat!(
"The latest update for the delegated request could not be delivered. Tell the user you ",
"couldn't bring the update into this voice conversation and ask them to try again. Do not ",
"say whether the delegated work succeeded or failed."
);
type LiveVoiceInteractionControls = Arc<Mutex<HashMap<String, Arc<LiveVoiceInteractionControl>>>>;
pub(crate) type LiveVoiceTranscriptPublisher = Arc<dyn Fn(Message) + Send + Sync>;
type LiveVoiceResolver =
Arc<dyn Fn() -> Result<Arc<dyn LiveVoiceProvider>, &'static str> + Send + Sync>;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum LiveVoiceInteractionCompletion {
Stopped,
Failed,
}
pub(crate) struct StartLiveVoiceInteractionResult {
pub(crate) interaction_id: LiveVoiceInteractionId,
pub(crate) answer: WebRtcAnswer,
pub(crate) completion_rx: watch::Receiver<Option<LiveVoiceInteractionCompletion>>,
}
#[derive(Debug)]
pub(crate) enum LiveVoiceError {
Unavailable,
StartFailed,
StopFailed,
}
struct LiveVoiceInteractionControl {
interaction_id: LiveVoiceInteractionId,
stop_requested: CancellationToken,
cleanup_finished: CancellationToken,
completion_tx: watch::Sender<Option<LiveVoiceInteractionCompletion>>,
}
impl LiveVoiceInteractionControl {
fn request_stop(&self) -> watch::Receiver<Option<LiveVoiceInteractionCompletion>> {
self.stop_requested.cancel();
self.completion_tx.subscribe()
}
fn request_cleanup(&self) -> CancellationToken {
self.stop_requested.cancel();
self.cleanup_finished.clone()
}
fn publish_completion(&self, completion: LiveVoiceInteractionCompletion) {
if self.completion_tx.borrow().is_none() {
self.completion_tx.send_replace(Some(completion));
}
}
}
/// Owns the interaction reservation from before agent preparation until cleanup finishes.
pub(crate) struct LiveVoiceInteractionGuard {
active_runs: Arc<ActiveRunRegistry>,
interactions_by_session: LiveVoiceInteractionControls,
session_id: String,
control: Arc<LiveVoiceInteractionControl>,
completion_on_drop: Option<LiveVoiceInteractionCompletion>,
}
impl LiveVoiceInteractionGuard {
fn new(
active_runs: Arc<ActiveRunRegistry>,
interactions_by_session: LiveVoiceInteractionControls,
session_id: &str,
control: Arc<LiveVoiceInteractionControl>,
) -> Self {
Self {
active_runs,
interactions_by_session,
session_id: session_id.to_string(),
control,
completion_on_drop: None,
}
}
pub(super) fn publish_completion(&self, completion: LiveVoiceInteractionCompletion) {
self.control.publish_completion(completion);
}
pub(super) fn publish_completion_after_cleanup(
&mut self,
completion: LiveVoiceInteractionCompletion,
) {
self.completion_on_drop = Some(completion);
}
pub(super) fn stop_requested(&self) -> &CancellationToken {
&self.control.stop_requested
}
pub(super) fn session_id(&self) -> &str {
&self.session_id
}
#[cfg(test)]
pub(super) fn for_test(session_id: &str) -> Self {
let (completion_tx, _) = watch::channel(None);
Self::new(
Arc::new(ActiveRunRegistry::default()),
Arc::new(Mutex::new(HashMap::new())),
session_id,
Arc::new(LiveVoiceInteractionControl {
interaction_id: LiveVoiceInteractionId::new(),
stop_requested: CancellationToken::new(),
cleanup_finished: CancellationToken::new(),
completion_tx,
}),
)
}
}
impl Drop for LiveVoiceInteractionGuard {
fn drop(&mut self) {
remove_interaction_if_current(
&self.interactions_by_session,
&self.session_id,
&self.control.interaction_id,
);
self.active_runs.finish_live(&self.session_id);
self.publish_completion(
self.completion_on_drop
.unwrap_or(LiveVoiceInteractionCompletion::Failed),
);
self.control.cleanup_finished.cancel();
}
}
pub struct LiveVoiceService {
live_voice_resolver: LiveVoiceResolver,
interactions_by_session: LiveVoiceInteractionControls,
active_runs: Arc<ActiveRunRegistry>,
}
impl LiveVoiceService {
pub fn from_config(active_runs: Arc<ActiveRunRegistry>) -> Self {
Self::new(Arc::new(configured_live_voice), active_runs)
}
fn new(live_voice_resolver: LiveVoiceResolver, active_runs: Arc<ActiveRunRegistry>) -> Self {
Self {
live_voice_resolver,
interactions_by_session: Arc::new(Mutex::new(HashMap::new())),
active_runs,
}
}
pub(crate) fn availability(
&self,
session_id: Option<&str>,
mode: GooseMode,
) -> Result<(), &'static str> {
self.eligible_provider(session_id, mode).map(|_| ())
}
fn eligible_provider(
&self,
session_id: Option<&str>,
mode: GooseMode,
) -> Result<Arc<dyn LiveVoiceProvider>, &'static str> {
let provider = self.provider_for_mode(mode)?;
if session_id.is_some_and(|session_id| self.active_runs.is_active(session_id)) {
Err("Live voice is unavailable while this session is busy")
} else {
Ok(provider)
}
}
fn provider_for_mode(
&self,
mode: GooseMode,
) -> Result<Arc<dyn LiveVoiceProvider>, &'static str> {
let provider = (self.live_voice_resolver)()?;
if mode != GooseMode::Auto {
Err("Live voice requires Autonomous mode")
} else {
Ok(provider)
}
}
pub(crate) fn reserve_interaction(
&self,
session_id: &str,
mode: GooseMode,
) -> Result<LiveVoiceInteractionGuard, LiveVoiceError> {
self.eligible_provider(Some(session_id), mode)
.map_err(|_| LiveVoiceError::Unavailable)?;
let (completion_tx, _) = watch::channel(None);
let control = Arc::new(LiveVoiceInteractionControl {
interaction_id: LiveVoiceInteractionId::new(),
stop_requested: CancellationToken::new(),
cleanup_finished: CancellationToken::new(),
completion_tx,
});
{
let mut interactions = self
.interactions_by_session
.lock()
.expect("live voice lock poisoned");
if !self.active_runs.start_live(session_id) {
return Err(LiveVoiceError::Unavailable);
}
interactions.insert(session_id.to_string(), control.clone());
}
Ok(LiveVoiceInteractionGuard::new(
self.active_runs.clone(),
self.interactions_by_session.clone(),
session_id,
control,
))
}
pub(crate) async fn start_interaction(
&self,
interaction_guard: LiveVoiceInteractionGuard,
offer: WebRtcOffer,
session_manager: Arc<SessionManager>,
transcript_publisher: LiveVoiceTranscriptPublisher,
main_agent: LiveMainAgent,
) -> Result<StartLiveVoiceInteractionResult, LiveVoiceError> {
let session = session_manager
.get_session(&interaction_guard.session_id, true)
.await
.map_err(|_| LiveVoiceError::Unavailable)?;
let provider = self
.provider_for_mode(session.goose_mode)
.map_err(|_| LiveVoiceError::Unavailable)?;
let input_messages =
live_voice_input_messages(&session.conversation.unwrap_or_default()).await?;
if interaction_guard.stop_requested().is_cancelled() {
return Err(LiveVoiceError::Unavailable);
}
let (answer, provider_connection) = provider
.start(offer, input_messages)
.await
.map_err(|_| LiveVoiceError::StartFailed)?;
let interaction_id = interaction_guard.control.interaction_id.clone();
let completion_rx = interaction_guard.control.completion_tx.subscribe();
let interaction = LiveVoiceInteraction::new(
provider_connection,
session_manager,
transcript_publisher,
main_agent,
interaction_guard,
);
tokio::spawn(interaction.run());
Ok(StartLiveVoiceInteractionResult {
interaction_id,
answer,
completion_rx,
})
}
pub(crate) async fn stop_interaction(
&self,
session_id: &str,
interaction_id: &LiveVoiceInteractionId,
) -> Result<(), LiveVoiceError> {
let completion_rx = {
let interactions = self
.interactions_by_session
.lock()
.expect("live voice lock poisoned");
let Some(control) = interactions.get(session_id) else {
return Ok(());
};
if &control.interaction_id != interaction_id {
return Ok(());
}
control.request_stop()
};
match wait_for_completion(completion_rx).await? {
LiveVoiceInteractionCompletion::Stopped => Ok(()),
LiveVoiceInteractionCompletion::Failed => Err(LiveVoiceError::StopFailed),
}
}
pub(crate) async fn stop_session_interaction(&self, session_id: &str) {
let cleanup_finished = {
let interactions = self
.interactions_by_session
.lock()
.expect("live voice lock poisoned");
interactions
.get(session_id)
.map(|control| control.request_cleanup())
};
if let Some(cleanup_finished) = cleanup_finished {
cleanup_finished.cancelled().await;
}
}
}
#[cfg(feature = "live-voice")]
fn configured_live_voice_enabled() -> bool {
crate::config::Config::global()
.get_param::<serde_json::Value>(LIVE_VOICE_ENABLED_CONFIG_KEY)
.is_ok_and(|value| match value {
serde_json::Value::Bool(enabled) => enabled,
serde_json::Value::Number(enabled) => enabled.as_u64() == Some(1),
serde_json::Value::String(enabled) => {
enabled == "1" || enabled.eq_ignore_ascii_case("true")
}
_ => false,
})
}
#[cfg(feature = "live-voice")]
fn configured_live_voice() -> Result<Arc<dyn LiveVoiceProvider>, &'static str> {
if !configured_live_voice_enabled() {
return Err("Live voice is disabled");
}
let config = crate::config::Config::global();
let api_key = config
.get_secret::<String>("OPENAI_API_KEY")
.map_err(|_| "Live voice provider is not configured")?;
let voice = config
.get_param::<String>(LIVE_VOICE_CONFIG_KEY)
.unwrap_or_else(|_| DEFAULT_OPENAI_LIVE_VOICE.into());
OpenAiLiveVoiceProvider::new(
api_key,
voice,
LIVE_SESSION_INSTRUCTIONS.into(),
DELEGATION_DELIVERY_FAILURE_INSTRUCTIONS.into(),
)
.map(|provider| Arc::new(provider) as Arc<dyn LiveVoiceProvider>)
.map_err(|_| "Live voice provider is not configured")
}
#[cfg(not(feature = "live-voice"))]
fn configured_live_voice() -> Result<Arc<dyn LiveVoiceProvider>, &'static str> {
Err("Live voice is disabled")
}
async fn live_voice_input_messages(
conversation: &Conversation,
) -> Result<Vec<LiveVoiceInputMessage>, LiveVoiceError> {
let messages = conversation
.messages()
.iter()
.filter(|message| message.is_user_visible())
.filter_map(|message| {
let text = message
.user_visible_content()
.content
.iter()
.filter_map(MessageContent::as_text)
.collect::<String>();
if text.trim().is_empty() {
return None;
}
Some(LiveVoiceInputMessage {
role: message.role.clone(),
text,
})
})
.collect::<Vec<_>>();
if messages.is_empty() {
return Ok(messages);
}
let token_counter = TokenCounter::new()
.await
.map_err(|_| LiveVoiceError::StartFailed)?;
let mut start = messages.len();
let mut tokens = 0;
for (index, message) in messages.iter().enumerate().rev() {
if messages.len() - start == LIVE_VOICE_INPUT_MESSAGE_LIMIT {
break;
}
let message_tokens = token_counter.count_tokens(&message.text);
if message_tokens > LIVE_VOICE_INPUT_TOKEN_LIMIT - tokens {
break;
}
tokens += message_tokens;
start = index;
}
Ok(messages.into_iter().skip(start).collect())
}
pub(crate) async fn wait_for_completion(
mut completion_rx: watch::Receiver<Option<LiveVoiceInteractionCompletion>>,
) -> Result<LiveVoiceInteractionCompletion, LiveVoiceError> {
loop {
if let Some(completion) = *completion_rx.borrow() {
return Ok(completion);
}
completion_rx
.changed()
.await
.map_err(|_| LiveVoiceError::StopFailed)?;
}
}
fn remove_interaction_if_current(
interactions_by_session: &LiveVoiceInteractionControls,
session_id: &str,
interaction_id: &LiveVoiceInteractionId,
) {
let mut interactions = interactions_by_session
.lock()
.expect("live voice lock poisoned");
if matches!(
interactions.get(session_id),
Some(control) if &control.interaction_id == interaction_id
) {
interactions.remove(session_id);
}
}
#[cfg(test)]
mod tests;

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,135 @@
use anyhow::{anyhow, Error, Result};
use async_trait::async_trait;
use goose_providers::live_voice_provider::{
DelegationUpdate, DelegationUpdateDelivery, LiveVoiceInputMessage, LiveVoiceProvider,
ProviderConnection, ProviderConnectionEvent, WebRtcAnswer, WebRtcOffer,
};
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
pub(super) fn provider_channel() -> (
Arc<FakeLiveVoiceProvider>,
mpsc::UnboundedReceiver<FakeStartRequest>,
) {
let (start_tx, start_rx) = mpsc::unbounded_channel();
(Arc::new(FakeLiveVoiceProvider { start_tx }), start_rx)
}
pub(super) struct FakeLiveVoiceProvider {
start_tx: mpsc::UnboundedSender<FakeStartRequest>,
}
#[async_trait]
impl LiveVoiceProvider for FakeLiveVoiceProvider {
async fn start(
&self,
_offer: WebRtcOffer,
input_messages: Vec<LiveVoiceInputMessage>,
) -> Result<(WebRtcAnswer, Box<dyn ProviderConnection>)> {
let (response_tx, response_rx) = oneshot::channel();
self.start_tx
.send(FakeStartRequest {
input_messages,
response_tx,
})
.map_err(|_| anyhow!("fake provider driver dropped"))?;
response_rx
.await
.map_err(|_| anyhow!("fake provider start response dropped"))?
}
}
pub(super) struct FakeStartRequest {
pub(super) input_messages: Vec<LiveVoiceInputMessage>,
response_tx: oneshot::Sender<Result<(WebRtcAnswer, Box<dyn ProviderConnection>)>>,
}
impl FakeStartRequest {
pub(super) fn accept(self, answer: WebRtcAnswer) -> Result<FakeConnectionDriver> {
let (stop_request_tx, stop_request_rx) = mpsc::unbounded_channel();
let (event_tx, event_rx) = mpsc::unbounded_channel();
let (delegation_update_tx, delegation_update_rx) = mpsc::unbounded_channel();
self.response_tx
.send(Ok((
answer,
Box::new(FakeProviderConnection {
stop_request_tx,
event_rx,
delegation_update_tx,
}),
)))
.map_err(|_| anyhow!("fake provider start caller dropped"))?;
Ok(FakeConnectionDriver {
stop_request_rx,
event_tx,
delegation_update_rx,
})
}
pub(super) fn reject(self, message: impl Into<String>) -> Result<()> {
self.response_tx
.send(Err(anyhow!(message.into())))
.map_err(|_| anyhow!("fake provider start caller dropped"))
}
}
pub(super) struct FakeConnectionDriver {
stop_request_rx: mpsc::UnboundedReceiver<oneshot::Sender<Result<(), String>>>,
event_tx: mpsc::UnboundedSender<ProviderConnectionEvent>,
delegation_update_rx: mpsc::UnboundedReceiver<DelegationUpdate>,
}
impl FakeConnectionDriver {
pub(super) async fn next_stop_request(
&mut self,
) -> Option<oneshot::Sender<Result<(), String>>> {
self.stop_request_rx.recv().await
}
pub(super) fn send_event(&self, event: ProviderConnectionEvent) -> Result<()> {
self.event_tx
.send(event)
.map_err(|_| anyhow!("fake provider event receiver dropped"))
}
pub(super) async fn next_delegation_update(&mut self) -> Option<DelegationUpdate> {
self.delegation_update_rx.recv().await
}
}
struct FakeProviderConnection {
stop_request_tx: mpsc::UnboundedSender<oneshot::Sender<Result<(), String>>>,
event_rx: mpsc::UnboundedReceiver<ProviderConnectionEvent>,
delegation_update_tx: mpsc::UnboundedSender<DelegationUpdate>,
}
#[async_trait]
impl ProviderConnection for FakeProviderConnection {
async fn next_event(&mut self) -> ProviderConnectionEvent {
self.event_rx
.recv()
.await
.unwrap_or(ProviderConnectionEvent::Failed)
}
async fn send_delegation_update(
&mut self,
update: DelegationUpdate,
) -> Result<DelegationUpdateDelivery> {
self.delegation_update_tx
.send(update)
.map_err(|_| anyhow!("fake provider driver dropped"))?;
Ok(DelegationUpdateDelivery::Delivered)
}
async fn stop(&mut self) -> Result<()> {
let (response_tx, response_rx) = oneshot::channel();
self.stop_request_tx
.send(response_tx)
.map_err(|_| anyhow!("fake provider driver dropped"))?;
response_rx
.await
.map_err(|_| anyhow!("fake provider stop response dropped"))?
.map_err(Error::msg)
}
}

View file

@ -0,0 +1,171 @@
use crate::conversation::message::{Message, MessageContent};
use chrono::Utc;
use rmcp::model::Role;
use uuid::Uuid;
#[derive(Default)]
pub(super) struct LiveTranscript {
// Consecutive deltas with the same role form one saved transcript entry.
transcript_entry_being_built: Option<Message>,
raw_transcript_entries_waiting_to_save: Vec<Message>,
// Delegation offsets address individual provider deltas, not grouped messages.
context_waiting_for_main_agent: Vec<ContextDelta>,
// A delegation before this prior position would resend old context.
last_delegation_position_ms: Option<u64>,
}
pub(super) enum DelegationContext {
Stale,
MissingUserInput,
Available(String),
}
struct ContextDelta {
role: Role,
text: String,
end_ms: u64,
}
impl LiveTranscript {
pub(super) fn append(&mut self, role: Role, delta_text: &str, end_ms: u64) -> Option<Message> {
if delta_text.is_empty() {
return None;
}
self.context_waiting_for_main_agent.push(ContextDelta {
role: role.clone(),
text: delta_text.to_string(),
end_ms,
});
if self
.transcript_entry_being_built
.as_ref()
.is_some_and(|message| message.role == role)
{
let message = self
.transcript_entry_being_built
.as_mut()
.expect("message exists");
let [MessageContent::Text(content)] = message.content.as_mut_slice() else {
unreachable!("Live transcript messages contain one text block");
};
content.text.push_str(delta_text);
return Some(transcript_delta(message, delta_text));
}
if let Some(message) = self.transcript_entry_being_built.take() {
self.raw_transcript_entries_waiting_to_save.push(message);
}
let message = transcript_message(role, delta_text.to_string());
let transcript_delta_to_display = transcript_delta(&message, delta_text);
self.transcript_entry_being_built = Some(message);
Some(transcript_delta_to_display)
}
pub(super) fn finish_transcript_entry_being_built(&mut self) {
if let Some(message) = self.transcript_entry_being_built.take() {
self.raw_transcript_entries_waiting_to_save.push(message);
}
}
pub(super) fn raw_transcript_entries_waiting_to_save(&self) -> &[Message] {
&self.raw_transcript_entries_waiting_to_save
}
pub(super) fn mark_raw_transcript_entries_saved(&mut self) {
self.raw_transcript_entries_waiting_to_save.clear();
}
pub(super) fn context_for_delegation(&self, offset_ms: u64) -> DelegationContext {
if self
.last_delegation_position_ms
.is_some_and(|last_position| offset_ms < last_position)
{
return DelegationContext::Stale;
}
let deltas = self
.context_waiting_for_main_agent
.iter()
.filter(|delta| delta.end_ms <= offset_ms)
.collect::<Vec<_>>();
if !deltas
.iter()
.any(|delta| delta.role == Role::User && !delta.text.trim().is_empty())
{
return DelegationContext::MissingUserInput;
}
DelegationContext::Available(
format_context(deltas).expect("delegation context contains transcript"),
)
}
pub(super) fn mark_context_sent_to_main_agent_through(&mut self, offset_ms: u64) {
self.context_waiting_for_main_agent
.retain(|delta| delta.end_ms > offset_ms);
self.last_delegation_position_ms = Some(offset_ms);
}
pub(super) fn context_waiting_for_main_agent(&self) -> Option<String> {
format_context(&self.context_waiting_for_main_agent)
}
pub(super) fn agent_only_context_message_waiting_to_save(&self) -> Option<Message> {
self.context_waiting_for_main_agent().map(|context| {
Message::user()
.with_id(format!("msg_live_context_{}", Uuid::now_v7()))
.with_text(context)
.agent_only()
})
}
pub(super) fn mark_context_saved_for_main_agent(&mut self) {
self.context_waiting_for_main_agent.clear();
}
}
fn format_context<'a>(deltas: impl IntoIterator<Item = &'a ContextDelta>) -> Option<String> {
let mut input = String::from("Live conversation context:\n");
let mut previous_role = None;
for delta in deltas {
if previous_role.as_ref() == Some(&delta.role) {
input.push_str(&delta.text);
} else {
if previous_role.is_some() {
input.push('\n');
}
input.push_str(speaker(&delta.role));
input.push_str(": ");
input.push_str(delta.text.trim_start());
previous_role = Some(delta.role.clone());
}
}
previous_role?;
Some(input)
}
fn speaker(role: &Role) -> &'static str {
match role {
Role::User => "User",
Role::Assistant => "Voice assistant",
}
}
fn transcript_message(role: Role, text: String) -> Message {
Message::new(role, Utc::now().timestamp(), vec![])
.with_id(format!("msg_live_{}", Uuid::now_v7()))
.with_text(text)
.user_only()
}
fn transcript_delta(message: &Message, text: &str) -> Message {
Message {
id: message.id.clone(),
role: message.role.clone(),
created: message.created,
content: vec![MessageContent::text(text)],
metadata: message.metadata.clone(),
}
}

View file

@ -1,6 +1,6 @@
use crate::config::paths::Paths;
use crate::config::GooseMode;
use crate::conversation::message::{Message, MessageUsage, TokenState};
use crate::conversation::message::{Message, MessageMetadata, MessageUsage, TokenState};
use crate::conversation::Conversation;
use crate::providers::base::CostSource;
use crate::providers::base::Provider;
@ -460,6 +460,16 @@ impl SessionManager {
self.storage.replace_conversation(id, conversation).await
}
pub(crate) async fn save_compacted_conversation(
&self,
id: &str,
conversation: &Conversation,
) -> Result<()> {
self.storage
.save_compacted_conversation(id, conversation)
.await
}
pub async fn list_sessions(&self) -> Result<Vec<Session>> {
self.storage.list_sessions().await
}
@ -2000,6 +2010,62 @@ impl SessionStorage {
Self::replace_conversation_inner(pool, session_id, conversation).await
}
async fn save_compacted_conversation(
&self,
session_id: &str,
conversation: &Conversation,
) -> Result<()> {
let pool = self.pool().await?;
let mut tx = pool.begin_with("BEGIN IMMEDIATE").await?;
for message in conversation.messages() {
let message_id = message
.id
.as_ref()
.ok_or_else(|| anyhow::anyhow!("compacted conversation message has no id"))?;
let stored_metadata_json = sqlx::query_scalar::<_, Option<String>>(
"SELECT metadata_json FROM messages WHERE session_id = ? AND message_id = ?",
)
.bind(session_id)
.bind(message_id)
.fetch_optional(&mut *tx)
.await?;
if let Some(stored_metadata_json) = stored_metadata_json {
let mut metadata = stored_metadata_json
.and_then(|json| serde_json::from_str::<MessageMetadata>(&json).ok())
.unwrap_or_default();
metadata.agent_visible = message.metadata.agent_visible;
sqlx::query(
"UPDATE messages SET metadata_json = ? WHERE session_id = ? AND message_id = ?",
)
.bind(serde_json::to_string(&metadata)?)
.bind(session_id)
.bind(message_id)
.execute(&mut *tx)
.await?;
} else {
sqlx::query(
"INSERT INTO messages (message_id, session_id, role, content_json, created_timestamp, metadata_json) VALUES (?, ?, ?, ?, ?, ?)",
)
.bind(message_id)
.bind(session_id)
.bind(role_to_string(&message.role))
.bind(serde_json::to_string(&message.content)?)
.bind(message.created)
.bind(serde_json::to_string(&message.metadata)?)
.execute(&mut *tx)
.await?;
}
}
sqlx::query("UPDATE sessions SET updated_at = datetime('now') WHERE id = ?")
.bind(session_id)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
async fn list_sessions_matching(&self, query: SessionListQuery<'_>) -> Result<Vec<Session>> {
let filters = &query.filters;
if matches!(filters.types, Some(types) if types.is_empty()) {

View file

@ -187,6 +187,98 @@ fn test_custom_get_tools() {
});
}
#[test]
#[serial]
#[cfg(feature = "live-voice")]
fn test_live_voice_availability_is_bound_to_an_accessible_main_session() {
let _guard = env_lock::lock_env([
("GOOSE_LIVE_VOICE_ENABLED", None::<&str>),
("OPENAI_API_KEY", None::<&str>),
]);
write_acp_global_config(DEFAULT_ACP_TEST_CONFIG);
run_test(async move {
let openai = OpenAiFixture::new(vec![], Arc::new(EnforceSessionId::default())).await;
let mut conn = AcpServerConnection::new(TestConnectionConfig::default(), openai).await;
let SessionData { session, .. } = conn.new_session().await.unwrap();
let disabled = send_custom(
conn.cx(),
"_goose/unstable/session/live-voice/availability",
serde_json::json!({
"sessionId": session.session_id().0,
"_meta": { "goose": { "unrolledAgentLoop": true } }
}),
)
.await
.unwrap();
assert_eq!(disabled["status"], "unavailable");
assert_eq!(disabled["message"], "Live voice is disabled");
send_custom(
conn.cx(),
"_goose/unstable/config/upsert",
serde_json::json!({
"key": "GOOSE_LIVE_VOICE_ENABLED",
"value": true,
"isSecret": false
}),
)
.await
.unwrap();
send_custom(
conn.cx(),
"_goose/unstable/config/upsert",
serde_json::json!({
"key": "OPENAI_API_KEY",
"value": "test-live-key",
"isSecret": true
}),
)
.await
.unwrap();
let legacy_loop = send_custom(
conn.cx(),
"_goose/unstable/session/live-voice/availability",
serde_json::json!({
"sessionId": session.session_id().0,
"_meta": { "goose": { "unrolledAgentLoop": false } }
}),
)
.await
.unwrap();
assert_eq!(legacy_loop["status"], "unavailable");
assert_eq!(
legacy_loop["message"],
"Live voice is unavailable while Use Legacy Agent Loop is enabled"
);
let response = send_custom(
conn.cx(),
"_goose/unstable/session/live-voice/availability",
serde_json::json!({
"sessionId": session.session_id().0,
"_meta": { "goose": { "unrolledAgentLoop": true } }
}),
)
.await
.unwrap();
assert_eq!(response["status"], "ready");
assert_eq!(response["message"], "Start Live voice");
let inaccessible = send_custom(
conn.cx(),
"_goose/unstable/session/live-voice/availability",
serde_json::json!({
"sessionId": "not-loaded-on-this-connection",
"_meta": { "goose": { "unrolledAgentLoop": true } }
}),
)
.await;
assert!(inaccessible.is_err());
});
}
#[test]
#[serial]
fn test_custom_get_extensions() {

View file

@ -389,6 +389,10 @@ pub async fn spawn_acp_server_in_process(
)
});
let active_runs = Arc::new(goose::acp::server::ActiveRunRegistry::default());
let live_voice = Arc::new(goose::acp::server::LiveVoiceService::from_config(
active_runs.clone(),
));
let agent = GooseAcpAgent::new(GooseAcpAgentOptions {
provider_factory,
builtin_selection: goose::acp::server::AcpBuiltinSelection {
@ -402,7 +406,8 @@ pub async fn spawn_acp_server_in_process(
additional_source_roots: Vec::new(),
session_cwd: None,
scheduler: Some(Arc::new(FixtureScheduler::new())),
active_prompt_runs: Default::default(),
active_runs,
live_voice,
})
.await
.unwrap();

View file

@ -69,8 +69,10 @@ module.exports = [
setInterval: 'readonly',
clearTimeout: 'readonly',
CustomEvent: 'readonly',
EventTarget: 'readonly',
Element: 'readonly',
HTMLElement: 'readonly',
HTMLAudioElement: 'readonly',
HTMLInputElement: 'readonly',
HTMLSelectElement: 'readonly',
HTMLTextAreaElement: 'readonly',
@ -110,6 +112,13 @@ module.exports = [
AnalyserNode: 'readonly',
MediaRecorder: 'readonly',
MediaStream: 'readonly',
RTCDataChannel: 'readonly',
RTCDataChannelState: 'readonly',
RTCIceGatheringState: 'readonly',
RTCPeerConnection: 'readonly',
RTCPeerConnectionState: 'readonly',
RTCSessionDescriptionInit: 'readonly',
RTCTrackEvent: 'readonly',
AudioWorkletNode: 'readonly',
DOMException: 'readonly',
MediaDeviceInfo: 'readonly',

View file

@ -57,6 +57,7 @@ import { trackErrorWithContext } from './utils/analytics';
import { AppEvents } from './constants/events';
import { registerPlatformEventHandlers } from './utils/platform_events';
import { reconnectAcpAfterSystemResume } from './acp/acpConnection';
import { useLiveVoice, type LiveVoiceController } from './liveVoice/useLiveVoice';
function PageViewTracker() {
usePageViewTracking();
@ -64,9 +65,15 @@ function PageViewTracker() {
}
// Route Components
const HubRouteWrapper = ({ draftRef }: { draftRef: RefObject<string> }) => {
const HubRouteWrapper = ({
draftRef,
liveVoice,
}: {
draftRef: RefObject<string>;
liveVoice: LiveVoiceController;
}) => {
const setView = useNavigation();
return <Hub setView={setView} draftRef={draftRef} />;
return <Hub setView={setView} draftRef={draftRef} liveVoice={liveVoice} />;
};
export function resolveSessionInitialMessage(
@ -328,7 +335,17 @@ export function AppInner() {
const nostrImportInFlight = useRef<string | null>(null);
const navigate = useNavigate();
const location = useLocation();
const setView = useNavigation();
const liveVoice = useLiveVoice();
const { activeSessionId: activeLiveVoiceSessionId, stop: stopLiveVoice } = liveVoice;
useEffect(() => {
const hasLiveVoiceEntryPoint = location.pathname === '/' || location.pathname === '/pair';
if (!hasLiveVoiceEntryPoint && activeLiveVoiceSessionId) {
void stopLiveVoice();
}
}, [activeLiveVoiceSessionId, location.pathname, stopLiveVoice]);
const [chat, setChat] = useState<ChatType>({
sessionId: '',
@ -662,12 +679,15 @@ export function AppInner() {
element={
<OnboardingGuard>
<ChatProvider chat={chat} setChat={setChat} contextKey="hub">
<AppLayout activeSessions={activeSessions} />
<AppLayout activeSessions={activeSessions} liveVoice={liveVoice} />
</ChatProvider>
</OnboardingGuard>
}
>
<Route index element={<HubRouteWrapper draftRef={hubDraftRef} />} />
<Route
index
element={<HubRouteWrapper draftRef={hubDraftRef} liveVoice={liveVoice} />}
/>
<Route
path="pair"
element={

View file

@ -4,9 +4,13 @@ import { AppEvents } from '../../constants/events';
import { ChatState } from '../../types/chatState';
import type { Session } from '../../types/session';
import { maybeHandlePlatformEvent } from '../../utils/platform_events';
import { handleAcpSessionNotification } from '../chatNotifications';
import {
handleAcpGooseSessionNotification,
handleAcpSessionNotification,
} from '../chatNotifications';
import type { AcpChatSessionSnapshot } from '../chatSessionStore';
import { acpChatSessionActions, acpChatSessionStore } from '../chatSessionStore';
import { subscribeToLiveVoiceInteractionEnded } from '../liveVoiceNotifications';
vi.mock('../chatSessionStore', () => ({
acpChatSessionStore: {
@ -184,4 +188,29 @@ describe('handleAcpSessionNotification', () => {
expect(maybeHandlePlatformEvent).not.toHaveBeenCalled();
});
it('routes Live voice interaction endings outside the conversation store', async () => {
const listener = vi.fn();
const unsubscribe = subscribeToLiveVoiceInteractionEnded(listener);
await handleAcpGooseSessionNotification({
sessionId: SESSION_ID,
update: {
sessionUpdate: 'live_voice_interaction_ended',
interactionId: 'live-opaque',
outcome: 'failed',
},
});
expect(listener).toHaveBeenCalledWith({
sessionId: SESSION_ID,
update: {
sessionUpdate: 'live_voice_interaction_ended',
interactionId: 'live-opaque',
outcome: 'failed',
},
});
expect(acpChatSessionActions.applyAcpGooseSessionNotification).not.toHaveBeenCalled();
unsubscribe();
});
});

View file

@ -0,0 +1,102 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { getAcpClient } from '../acpConnection';
import { acpGetLiveVoiceAvailability, acpStartLiveVoice, acpStopLiveVoice } from '../liveVoice';
vi.mock('../acpConnection', () => ({ getAcpClient: vi.fn() }));
describe('ACP Live voice', () => {
beforeEach(() => {
vi.clearAllMocks();
vi.mocked(window.electron.getSetting).mockResolvedValue(false);
});
it('uses the generated availability client for the displayed session', async () => {
const sessionLiveVoiceAvailability = vi.fn().mockResolvedValue({
status: 'ready',
message: 'Start Live voice',
});
vi.mocked(getAcpClient).mockResolvedValue({
goose: {
sessionLiveVoiceAvailability_unstable: sessionLiveVoiceAvailability,
},
} as unknown as Awaited<ReturnType<typeof getAcpClient>>);
await expect(acpGetLiveVoiceAvailability('main-session')).resolves.toEqual({
status: 'ready',
message: 'Start Live voice',
});
expect(sessionLiveVoiceAvailability).toHaveBeenCalledWith({
sessionId: 'main-session',
_meta: { goose: { unrolledAgentLoop: true } },
});
});
it('checks availability without a session for a new chat', async () => {
const sessionLiveVoiceAvailability = vi.fn().mockResolvedValue({
status: 'unavailable',
message: 'Live voice is disabled',
});
vi.mocked(getAcpClient).mockResolvedValue({
goose: {
sessionLiveVoiceAvailability_unstable: sessionLiveVoiceAvailability,
},
} as unknown as Awaited<ReturnType<typeof getAcpClient>>);
await expect(acpGetLiveVoiceAvailability()).resolves.toEqual({
status: 'unavailable',
message: 'Live voice is disabled',
});
expect(sessionLiveVoiceAvailability).toHaveBeenCalledWith({
_meta: { goose: { unrolledAgentLoop: true } },
});
});
it('passes the legacy loop selection to availability', async () => {
vi.mocked(window.electron.getSetting).mockResolvedValue(true);
const sessionLiveVoiceAvailability = vi.fn().mockResolvedValue({
status: 'unavailable',
message: 'Live voice is unavailable while Use Legacy Agent Loop is enabled',
});
vi.mocked(getAcpClient).mockResolvedValue({
goose: {
sessionLiveVoiceAvailability_unstable: sessionLiveVoiceAvailability,
},
} as unknown as Awaited<ReturnType<typeof getAcpClient>>);
await acpGetLiveVoiceAvailability('main-session');
expect(sessionLiveVoiceAvailability).toHaveBeenCalledWith({
sessionId: 'main-session',
_meta: { goose: { unrolledAgentLoop: false } },
});
});
it('uses generated start and stop clients with the interaction ID', async () => {
const start = vi.fn().mockResolvedValue({
interactionId: 'live-opaque',
answerSdp: 'answer',
});
const stop = vi.fn().mockResolvedValue({});
vi.mocked(getAcpClient).mockResolvedValue({
goose: {
sessionLiveVoiceStart_unstable: start,
sessionLiveVoiceStop_unstable: stop,
},
} as unknown as Awaited<ReturnType<typeof getAcpClient>>);
await expect(acpStartLiveVoice('main-session', 'offer')).resolves.toMatchObject({
interactionId: 'live-opaque',
});
await acpStopLiveVoice('main-session', 'live-opaque');
expect(start).toHaveBeenCalledWith({
sessionId: 'main-session',
offerSdp: 'offer',
_meta: { goose: { unrolledAgentLoop: true } },
});
expect(stop).toHaveBeenCalledWith({
sessionId: 'main-session',
interactionId: 'live-opaque',
});
});
});

View file

@ -7,11 +7,11 @@ import { AppEvents } from '../constants/events';
import { maybeHandlePlatformEvent } from '../utils/platform_events';
import { toolNotificationEvent } from './adapter/toolNotifications';
import { acpChatSessionActions, acpChatSessionStore } from './chatSessionStore';
import { publishLiveVoiceInteractionEnded } from './liveVoiceNotifications';
export function handleAcpSessionNotification(notification: SessionNotification): Promise<void> {
const sessionNameBeforeNotification = acpChatSessionStore.getSnapshot(
notification.sessionId
)?.session?.name;
const sessionNameBeforeNotification = acpChatSessionStore.getSnapshot(notification.sessionId)
?.session?.name;
const updatedName =
notification.update.sessionUpdate === 'session_info_update'
? notification.update.title
@ -49,6 +49,14 @@ function maybeHandleLivePlatformEvent(notification: SessionNotification): void {
export function handleAcpGooseSessionNotification(
notification: GooseSessionNotification_unstable
): Promise<void> {
if (notification.update.sessionUpdate === 'live_voice_interaction_ended') {
publishLiveVoiceInteractionEnded({
sessionId: notification.sessionId,
update: notification.update,
});
return Promise.resolve();
}
acpChatSessionActions.applyAcpGooseSessionNotification(notification);
return Promise.resolve();
}

View file

@ -0,0 +1,34 @@
import type {
LiveVoiceAvailabilityResponse_unstable,
LiveVoiceStartResponse_unstable,
} from '@aaif/goose-acp-client';
import { getAcpClient } from './acpConnection';
export async function acpGetLiveVoiceAvailability(
sessionId?: string
): Promise<LiveVoiceAvailabilityResponse_unstable> {
const { goose } = await getAcpClient();
const useLegacyAgentLoop = await window.electron.getSetting('useLegacyAgentLoop');
return goose.sessionLiveVoiceAvailability_unstable({
...(sessionId ? { sessionId } : {}),
_meta: { goose: { unrolledAgentLoop: !useLegacyAgentLoop } },
});
}
export async function acpStartLiveVoice(
sessionId: string,
offerSdp: string
): Promise<LiveVoiceStartResponse_unstable> {
const { goose } = await getAcpClient();
const useLegacyAgentLoop = await window.electron.getSetting('useLegacyAgentLoop');
return goose.sessionLiveVoiceStart_unstable({
sessionId,
offerSdp,
_meta: { goose: { unrolledAgentLoop: !useLegacyAgentLoop } },
});
}
export async function acpStopLiveVoice(sessionId: string, interactionId: string): Promise<void> {
const { goose } = await getAcpClient();
await goose.sessionLiveVoiceStop_unstable({ sessionId, interactionId });
}

View file

@ -0,0 +1,30 @@
import type { GooseSessionNotification_unstable } from '@aaif/goose-acp-client';
export type LiveVoiceInteractionEndedNotification = {
sessionId: string;
update: Extract<
GooseSessionNotification_unstable['update'],
{ sessionUpdate: 'live_voice_interaction_ended' }
>;
};
type LiveVoiceInteractionEndedListener = (
notification: LiveVoiceInteractionEndedNotification
) => void;
const listeners = new Set<LiveVoiceInteractionEndedListener>();
export function subscribeToLiveVoiceInteractionEnded(
listener: LiveVoiceInteractionEndedListener
): () => void {
listeners.add(listener);
return () => listeners.delete(listener);
}
export function publishLiveVoiceInteractionEnded(
notification: LiveVoiceInteractionEndedNotification
): void {
for (const listener of listeners) {
listener(notification);
}
}

View file

@ -34,6 +34,11 @@ import { Goose } from './icons';
import EnvironmentBadge from './GooseSidebar/EnvironmentBadge';
import SessionActionsHeader from './SessionActionsHeader';
import { isAcpRecovering, subscribeToAcpRecovery } from '../acp/acpConnection';
import type { LiveVoiceAvailabilityResponse_unstable } from '@aaif/goose-acp-client';
import { acpGetLiveVoiceAvailability } from '../acp/liveVoice';
import type { LiveVoiceController } from '../liveVoice/useLiveVoice';
const NEW_LIVE_VOICE_GREETING = 'Hello! What can I help you with?';
const i18n = defineMessages({
failedToLoadSession: {
@ -69,6 +74,7 @@ interface BaseChatProps {
isActiveSession: boolean;
initialMessage?: UserInput;
noAutoSubmit?: boolean;
liveVoice: LiveVoiceController;
}
export default function BaseChat({
@ -80,6 +86,7 @@ export default function BaseChat({
initialMessage,
noAutoSubmit,
isActiveSession,
liveVoice,
}: BaseChatProps) {
const intl = useIntl();
const location = useLocation();
@ -87,11 +94,32 @@ export default function BaseChat({
const scrollRef = useRef<ScrollAreaHandle>(null);
const chatInputRef = useRef<HTMLTextAreaElement>(null);
const disableAnimation = location.state?.disableAnimation || false;
const shouldStartLiveVoice = location.state?.startLiveVoice === true;
const [hasStartedUsingRecipe, setHasStartedUsingRecipe] = React.useState(false);
const [acpRecovering, setAcpRecovering] = useState(isAcpRecovering);
const [liveVoiceAvailability, setLiveVoiceAvailability] =
useState<LiveVoiceAvailabilityResponse_unstable | null>(null);
const isMobile = useIsMobile();
const navContext = useNavigationContextSafe();
const setView = useNavigation();
const {
activeSessionId: activeLiveVoiceSessionId,
liveVoiceSessionId,
start: startSharedLiveVoice,
} = liveVoice;
const ownsLiveVoice = liveVoiceSessionId === sessionId;
const liveVoiceActiveInAnotherSession =
activeLiveVoiceSessionId !== null && activeLiveVoiceSessionId !== sessionId;
const startLiveVoice = useCallback(
async (initialCommentary?: string) => {
if (activeLiveVoiceSessionId && activeLiveVoiceSessionId !== sessionId) {
setView('pair', { resumeSessionId: activeLiveVoiceSessionId });
return;
}
await startSharedLiveVoice(sessionId, initialCommentary);
},
[activeLiveVoiceSessionId, sessionId, setView, startSharedLiveVoice]
);
const isNavCollapsed = !navContext?.isNavExpanded;
const contentClassName = cn('pr-1 pb-10 pt-12', (isMobile || isNavCollapsed) && 'pt-16');
const { droppedFiles, setDroppedFiles, handleDrop, handleDragOver } = useFileDrop();
@ -115,6 +143,7 @@ export default function BaseChat({
notifications: toolCallNotifications,
pauseQueueOnStop,
queueProcessingBlocked,
hasActiveRun,
onMessageUpdate,
} = useChatSession({
sessionId,
@ -125,6 +154,66 @@ export default function BaseChat({
[handleSubmit]
);
const sessionLoaded = session !== undefined;
const liveVoiceChatBusy = chatState !== ChatState.Idle;
useEffect(() => {
if (!isActiveSession || !shouldStartLiveVoice || liveVoiceAvailability === null) {
return;
}
if (liveVoiceAvailability.status === 'ready') {
void startLiveVoice(NEW_LIVE_VOICE_GREETING);
}
navigate(location, {
replace: true,
state: { ...location.state, startLiveVoice: undefined },
});
}, [
isActiveSession,
shouldStartLiveVoice,
liveVoiceAvailability,
startLiveVoice,
location,
navigate,
]);
useEffect(() => {
if (!isActiveSession || !sessionLoaded || acpRecovering || liveVoiceActiveInAnotherSession) {
setLiveVoiceAvailability(null);
return;
}
let current = true;
setLiveVoiceAvailability(null);
void acpGetLiveVoiceAvailability(sessionId).then(
(response) => {
if (current) {
setLiveVoiceAvailability(response);
}
},
() => {
if (current) {
setLiveVoiceAvailability(null);
}
}
);
return () => {
current = false;
};
}, [
acpRecovering,
hasActiveRun,
isActiveSession,
liveVoice.phase,
liveVoiceActiveInAnotherSession,
liveVoiceChatBusy,
session?.goose_mode,
sessionId,
sessionLoaded,
]);
const handleWorkingDirChange = useCallback(
async (newDir: string) => {
if (!session) {
@ -214,7 +303,6 @@ export default function BaseChat({
const sessionModel = session?.model_config?.model_name ?? null;
const sessionProvider = session?.provider_name ?? null;
const sessionLoaded = session !== undefined;
const latestInference = useMemo(() => {
for (let i = messages.length - 1; i >= 0; i--) {
const message = messages[i];
@ -475,6 +563,7 @@ export default function BaseChat({
sessionId={sessionId}
handleSubmit={chatInputSubmit}
chatState={chatState}
hasActiveRun={hasActiveRun}
onStop={stopStreaming}
onSteerQueuedMessage={onSteerQueuedMessage}
pauseQueueOnStop={pauseQueueOnStop}
@ -507,6 +596,15 @@ export default function BaseChat({
workingDir={session?.working_dir}
onWorkingDirChange={handleWorkingDirChange}
latestInference={latestInference}
liveVoice={{
availability: liveVoiceAvailability,
phase: ownsLiveVoice ? liveVoice.phase : 'idle',
muted: ownsLiveVoice ? liveVoice.muted : false,
activeInAnotherSession: liveVoiceActiveInAnotherSession,
start: () => startLiveVoice(),
stop: liveVoice.stop,
toggleMute: liveVoice.toggleMute,
}}
{...customChatInputProps}
/>
</ChatInputCard>

View file

@ -41,6 +41,9 @@ import { getTextDirection } from '../utils/textDirection';
import { defineMessages, useIntl } from '../i18n';
import TurndownService from 'turndown';
import type { NextChatExtensionDraft } from '../utils/nextChatExtensions';
import { LiveVoiceButton } from './LiveVoiceButton';
import type { LiveVoiceAvailabilityResponse_unstable } from '@aaif/goose-acp-client';
import { isLiveVoiceActive, type LiveVoiceController } from '../liveVoice/useLiveVoice';
const turndown = new TurndownService({
headingStyle: 'atx',
@ -69,6 +72,15 @@ interface PastedImage {
error?: string;
}
type ChatInputLiveVoice = Pick<
LiveVoiceController,
'phase' | 'muted' | 'stop' | 'toggleMute'
> & {
availability: LiveVoiceAvailabilityResponse_unstable | null;
activeInAnotherSession: boolean;
start: () => Promise<void>;
};
const moveQueuedMessageToFront = (
messages: QueuedMessage[],
messageId: string
@ -163,6 +175,7 @@ interface ChatInputProps {
sessionId: string | null;
handleSubmit: (input: UserInput) => void;
chatState: ChatState;
hasActiveRun: boolean;
onStop?: () => void;
onSteerQueuedMessage?: (input: UserInput) => Promise<boolean>;
pauseQueueOnStop?: boolean;
@ -198,12 +211,14 @@ interface ChatInputProps {
latestInference?: Message['metadata']['inference'] | null;
nextChatExtensionDraft?: NextChatExtensionDraft;
onNextChatExtensionDraftChange?: (draft: NextChatExtensionDraft) => void;
liveVoice?: ChatInputLiveVoice;
}
export default function ChatInput({
sessionId,
handleSubmit,
chatState = ChatState.Idle,
hasActiveRun,
onStop,
onSteerQueuedMessage,
pauseQueueOnStop = false,
@ -234,6 +249,7 @@ export default function ChatInput({
latestInference,
nextChatExtensionDraft,
onNextChatExtensionDraftChange,
liveVoice,
}: ChatInputProps) {
const [_value, setValue] = useState(initialValue);
const [displayValue, setDisplayValue] = useState(initialValue); // For immediate visual feedback
@ -256,13 +272,14 @@ export default function ChatInput({
// Derived state - chatState != Idle means we're in some form of loading state
const isLoading = chatState !== ChatState.Idle;
const isLoadingRef = useRef(isLoading);
const liveVoiceBlocksSubmission = liveVoice ? isLiveVoiceActive(liveVoice.phase) : false;
const isSubmissionBusy = isLoading || hasActiveRun || liveVoiceBlocksSubmission;
const isSubmissionBusyRef = useRef(isSubmissionBusy);
const composerDir = useMemo(() => getTextDirection(displayValue) ?? undefined, [displayValue]);
const queueProcessingBlockedRef = useRef(queueProcessingBlocked);
const wasLoadingRef = useRef(isLoading);
const wasSubmissionBusyRef = useRef(isSubmissionBusy);
const wasQueueProcessingBlockedRef = useRef(queueProcessingBlocked);
isLoadingRef.current = isLoading;
isSubmissionBusyRef.current = isSubmissionBusy;
queueProcessingBlockedRef.current = queueProcessingBlocked;
// Queue functionality - ephemeral, only exists in memory for this chat instance
@ -393,12 +410,12 @@ export default function ChatInput({
// Queue processing
useEffect(() => {
const becameIdle = wasLoadingRef.current && !isLoading;
const becameAvailable = wasSubmissionBusyRef.current && !isSubmissionBusy;
const becameUnblocked = wasQueueProcessingBlockedRef.current && !queueProcessingBlocked;
const hasSendNowInFlight = sendNowInFlightMessageIdsRef.current.size > 0;
if (
(becameIdle || (becameUnblocked && !isLoading)) &&
(becameAvailable || (becameUnblocked && !isSubmissionBusy)) &&
!queueProcessingBlocked &&
!hasSendNowInFlight &&
queuedMessages.length > 0
@ -410,13 +427,13 @@ export default function ChatInput({
if (pendingSendAfterStopId && !messageToSend) {
clearPendingSendAfterStop(pendingSendAfterStopId);
wasLoadingRef.current = isLoading;
wasSubmissionBusyRef.current = isSubmissionBusy;
wasQueueProcessingBlockedRef.current = queueProcessingBlocked;
return;
}
if (!messageToSend) {
wasLoadingRef.current = isLoading;
wasSubmissionBusyRef.current = isSubmissionBusy;
wasQueueProcessingBlockedRef.current = queueProcessingBlocked;
return;
}
@ -452,10 +469,10 @@ export default function ChatInput({
}
}
}
wasLoadingRef.current = isLoading;
wasSubmissionBusyRef.current = isSubmissionBusy;
wasQueueProcessingBlockedRef.current = queueProcessingBlocked;
}, [
isLoading,
isSubmissionBusy,
queueProcessingBlocked,
queuedMessages,
handleSubmit,
@ -1072,14 +1089,14 @@ export default function ChatInput({
};
const handleInterruptionAndQueue = () => {
if (!isLoading || !hasSubmittableContent) {
if (!isSubmissionBusy || !hasSubmittableContent) {
return false;
}
const imageData = convertImagesToImageData();
const contentToQueue = appendDroppedFilePaths(displayValue.trim());
const interruptionMatch = detectInterruption(displayValue.trim());
const interruptionMatch = isLoading ? detectInterruption(displayValue.trim()) : null;
if (interruptionMatch && interruptionMatch.shouldInterrupt) {
setLastInterruption(interruptionMatch.matchedText);
@ -1122,7 +1139,7 @@ export default function ChatInput({
};
const canSubmit =
!isLoading &&
!isSubmissionBusy &&
!queueProcessingBlocked &&
(displayValue.trim() ||
pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) ||
@ -1244,12 +1261,12 @@ export default function ChatInput({
if (queueProcessingBlocked) {
return;
}
if (isLoading && hasSubmittableContent) {
if (isSubmissionBusy && hasSubmittableContent) {
handleInterruptionAndQueue();
return;
}
const canSubmit =
!isLoading &&
!isSubmissionBusy &&
!queueProcessingBlocked &&
(displayValue.trim() ||
pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) ||
@ -1359,7 +1376,6 @@ export default function ChatInput({
allDroppedFiles.some((file) => !file.error && !file.isLoading);
const isAnyImageLoading = pastedImages.some((img) => img.isLoading);
const isAnyDroppedFileLoading = allDroppedFiles.some((file) => file.isLoading);
const isSubmitButtonDisabled =
!hasSubmittableContent ||
isAnyImageLoading ||
@ -1412,7 +1428,7 @@ export default function ChatInput({
if (!messageToSend) return;
if (queueProcessingBlocked) return;
if (!isLoading) {
if (!isSubmissionBusy) {
setQueuedMessages((prev) => removeQueuedMessage(prev, messageId));
LocalMessageStorage.addMessage(messageToSend.content);
handleSubmit({ msg: messageToSend.content, images: messageToSend.images });
@ -1451,7 +1467,7 @@ export default function ChatInput({
setSendNowInFlightMessage(messageId, false);
}
if (!isLoadingRef.current && !queueProcessingBlockedRef.current) {
if (!isSubmissionBusyRef.current && !queueProcessingBlockedRef.current) {
queuePausedRef.current = wasQueuePausedBeforeSteer;
setQueuedMessages((prev) => {
const newQueue = removeQueuedMessage(prev, messageId);
@ -1482,7 +1498,7 @@ export default function ChatInput({
const handleResumeQueue = () => {
queuePausedRef.current = false;
setLastInterruption(null);
if (!isLoading && !queueProcessingBlocked && queuedMessages.length > 0) {
if (!isSubmissionBusy && !queueProcessingBlocked && queuedMessages.length > 0) {
const nextMessage = queuedMessages[0];
LocalMessageStorage.addMessage(nextMessage.content);
handleSubmit({ msg: nextMessage.content, images: nextMessage.images });
@ -1553,7 +1569,7 @@ export default function ChatInput({
onBlur={() => setIsFocused(false)}
ref={textAreaRef}
rows={1}
readOnly={isRecording}
readOnly={isRecording || liveVoiceBlocksSubmission}
style={{
minHeight: `${minTextareaHeight}px`,
maxHeight: `${maxHeight}px`,
@ -1800,6 +1816,23 @@ export default function ChatInput({
</>
)}
{liveVoice && (
<LiveVoiceButton
availability={liveVoice.availability}
phase={liveVoice.phase}
muted={liveVoice.muted}
activeInAnotherSession={liveVoice.activeInAnotherSession}
onStart={() => void liveVoice.start()}
onStop={() => void liveVoice.stop()}
onToggleMute={liveVoice.toggleMute}
composerEmpty={
displayValue.trim().length === 0 &&
pastedImages.length === 0 &&
allDroppedFiles.length === 0
}
/>
)}
{/* Right: mic — ghost icon, no background when idle */}
{dictationProvider && (
<Tooltip>

View file

@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
import ChatSessionsContainer from './ChatSessionsContainer';
import { subscribeToAcpRecovery } from '../acp/acpConnection';
import { acpChatSessionController } from '../acp/chatSessionController';
import type { LiveVoiceController } from '../liveVoice/useLiveVoice';
vi.mock('react-router', () => ({
useSearchParams: () => [new URLSearchParams('resumeSessionId=session-1')],
@ -38,6 +39,17 @@ describe('ChatSessionsContainer', () => {
<ChatSessionsContainer
setChat={vi.fn()}
activeSessions={[{ sessionId: 'session-1' }, { sessionId: 'session-2' }]}
liveVoice={
{
activeSessionId: null,
liveVoiceSessionId: null,
phase: 'idle',
muted: false,
start: vi.fn(),
stop: vi.fn(),
toggleMute: vi.fn(),
} satisfies LiveVoiceController
}
/>
);

View file

@ -5,6 +5,7 @@ import { ChatType } from '../types/chat';
import { UserInput } from '../types/message';
import { subscribeToAcpRecovery } from '../acp/acpConnection';
import { acpChatSessionController } from '../acp/chatSessionController';
import type { LiveVoiceController } from '../liveVoice/useLiveVoice';
interface ChatSessionsContainerProps {
setChat: (chat: ChatType) => void;
@ -13,6 +14,7 @@ interface ChatSessionsContainerProps {
initialMessage?: UserInput;
noAutoSubmit?: boolean;
}>;
liveVoice: LiveVoiceController;
}
/**
@ -23,6 +25,7 @@ interface ChatSessionsContainerProps {
export default function ChatSessionsContainer({
setChat,
activeSessions,
liveVoice,
}: ChatSessionsContainerProps) {
const [searchParams] = useSearchParams();
const currentSessionId = searchParams.get('resumeSessionId') ?? undefined;
@ -72,6 +75,7 @@ export default function ChatSessionsContainer({
noAutoSubmit={session.noAutoSubmit}
suppressEmptyState={false}
isActiveSession={isVisible}
liveVoice={liveVoice}
/>
</div>
);

View file

@ -1,21 +1,38 @@
/**
* @vitest-environment jsdom
*/
import { act, render } from '@testing-library/react';
import { act, render, waitFor } from '@testing-library/react';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import Hub from './Hub';
import { IntlTestWrapper } from '../i18n/test-utils';
import { createSession } from '../sessions';
import { UserInput } from '../types/message';
import { acpGetLiveVoiceAvailability } from '../acp/liveVoice';
import { subscribeToAcpRecovery } from '../acp/acpConnection';
import type { LiveVoiceController } from '../liveVoice/useLiveVoice';
type ChatInputCapture = {
draftRef?: { current: string };
handleSubmit: (input: UserInput) => void;
liveVoice?: {
availability: { status: string; message: string } | null;
start: () => Promise<void>;
};
onNextChatExtensionDraftChange?: (draft: { selectedNames: Set<string> }) => void;
};
type Session = Awaited<ReturnType<typeof createSession>>;
const liveVoice: LiveVoiceController = {
activeSessionId: null,
liveVoiceSessionId: null,
phase: 'idle',
muted: false,
start: vi.fn(),
stop: vi.fn(),
toggleMute: vi.fn(),
};
const captured = vi.hoisted(() => ({ chatInput: null as ChatInputCapture | null }));
vi.mock('./ChatInput', () => ({
@ -47,6 +64,10 @@ vi.mock('../acp/errors', () => ({ formatAcpError: (error: unknown) => String(err
vi.mock('../toasts', () => ({ toastError: vi.fn() }));
vi.mock('../acp/liveVoice', () => ({ acpGetLiveVoiceAvailability: vi.fn() }));
vi.mock('../acp/acpConnection', () => ({ subscribeToAcpRecovery: vi.fn() }));
const DRAFT = 'a half-written thought';
const TYPED_WHILE_STARTING = 'and one more thought';
@ -63,10 +84,10 @@ function pendingSession() {
return settle;
}
function renderHub(draftRef: { current: string }) {
function renderHub(draftRef: { current: string }, setView = vi.fn()) {
return render(
<IntlTestWrapper>
<Hub setView={vi.fn()} draftRef={draftRef} />
<Hub setView={setView} draftRef={draftRef} liveVoice={liveVoice} />
</IntlTestWrapper>
);
}
@ -80,7 +101,48 @@ async function submit() {
describe('Hub', () => {
beforeEach(() => {
vi.clearAllMocks();
liveVoice.activeSessionId = null;
liveVoice.liveVoiceSessionId = null;
liveVoice.phase = 'idle';
captured.chatInput = null;
vi.mocked(acpGetLiveVoiceAvailability).mockRejectedValue(new Error('ACP unavailable'));
vi.mocked(subscribeToAcpRecovery).mockReturnValue(() => undefined);
});
it('requests Live voice availability again after ACP recovers', async () => {
let recoveryChanged: ((recovering: boolean) => void) | undefined;
const available = { status: 'ready' as const, message: 'Start Live voice' };
vi.mocked(acpGetLiveVoiceAvailability)
.mockRejectedValueOnce(new Error('ACP disconnected'))
.mockResolvedValueOnce(available);
vi.mocked(subscribeToAcpRecovery).mockImplementation((listener) => {
recoveryChanged = listener;
return () => undefined;
});
renderHub({ current: '' });
await waitFor(() => expect(acpGetLiveVoiceAvailability).toHaveBeenCalledTimes(1));
act(() => recoveryChanged?.(true));
act(() => recoveryChanged?.(false));
await waitFor(() => {
expect(acpGetLiveVoiceAvailability).toHaveBeenCalledTimes(2);
expect(captured.chatInput?.liveVoice?.availability).toEqual(available);
});
});
it('returns to the session with the active Live voice interaction', async () => {
const setView = vi.fn();
liveVoice.activeSessionId = 'session-with-live-voice';
renderHub({ current: '' }, setView);
await act(async () => captured.chatInput?.liveVoice?.start?.());
expect(setView).toHaveBeenCalledWith('pair', {
resumeSessionId: 'session-with-live-voice',
});
expect(createSession).not.toHaveBeenCalled();
});
it('starts a chat with no extensions when the user cleared the picker', async () => {

View file

@ -28,6 +28,10 @@ import {
import { formatAcpError } from '../acp/errors';
import { toastError } from '../toasts';
import { formatClockDisplay } from '../utils/timeUtils';
import { acpGetLiveVoiceAvailability } from '../acp/liveVoice';
import type { LiveVoiceAvailabilityResponse_unstable } from '@aaif/goose-acp-client';
import { subscribeToAcpRecovery } from '../acp/acpConnection';
import type { LiveVoiceController } from '../liveVoice/useLiveVoice';
const i18n = defineMessages({
goodMorning: { id: 'hub.goodMorning', defaultMessage: 'Good morning' },
@ -48,16 +52,20 @@ function useClock() {
export default function Hub({
setView,
draftRef,
liveVoice,
}: {
setView: (view: View, viewOptions?: ViewOptions) => void;
/** Unsent input of this screen, kept above the route outlet across the unmount. */
draftRef: RefObject<string>;
liveVoice: LiveVoiceController;
}) {
const intl = useIntl();
const { extensionsList } = useConfig();
const [workingDir, setWorkingDir] = useState(getInitialWorkingDir());
const userSelectedWorkingDirRef = useRef(false);
const [isCreatingSession, setIsCreatingSession] = useState(false);
const [liveVoiceAvailability, setLiveVoiceAvailability] =
useState<LiveVoiceAvailabilityResponse_unstable | null>(null);
const [nextChatExtensionDraft, setNextChatExtensionDraft] =
useState<NextChatExtensionDraft | null>(null);
const inputRef = useRef<HTMLTextAreaElement>(null);
@ -75,6 +83,35 @@ export default function Hub({
};
}, []);
useEffect(() => {
let requestGeneration = 0;
const requestAvailability = async () => {
const generation = ++requestGeneration;
try {
const availability = await acpGetLiveVoiceAvailability();
if (generation === requestGeneration) {
setLiveVoiceAvailability(availability);
}
} catch {
if (generation === requestGeneration) {
setLiveVoiceAvailability(null);
}
}
};
void requestAvailability();
const unsubscribe = subscribeToAcpRecovery((recovering) => {
if (!recovering) {
void requestAvailability();
}
});
return () => {
requestGeneration += 1;
unsubscribe();
};
}, []);
const greeting = useMemo(() => {
if (hour < 12) return intl.formatMessage(i18n.goodMorning);
if (hour < 18) return intl.formatMessage(i18n.goodAfternoon);
@ -103,11 +140,8 @@ export default function Hub({
setWorkingDir(dir);
}, []);
const handleSubmit = async (input: UserInput) => {
const { msg: userMessage, images } = input;
if (!(images.length > 0 || userMessage.trim()) || isCreatingSession) return;
const draftAtSubmit = draftRef.current;
const createHubSession = async () => {
if (isCreatingSession) return null;
setIsCreatingSession(true);
try {
@ -124,36 +158,68 @@ export default function Hub({
const dir = userSelectedWorkingDirRef.current ? workingDir : await getEffectiveWorkingDir();
const session = await createSession(dir, sessionOptions);
setNextChatExtensionDraft(null);
window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED));
window.dispatchEvent(
new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, {
detail: { sessionId: session.id, initialMessage: { msg: userMessage, images } },
})
);
// The draft is this screen's own, so it is dropped once the session exists.
// Comparing it against the value at submit leaves an edit made while the
// session was starting alone, including one that emptied the input.
if (draftRef.current === draftAtSubmit) {
draftRef.current = '';
}
setView('pair', {
disableAnimation: true,
resumeSessionId: session.id,
initialMessage: { msg: userMessage, images },
});
return session;
} catch (error) {
console.error('Failed to create session:', error);
toastError({ title: "Couldn't start chat", msg: formatAcpError(error) });
setIsCreatingSession(false);
return null;
}
};
const handleSubmit = async (input: UserInput) => {
const { msg: userMessage, images } = input;
if (!(images.length > 0 || userMessage.trim())) return;
const draftAtSubmit = draftRef.current;
const session = await createHubSession();
if (!session) return;
window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED));
window.dispatchEvent(
new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, {
detail: { sessionId: session.id, initialMessage: { msg: userMessage, images } },
})
);
// Preserve edits made while the session was being created.
if (draftRef.current === draftAtSubmit) {
draftRef.current = '';
}
setView('pair', {
disableAnimation: true,
resumeSessionId: session.id,
initialMessage: { msg: userMessage, images },
});
};
const handleStartLiveVoice = async () => {
if (liveVoice.activeSessionId) {
setView('pair', { resumeSessionId: liveVoice.activeSessionId });
return;
}
const session = await createHubSession();
if (!session) return;
window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED));
window.dispatchEvent(
new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, {
detail: { sessionId: session.id },
})
);
setView('pair', {
disableAnimation: true,
resumeSessionId: session.id,
startLiveVoice: true,
});
};
return (
<div className="flex flex-col h-full min-h-0 items-center justify-center px-6 relative">
<div className="w-full max-w-2xl">
<div className="w-full max-w-3xl">
<div className="flex items-baseline gap-2 mb-1">
<span className="text-6xl font-light text-text-primary tracking-tight tabular-nums">
{time}
@ -170,6 +236,7 @@ export default function Hub({
draftRef={draftRef}
handleSubmit={handleSubmit}
chatState={isCreatingSession ? ChatState.LoadingConversation : ChatState.Idle}
hasActiveRun={false}
onStop={() => {}}
initialValue=""
setView={setView}
@ -185,6 +252,15 @@ export default function Hub({
inputRef={inputRef}
nextChatExtensionDraft={draftForMenu}
onNextChatExtensionDraftChange={handleNextChatExtensionDraftChange}
liveVoice={{
availability: isCreatingSession ? null : liveVoiceAvailability,
phase: 'idle',
muted: false,
activeInAnotherSession: liveVoice.activeSessionId !== null,
start: handleStartLiveVoice,
stop: liveVoice.stop,
toggleMute: liveVoice.toggleMute,
}}
/>
</ChatInputCard>
</div>

View file

@ -12,6 +12,7 @@ import { Navigation } from './NavigationPanel';
import { Z_INDEX } from './constants';
import { cn } from '../../utils';
import { UserInput } from '../../types/message';
import type { LiveVoiceController } from '../../liveVoice/useLiveVoice';
const i18n = defineMessages({
openNavigation: {
@ -30,9 +31,10 @@ interface AppLayoutContentProps {
initialMessage?: UserInput;
noAutoSubmit?: boolean;
}>;
liveVoice: LiveVoiceController;
}
const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions }) => {
const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions, liveVoice }) => {
const intl = useIntl();
const location = useLocation();
const safeIsMacOS = (window?.electron?.platform || 'darwin') === 'darwin';
@ -102,7 +104,11 @@ const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions }) =
<div className="relative flex h-full w-full flex-1 bg-background-secondary">
<Outlet />
<div className="hidden">
<ChatSessionsContainer setChat={setChat} activeSessions={activeSessions} />
<ChatSessionsContainer
setChat={setChat}
activeSessions={activeSessions}
liveVoice={liveVoice}
/>
</div>
</div>
);
@ -147,7 +153,7 @@ const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions }) =
className="relative flex-shrink-0 overflow-hidden h-full p-2"
>
<div className="w-full h-full overflow-hidden rounded-xl border border-border-primary">
<Navigation />
<Navigation activeLiveVoiceSessionId={liveVoice.activeSessionId} />
</div>
{isNavExpanded && (
<div
@ -163,7 +169,11 @@ const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions }) =
{/* Always render ChatSessionsContainer to keep SSE connections alive.
When navigating away from /pair, hide it with CSS */}
<div className={isOnPairRoute ? 'contents' : 'hidden'}>
<ChatSessionsContainer setChat={setChat} activeSessions={activeSessions} />
<ChatSessionsContainer
setChat={setChat}
activeSessions={activeSessions}
liveVoice={liveVoice}
/>
</div>
</div>
</div>
@ -177,12 +187,13 @@ interface AppLayoutProps {
initialMessage?: UserInput;
noAutoSubmit?: boolean;
}>;
liveVoice: LiveVoiceController;
}
export const AppLayout: React.FC<AppLayoutProps> = ({ activeSessions }) => {
export const AppLayout: React.FC<AppLayoutProps> = ({ activeSessions, liveVoice }) => {
return (
<NavigationProvider>
<AppLayoutContent activeSessions={activeSessions} />
<AppLayoutContent activeSessions={activeSessions} liveVoice={liveVoice} />
</NavigationProvider>
);
};

View file

@ -1,6 +1,6 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useLocation } from 'react-router';
import { ChevronDown, ChevronRight } from 'lucide-react';
import { AudioLines, ChevronDown, ChevronRight } from 'lucide-react';
import { motion } from 'framer-motion';
import { useNavigationContext } from './NavigationContext';
import { useConfig } from '../ConfigContext';
@ -77,6 +77,10 @@ const i18n = defineMessages({
id: 'navigationPanel.statusIdle',
defaultMessage: 'Idle',
},
returnToActiveLiveVoice: {
id: 'liveVoice.returnToActive',
defaultMessage: 'Return to active Live voice',
},
});
const navItemClass = (active: boolean) =>
@ -111,6 +115,7 @@ const NavRow: React.FC<NavRowProps> = ({ item, active, onClick }) => {
interface SessionRowProps {
session: SessionListItem;
active: boolean;
isLiveVoiceActive: boolean;
status: SessionStatus | undefined;
onClick: () => void;
onRenamed: () => void;
@ -163,7 +168,14 @@ const SessionTooltipContent: React.FC<SessionTooltipContentProps> = ({ session,
);
};
const SessionRow: React.FC<SessionRowProps> = ({ session, active, status, onClick, onRenamed }) => {
const SessionRow: React.FC<SessionRowProps> = ({
session,
active,
isLiveVoiceActive,
status,
onClick,
onRenamed,
}) => {
const intl = useIntl();
const [isEditing, setIsEditing] = useState(false);
const [tooltipOpen, setTooltipOpen] = useState(false);
@ -209,6 +221,12 @@ const SessionRow: React.FC<SessionRowProps> = ({ session, active, status, onClic
onEditStart={() => setIsEditing(true)}
onEditEnd={() => setIsEditing(false)}
/>
{isLiveVoiceActive && (
<AudioLines
className="w-3.5 h-3.5 flex-shrink-0 text-blue-500"
aria-label={intl.formatMessage(i18n.returnToActiveLiveVoice)}
/>
)}
<SessionIndicators isStreaming={isStreaming} hasUnread={hasUnread} hasError={hasError} />
</div>
</TooltipTrigger>
@ -219,7 +237,10 @@ const SessionRow: React.FC<SessionRowProps> = ({ session, active, status, onClic
);
};
export const Navigation: React.FC<{ className?: string }> = ({ className }) => {
export const Navigation: React.FC<{
className?: string;
activeLiveVoiceSessionId: string | null;
}> = ({ className, activeLiveVoiceSessionId }) => {
const intl = useIntl();
const { isNavExpanded } = useNavigationContext();
const location = useLocation();
@ -369,6 +390,7 @@ export const Navigation: React.FC<{ className?: string }> = ({ className }) => {
key={session.id}
session={session}
active={session.id === activeSessionId}
isLiveVoiceActive={session.id === activeLiveVoiceSessionId}
status={sessionStatuses.get(session.id)}
onClick={() => {
clearUnread(session.id);
@ -386,6 +408,7 @@ export const Navigation: React.FC<{ className?: string }> = ({ className }) => {
key={session.id}
session={session}
active={session.id === activeSessionId}
isLiveVoiceActive={session.id === activeLiveVoiceSessionId}
status={sessionStatuses.get(session.id)}
onClick={() => {
clearUnread(session.id);

View file

@ -0,0 +1,113 @@
import { render, screen } from '@testing-library/react';
import type { ComponentProps } from 'react';
import { describe, expect, it, vi } from 'vitest';
import { IntlTestWrapper } from '../i18n/test-utils';
import { LiveVoiceButton } from './LiveVoiceButton';
const baseProps: ComponentProps<typeof LiveVoiceButton> = {
availability: { status: 'ready', message: 'Start Live voice' },
composerEmpty: true,
phase: 'idle',
muted: false,
activeInAnotherSession: false,
onStart: vi.fn(),
onStop: vi.fn(),
onToggleMute: vi.fn(),
};
function renderButton(props: Partial<typeof baseProps> = {}) {
return render(<LiveVoiceButton {...baseProps} {...props} />, { wrapper: IntlTestWrapper });
}
describe('LiveVoiceButton', () => {
it('shows an eligible state for an empty composer', () => {
renderButton();
const button = screen.getByTestId('live-voice-button');
expect(button).toBeEnabled();
expect(button).toHaveAttribute('data-eligible', 'true');
expect(button).toHaveAccessibleName('Start Live voice');
});
it.each(['idle', 'error'] as const)('shows the unavailable reason while %s', (phase) => {
renderButton({
availability: {
status: 'unavailable',
message: 'Live voice requires Autonomous mode',
},
phase,
});
const button = screen.getByTestId('live-voice-button');
expect(button).toBeDisabled();
expect(button).toHaveAttribute('data-eligible', 'false');
expect(button).toHaveAccessibleName('Live voice requires Autonomous mode');
});
it.each(['idle', 'error'] as const)('applies empty-composer gate while %s', (phase) => {
renderButton({ composerEmpty: false, phase });
const button = screen.getByTestId('live-voice-button');
expect(button).toBeDisabled();
expect(button).toHaveAttribute('data-eligible', 'false');
expect(button).toHaveAccessibleName('Clear the message and attachments to use Live voice');
});
it('does not render without eligibility for the displayed session', () => {
renderButton({ availability: null });
expect(screen.queryByTestId('live-voice-button')).not.toBeInTheDocument();
});
it('returns to an interaction active in another session', () => {
const onStart = vi.fn();
renderButton({
availability: null,
composerEmpty: false,
activeInAnotherSession: true,
onStart,
});
const button = screen.getByTestId('live-voice-button');
expect(button).toBeEnabled();
expect(button).toHaveAccessibleName('Return to active Live voice');
button.click();
expect(onStart).toHaveBeenCalledOnce();
});
it('starts when idle and keeps stop controls available while availability refreshes', () => {
const onStart = vi.fn();
const onStop = vi.fn();
const activeProps = { ...baseProps, availability: null, onStart, onStop };
const { container, rerender } = renderButton({ onStart, onStop });
screen.getByTestId('live-voice-button').click();
expect(onStart).toHaveBeenCalledOnce();
rerender(<LiveVoiceButton {...activeProps} phase="connecting" />);
expect(container.querySelector('.animate-spin')).toBeInTheDocument();
screen.getByTestId('live-voice-button').click();
expect(onStop).toHaveBeenCalledOnce();
rerender(<LiveVoiceButton {...activeProps} phase="live" />);
screen.getByTestId('live-voice-button').click();
expect(onStop).toHaveBeenCalledTimes(2);
});
it('disables the stopping state', () => {
renderButton({ availability: null, phase: 'stopping' });
expect(screen.getByTestId('live-voice-button')).toBeDisabled();
});
it('toggles microphone mute while live', () => {
const onToggleMute = vi.fn();
const liveProps = { availability: null, phase: 'live', onToggleMute } as const;
const { rerender } = renderButton(liveProps);
screen.getByTestId('live-voice-mute-button').click();
expect(onToggleMute).toHaveBeenCalledOnce();
expect(screen.getByTestId('live-voice-mute-button')).toHaveAccessibleName('Mute microphone');
rerender(<LiveVoiceButton {...baseProps} {...liveProps} muted />);
expect(screen.getByTestId('live-voice-mute-button')).toHaveAccessibleName('Unmute microphone');
});
});

View file

@ -0,0 +1,145 @@
import type { LiveVoiceAvailabilityResponse_unstable } from '@aaif/goose-acp-client';
import { AudioLines, LoaderCircle, Mic, MicOff, Square } from 'lucide-react';
import { defineMessages, useIntl } from '../i18n';
import { cn } from '../utils';
import { Button } from './ui/button';
import { Tooltip, TooltipContent, TooltipTrigger } from './ui/Tooltip';
import { isLiveVoiceActive, type LiveVoicePhase } from '../liveVoice/useLiveVoice';
const i18n = defineMessages({
emptyComposerRequired: {
id: 'liveVoice.emptyComposerRequired',
defaultMessage: 'Clear the message and attachments to use Live voice',
},
connecting: {
id: 'liveVoice.connecting',
defaultMessage: 'Connecting Live voice',
},
live: {
id: 'liveVoice.live',
defaultMessage: 'Stop Live voice',
},
stopping: {
id: 'liveVoice.stopping',
defaultMessage: 'Stopping Live voice',
},
error: {
id: 'liveVoice.error',
defaultMessage: 'Live voice failed. Try again',
},
mute: {
id: 'liveVoice.mute',
defaultMessage: 'Mute microphone',
},
unmute: {
id: 'liveVoice.unmute',
defaultMessage: 'Unmute microphone',
},
returnToActive: {
id: 'liveVoice.returnToActive',
defaultMessage: 'Return to active Live voice',
},
});
interface LiveVoiceButtonProps {
availability: LiveVoiceAvailabilityResponse_unstable | null;
composerEmpty: boolean;
phase: LiveVoicePhase;
muted: boolean;
activeInAnotherSession: boolean;
onStart: () => void;
onStop: () => void;
onToggleMute: () => void;
}
export function LiveVoiceButton({
availability,
composerEmpty,
phase,
muted,
activeInAnotherSession,
onStart,
onStop,
onToggleMute,
}: LiveVoiceButtonProps) {
const intl = useIntl();
if (availability === null && !isLiveVoiceActive(phase) && !activeInAnotherSession) return null;
const eligible = activeInAnotherSession || (availability?.status === 'ready' && composerEmpty);
const stopping = phase === 'stopping';
const canStart = phase === 'idle' || phase === 'error';
const canStop = phase === 'connecting' || phase === 'live';
let label = availability?.message;
if (activeInAnotherSession) {
label = intl.formatMessage(i18n.returnToActive);
} else if (phase === 'connecting') {
label = intl.formatMessage(i18n.connecting);
} else if (phase === 'live') {
label = intl.formatMessage(i18n.live);
} else if (phase === 'stopping') {
label = intl.formatMessage(i18n.stopping);
} else if (!composerEmpty) {
label = intl.formatMessage(i18n.emptyComposerRequired);
} else if (availability?.status === 'ready' && phase === 'error') {
label = intl.formatMessage(i18n.error);
}
const disabled = activeInAnotherSession ? false : canStart ? !eligible : stopping;
return (
<>
{phase === 'live' && (
<Tooltip>
<TooltipTrigger asChild>
<Button
type="button"
variant="ghost"
size="sm"
shape="round"
onClick={onToggleMute}
aria-label={intl.formatMessage(muted ? i18n.unmute : i18n.mute)}
data-testid="live-voice-mute-button"
aria-pressed={muted}
>
{muted ? <MicOff className="w-4 h-4" /> : <Mic className="w-4 h-4" />}
</Button>
</TooltipTrigger>
<TooltipContent>{intl.formatMessage(muted ? i18n.unmute : i18n.mute)}</TooltipContent>
</Tooltip>
)}
<Tooltip>
<TooltipTrigger asChild>
<span>
<Button
type="button"
variant="ghost"
size="sm"
shape="round"
disabled={disabled}
onClick={activeInAnotherSession ? onStart : canStop ? onStop : onStart}
aria-label={label}
data-testid="live-voice-button"
data-eligible={eligible}
data-phase={phase}
className={cn(
'transition-colors',
canStop && 'text-red-500 hover:text-red-600 cursor-pointer',
phase === 'error' && 'text-red-500 hover:text-red-600 cursor-pointer',
phase === 'idle' && eligible && 'text-text-primary/70 cursor-pointer',
disabled && 'text-text-secondary opacity-50'
)}
>
{phase === 'connecting' || stopping ? (
<LoaderCircle className="w-4 h-4 animate-spin" />
) : canStop ? (
<Square className="w-4 h-4" />
) : (
<AudioLines className="w-4 h-4" />
)}
</Button>
</span>
</TooltipTrigger>
<TooltipContent>{label}</TooltipContent>
</Tooltip>
</>
);
}

View file

@ -3,6 +3,7 @@ import { DictationSettings } from '../dictation/DictationSettings';
import { SecurityToggle } from '../security/SecurityToggle';
import { ResponseStylesSection } from '../response_styles/ResponseStylesSection';
import { GoosehintsSection } from './GoosehintsSection';
import { LiveVoiceSettings } from './LiveVoiceSettings';
import { SpellcheckToggle } from './SpellcheckToggle';
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '../../ui/card';
import { defineMessages, useIntl } from '../../../i18n';
@ -50,6 +51,7 @@ export default function ChatSettingsSection() {
<Card className="pb-2 rounded-lg">
<CardContent className="px-2">
<LiveVoiceSettings />
<DictationSettings />
<SpellcheckToggle />
</CardContent>

View file

@ -0,0 +1,35 @@
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { IntlTestWrapper } from '../../../i18n/test-utils';
import { useConfig } from '../../ConfigContext';
import { LiveVoiceSettings } from './LiveVoiceSettings';
vi.mock('../../ConfigContext', () => ({ useConfig: vi.fn() }));
describe('LiveVoiceSettings', () => {
const read = vi.fn();
const upsert = vi.fn();
beforeEach(() => {
vi.clearAllMocks();
read.mockResolvedValue(true);
vi.mocked(useConfig).mockReturnValue({ read, upsert } as unknown as ReturnType<
typeof useConfig
>);
});
it('loads and persists the Live voice toggle', async () => {
render(<LiveVoiceSettings />, { wrapper: IntlTestWrapper });
const toggle = screen.getByRole('switch', { name: 'Live voice' });
await waitFor(() => expect(toggle).toBeChecked());
expect(read).toHaveBeenCalledWith('GOOSE_LIVE_VOICE_ENABLED', false);
fireEvent.click(toggle);
await waitFor(() =>
expect(upsert).toHaveBeenCalledWith('GOOSE_LIVE_VOICE_ENABLED', false, false)
);
await waitFor(() => expect(toggle).not.toBeChecked());
});
});

View file

@ -0,0 +1,52 @@
import { useEffect, useState } from 'react';
import { useConfig } from '../../ConfigContext';
import { Switch } from '../../ui/switch';
import { defineMessages, useIntl } from '../../../i18n';
const LIVE_VOICE_ENABLED_CONFIG_KEY = 'GOOSE_LIVE_VOICE_ENABLED';
const i18n = defineMessages({
title: {
id: 'liveVoiceSettings.title',
defaultMessage: 'Live voice',
},
description: {
id: 'liveVoiceSettings.description',
defaultMessage: 'Have a real-time voice conversation with Goose',
},
});
export const LiveVoiceSettings = () => {
const intl = useIntl();
const { read, upsert } = useConfig();
const [enabled, setEnabled] = useState<boolean | null>(null);
useEffect(() => {
void read(LIVE_VOICE_ENABLED_CONFIG_KEY, false).then((value) => {
setEnabled(value === true || value === 1 || value === '1' || value === 'true');
});
}, [read]);
const handleToggle = async (checked: boolean) => {
await upsert(LIVE_VOICE_ENABLED_CONFIG_KEY, checked, false);
setEnabled(checked);
};
return (
<div className="flex items-center justify-between py-2 px-2 hover:bg-background-secondary rounded-lg transition-all">
<div>
<h3 className="text-text-primary">{intl.formatMessage(i18n.title)}</h3>
<p className="text-xs text-text-secondary max-w-md mt-[2px]">
{intl.formatMessage(i18n.description)}
</p>
</div>
<Switch
checked={enabled ?? false}
disabled={enabled === null}
onCheckedChange={handleToggle}
variant="mono"
aria-label={intl.formatMessage(i18n.title)}
/>
</div>
);
};

View file

@ -68,6 +68,7 @@ export function useChatSession({
const sessionLoadError = acpSnapshot?.sessionLoadError;
const tokenState = acpSnapshot?.tokenState ?? initialTokenState;
const queueProcessingBlocked = acpSnapshot?.pendingCancelPromptAttemptId != null;
const hasActiveRun = acpSnapshot?.activeRunId != null;
const snapshotRef = useRef(acpSnapshot);
snapshotRef.current = acpSnapshot;
@ -354,6 +355,7 @@ export function useChatSession({
notifications: notificationsMap,
pauseQueueOnStop: false,
queueProcessingBlocked,
hasActiveRun,
onMessageUpdate,
};
}

View file

@ -28,6 +28,7 @@ export interface UseChatSessionResult {
notifications: Map<string, NotificationEvent[]>;
pauseQueueOnStop: boolean;
queueProcessingBlocked: boolean;
hasActiveRun: boolean;
onMessageUpdate: (
messageId: string,
newContent: string,

View file

@ -1733,6 +1733,36 @@
"launcher.placeholder": {
"defaultMessage": "Frag goose irgendetwas..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Führe ein Echtzeit-Sprachgespräch mit Goose"
},
"liveVoiceSettings.title": {
"defaultMessage": "Live-Sprachmodus"
},
"loadingGoose.compacting": {
"defaultMessage": "goose komprimiert die Unterhaltung..."
},

View file

@ -1754,6 +1754,36 @@
"launcher.placeholder": {
"defaultMessage": "Ask goose anything..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Have a real-time voice conversation with Goose"
},
"liveVoiceSettings.title": {
"defaultMessage": "Live voice"
},
"loadingGoose.compacting": {
"defaultMessage": "goose is compacting the conversation..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "No se pudo actualizar el directorio de trabajo"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "Pregúntale lo que sea a goose..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Mantén una conversación de voz en tiempo real con Goose"
},
"liveVoiceSettings.title": {
"defaultMessage": "Voz en directo"
},
"loadingGoose.compacting": {
"defaultMessage": "goose está compactando la conversación..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "Échec de la mise à jour du répertoire de travail"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "Demandez n'importe quoi à goose..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Discutez avec Goose en temps réel par la voix"
},
"liveVoiceSettings.title": {
"defaultMessage": "Voix en direct"
},
"loadingGoose.compacting": {
"defaultMessage": "goose compacte la conversation..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "कार्यशील निर्देशिका अद्यतन करने में विफल"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "goose से कुछ भी पूछें..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Goose के साथ रीयल-टाइम वॉइस बातचीत करें"
},
"liveVoiceSettings.title": {
"defaultMessage": "लाइव वॉइस"
},
"loadingGoose.compacting": {
"defaultMessage": "goose बातचीत को संक्षिप्त कर रहा है..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "Gagal memperbarui direktori kerja"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "Tanyakan apa saja kepada goose..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Lakukan percakapan suara real-time dengan Goose"
},
"liveVoiceSettings.title": {
"defaultMessage": "Suara langsung"
},
"loadingGoose.compacting": {
"defaultMessage": "goose sedang meringkas percakapan..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "Impossibile aggiornare la cartella di lavoro"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "Chiedi qualsiasi cosa a goose..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Parla con Goose in tempo reale"
},
"liveVoiceSettings.title": {
"defaultMessage": "Voce in tempo reale"
},
"loadingGoose.compacting": {
"defaultMessage": "goose sta compattando la conversazione..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "作業ディレクトリの更新に失敗しました"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "gooseに何でも聞いて..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Goose とリアルタイムで音声会話します"
},
"liveVoiceSettings.title": {
"defaultMessage": "ライブ音声"
},
"loadingGoose.compacting": {
"defaultMessage": "gooseが会話を圧縮しています..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "작업 디렉터리를 업데이트하지 못했습니다."
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "goose에게 무엇이든 물어보세요..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Goose와 실시간으로 음성 대화"
},
"liveVoiceSettings.title": {
"defaultMessage": "실시간 음성"
},
"loadingGoose.compacting": {
"defaultMessage": "goose가 대화를 압축하고 있어요..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "Gagal mengemas kini direktori kerja"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "Tanya goose apa sahaja..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Adakan perbualan suara masa nyata dengan Goose"
},
"liveVoiceSettings.title": {
"defaultMessage": "Suara langsung"
},
"loadingGoose.compacting": {
"defaultMessage": "goose sedang memadatkan perbualan..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "Falha ao atualizar o diretório de trabalho"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "Pergunte qualquer coisa ao goose..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Tenha uma conversa por voz em tempo real com o Goose"
},
"liveVoiceSettings.title": {
"defaultMessage": "Voz em tempo real"
},
"loadingGoose.compacting": {
"defaultMessage": "o goose está compactando a conversa..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "Не удалось обновить рабочий каталог"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "Спросите goose о чем угодно..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Общайтесь с Goose голосом в реальном времени"
},
"liveVoiceSettings.title": {
"defaultMessage": "Голосовой режим"
},
"loadingGoose.compacting": {
"defaultMessage": "goose сжимает разговор..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "Çalışma dizini güncellenemedi"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "goose'a herhangi bir şey sorun..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Goose ile gerçek zamanlı sesli sohbet edin"
},
"liveVoiceSettings.title": {
"defaultMessage": "Canlı ses"
},
"loadingGoose.compacting": {
"defaultMessage": "goose konuşmayı sıkıştırıyor..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "Không thể cập nhật thư mục làm việc"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "Hỏi goose bất cứ điều gì..."
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "Trò chuyện bằng giọng nói theo thời gian thực với Goose"
},
"liveVoiceSettings.title": {
"defaultMessage": "Giọng nói trực tiếp"
},
"loadingGoose.compacting": {
"defaultMessage": "goose đang nén cuộc trò chuyện..."
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "更新工作目录失败"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "问 goose 任何问题…"
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "与 Goose 进行实时语音对话"
},
"liveVoiceSettings.title": {
"defaultMessage": "实时语音"
},
"loadingGoose.compacting": {
"defaultMessage": "goose 正在压缩对话…"
},

View file

@ -929,6 +929,10 @@
"dirSwitcher.enterPathInvalid": {
"defaultMessage": "Working directory must be an absolute path"
},
"dirSwitcher.failedToUpdateWorkingDir": {
"defaultMessage": "無法更新工作目錄"
},
@ -1733,6 +1737,36 @@
"launcher.placeholder": {
"defaultMessage": "向 goose 詢問任何問題…"
},
"liveVoice.connecting": {
"defaultMessage": "Connecting Live voice"
},
"liveVoice.emptyComposerRequired": {
"defaultMessage": "Clear the message and attachments to use Live voice"
},
"liveVoice.error": {
"defaultMessage": "Live voice failed. Try again"
},
"liveVoice.live": {
"defaultMessage": "Stop Live voice"
},
"liveVoice.mute": {
"defaultMessage": "Mute microphone"
},
"liveVoice.returnToActive": {
"defaultMessage": "Return to active Live voice"
},
"liveVoice.stopping": {
"defaultMessage": "Stopping Live voice"
},
"liveVoice.unmute": {
"defaultMessage": "Unmute microphone"
},
"liveVoiceSettings.description": {
"defaultMessage": "與 Goose 進行即時語音對話"
},
"liveVoiceSettings.title": {
"defaultMessage": "即時語音"
},
"loadingGoose.compacting": {
"defaultMessage": "goose 正在壓縮對話…"
},

View file

@ -0,0 +1,191 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { LiveVoiceMediaSession } from './LiveVoiceMediaSession';
const createElement = document.createElement.bind(document);
class FakeTrack extends EventTarget {
enabled = true;
stop = vi.fn();
}
class FakeStream {
constructor(private readonly tracks: FakeTrack[]) {}
getTracks = () => this.tracks;
getAudioTracks = () => this.tracks;
}
class FakeDataChannel extends EventTarget {
readyState: RTCDataChannelState = 'open';
close = vi.fn();
}
class FakePeerConnection extends EventTarget {
iceGatheringState: RTCIceGatheringState = 'complete';
connectionState: RTCPeerConnectionState = 'connected';
localDescription: RTCSessionDescriptionInit | null = null;
ontrack: ((event: RTCTrackEvent) => void) | null = null;
dataChannel = new FakeDataChannel();
sessionEvent = { type: 'session.started' };
close = vi.fn();
addTrack = vi.fn();
createDataChannel = vi.fn(() => this.dataChannel as unknown as RTCDataChannel);
createOffer = vi.fn(async () => ({ type: 'offer' as const, sdp: 'bounded-offer' }));
setLocalDescription = vi.fn(async (description: RTCSessionDescriptionInit) => {
this.localDescription = description;
});
setRemoteDescription = vi.fn(async () => {
this.ontrack?.({
streams: [new FakeStream([new FakeTrack()])],
track: new FakeTrack(),
} as unknown as RTCTrackEvent);
this.dataChannel.dispatchEvent(
new MessageEvent('message', { data: JSON.stringify(this.sessionEvent) })
);
});
}
describe('LiveVoiceMediaSession', () => {
let localTrack: FakeTrack;
let peerConnection: FakePeerConnection;
let play: ReturnType<typeof vi.fn<() => Promise<void>>>;
beforeEach(() => {
localTrack = new FakeTrack();
peerConnection = new FakePeerConnection();
play = vi.fn<() => Promise<void>>().mockResolvedValue(undefined);
vi.stubGlobal('MediaStream', FakeStream);
vi.stubGlobal(
'RTCPeerConnection',
vi.fn(function RTCPeerConnectionMock() {
return peerConnection;
})
);
Object.defineProperty(navigator, 'mediaDevices', {
configurable: true,
value: {
getUserMedia: vi.fn(async () => new FakeStream([localTrack])),
},
});
vi.spyOn(document, 'createElement').mockImplementation(((tagName: string) => {
const element = createElement(tagName);
if (tagName === 'audio') {
vi.spyOn(element as HTMLAudioElement, 'play').mockImplementation(play);
vi.spyOn(element as HTMLAudioElement, 'pause').mockImplementation(() => undefined);
}
return element;
}) as typeof document.createElement);
});
it('keeps the microphone disabled until media is connected, then tears down once', async () => {
const media = new LiveVoiceMediaSession(vi.fn());
await expect(media.createOffer()).resolves.toBe('bounded-offer');
expect(localTrack.enabled).toBe(false);
expect(peerConnection.createDataChannel).toHaveBeenCalledWith('oai-events');
await media.applyAnswer('bounded-answer');
media.setMuted(false);
expect(localTrack.enabled).toBe(true);
media.setMuted(true);
media.setMuted(false);
media.setMuted(true);
expect(localTrack.enabled).toBe(false);
media.teardown();
media.teardown();
expect(localTrack.stop).toHaveBeenCalledOnce();
expect(peerConnection.dataChannel.close).toHaveBeenCalledOnce();
expect(peerConnection.close).toHaveBeenCalledOnce();
});
it('returns a product-safe error when microphone acquisition fails', async () => {
vi.mocked(navigator.mediaDevices.getUserMedia).mockRejectedValueOnce(
new Error('native device identifier')
);
await expect(new LiveVoiceMediaSession(vi.fn()).createOffer()).rejects.toThrow(
'Microphone or media setup failed'
);
});
it('rejects setup when the data channel reports a startup error', async () => {
const media = new LiveVoiceMediaSession(vi.fn());
peerConnection.sessionEvent = { type: 'error' };
await media.createOffer();
await expect(media.applyAnswer('bounded-answer')).rejects.toThrow(
'Live voice could not connect media'
);
expect(localTrack.stop).toHaveBeenCalledOnce();
});
it('rejects setup when audio playback cannot start', async () => {
const media = new LiveVoiceMediaSession(vi.fn());
play.mockRejectedValueOnce(new Error('autoplay blocked'));
await media.createOffer();
await expect(media.applyAnswer('bounded-answer')).rejects.toThrow(
'Live voice could not connect media'
);
expect(localTrack.stop).toHaveBeenCalledOnce();
});
it('stops a microphone stream acquired after teardown', async () => {
let provideStream!: (stream: MediaStream) => void;
vi.mocked(navigator.mediaDevices.getUserMedia).mockReturnValueOnce(
new Promise((resolve) => {
provideStream = resolve;
})
);
const lateTrack = new FakeTrack();
const media = new LiveVoiceMediaSession(vi.fn());
const offer = media.createOffer();
media.teardown();
provideStream(new FakeStream([lateTrack]) as unknown as MediaStream);
await expect(offer).rejects.toThrow('Microphone or media setup failed');
expect(lateTrack.stop).toHaveBeenCalledOnce();
expect(peerConnection.createOffer).not.toHaveBeenCalled();
});
it('reports terminal microphone failure after media connects', async () => {
const onMediaFailure = vi.fn();
const media = new LiveVoiceMediaSession(onMediaFailure);
await media.createOffer();
await media.applyAnswer('bounded-answer');
localTrack.dispatchEvent(new Event('ended'));
expect(onMediaFailure).toHaveBeenCalledOnce();
});
it('reports terminal peer failure after media connects', async () => {
const onMediaFailure = vi.fn();
const media = new LiveVoiceMediaSession(onMediaFailure);
await media.createOffer();
await media.applyAnswer('bounded-answer');
peerConnection.connectionState = 'failed';
peerConnection.dispatchEvent(new Event('connectionstatechange'));
expect(onMediaFailure).toHaveBeenCalledOnce();
});
it('does not report its own teardown as a media failure', async () => {
const onMediaFailure = vi.fn();
const media = new LiveVoiceMediaSession(onMediaFailure);
await media.createOffer();
await media.applyAnswer('bounded-answer');
media.teardown();
localTrack.dispatchEvent(new Event('ended'));
peerConnection.connectionState = 'failed';
peerConnection.dispatchEvent(new Event('connectionstatechange'));
expect(onMediaFailure).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,275 @@
const MEDIA_SETUP_TIMEOUT_MS = 15_000;
export class LiveVoiceMediaSession {
private peerConnection: RTCPeerConnection | null = null;
private localStream: MediaStream | null = null;
private remoteStream: MediaStream | null = null;
private dataChannel: RTCDataChannel | null = null;
private audioElement: HTMLAudioElement | null = null;
private playbackStarted: Promise<void> | null = null;
private tornDown = false;
constructor(private readonly onMediaFailure: () => void) {}
async createOffer(): Promise<string> {
try {
const localStream = await navigator.mediaDevices.getUserMedia({ audio: true });
if (this.tornDown) {
for (const track of localStream.getTracks()) {
track.enabled = false;
track.stop();
}
throw new Error('Live voice media is unavailable');
}
this.localStream = localStream;
for (const track of localStream.getAudioTracks()) {
track.enabled = false;
}
const peerConnection = new RTCPeerConnection();
this.peerConnection = peerConnection;
this.audioElement = document.createElement('audio');
this.audioElement.autoplay = true;
this.dataChannel = peerConnection.createDataChannel('oai-events');
peerConnection.ontrack = (event) => {
const stream = event.streams[0] ?? new MediaStream([event.track]);
this.remoteStream = stream;
if (this.audioElement) {
this.audioElement.srcObject = stream;
this.playbackStarted = this.audioElement.play();
void this.playbackStarted.catch(() => undefined);
}
};
for (const track of localStream.getTracks()) {
peerConnection.addTrack(track, localStream);
}
const offer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(offer);
await waitForIceGathering(peerConnection);
const offerSdp = peerConnection.localDescription?.sdp;
if (!offerSdp) {
throw new Error('Live voice could not create a media offer');
}
return offerSdp;
} catch {
this.teardown();
throw new Error('Microphone or media setup failed');
}
}
async applyAnswer(answerSdp: string): Promise<void> {
const peerConnection = this.peerConnection;
const dataChannel = this.dataChannel;
if (!peerConnection || !dataChannel || this.tornDown) {
throw new Error('Live voice media is unavailable');
}
try {
const sessionStarted = waitForSessionStarted(dataChannel);
try {
await peerConnection.setRemoteDescription({ type: 'answer', sdp: answerSdp });
await Promise.all([
waitForPeerConnection(peerConnection),
waitForDataChannel(dataChannel),
waitForRemoteTrack(this),
sessionStarted.promise,
]);
await this.playbackStarted;
this.observeMediaFailures(peerConnection);
} finally {
sessionStarted.cancel();
}
} catch {
this.teardown();
throw new Error('Live voice could not connect media');
}
}
setMuted(muted: boolean): void {
if (this.tornDown) return;
for (const track of this.localStream?.getAudioTracks() ?? []) {
track.enabled = !muted;
}
}
sendCommentary(text: string): void {
const content = text.trim();
if (this.tornDown || !content || this.dataChannel?.readyState !== 'open') return;
try {
this.dataChannel.send(
JSON.stringify({
type: 'session.commentary.append',
event_id: `event_${globalThis.crypto.randomUUID()}`,
delegation_id: null,
content,
})
);
} catch {
// Greeting failure must not end an otherwise healthy Live voice interaction.
}
}
hasRemoteTrack(): boolean {
return this.remoteStream !== null;
}
teardown(): void {
if (this.tornDown) return;
this.tornDown = true;
for (const track of this.localStream?.getTracks() ?? []) {
track.enabled = false;
track.stop();
}
for (const track of this.remoteStream?.getTracks() ?? []) {
track.stop();
}
this.dataChannel?.close();
this.peerConnection?.close();
if (this.audioElement) {
this.audioElement.pause();
this.audioElement.srcObject = null;
}
this.localStream = null;
this.remoteStream = null;
this.dataChannel = null;
this.peerConnection = null;
this.audioElement = null;
this.playbackStarted = null;
}
private observeMediaFailures(peerConnection: RTCPeerConnection): void {
for (const track of this.localStream?.getAudioTracks() ?? []) {
track.addEventListener('ended', this.reportMediaFailure, { once: true });
}
peerConnection.addEventListener('connectionstatechange', () => {
if (peerConnection.connectionState === 'failed') {
this.reportMediaFailure();
}
});
}
private reportMediaFailure = (): void => {
if (this.tornDown) return;
this.onMediaFailure();
};
}
function waitForIceGathering(peerConnection: RTCPeerConnection): Promise<void> {
if (peerConnection.iceGatheringState === 'complete') return Promise.resolve();
return waitForEvent(
peerConnection,
'icegatheringstatechange',
() => peerConnection.iceGatheringState === 'complete'
);
}
function waitForPeerConnection(peerConnection: RTCPeerConnection): Promise<void> {
if (peerConnection.connectionState === 'connected') return Promise.resolve();
return waitForEvent(peerConnection, 'connectionstatechange', () => {
if (
peerConnection.connectionState === 'failed' ||
peerConnection.connectionState === 'closed'
) {
throw new Error('Peer connection failed');
}
return peerConnection.connectionState === 'connected';
});
}
function waitForDataChannel(dataChannel: RTCDataChannel): Promise<void> {
if (dataChannel.readyState === 'open') return Promise.resolve();
return waitForEvent(dataChannel, 'open', () => true);
}
function waitForSessionStarted(dataChannel: RTCDataChannel): {
promise: Promise<void>;
cancel: () => void;
} {
let cancel: () => void = () => undefined;
const promise = new Promise<void>((resolve, reject) => {
const timeoutId = setTimeout(
() => finish(new Error('Live session startup timed out')),
MEDIA_SETUP_TIMEOUT_MS
);
const listener = (event: MessageEvent) => {
if (typeof event.data !== 'string') return;
let message: { type?: unknown };
try {
message = JSON.parse(event.data) as { type?: unknown };
} catch {
return;
}
if (message.type === 'session.started') {
finish();
} else if (message.type === 'error') {
finish(new Error('Live session failed to start'));
}
};
const finish = (error?: Error) => {
clearTimeout(timeoutId);
dataChannel.removeEventListener('message', listener);
if (error) {
reject(error);
} else {
resolve();
}
};
cancel = finish;
dataChannel.addEventListener('message', listener);
});
return { promise, cancel };
}
function waitForRemoteTrack(media: LiveVoiceMediaSession): Promise<void> {
if (media.hasRemoteTrack()) return Promise.resolve();
return pollUntil(() => media.hasRemoteTrack());
}
function waitForEvent(
target: EventTarget,
eventName: string,
isReady: () => boolean
): Promise<void> {
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(
() => finish(new Error('Media setup timed out')),
MEDIA_SETUP_TIMEOUT_MS
);
const listener = () => {
try {
if (isReady()) finish();
} catch (error) {
finish(error instanceof Error ? error : new Error('Media setup failed'));
}
};
const finish = (error?: Error) => {
clearTimeout(timeoutId);
target.removeEventListener(eventName, listener);
if (error) {
reject(error);
} else {
resolve();
}
};
target.addEventListener(eventName, listener);
});
}
function pollUntil(isReady: () => boolean): Promise<void> {
return new Promise((resolve, reject) => {
const startedAt = Date.now();
const poll = () => {
if (isReady()) {
resolve();
} else if (Date.now() - startedAt >= MEDIA_SETUP_TIMEOUT_MS) {
reject(new Error('Media setup timed out'));
} else {
setTimeout(poll, 25);
}
};
poll();
});
}

View file

@ -0,0 +1,317 @@
import { act, renderHook, waitFor } from '@testing-library/react';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { isAcpRecovering, subscribeToAcpRecovery } from '../acp/acpConnection';
import { acpStartLiveVoice, acpStopLiveVoice } from '../acp/liveVoice';
import { publishLiveVoiceInteractionEnded } from '../acp/liveVoiceNotifications';
import { LiveVoiceMediaSession } from './LiveVoiceMediaSession';
import { useLiveVoice } from './useLiveVoice';
vi.mock('../acp/liveVoice', () => ({
acpStartLiveVoice: vi.fn(),
acpStopLiveVoice: vi.fn(),
}));
vi.mock('../acp/acpConnection', () => ({
isAcpRecovering: vi.fn(() => false),
subscribeToAcpRecovery: vi.fn(),
}));
vi.mock('./LiveVoiceMediaSession', () => ({
LiveVoiceMediaSession: vi.fn(),
}));
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>((done) => {
resolve = done;
});
return { promise, resolve };
}
describe('useLiveVoice', () => {
let mediaFailure: () => void;
let recoveryChanged: (recovering: boolean) => void;
const media = {
createOffer: vi.fn(),
applyAnswer: vi.fn(),
setMuted: vi.fn(),
teardown: vi.fn(),
};
beforeEach(() => {
vi.clearAllMocks();
media.createOffer.mockResolvedValue('offer');
media.applyAnswer.mockResolvedValue(undefined);
vi.mocked(isAcpRecovering).mockReturnValue(false);
vi.mocked(subscribeToAcpRecovery).mockImplementation((listener) => {
recoveryChanged = listener;
return () => undefined;
});
vi.mocked(LiveVoiceMediaSession).mockImplementation(
function LiveVoiceMediaSessionMock(onFailure) {
mediaFailure = onFailure;
return media as unknown as LiveVoiceMediaSession;
}
);
vi.mocked(acpStartLiveVoice).mockResolvedValue({
interactionId: 'live-opaque',
answerSdp: 'answer',
});
vi.mocked(acpStopLiveVoice).mockResolvedValue(undefined);
});
it('starts media from the start response and stops the same connection', async () => {
const { result } = renderHook(() => useLiveVoice());
await act(async () => result.current.start('main-session'));
expect(media.applyAnswer).toHaveBeenCalledWith('answer');
expect(media.setMuted).toHaveBeenCalledWith(false);
expect(result.current.phase).toBe('live');
expect(result.current.activeSessionId).toBe('main-session');
await act(async () => result.current.stop());
expect(media.teardown).toHaveBeenCalledOnce();
expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'live-opaque');
expect(result.current.phase).toBe('idle');
expect(result.current.activeSessionId).toBeNull();
});
it('blocks retry until the server interaction stops after media setup fails', async () => {
const cleanup = deferred<void>();
media.applyAnswer.mockRejectedValueOnce(new Error('media failed'));
vi.mocked(acpStopLiveVoice).mockReturnValueOnce(cleanup.promise);
const { result } = renderHook(() => useLiveVoice());
let startPromise!: Promise<void>;
act(() => {
startPromise = result.current.start('main-session');
});
await waitFor(() => expect(acpStopLiveVoice).toHaveBeenCalledOnce());
expect(media.teardown).toHaveBeenCalledOnce();
expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'live-opaque');
expect(result.current.phase).toBe('stopping');
await act(async () => result.current.start('main-session'));
expect(acpStartLiveVoice).toHaveBeenCalledOnce();
await act(async () => {
cleanup.resolve();
await startPromise;
});
expect(result.current.phase).toBe('error');
});
it('tears down and stops an active interaction when unmounted', async () => {
const { result, unmount } = renderHook(() => useLiveVoice());
await act(async () => result.current.start('main-session'));
unmount();
expect(media.teardown).toHaveBeenCalledOnce();
expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'live-opaque');
});
it('keeps one interaction when another session tries to start', async () => {
const { result } = renderHook(() => useLiveVoice());
await act(async () => result.current.start('main-session'));
await act(async () => result.current.start('other-session'));
expect(result.current.activeSessionId).toBe('main-session');
expect(media.teardown).not.toHaveBeenCalled();
expect(acpStopLiveVoice).not.toHaveBeenCalled();
expect(LiveVoiceMediaSession).toHaveBeenCalledOnce();
expect(acpStartLiveVoice).toHaveBeenCalledOnce();
});
it.each(['unmount', 'stop'] as const)(
'closes an interaction that finishes after %s while connecting',
async (action) => {
const pending = deferred<{ interactionId: string; answerSdp: string }>();
vi.mocked(acpStartLiveVoice).mockReturnValueOnce(pending.promise);
const { result, unmount } = renderHook(() => useLiveVoice());
act(() => {
void result.current.start('main-session');
});
await waitFor(() => expect(acpStartLiveVoice).toHaveBeenCalledOnce());
if (action === 'unmount') {
unmount();
} else {
await act(async () => result.current.stop());
expect(result.current.phase).toBe('stopping');
}
await act(async () => {
pending.resolve({ interactionId: 'late-interaction', answerSdp: 'late-answer' });
});
expect(media.teardown).toHaveBeenCalledOnce();
expect(media.applyAnswer).not.toHaveBeenCalled();
expect(media.setMuted).not.toHaveBeenCalled();
expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'late-interaction');
if (action === 'stop') expect(result.current.phase).toBe('idle');
}
);
it('does not become live after stopping during answer setup', async () => {
const pending = deferred<void>();
media.applyAnswer.mockReturnValueOnce(pending.promise);
const { result } = renderHook(() => useLiveVoice());
act(() => {
void result.current.start('main-session');
});
await waitFor(() => expect(media.applyAnswer).toHaveBeenCalledOnce());
await act(async () => result.current.stop());
await act(async () => pending.resolve(undefined));
expect(media.setMuted).not.toHaveBeenCalled();
expect(result.current.phase).toBe('idle');
});
it('applies rapid mute changes to the current media session', async () => {
const { result } = renderHook(() => useLiveVoice());
await act(async () => result.current.start('main-session'));
act(() => {
result.current.toggleMute();
result.current.toggleMute();
result.current.toggleMute();
});
expect(media.setMuted.mock.calls).toEqual([[false], [true], [false], [true]]);
expect(result.current.muted).toBe(true);
});
it('cleans up when the backend reports that the current interaction failed', async () => {
const { result } = renderHook(() => useLiveVoice());
await act(async () => result.current.start('main-session'));
act(() => {
publishLiveVoiceInteractionEnded({
sessionId: 'main-session',
update: {
sessionUpdate: 'live_voice_interaction_ended',
interactionId: 'live-opaque',
outcome: 'failed',
},
});
publishLiveVoiceInteractionEnded({
sessionId: 'main-session',
update: {
sessionUpdate: 'live_voice_interaction_ended',
interactionId: 'live-opaque',
outcome: 'failed',
},
});
});
expect(media.teardown).toHaveBeenCalledOnce();
expect(acpStopLiveVoice).not.toHaveBeenCalled();
expect(result.current.phase).toBe('error');
});
it('ignores stale session and interaction endings', async () => {
const { result } = renderHook(() => useLiveVoice());
await act(async () => result.current.start('main-session'));
act(() => {
publishLiveVoiceInteractionEnded({
sessionId: 'other-session',
update: {
sessionUpdate: 'live_voice_interaction_ended',
interactionId: 'live-opaque',
outcome: 'failed',
},
});
publishLiveVoiceInteractionEnded({
sessionId: 'main-session',
update: {
sessionUpdate: 'live_voice_interaction_ended',
interactionId: 'old-interaction',
outcome: 'failed',
},
});
});
expect(media.teardown).not.toHaveBeenCalled();
expect(result.current.phase).toBe('live');
});
it('reconciles a terminal update that arrives before the start response', async () => {
const pending = deferred<{ interactionId: string; answerSdp: string }>();
vi.mocked(acpStartLiveVoice).mockReturnValueOnce(pending.promise);
const { result } = renderHook(() => useLiveVoice());
act(() => {
void result.current.start('main-session');
});
await waitFor(() => expect(acpStartLiveVoice).toHaveBeenCalledOnce());
act(() => {
publishLiveVoiceInteractionEnded({
sessionId: 'main-session',
update: {
sessionUpdate: 'live_voice_interaction_ended',
interactionId: 'early-interaction',
outcome: 'failed',
},
});
});
await act(async () =>
pending.resolve({ interactionId: 'early-interaction', answerSdp: 'answer' })
);
expect(media.applyAnswer).not.toHaveBeenCalled();
expect(media.teardown).toHaveBeenCalledOnce();
expect(result.current.phase).toBe('error');
});
it('cleans up locally without stopping through a recovering ACP connection', async () => {
const { result } = renderHook(() => useLiveVoice());
await act(async () => result.current.start('main-session'));
act(() => recoveryChanged(true));
expect(media.teardown).toHaveBeenCalledOnce();
expect(acpStopLiveVoice).not.toHaveBeenCalled();
expect(result.current.phase).toBe('idle');
});
it('does not stop a late interaction response through a recovered ACP connection', async () => {
const pending = deferred<{ interactionId: string; answerSdp: string }>();
vi.mocked(acpStartLiveVoice).mockReturnValueOnce(pending.promise);
const { result } = renderHook(() => useLiveVoice());
act(() => {
void result.current.start('main-session');
});
await waitFor(() => expect(acpStartLiveVoice).toHaveBeenCalledOnce());
act(() => recoveryChanged(true));
await act(async () =>
pending.resolve({ interactionId: 'old-interaction', answerSdp: 'answer' })
);
expect(media.teardown).toHaveBeenCalledOnce();
expect(acpStopLiveVoice).not.toHaveBeenCalled();
expect(result.current.phase).toBe('idle');
});
it('blocks retry until the server interaction stops after active media fails', async () => {
const cleanup = deferred<void>();
const { result } = renderHook(() => useLiveVoice());
await act(async () => result.current.start('main-session'));
vi.mocked(acpStopLiveVoice).mockReturnValueOnce(cleanup.promise);
act(() => mediaFailure());
expect(media.teardown).toHaveBeenCalledOnce();
expect(acpStopLiveVoice).toHaveBeenCalledWith('main-session', 'live-opaque');
expect(result.current.phase).toBe('stopping');
await act(async () => result.current.start('main-session'));
expect(acpStartLiveVoice).toHaveBeenCalledOnce();
await act(async () => cleanup.resolve());
expect(result.current.phase).toBe('error');
});
});

View file

@ -0,0 +1,242 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import { isAcpRecovering, subscribeToAcpRecovery } from '../acp/acpConnection';
import { acpStartLiveVoice, acpStopLiveVoice } from '../acp/liveVoice';
import {
subscribeToLiveVoiceInteractionEnded,
type LiveVoiceInteractionEndedNotification,
} from '../acp/liveVoiceNotifications';
import { LiveVoiceMediaSession } from './LiveVoiceMediaSession';
export type LiveVoicePhase = 'idle' | 'connecting' | 'live' | 'stopping' | 'error';
export function isLiveVoiceActive(phase: LiveVoicePhase): boolean {
return phase === 'connecting' || phase === 'live' || phase === 'stopping';
}
export interface LiveVoiceController {
activeSessionId: string | null;
liveVoiceSessionId: string | null;
phase: LiveVoicePhase;
muted: boolean;
start: (sessionId: string, initialCommentary?: string) => Promise<void>;
stop: () => Promise<void>;
toggleMute: () => void;
}
interface LiveVoiceInteraction {
sessionId: string;
interactionId?: string;
remoteStartPending: boolean;
media: LiveVoiceMediaSession;
mediaReady: boolean;
invalidated: boolean;
acpConnectionLost: boolean;
pendingOutcomesByInteractionId: Map<
string,
LiveVoiceInteractionEndedNotification['update']['outcome']
>;
}
async function stopRemoteInteraction(
interaction: LiveVoiceInteraction
): Promise<'stopped' | 'failed'> {
if (!interaction.interactionId || interaction.acpConnectionLost) return 'stopped';
try {
await acpStopLiveVoice(interaction.sessionId, interaction.interactionId);
return 'stopped';
} catch {
return 'failed';
}
}
export function useLiveVoice(): LiveVoiceController {
const [liveVoiceSessionId, setLiveVoiceSessionId] = useState<string | null>(null);
const [phase, setPhase] = useState<LiveVoicePhase>('idle');
const [muted, setMuted] = useState(false);
const mutedRef = useRef(false);
const interactionRef = useRef<LiveVoiceInteraction | null>(null);
const invalidateInteractionAndReleaseMedia = useCallback((interaction: LiveVoiceInteraction) => {
if (interaction.invalidated) return;
interaction.invalidated = true;
interaction.media.teardown();
mutedRef.current = false;
}, []);
const finishCurrentInteraction = useCallback(
(
interaction: LiveVoiceInteraction,
outcome: LiveVoiceInteractionEndedNotification['update']['outcome']
) => {
if (interactionRef.current !== interaction) return false;
interactionRef.current = null;
invalidateInteractionAndReleaseMedia(interaction);
if (outcome !== 'failed') {
setLiveVoiceSessionId(null);
}
setMuted(false);
setPhase(outcome === 'failed' ? 'error' : 'idle');
return true;
},
[invalidateInteractionAndReleaseMedia]
);
const failCurrentInteraction = useCallback(
async (interaction: LiveVoiceInteraction) => {
if (interactionRef.current !== interaction || interaction.invalidated) return;
invalidateInteractionAndReleaseMedia(interaction);
setMuted(false);
if (interaction.interactionId) {
setPhase('stopping');
await stopRemoteInteraction(interaction);
}
finishCurrentInteraction(interaction, 'failed');
},
[finishCurrentInteraction, invalidateInteractionAndReleaseMedia]
);
useEffect(() => {
return () => {
const interaction = interactionRef.current;
if (!interaction) return;
interactionRef.current = null;
invalidateInteractionAndReleaseMedia(interaction);
void stopRemoteInteraction(interaction);
};
}, [invalidateInteractionAndReleaseMedia]);
useEffect(() => {
return subscribeToLiveVoiceInteractionEnded((notification) => {
const interaction = interactionRef.current;
if (!interaction || interaction.sessionId !== notification.sessionId) return;
if (!interaction.interactionId) {
interaction.pendingOutcomesByInteractionId.set(
notification.update.interactionId,
notification.update.outcome
);
return;
}
if (
!interaction.invalidated &&
interaction.interactionId === notification.update.interactionId
) {
finishCurrentInteraction(interaction, notification.update.outcome);
}
});
}, [finishCurrentInteraction]);
useEffect(() => {
return subscribeToAcpRecovery((recovering) => {
if (!recovering) return;
const interaction = interactionRef.current;
if (interaction) {
interaction.acpConnectionLost = true;
finishCurrentInteraction(interaction, 'stopped');
}
});
}, [finishCurrentInteraction]);
const start = useCallback(
async (sessionId: string, initialCommentary?: string) => {
if (interactionRef.current || isAcpRecovering()) return;
setLiveVoiceSessionId(sessionId);
mutedRef.current = false;
setMuted(false);
setPhase('connecting');
let interaction: LiveVoiceInteraction;
const media = new LiveVoiceMediaSession(() => {
void failCurrentInteraction(interaction);
});
interaction = {
sessionId,
remoteStartPending: false,
media,
mediaReady: false,
invalidated: false,
acpConnectionLost: false,
pendingOutcomesByInteractionId: new Map(),
};
interactionRef.current = interaction;
const isCurrent = () => interactionRef.current === interaction && !interaction.invalidated;
try {
const offerSdp = await interaction.media.createOffer();
if (!isCurrent()) return;
interaction.remoteStartPending = true;
const response = await acpStartLiveVoice(sessionId, offerSdp);
interaction.remoteStartPending = false;
interaction.interactionId = response.interactionId;
const pendingOutcome = interaction.pendingOutcomesByInteractionId.get(
interaction.interactionId
);
interaction.pendingOutcomesByInteractionId.clear();
if (pendingOutcome) {
finishCurrentInteraction(interaction, pendingOutcome);
return;
}
if (!isCurrent()) {
finishCurrentInteraction(interaction, await stopRemoteInteraction(interaction));
return;
}
await interaction.media.applyAnswer(response.answerSdp);
if (!isCurrent()) return;
interaction.mediaReady = true;
interaction.media.setMuted(mutedRef.current);
setPhase('live');
if (initialCommentary) {
interaction.media.sendCommentary(initialCommentary);
}
} catch {
interaction.remoteStartPending = false;
if (interaction.invalidated) {
if (!interaction.interactionId) finishCurrentInteraction(interaction, 'stopped');
return;
}
await failCurrentInteraction(interaction);
}
},
[failCurrentInteraction, finishCurrentInteraction]
);
const toggleMute = useCallback(() => {
const interaction = interactionRef.current;
if (!interaction || interaction.invalidated || !interaction.mediaReady) return;
mutedRef.current = !mutedRef.current;
interaction.media.setMuted(mutedRef.current);
setMuted(mutedRef.current);
}, []);
const stop = useCallback(async () => {
const interaction = interactionRef.current;
if (!interaction || interaction.invalidated) return;
invalidateInteractionAndReleaseMedia(interaction);
setMuted(false);
if (!interaction.interactionId) {
if (interaction.remoteStartPending) {
setPhase('stopping');
return;
}
interactionRef.current = null;
setLiveVoiceSessionId(null);
setPhase('idle');
return;
}
setPhase('stopping');
finishCurrentInteraction(interaction, await stopRemoteInteraction(interaction));
}, [finishCurrentInteraction, invalidateInteractionAndReleaseMedia]);
const activeSessionId = isLiveVoiceActive(phase) ? liveVoiceSessionId : null;
return { activeSessionId, liveVoiceSessionId, phase, muted, start, stop, toggleMute };
}

View file

@ -29,6 +29,7 @@ export type ViewOptions = {
disableAnimation?: boolean;
initialMessage?: UserInput;
resumeSessionId?: string;
startLiveVoice?: boolean;
pendingScheduleDeepLink?: string;
};

View file

@ -97,6 +97,11 @@ import type {
ListSlashCommandsResponse_unstable,
ListSourcesRequest_unstable,
ListSourcesResponse_unstable,
LiveVoiceAvailabilityRequest_unstable,
LiveVoiceAvailabilityResponse_unstable,
LiveVoiceStartRequest_unstable,
LiveVoiceStartResponse_unstable,
LiveVoiceStopRequest_unstable,
LocalInferenceBuiltinChatTemplatesListRequest_unstable,
LocalInferenceBuiltinChatTemplatesListResponse_unstable,
LocalInferenceHuggingFaceRepoVariantsRequest_unstable,
@ -227,6 +232,8 @@ import {
zListSchedulesResponse_unstable,
zListSlashCommandsResponse_unstable,
zListSourcesResponse_unstable,
zLiveVoiceAvailabilityResponse_unstable,
zLiveVoiceStartResponse_unstable,
zLocalInferenceBuiltinChatTemplatesListResponse_unstable,
zLocalInferenceHuggingFaceRepoVariantsResponse_unstable,
zLocalInferenceHuggingFaceSearchResponse_unstable,
@ -384,6 +391,36 @@ export class GooseExtClient {
) as SteerSessionResponse_unstable;
}
async sessionLiveVoiceAvailability_unstable(
params: LiveVoiceAvailabilityRequest_unstable,
): Promise<LiveVoiceAvailabilityResponse_unstable> {
const raw = await this.conn.request(
"_goose/unstable/session/live-voice/availability",
params,
);
return zLiveVoiceAvailabilityResponse_unstable.parse(
raw,
) as LiveVoiceAvailabilityResponse_unstable;
}
async sessionLiveVoiceStart_unstable(
params: LiveVoiceStartRequest_unstable,
): Promise<LiveVoiceStartResponse_unstable> {
const raw = await this.conn.request(
"_goose/unstable/session/live-voice/start",
params,
);
return zLiveVoiceStartResponse_unstable.parse(
raw,
) as LiveVoiceStartResponse_unstable;
}
async sessionLiveVoiceStop_unstable(
params: LiveVoiceStopRequest_unstable,
): Promise<void> {
await this.conn.request("_goose/unstable/session/live-voice/stop", params);
}
async diagnosticsGet_unstable(
params: DiagnosticsGetRequest_unstable,
): Promise<DiagnosticsGetResponse_unstable> {

File diff suppressed because one or more lines are too long

View file

@ -678,9 +678,44 @@ export type SteerSessionResponse_unstable = {
messageId: string;
};
/**
* Ask whether a new or existing chat can enter direct Live voice.
*/
export type LiveVoiceAvailabilityRequest_unstable = {
sessionId?: string | null;
_meta?: {
[key: string]: unknown;
} | null;
};
export type LiveVoiceAvailabilityResponse_unstable = {
status: LiveVoiceStatus;
message: string;
};
export type LiveVoiceStatus = 'ready' | 'unavailable';
/**
* Get a diagnostic report for a session.
*/
export type LiveVoiceStartRequest_unstable = {
sessionId: string;
offerSdp: string;
_meta?: {
[key: string]: unknown;
} | null;
};
export type LiveVoiceStartResponse_unstable = {
interactionId: string;
answerSdp: string;
};
export type LiveVoiceStopRequest_unstable = {
sessionId: string;
interactionId: string;
};
export type DiagnosticsGetRequest_unstable = {
sessionId: string;
level?: DiagnosticsReportLevel;
@ -2646,7 +2681,9 @@ export type GooseSessionUpdate = ({
sessionUpdate: 'status_message';
} & StatusMessageUpdate) | ({
sessionUpdate: 'message_usage';
} & MessageUsageUpdate);
} & MessageUsageUpdate) | ({
sessionUpdate: 'live_voice_interaction_ended';
} & LiveVoiceInteractionEndedUpdate);
/**
* Streaming context-window usage update for a session.
@ -2712,6 +2749,13 @@ export type MessageUsageUpdate = {
usage: MessageUsageData;
};
export type LiveVoiceInteractionOutcome = 'stopped' | 'failed';
export type LiveVoiceInteractionEndedUpdate = {
interactionId: string;
outcome: LiveVoiceInteractionOutcome;
};
/**
* Dedicated provider notification for OAuth device-code flow.
* Sent during provider authentication when the ACP client supports
@ -2745,14 +2789,14 @@ export type RecipeParamsAction = 'submit' | 'cancel';
export type ExtRequest = {
id: string;
method: string;
params?: AddSessionExtensionRequest_unstable | RemoveSessionExtensionRequest_unstable | GetToolsRequest_unstable | SetToolPermissionsRequest_unstable | GooseToolCallRequest_unstable | ReadResourceRequest_unstable | AppsListRequest_unstable | AppsExportRequest_unstable | AppsImportRequest_unstable | AppsDeleteRequest_unstable | UpdateWorkingDirRequest_unstable | SetSessionSystemPromptRequest_unstable | SteerSessionRequest_unstable | DiagnosticsGetRequest_unstable | ListPromptsRequest_unstable | GetPromptRequest_unstable | SavePromptRequest_unstable | ResetPromptRequest_unstable | GetConfigExtensionsRequest_unstable | AddConfigExtensionRequest_unstable | RemoveConfigExtensionRequest_unstable | SetConfigExtensionEnabledRequest_unstable | GetSessionExtensionsRequest_unstable | ListProvidersRequest_unstable | ProviderSupportedModelsListRequest_unstable | ProviderCatalogListRequest_unstable | ProviderSetupCatalogListRequest_unstable | ProviderCatalogTemplateRequest_unstable | CustomProviderCreateRequest_unstable | CustomProviderReadRequest_unstable | CustomProviderUpdateRequest_unstable | CustomProviderDeleteRequest_unstable | RefreshProviderInventoryRequest_unstable | ProviderReadinessCheckRequest_unstable | ProviderConfigReadRequest_unstable | ProviderConfigStatusRequest_unstable | ProviderConfigSaveRequest_unstable | ProviderConfigDeleteRequest_unstable | ProviderConfigAuthenticateRequest_unstable | ProviderSecretsListRequest_unstable | ProviderSecretDeleteRequest_unstable | CanonicalModelInfoRequest_unstable | PreferencesReadRequest_unstable | PreferencesSaveRequest_unstable | ConfigReadRequest_unstable | ConfigUpsertRequest_unstable | ConfigRemoveRequest_unstable | ConfigReadAllRequest_unstable | DefaultsReadRequest_unstable | DefaultsSaveRequest_unstable | DefaultsClearRequest_unstable | OnboardingImportScanRequest_unstable | OnboardingImportApplyRequest_unstable | ExportSessionRequest_unstable | ImportSessionRequest_unstable | ShareSessionNostrRequest_unstable | EncodeRecipeRequest_unstable | DecodeRecipeRequest_unstable | ScanRecipeRequest_unstable | ListRecipesRequest_unstable | DeleteRecipeRequest_unstable | ScheduleRecipeRequest_unstable | SetRecipeSlashCommandRequest_unstable | SaveRecipeRequest_unstable | ParseRecipeRequest_unstable | RecipeToYamlRequest_unstable | ListSchedulesRequest_unstable | ListScheduleSessionsRequest_unstable | CreateScheduleRequest_unstable | DeleteScheduleRequest_unstable | PauseScheduleRequest_unstable | UnpauseScheduleRequest_unstable | UpdateScheduleRequest_unstable | RunScheduleNowRequest_unstable | KillRunningJobRequest_unstable | InspectRunningJobRequest_unstable | GetSessionInfoRequest_unstable | TruncateSessionConversationRequest_unstable | UpdateSessionProjectRequest_unstable | RenameSessionRequest_unstable | ArchiveSessionRequest_unstable | UnarchiveSessionRequest_unstable | CreateSourceRequest_unstable | ListSourcesRequest_unstable | ListAgentMentionsRequest_unstable | ListSlashCommandsRequest_unstable | UpdateSourceRequest_unstable | DeleteSourceRequest_unstable | ExportSourceRequest_unstable | ImportSourcesRequest_unstable | DictationTranscribeRequest_unstable | DictationConfigRequest_unstable | DictationModelsListRequest_unstable | DictationModelDownloadRequest_unstable | DictationModelDownloadProgressRequest_unstable | DictationModelCancelRequest_unstable | DictationModelDeleteRequest_unstable | LocalInferenceModelsListRequest_unstable | LocalInferenceModelDownloadRequest_unstable | LocalInferenceModelDownloadProgressRequest_unstable | LocalInferenceModelDownloadCancelRequest_unstable | LocalInferenceModelDeleteRequest_unstable | LocalInferenceModelEvictRequest_unstable | LocalInferenceModelSettingsReadRequest_unstable | LocalInferenceModelSettingsUpdateRequest_unstable | LocalInferenceHuggingFaceSearchRequest_unstable | LocalInferenceHuggingFaceRepoVariantsRequest_unstable | LocalInferenceBuiltinChatTemplatesListRequest_unstable | {
params?: AddSessionExtensionRequest_unstable | RemoveSessionExtensionRequest_unstable | GetToolsRequest_unstable | SetToolPermissionsRequest_unstable | GooseToolCallRequest_unstable | ReadResourceRequest_unstable | AppsListRequest_unstable | AppsExportRequest_unstable | AppsImportRequest_unstable | AppsDeleteRequest_unstable | UpdateWorkingDirRequest_unstable | SetSessionSystemPromptRequest_unstable | SteerSessionRequest_unstable | LiveVoiceAvailabilityRequest_unstable | LiveVoiceStartRequest_unstable | LiveVoiceStopRequest_unstable | DiagnosticsGetRequest_unstable | ListPromptsRequest_unstable | GetPromptRequest_unstable | SavePromptRequest_unstable | ResetPromptRequest_unstable | GetConfigExtensionsRequest_unstable | AddConfigExtensionRequest_unstable | RemoveConfigExtensionRequest_unstable | SetConfigExtensionEnabledRequest_unstable | GetSessionExtensionsRequest_unstable | ListProvidersRequest_unstable | ProviderSupportedModelsListRequest_unstable | ProviderCatalogListRequest_unstable | ProviderSetupCatalogListRequest_unstable | ProviderCatalogTemplateRequest_unstable | CustomProviderCreateRequest_unstable | CustomProviderReadRequest_unstable | CustomProviderUpdateRequest_unstable | CustomProviderDeleteRequest_unstable | RefreshProviderInventoryRequest_unstable | ProviderReadinessCheckRequest_unstable | ProviderConfigReadRequest_unstable | ProviderConfigStatusRequest_unstable | ProviderConfigSaveRequest_unstable | ProviderConfigDeleteRequest_unstable | ProviderConfigAuthenticateRequest_unstable | ProviderSecretsListRequest_unstable | ProviderSecretDeleteRequest_unstable | CanonicalModelInfoRequest_unstable | PreferencesReadRequest_unstable | PreferencesSaveRequest_unstable | ConfigReadRequest_unstable | ConfigUpsertRequest_unstable | ConfigRemoveRequest_unstable | ConfigReadAllRequest_unstable | DefaultsReadRequest_unstable | DefaultsSaveRequest_unstable | DefaultsClearRequest_unstable | OnboardingImportScanRequest_unstable | OnboardingImportApplyRequest_unstable | ExportSessionRequest_unstable | ImportSessionRequest_unstable | ShareSessionNostrRequest_unstable | EncodeRecipeRequest_unstable | DecodeRecipeRequest_unstable | ScanRecipeRequest_unstable | ListRecipesRequest_unstable | DeleteRecipeRequest_unstable | ScheduleRecipeRequest_unstable | SetRecipeSlashCommandRequest_unstable | SaveRecipeRequest_unstable | ParseRecipeRequest_unstable | RecipeToYamlRequest_unstable | ListSchedulesRequest_unstable | ListScheduleSessionsRequest_unstable | CreateScheduleRequest_unstable | DeleteScheduleRequest_unstable | PauseScheduleRequest_unstable | UnpauseScheduleRequest_unstable | UpdateScheduleRequest_unstable | RunScheduleNowRequest_unstable | KillRunningJobRequest_unstable | InspectRunningJobRequest_unstable | GetSessionInfoRequest_unstable | TruncateSessionConversationRequest_unstable | UpdateSessionProjectRequest_unstable | RenameSessionRequest_unstable | ArchiveSessionRequest_unstable | UnarchiveSessionRequest_unstable | CreateSourceRequest_unstable | ListSourcesRequest_unstable | ListAgentMentionsRequest_unstable | ListSlashCommandsRequest_unstable | UpdateSourceRequest_unstable | DeleteSourceRequest_unstable | ExportSourceRequest_unstable | ImportSourcesRequest_unstable | DictationTranscribeRequest_unstable | DictationConfigRequest_unstable | DictationModelsListRequest_unstable | DictationModelDownloadRequest_unstable | DictationModelDownloadProgressRequest_unstable | DictationModelCancelRequest_unstable | DictationModelDeleteRequest_unstable | LocalInferenceModelsListRequest_unstable | LocalInferenceModelDownloadRequest_unstable | LocalInferenceModelDownloadProgressRequest_unstable | LocalInferenceModelDownloadCancelRequest_unstable | LocalInferenceModelDeleteRequest_unstable | LocalInferenceModelEvictRequest_unstable | LocalInferenceModelSettingsReadRequest_unstable | LocalInferenceModelSettingsUpdateRequest_unstable | LocalInferenceHuggingFaceSearchRequest_unstable | LocalInferenceHuggingFaceRepoVariantsRequest_unstable | LocalInferenceBuiltinChatTemplatesListRequest_unstable | {
[key: string]: unknown;
} | null;
};
export type ExtResponse = {
id: string;
result?: EmptyResponse | GetToolsResponse_unstable | SetToolPermissionsResponse_unstable | GooseToolCallResponse_unstable | ReadResourceResponse_unstable | AppsListResponse_unstable | AppsExportResponse_unstable | AppsImportResponse_unstable | AppsDeleteResponse_unstable | SteerSessionResponse_unstable | DiagnosticsGetResponse_unstable | ListPromptsResponse_unstable | GetPromptResponse_unstable | PromptOperationResponse_unstable | GetConfigExtensionsResponse_unstable | GetSessionExtensionsResponse_unstable | ListProvidersResponse_unstable | ProviderSupportedModelsListResponse_unstable | ProviderCatalogListResponse_unstable | ProviderSetupCatalogListResponse_unstable | ProviderCatalogTemplateResponse_unstable | CustomProviderCreateResponse_unstable | CustomProviderReadResponse_unstable | CustomProviderUpdateResponse_unstable | CustomProviderDeleteResponse_unstable | RefreshProviderInventoryResponse_unstable | ProviderReadinessCheckResponse_unstable | ProviderConfigReadResponse_unstable | ProviderConfigStatusResponse_unstable | ProviderConfigChangeResponse_unstable | ProviderSecretsListResponse_unstable | CanonicalModelInfoResponse_unstable | PreferencesReadResponse_unstable | ConfigReadResponse_unstable | ConfigReadAllResponse_unstable | DefaultsReadResponse_unstable | OnboardingImportScanResponse_unstable | OnboardingImportApplyResponse_unstable | ExportSessionResponse_unstable | ImportSessionResponse_unstable | ShareSessionNostrResponse_unstable | EncodeRecipeResponse_unstable | DecodeRecipeResponse_unstable | ScanRecipeResponse_unstable | ListRecipesResponse_unstable | SaveRecipeResponse_unstable | ParseRecipeResponse_unstable | RecipeToYamlResponse_unstable | ListSchedulesResponse_unstable | ListScheduleSessionsResponse_unstable | CreateScheduleResponse_unstable | UpdateScheduleResponse_unstable | RunScheduleNowResponse_unstable | KillRunningJobResponse_unstable | InspectRunningJobResponse_unstable | GetSessionInfoResponse_unstable | CreateSourceResponse_unstable | ListSourcesResponse_unstable | ListAgentMentionsResponse_unstable | ListSlashCommandsResponse_unstable | UpdateSourceResponse_unstable | ExportSourceResponse_unstable | ImportSourcesResponse_unstable | DictationTranscribeResponse_unstable | DictationConfigResponse_unstable | DictationModelsListResponse_unstable | DictationModelDownloadProgressResponse_unstable | LocalInferenceModelsListResponse_unstable | LocalInferenceModelDownloadResponse_unstable | LocalInferenceModelDownloadProgressResponse_unstable | LocalInferenceModelSettingsReadResponse_unstable | LocalInferenceModelSettingsUpdateResponse_unstable | LocalInferenceHuggingFaceSearchResponse_unstable | LocalInferenceHuggingFaceRepoVariantsResponse_unstable | LocalInferenceBuiltinChatTemplatesListResponse_unstable | unknown;
result?: EmptyResponse | GetToolsResponse_unstable | SetToolPermissionsResponse_unstable | GooseToolCallResponse_unstable | ReadResourceResponse_unstable | AppsListResponse_unstable | AppsExportResponse_unstable | AppsImportResponse_unstable | AppsDeleteResponse_unstable | SteerSessionResponse_unstable | LiveVoiceAvailabilityResponse_unstable | LiveVoiceStartResponse_unstable | DiagnosticsGetResponse_unstable | ListPromptsResponse_unstable | GetPromptResponse_unstable | PromptOperationResponse_unstable | GetConfigExtensionsResponse_unstable | GetSessionExtensionsResponse_unstable | ListProvidersResponse_unstable | ProviderSupportedModelsListResponse_unstable | ProviderCatalogListResponse_unstable | ProviderSetupCatalogListResponse_unstable | ProviderCatalogTemplateResponse_unstable | CustomProviderCreateResponse_unstable | CustomProviderReadResponse_unstable | CustomProviderUpdateResponse_unstable | CustomProviderDeleteResponse_unstable | RefreshProviderInventoryResponse_unstable | ProviderReadinessCheckResponse_unstable | ProviderConfigReadResponse_unstable | ProviderConfigStatusResponse_unstable | ProviderConfigChangeResponse_unstable | ProviderSecretsListResponse_unstable | CanonicalModelInfoResponse_unstable | PreferencesReadResponse_unstable | ConfigReadResponse_unstable | ConfigReadAllResponse_unstable | DefaultsReadResponse_unstable | OnboardingImportScanResponse_unstable | OnboardingImportApplyResponse_unstable | ExportSessionResponse_unstable | ImportSessionResponse_unstable | ShareSessionNostrResponse_unstable | EncodeRecipeResponse_unstable | DecodeRecipeResponse_unstable | ScanRecipeResponse_unstable | ListRecipesResponse_unstable | SaveRecipeResponse_unstable | ParseRecipeResponse_unstable | RecipeToYamlResponse_unstable | ListSchedulesResponse_unstable | ListScheduleSessionsResponse_unstable | CreateScheduleResponse_unstable | UpdateScheduleResponse_unstable | RunScheduleNowResponse_unstable | KillRunningJobResponse_unstable | InspectRunningJobResponse_unstable | GetSessionInfoResponse_unstable | CreateSourceResponse_unstable | ListSourcesResponse_unstable | ListAgentMentionsResponse_unstable | ListSlashCommandsResponse_unstable | UpdateSourceResponse_unstable | ExportSourceResponse_unstable | ImportSourcesResponse_unstable | DictationTranscribeResponse_unstable | DictationConfigResponse_unstable | DictationModelsListResponse_unstable | DictationModelDownloadProgressResponse_unstable | LocalInferenceModelsListResponse_unstable | LocalInferenceModelDownloadResponse_unstable | LocalInferenceModelDownloadProgressResponse_unstable | LocalInferenceModelSettingsReadResponse_unstable | LocalInferenceModelSettingsUpdateResponse_unstable | LocalInferenceHuggingFaceSearchResponse_unstable | LocalInferenceHuggingFaceRepoVariantsResponse_unstable | LocalInferenceBuiltinChatTemplatesListResponse_unstable | unknown;
} | {
error: {
code: number;

View file

@ -423,11 +423,42 @@ export const zSteerSessionResponse_unstable = z.object({
messageId: z.string()
});
export const zDiagnosticsReportLevel = z.enum(['summary', 'full']);
/**
* Ask whether a new or existing chat can enter direct Live voice.
*/
export const zLiveVoiceAvailabilityRequest_unstable = z.object({
sessionId: z.string().nullish(),
_meta: z.record(z.string(), z.unknown()).nullish()
});
export const zLiveVoiceStatus = z.enum(['ready', 'unavailable']);
export const zLiveVoiceAvailabilityResponse_unstable = z.object({
status: zLiveVoiceStatus,
message: z.string()
});
/**
* Get a diagnostic report for a session.
*/
export const zLiveVoiceStartRequest_unstable = z.object({
sessionId: z.string(),
offerSdp: z.string(),
_meta: z.record(z.string(), z.unknown()).nullish()
});
export const zLiveVoiceStartResponse_unstable = z.object({
interactionId: z.string(),
answerSdp: z.string()
});
export const zLiveVoiceStopRequest_unstable = z.object({
sessionId: z.string(),
interactionId: z.string()
});
export const zDiagnosticsReportLevel = z.enum(['summary', 'full']);
export const zDiagnosticsGetRequest_unstable = z.object({
sessionId: z.string(),
level: zDiagnosticsReportLevel.optional().default('summary')
@ -2234,6 +2265,13 @@ export const zMessageUsageUpdate = z.object({
usage: zMessageUsageData
});
export const zLiveVoiceInteractionOutcome = z.enum(['stopped', 'failed']);
export const zLiveVoiceInteractionEndedUpdate = z.object({
interactionId: z.string(),
outcome: zLiveVoiceInteractionOutcome
});
/**
* Discriminated union of goose-specific session update payloads.
* Variant tag matches ACP's convention (`sessionUpdate: "<snake_case>"`).
@ -2245,7 +2283,8 @@ export const zMessageUsageUpdate = z.object({
export const zGooseSessionUpdate = z.discriminatedUnion('sessionUpdate', [
zSessionUsageUpdate.extend({ sessionUpdate: z.literal('usage_update') }),
zStatusMessageUpdate.extend({ sessionUpdate: z.literal('status_message') }),
zMessageUsageUpdate.extend({ sessionUpdate: z.literal('message_usage') })
zMessageUsageUpdate.extend({ sessionUpdate: z.literal('message_usage') }),
zLiveVoiceInteractionEndedUpdate.extend({ sessionUpdate: z.literal('live_voice_interaction_ended') })
]);
/**
@ -2303,6 +2342,9 @@ export const zExtRequest = z.object({
zUpdateWorkingDirRequest_unstable,
zSetSessionSystemPromptRequest_unstable,
zSteerSessionRequest_unstable,
zLiveVoiceAvailabilityRequest_unstable,
zLiveVoiceStartRequest_unstable,
zLiveVoiceStopRequest_unstable,
zDiagnosticsGetRequest_unstable,
zListPromptsRequest_unstable,
zGetPromptRequest_unstable,
@ -2418,6 +2460,8 @@ export const zExtResponse = z.union([
zAppsImportResponse_unstable,
zAppsDeleteResponse_unstable,
zSteerSessionResponse_unstable,
zLiveVoiceAvailabilityResponse_unstable,
zLiveVoiceStartResponse_unstable,
zDiagnosticsGetResponse_unstable,
zListPromptsResponse_unstable,
zGetPromptResponse_unstable,