From 2ee8e19137c1f4e40701925cab4a722da3120bdc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:49:52 +0530 Subject: [PATCH] feat(local): voice podcasts with server audio models Remove the local-only gate on the audio slot, so a podcast can be voiced by an audio model on an OpenAI-compatible server as well as by audio.cpp on this computer. - Speech client: one POST /audio/speech per line. Asks for WAV, switches the episode to MP3 when the server refuses (OpenRouter), and joins the lines into one file, splitting replies that hold several WAVs back to back. A failure is not retried, since a retry would bill the episode again. - Model types: audio output follows models.json and the endpoint's declared outputs, as for image and video. OpenRouter's `speech` output maps to the audio slot, and the extra listing request asks for output_modalities=all. - Voices come from one place, llm/voices/: the local model's own list, the server's GET /audio/voices, or voices the user adds in Settings. Each added voice is played once before it is kept, in a new general `settings` JSON column on selected_models (migration 0019), cleared when the slot takes another model. No voice is guessed. - Settings: "Voices for this model" under the model in use, with play and remove, and a link to the model's page for OpenRouter only. The Try dialog's voice field is optional; empty uses the server's default. - Podcast brief: a "Voiced by {model} on {server}" line, otherwise the local form's labels. A server model with no voices shows only "Set up voices", and Generate is disabled. - Test lines are spoken in the interface's language. - Docs: studio, connections and data-model. --- .agents/skills/translate/glossary.md | 1 + docs/architecture/connections.md | 8 +- docs/architecture/data-model.md | 3 +- docs/architecture/local-models/selection.md | 2 +- docs/architecture/studio.md | 19 +- .../versions/0019_selection_settings.py | 29 +++ .../modules/artifacts/podcast/brief.py | 49 +++- .../modules/artifacts/podcast/router.py | 25 +- .../modules/artifacts/podcast/service.py | 12 +- .../backend/modules/artifacts/service.py | 7 +- .../backend/modules/llm/connections/router.py | 34 +++ .../modules/llm/connections/service.py | 29 ++- surfsense_local/backend/modules/llm/models.py | 6 +- .../llm/providers/audiocpp/joined_wav.py | 4 +- .../llm/providers/openai_compatible/image.py | 16 +- .../openai_compatible/joined_replies.py | 93 +++++++ .../llm/providers/openai_compatible/speech.py | 113 +++++++++ .../providers/openai_compatible/voice_list.py | 58 +++++ .../modules/llm/providers/protocols.py | 7 +- .../backend/modules/llm/resolution.py | 48 +++- surfsense_local/backend/modules/llm/router.py | 2 + .../backend/modules/llm/schemas.py | 2 + .../backend/modules/llm/selection.py | 7 + .../backend/modules/llm/voices/__init__.py | 3 + .../backend/modules/llm/voices/roster.py | 75 ++++++ .../backend/modules/llm/voices/router.py | 95 ++++++++ .../backend/modules/llm/voices/saved.py | 41 ++++ .../integration/artifacts/test_routes.py | 115 ++++++++- .../backend/tests/integration/llm/conftest.py | 38 ++- .../tests/integration/llm/test_connections.py | 48 ++++ .../tests/integration/llm/test_voices.py | 162 ++++++++++++ .../tests/integration/worker/test_studio.py | 27 ++ .../llm/catalog/remote/test_classifier.py | 7 + .../providers/openai_compatible/__init__.py | 0 .../openai_compatible/test_speech.py | 199 +++++++++++++++ .../openai_compatible/test_voice_list.py | 83 +++++++ .../tests/unit/llm/test_openai_compatible.py | 28 +++ surfsense_local/backend/worker/studio/job.py | 6 +- .../frontend/src/components/ui/icons.tsx | 2 + .../src/features/dashboard/dashboard-page.tsx | 4 + .../src/features/models/remote/models/api.ts | 22 ++ .../models/server-model-picker.test.tsx | 95 ++++++++ .../models/remote/models/server-models.tsx | 7 + .../models/remote/models/try-model-dialog.tsx | 89 ++++++- .../models/remote/servers-can-serve.ts | 12 - .../src/features/models/remote/voices/api.ts | 41 ++++ .../remote/voices/server-voices.test.tsx | 106 ++++++++ .../models/remote/voices/server-voices.tsx | 230 ++++++++++++++++++ .../models/remote/voices/use-server-voices.ts | 27 ++ .../onboarding/model-step/model-step.tsx | 181 ++++++-------- .../onboarding/onboarding-page.test.tsx | 11 +- .../models/audio-models-settings.test.tsx | 3 +- .../settings/models/model-slot-settings.tsx | 71 +++--- .../frontend/src/features/studio/api.ts | 23 +- .../studio/podcast-brief-form.test.tsx | 164 ++++++++++++- .../features/studio/podcast-brief-form.tsx | 101 ++++++-- .../src/features/studio/studio-panel.test.tsx | 66 ++++- .../src/features/studio/studio-panel.tsx | 24 +- .../src/features/studio/use-podcast-brief.ts | 13 +- surfsense_local/frontend/translations/de.json | 43 +++- surfsense_local/frontend/translations/en.json | 43 +++- surfsense_local/frontend/translations/es.json | 43 +++- surfsense_local/frontend/translations/fr.json | 43 +++- surfsense_local/frontend/translations/hi.json | 43 +++- surfsense_local/frontend/translations/ja.json | 43 +++- surfsense_local/frontend/translations/ko.json | 43 +++- .../frontend/translations/pt-BR.json | 43 +++- surfsense_local/frontend/translations/ru.json | 43 +++- .../frontend/translations/zh-CN.json | 43 +++- 69 files changed, 2827 insertions(+), 396 deletions(-) create mode 100644 surfsense_local/backend/alembic/versions/0019_selection_settings.py create mode 100644 surfsense_local/backend/modules/llm/providers/openai_compatible/joined_replies.py create mode 100644 surfsense_local/backend/modules/llm/providers/openai_compatible/speech.py create mode 100644 surfsense_local/backend/modules/llm/providers/openai_compatible/voice_list.py create mode 100644 surfsense_local/backend/modules/llm/voices/__init__.py create mode 100644 surfsense_local/backend/modules/llm/voices/roster.py create mode 100644 surfsense_local/backend/modules/llm/voices/router.py create mode 100644 surfsense_local/backend/modules/llm/voices/saved.py create mode 100644 surfsense_local/backend/tests/integration/llm/test_voices.py create mode 100644 surfsense_local/backend/tests/unit/llm/providers/openai_compatible/__init__.py create mode 100644 surfsense_local/backend/tests/unit/llm/providers/openai_compatible/test_speech.py create mode 100644 surfsense_local/backend/tests/unit/llm/providers/openai_compatible/test_voice_list.py delete mode 100644 surfsense_local/frontend/src/features/models/remote/servers-can-serve.ts create mode 100644 surfsense_local/frontend/src/features/models/remote/voices/api.ts create mode 100644 surfsense_local/frontend/src/features/models/remote/voices/server-voices.test.tsx create mode 100644 surfsense_local/frontend/src/features/models/remote/voices/server-voices.tsx create mode 100644 surfsense_local/frontend/src/features/models/remote/voices/use-server-voices.ts diff --git a/.agents/skills/translate/glossary.md b/.agents/skills/translate/glossary.md index a7897957e..741839b49 100644 --- a/.agents/skills/translate/glossary.md +++ b/.agents/skills/translate/glossary.md @@ -31,6 +31,7 @@ SurfSense, Studio, llama.cpp, GGUF, Hugging Face, API, MCP, and every model and | quiz | クイズ | 퀴즈 | 测验 | क्विज़ | Quiz † | cuestionario | Quiz | Quiz | Тест | | study guide | 学習ガイド | 학습 가이드 | 学习指南 | स्टडी गाइड | Lernzettel † | guía de estudio | Guide de révision | Guia de estudos | конспект | | podcast | ポッドキャスト | 팟캐스트 | 播客 | पॉडकास्ट | Podcast | podcast | Podcast | Podcast | подкаст | +| speech (a model reading text aloud) | 音声合成 | 음성 | 语音 | वॉइस | Sprache | voz | voix | voz | речь | | workspace | ワークスペース | 워크스페이스 | 工作区 | वर्कस्पेस | Workspace | espacio de trabajo | espace de travail | Workspace | рабочая область | | artifact (a Studio output) | 生成物 | 생성물 | 产物 | आर्टिफ़ैक्ट | Artefakt | artefacto | artefact | artefato | артефакт | | chunk (a cited passage) | チャンク | 청크 | 片段 | चंक | Abschnitt | fragmento | extrait | trecho | фрагмент | diff --git a/docs/architecture/connections.md b/docs/architecture/connections.md index 22abe3fe9..38c905c58 100644 --- a/docs/architecture/connections.md +++ b/docs/architecture/connections.md @@ -57,6 +57,7 @@ A base URL must be `http` or `https` with a host, and may not carry credentials, | `GET` | `/llm/connections/{connection_id}/models` | the endpoint's live model list | | `POST` | `/llm/connections/{connection_id}/chat-test` | one short answer from a chosen model | | `POST` | `/llm/connections/{connection_id}/image-test` | one image from a chosen model | +| `POST` | `/llm/connections/{connection_id}/speech-test` | one spoken line from a chosen model | The write body: @@ -80,11 +81,11 @@ The write body: ## Live models -`GET .../models` calls `{base_url}/models` and, in parallel, `{base_url}/models?output_modalities=image`. A valid answer to the second is merged in and any failure of it is ignored: OpenRouter's default listing leaves out most of its image models, and an endpoint that ignores the parameter returns the same set, so no provider has to be recognised. Ids are deduplicated within the connection and sorted. A failed baseline call is a `502`, and the connection stays. +`GET .../models` calls `{base_url}/models` and, in parallel, `{base_url}/models?output_modalities=all`. A valid answer to the second is merged in and any failure of it is ignored: OpenRouter's default listing holds 464 of its 635 models and leaves out most of its image and all of its speech models, and an endpoint that ignores the parameter returns the same set, so no provider has to be recognised. Ids are deduplicated within the connection and sorted. A failed baseline call is a `502`, and the connection stays. Each model's `types` come from the first of three sources that knows it, and nothing is guessed; `capability_source` says which: -1. `declared`: output modalities the endpoint publishes: text is `text_gen`, image `image_gen`, video `video_gen` and audio `audio_gen`. +1. `declared`: output modalities the endpoint publishes: text is `text_gen`, image `image_gen`, video `video_gen`, and audio or OpenRouter's `speech` `audio_gen`. Any other word, such as OpenRouter's `transcription` or `embeddings`, is a known answer that fills no slot. 2. `catalog`: the remote model manifest, [`catalog/remote/manifest/models.json`](../../surfsense_local/backend/modules/llm/catalog/remote/manifest/models.json). `scripts/refresh_remote_manifest.py` builds it offline from models.dev, keyed provider then model, with the evidence the classifier reads rather than a verdict; a person reviews the diff and commits it, and nothing fetches models.dev at runtime. A connection with a `catalog_provider` reads that provider's entry first. Otherwise, and for an id its provider does not carry, the lookup reads the maker's own entry when the id's prefix names one, otherwise the types every provider carrying the id agrees on, trying the full id and then its last path segment ([`lookup.py`](../../surfsense_local/backend/modules/llm/catalog/remote/manifest/lookup.py)). A model found with no types, such as an embedder, is known to fill no slot. 3. `unknown`: neither source knows the id. @@ -142,6 +143,8 @@ Loading a connection runs the egress check for its host. The bundled sd-server s - `chat-test` streams one answer from the chosen model, by default to a prompt asking for one short sentence, capped at 1,024 tokens and 600 characters, so a model whose capability is `unknown` can be seen answering before it becomes the chat model. An empty reply is a `502`, which is what a reasoning model returns when it spends the whole budget thinking. - `image-test` generates one image, by default a blue circle on white, through the real image client and returns the bytes with `Cache-Control: no-store`. It creates no artifact. +- `speech-test` voices the `prompt` sent, the Try dialog's line in the interface's language, through the podcast's speech client, in the `voice` sent, and returns the audio with `Cache-Control: no-store`. Without one the request carries no `voice`, so the server speaks in its own default where it has one, and says so where it has none; no voice is assumed. The Try dialog has an optional voice field for audio models and plays the clip. +- `GET`, `POST` and `DELETE /llm/selection/audio_gen/voices` read the audio selection's voices (the server's list, else the user's), add one once the server has voiced the `text` sent (`502` with the server's words when it refuses), and remove one. A model on this computer is a `409`. - Selecting a model never runs inference. The server group offers a test before the model is used, and "Use without testing" for a trusted internal endpoint, whose first real request then reports any error normally. Image tests run only on an explicit action, because they are real inference and may cost money. ## Where keys live @@ -165,7 +168,6 @@ Keys are protected by envelope encryption ([ADR 0018](../adr/0018-keychain-envel - Each model section in Settings, Chat and Image, shows every connection as a group under the local models. A group loads its models only when opened, so a slow or failed endpoint does not hold up the others, and lists only those whose `selectable_for` includes that section's slot; the model in use, if it comes from that server, stays shown above the list whether the group is open or closed. Open, its list scrolls inside its own capped-height area rather than lengthening the page. A model's row shows its type chips, and a Vision chip when `reads_images` is true. A model is assigned after an optional test; an exact ID the listing lacks can be typed in. A new connection is added from **Use a server** on the section's **Add model** page. Adding and editing a connection happen in a dialog over the page. Saving a new connection returns to the list with its group open on its models, since choosing one is why it was added; saving an edit only refreshes. - Edit and Disconnect sit on each group. Edit opens the same form in that dialog. A connection serves every slot, so Disconnect names each model it will clear, Chat, Image or both, whichever section it is disconnected from. - Onboarding's model steps use the same groups and the same dialog; with nothing connected yet, their **Connect** opens the dialog directly, and saving opens the server page on the new connection's models. -- A model type no server model can fill yet offers no servers anywhere, in Settings or onboarding, and its copy drops the mention: audio today, since no server model voices podcasts. The list is `serversCanServe()` in `features/models/remote/servers-can-serve.ts`. - The connection form picks a provider from the remote manifest, through `GET /llm/catalog/remote`, or "Local or custom server". A ready provider fills its URL, which stays editable so a proxy in front of it still works; a provider that needs account details asks for each field and builds the URL from its template; a provider that needs a URL leaves it to the user; an unreachable one is listed, disabled, with its reason. A provider that takes no key, a loopback server, hides the key field. "Local or custom server" leaves the URL to the user, since its port is whatever its owner set, with `http://localhost:11434/v1` as placeholder text only; a loopback server the manifest lists, such as LM Studio, fills the manifest's URL like any ready provider. The save sends `catalog_provider`. ## Known gaps diff --git a/docs/architecture/data-model.md b/docs/architecture/data-model.md index 20a80461f..d4569598b 100644 --- a/docs/architecture/data-model.md +++ b/docs/architecture/data-model.md @@ -117,7 +117,7 @@ An artifact's searchable body is a `Document` with `document_type = ARTIFACT`; ` | Table | Columns | Notes | |---|---|---| | `provider_connections` | `id`, `label`, `provider`, `base_url`, `catalog_provider`, `api_key_ciphertext`, timestamps | `label` unique case-insensitively; `provider` must be `openai_compatible`; `catalog_provider` is a remote manifest provider id or `custom`, since `0014`; the key is Fernet ciphertext since `0007` | -| `selected_models` | `model_type`, `provider`, `connection_id`, `name`, `params_b`, `vendor`, `line`, `updated_at` | one row per model type: `text_gen`, `image_gen`, `image_edit`, `video_gen` or `audio_gen` | +| `selected_models` | `model_type`, `provider`, `connection_id`, `name`, `params_b`, `vendor`, `line`, `settings`, `updated_at` | one row per model type: `text_gen`, `image_gen`, `image_edit`, `video_gen` or `audio_gen`; `settings` is JSON for what the user set that no endpoint states, keyed by the slice that owns each entry, and cleared when the slot takes another model | | `onboarding_completion` | `id`, `completed_at` | a singleton (`CHECK id = 1`) whose presence means onboarding is done | - A CHECK on `selected_models` allows `llamacpp`, `sdcpp` and `audiocpp` only without a connection and `openai_compatible` only with one, and a second, `local_runtime_type`, lets `llamacpp` hold only `text_gen`, `sdcpp` only `image_gen`, `image_edit` and `video_gen`, and `audiocpp` only `audio_gen`. `connection_id` cascades, so deleting a connection clears exactly the selections that used it. @@ -303,6 +303,7 @@ erDiagram | `0014` | `0014_connection_catalog_provider.py` | `provider_connections.catalog_provider`, `custom` for every existing connection; downgrading drops the column in place, because a table rebuild would cascade into `selected_models` | | `0015` | `0015_image_selection_by_build.py` | a local `image_gen` selection is renamed from the old list's name to its curated build's id (`stable-diffusion-1.5` to `v1-5-pruned_Q4_0`, and the two SDXL models); the map is frozen in the migration, and downgrading reverses it | | `0016` | `0016_local_audio_provider.py` | `selected_models` rebuilt so `audiocpp` may hold `audio_gen`, and only that, without a connection; downgrading drops an `audiocpp` selection | +| `0019` | `0019_selection_settings.py` | `selected_models.settings`, a nullable JSON column added in place; its first entry is a server audio model's `voices` | - Migrations run on every API start and are idempotent. Autogenerate is off: it renders a rename as a drop plus an add, which deletes a column's data silently, and `env.py` carries no `target_metadata`, so it cannot be used by accident. - SQLite cannot alter a CHECK constraint in place, so `0004`, `0009`, `0012` and `0013` copy `selected_models` into a new table. diff --git a/docs/architecture/local-models/selection.md b/docs/architecture/local-models/selection.md index 643f99f44..80d3e9722 100644 --- a/docs/architecture/local-models/selection.md +++ b/docs/architecture/local-models/selection.md @@ -22,7 +22,7 @@ choosing or clearing a model never touches. | `text_gen` | `llamacpp`, the bundled runtime | `openai_compatible`, with a `connection_id` | chat, titles, Studio's writing | | `image_gen` | `sdcpp`, the bundled sd-server | `openai_compatible`, with a `connection_id` | Studio's `image` and `infographic` | | `image_edit` | `sdcpp`, a model whose entry names `edit` | `openai_compatible`, with a `connection_id` | nothing yet | -| `audio_gen` | `audiocpp`, the bundled audio.cpp server | `openai_compatible`, with a `connection_id`, which nothing reads yet | Studio's `podcast` | +| `audio_gen` | `audiocpp`, the bundled audio.cpp server | `openai_compatible`, with a `connection_id` | Studio's `podcast` | | `video_gen` | `sdcpp`, a model whose entry has a `video` block | `openai_compatible`, with a `connection_id` | nothing yet | A type no feature reads can still be chosen; the feature that first reads one diff --git a/docs/architecture/studio.md b/docs/architecture/studio.md index e3422785e..3f8ff29cd 100644 --- a/docs/architecture/studio.md +++ b/docs/architecture/studio.md @@ -39,7 +39,7 @@ The catalog is a tuple of twelve `Format` rows in [`formats.py`](../../surfsense | `infographic` | Infographic | image_gen, text_gen | `media/visual/infographic/` | the image | - `requires_model_types` is a tuple in the order the pipeline's `render()` takes its models. A format is available when every required model type has a selection; otherwise the reason names every missing one in a fixed reading order: "Needs a chat model", "Needs an image model", "Needs an audio model", or two of them joined, as "Needs a chat model and an audio model". Naming only the first missing type made selecting it look like the gate moving to the other. -- `podcast` needs its `audio_gen` model on this computer: an audio model chosen from a server is refused with "Needs an audio model on this computer", because nothing calls a remote speech endpoint yet. Opening a podcast brief without an audio model answers `409` with "Needs an audio model". The app ships Kokoro and chooses it at startup when no audio model is chosen, so that answer needs the choice cleared since the last start, or a build without audio.cpp ([`local-models/selection.md`](local-models/selection.md)). +- `podcast` runs with an `audio_gen` model on this computer or on a server. The format list and the brief read the model's voices without calling it, so neither needs the server's host allowed; the job does. Opening a podcast brief without an audio model answers `409` with "Needs an audio model". The app ships Kokoro and chooses it at startup when no audio model is chosen, so that answer needs the choice cleared since the last start, or a build without audio.cpp ([`local-models/selection.md`](local-models/selection.md)). - Which formats are available is the server's answer to what is selected, so the panel asks again whenever the chat selection changes or the settings dialog closes, since the image model is chosen inside settings and nothing else reports it. - An image selection can resolve to the bundled sd-server as well as to a remote connection, so needing an image model does not mean needing a key or a network. - [`tests/unit/worker/test_studio_job_router.py`](../../surfsense_local/backend/tests/unit/worker/test_studio_job_router.py) asserts that `job_router.py` names every catalog key and nothing else, that each key has a pipeline, and that each pipeline takes its models, the sources, the prompt and, for a format with options, the options. @@ -61,7 +61,7 @@ Eight formats follow it: Four formats break it. DOCX, PPTX, XLSX and PDF go through `office/`: the tier prompt asks the model to "write one standalone Python script" for the format's library (python-docx, python-pptx, xlsxwriter or ReportLab), with a `SKILL.md` of authoring guidance beside each format, and [`office/runner.py`](../../surfsense_local/backend/worker/studio/office/runner.py) runs the reply with `exec()` on a thread in the worker process. The script must leave the file's bytes in `output_bytes` and may set `title` and `summary`. A failing script goes back to the model with its error, for up to three attempts. The 120-second limit is a `thread.join`: it bounds how long the job waits, but it cannot stop a thread that ignores it, and the code runs with the worker's privileges. The module says so and names an out-of-process sandbox runner as the way up. -There is no Electron `printToPDF` and no ffmpeg in `surfsense_local`. A PDF is ReportLab through that path, and a podcast is one `audio/wav` joined from audio.cpp's WAV for each turn. +There is no Electron `printToPDF` and no ffmpeg in `surfsense_local`. A PDF is ReportLab through that path, and a podcast is one `audio/wav` joined from each turn's WAV, or one `audio/mpeg` from a server that sends only MP3. ## Voicing a podcast @@ -73,6 +73,19 @@ There is no Electron `printToPDF` and no ffmpeg in `surfsense_local`. A PDF is R - **A turn the server fails ends the episode with the server's own words**: "audio.cpp could not voice turn 3 of 40: unknown Kokoro voice id: nobody". The server was reached, so this is not "The model could not be reached", which stays for a server that does not answer. - **The model is unloaded when voicing ends**, with `POST /v1/tasks/unload_all_models`, success or failure, so its memory is back before the chat model's next job. The server's five-minute idle unload is the backstop. +### On a server + +[`providers/openai_compatible/speech.py`](../../surfsense_local/backend/modules/llm/providers/openai_compatible/speech.py) voices a podcast with an `audio_gen` model chosen from a connection: + +- **The voices are the server's, or the user's, never guessed.** The OpenAI API has no route that lists voices and models.dev carries none, so the server is asked `GET {base}/audio/voices`, Kokoro-FastAPI's route ([`voice_list.py`](../../surfsense_local/backend/modules/llm/providers/openai_compatible/voice_list.py)), held for the process and never stored. A server that lists none, as OpenAI, Groq and OpenRouter do, takes the voices the user adds in Settings › Audio generation models, inside that server's section under its model in use, and in onboarding's server path alike: each is voiced once before it is kept, under `voices` in the selection's `settings`, which is cleared when the slot takes another model. For OpenRouter alone the section links the model's page, whose "Request fields" name its voices; no other provider publishes them at an address the app can build. One slice, [`llm/voices/`](../../surfsense_local/backend/modules/llm/voices/), answers which voices the chosen audio model offers, local or server, for the brief and Settings alike. The server is asked only once its host is allowed, and the format list never asks. +- **Nothing says which languages a server's voice speaks**: not the API, not `/audio/voices`, not models.dev. So a voice is heard, in Settings and in the Try dialog, saying a line in the interface's language, the one the user likeliest writes in, and the brief offers a fixed list of script languages. A voice that cannot speak the chosen language reads it anyway, badly: no server reports it as an error. +- **The brief reads like the local one.** It adds one line, "Voiced by {model} on {server}. Each line is billed by the provider.", and otherwise keeps the local labels; only its lists differ, the voices one plain list with no gender the server did not state. With no voices yet, the brief shows only a Set up voices button, which opens Settings, and Generate is held. The brief response carries `voices_source` (`local`, `server` or `saved`) and `voiced_by`. +- **No voice is checked before drafting.** A listed voice came from the server and an added one was heard when it was added, so the brief's own check, that each speaker's voice is one of them, is what stands between the user and a voice the server refuses. +- **Only OpenAI's `/audio/speech` is spoken.** ElevenLabs, Gemini and other providers with their own speech APIs are reached through a gateway that speaks it for them, OpenRouter or a LiteLLM proxy, rather than through an adapter per provider here. +- **One `POST /audio/speech` per turn**, asking for `wav`, because only WAV takes a pause between speakers. When the first line's `wav` is refused, the episode asks for `mp3` instead: OpenRouter takes only `mp3` or `pcm`, and its `pcm` states no sample rate. The format is read from the bytes, not from what was asked. WAV turns join as audio.cpp's do, and a reply holding several WAVs back to back is one turn. MP3 turns join frame to frame without a pause, each turn's ID3 tag and Xing frame dropped so a player reads the whole episode's length. Anything else ends the episode with "the server answered, but not as WAV or MP3". +- **Nothing is checked for memory**, since the server spends it. +- **A failure is not retried.** A refused turn, an unreachable server or an unreadable answer ends the episode with the server's own words, and the job is not retried, because a retry would draft and bill the whole episode again. + ## Grounding Studio does not call `retrieve()`. [`shared/gather.py`](../../surfsense_local/backend/worker/studio/shared/gather.py) loads the selected documents' markdown in selection order, up to 24,000 characters in total, and [`shared/generate.py`](../../surfsense_local/backend/worker/studio/shared/generate.py) sends it as one user message, each document under its title as a heading, after the format's system prompt. The code marks the flat cap as a ceiling, fine for summarising a handful of local documents, with retrieval scoped to the selection as the way up. @@ -114,7 +127,7 @@ Every pipeline returns a `Built`: a `title`, the `markdown` that is always the i |---|---|---| | `GET` | `/workspaces/{workspace_id}/studio/formats` | the catalog, each format with `available` and `unavailable_reason` | | `POST` | `/workspaces/{workspace_id}/studio/jobs` | `{format, document_ids, prompt?, options?}`; `201` with the artifact | -| `GET` | `/workspaces/{workspace_id}/studio/podcast/brief` | the podcast brief and voices to review before submitting | +| `GET` | `/workspaces/{workspace_id}/studio/podcast/brief` | the podcast brief to review before submitting, the model's `voices` (`null` when they are typed) and the `languages` it may use | | `GET` | `/workspaces/{workspace_id}/artifacts` | the workspace's artifacts, newest first | | `GET` | `/artifacts/{artifact_id}` | detail: the body, the files, quiz or flashcard progress | | `POST` | `/artifacts/{artifact_id}/regenerate` | run the job again; `202` | diff --git a/surfsense_local/backend/alembic/versions/0019_selection_settings.py b/surfsense_local/backend/alembic/versions/0019_selection_settings.py new file mode 100644 index 000000000..2a7896b52 --- /dev/null +++ b/surfsense_local/backend/alembic/versions/0019_selection_settings.py @@ -0,0 +1,29 @@ +"""Keep what the user sets for a selected model that no endpoint states. + +A JSON column, general to every slot, keyed by the slice that owns each entry. +The first is a server audio model's voices, added by the user when the server +lists none. SQLite adds a nullable column in place, with no table rebuild. + +Revision ID: 0019 +Revises: 0018 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0019" +down_revision: str | Sequence[str] | None = "0018" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column("selected_models", sa.Column("settings", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("selected_models") as table: + table.drop_column("settings") diff --git a/surfsense_local/backend/modules/artifacts/podcast/brief.py b/surfsense_local/backend/modules/artifacts/podcast/brief.py index f38603cbf..988215b14 100644 --- a/surfsense_local/backend/modules/artifacts/podcast/brief.py +++ b/surfsense_local/backend/modules/artifacts/podcast/brief.py @@ -5,10 +5,17 @@ from pydantic import BaseModel, Field, StringConstraints, ValidationError from sqlalchemy.orm import Session from modules.llm.providers.protocols import Voice -from modules.llm.resolution import resolve_text_to_speech +from modules.llm.voices.roster import speech_voices MAX_SPEAKERS = 6 DEFAULT_LANGUAGE = "en-US" +# What a script may be written in when no voice states its languages, as a +# server's voices never do. The product's choice of languages to offer, not a +# claim about the model. +SCRIPT_LANGUAGES = ( + "en", "ar", "de", "es", "fr", "hi", "id", "it", "ja", "ko", "nl", "pl", + "pt", "ru", "sv", "tr", "uk", "vi", "zh", +) # fmt: skip class Style(StrEnum): @@ -44,10 +51,14 @@ Name = Annotated[ ] +# A voice id as the model takes it: from its roster, or a server's. +VoiceId = Annotated[str, StringConstraints(strip_whitespace=True, max_length=100)] + + class Speaker(BaseModel): name: Name role: Role - voice: str + voice: VoiceId class PodcastBrief(BaseModel): @@ -59,10 +70,22 @@ class PodcastBrief(BaseModel): speakers: list[Speaker] = Field(min_length=1, max_length=MAX_SPEAKERS) +def languages(voices: list[Voice]) -> list[str]: + """What the brief may be written in: what the roster speaks, in its order, + or the script languages when no voice says.""" + stated = [lang for voice in voices for lang in voice.languages] + return list(dict.fromkeys(stated)) if stated else list(SCRIPT_LANGUAGES) + + +def _speaks(voice: Voice, language: str) -> bool: + """A voice that states no language is not refused one.""" + return not voice.languages or language in voice.languages + + def default_language(voices: list[Voice]) -> str: """American English where a voice speaks it, else any English, else the first language the model's roster names: Kitten lists English as `en`.""" - spoken = [language for voice in voices for language in voice.languages] + spoken = languages(voices) if DEFAULT_LANGUAGE in spoken: return DEFAULT_LANGUAGE english = [language for language in spoken if language.split("-")[0] == "en"] @@ -70,27 +93,29 @@ def default_language(voices: list[Voice]) -> str: def proposed(voices: list[Voice], language: str | None = None) -> PodcastBrief: - """The defaults the form opens with: two speakers in a language they speak.""" + """The defaults the form opens with: two speakers in a language they speak. + A server model with no voices yet opens with one voiceless speaker, which + the form shows as voices to set up.""" language = language or default_language(voices) - spoken = [voice for voice in voices if language in voice.languages][:2] + chosen = [v.id for v in voices if _speaks(v, language)][:2] or [""] return PodcastBrief( language=language, speakers=[ - Speaker(name=_default_name(slot), role=_ROLE_BY_SLOT[slot], voice=voice.id) - for slot, voice in enumerate(spoken) + Speaker(name=_default_name(slot), role=_ROLE_BY_SLOT[slot], voice=voice) + for slot, voice in enumerate(chosen) ], ) def validate_options(session: Session, raw: dict | None) -> dict: """The Format hook: the brief, checked against the chosen model's voices.""" - return validated(resolve_text_to_speech(session).voices(), raw).model_dump( - mode="json" - ) + return validated(speech_voices(session).voices, raw).model_dump(mode="json") def validated(voices: list[Voice], raw: dict | None) -> PodcastBrief: """The brief a job may run with; raises ValueError with one readable line.""" + if not voices: + raise ValueError("add voices for the audio model in Settings first") if raw is None: return proposed(voices) try: @@ -100,10 +125,12 @@ def validated(voices: list[Voice], raw: dict | None) -> PodcastBrief: location = ".".join(str(part) for part in first["loc"]) raise ValueError(f"{location}: {first['msg']}") from error + if brief.language not in languages(voices): + raise ValueError(f"language: {brief.language} is not one to write in") by_id = {voice.id: voice for voice in voices} for speaker in brief.speakers: voice = by_id.get(speaker.voice) - if voice is None or brief.language not in voice.languages: + if voice is None or not _speaks(voice, brief.language): raise ValueError(f"{speaker.name}: pick a {brief.language} voice") if len({speaker.voice for speaker in brief.speakers}) < len(brief.speakers): raise ValueError("each speaker needs their own voice") diff --git a/surfsense_local/backend/modules/artifacts/podcast/router.py b/surfsense_local/backend/modules/artifacts/podcast/router.py index 0f1e1e5d2..f70fb447f 100644 --- a/surfsense_local/backend/modules/artifacts/podcast/router.py +++ b/surfsense_local/backend/modules/artifacts/podcast/router.py @@ -1,8 +1,10 @@ +from typing import Literal + from fastapi import APIRouter from pydantic import BaseModel from api.dependencies import SessionDep -from modules.artifacts.podcast.brief import PodcastBrief +from modules.artifacts.podcast.brief import PodcastBrief, languages from modules.artifacts.podcast.service import open_brief from modules.llm.providers.protocols import Voice from modules.workspaces.dependencies import WorkspaceDep @@ -10,9 +12,19 @@ from modules.workspaces.dependencies import WorkspaceDep router = APIRouter(tags=["studio"]) +class VoicedByRead(BaseModel): + server: str + model: str + + class BriefRead(BaseModel): brief: PodcastBrief + # Empty only for a server model with no voices added yet. voices: list[Voice] + voices_source: Literal["local", "server", "saved"] + # The server that voices it; None for a model on this computer. + voiced_by: VoicedByRead | None + languages: list[str] @router.get( @@ -21,5 +33,12 @@ class BriefRead(BaseModel): summary="The podcast brief to review before generating", ) def podcast_brief(workspace: WorkspaceDep, session: SessionDep) -> BriefRead: - brief, voices = open_brief(session, workspace) - return BriefRead(brief=brief, voices=voices) + brief, roster = open_brief(session, workspace) + by = roster.voiced_by + return BriefRead( + brief=brief, + voices=roster.voices, + voices_source=roster.source, + voiced_by=VoicedByRead(server=by.server, model=by.model) if by else None, + languages=languages(roster.voices), + ) diff --git a/surfsense_local/backend/modules/artifacts/podcast/service.py b/surfsense_local/backend/modules/artifacts/podcast/service.py index 010b9b2be..ffa7f0f54 100644 --- a/surfsense_local/backend/modules/artifacts/podcast/service.py +++ b/surfsense_local/backend/modules/artifacts/podcast/service.py @@ -4,8 +4,8 @@ from sqlalchemy.orm import Session from modules.artifacts.models import Artifact from modules.artifacts.podcast import brief -from modules.llm.providers.protocols import Voice -from modules.llm.resolution import ModelResolutionError, resolve_text_to_speech +from modules.llm.resolution import ModelResolutionError +from modules.llm.voices.roster import SpeechRoster, speech_voices from modules.workspaces.models import Workspace FORMAT = "podcast" @@ -13,14 +13,14 @@ FORMAT = "podcast" def open_brief( session: Session, workspace: Workspace -) -> tuple[brief.PodcastBrief, list[Voice]]: +) -> tuple[brief.PodcastBrief, SpeechRoster]: """The brief the form opens with, and the voices it may pick from. The last episode's brief comes back when it still fits the catalog, so a returning user changes only what differs; otherwise the defaults. """ try: - voices = resolve_text_to_speech(session).voices() + roster = speech_voices(session) except ModelResolutionError as error: raise HTTPException(status.HTTP_409_CONFLICT, "Needs an audio model") from error @@ -33,7 +33,7 @@ def open_brief( options = (last.artifact_metadata or {}).get("options") if last else None if options is not None: try: - return brief.validated(voices, options), voices + return brief.validated(roster.voices, options), roster except ValueError: pass - return brief.proposed(voices), voices + return brief.proposed(roster.voices), roster diff --git a/surfsense_local/backend/modules/artifacts/service.py b/surfsense_local/backend/modules/artifacts/service.py index 629145658..d2b5e59d2 100644 --- a/surfsense_local/backend/modules/artifacts/service.py +++ b/surfsense_local/backend/modules/artifacts/service.py @@ -13,8 +13,7 @@ from modules.llm.model_type import ModelType from modules.llm.models import SelectedModel from modules.llm.resolution import ( ModelResolutionError, - VoiceNotLocalError, - resolve_text_to_speech, + speech_selected, ) from modules.workspaces.models import Workspace from worker.jobs import cancel_studio_job @@ -185,9 +184,7 @@ def _availability(session: Session, fmt: Format) -> tuple[bool, str | None]: return False, _required(missing) if ModelType.AUDIO_GEN in fmt.requires_model_types: try: - resolve_text_to_speech(session) - except VoiceNotLocalError: - return False, "Needs an audio model on this computer" + speech_selected(session) except ModelResolutionError: return False, _required([ModelType.AUDIO_GEN]) return True, None diff --git a/surfsense_local/backend/modules/llm/connections/router.py b/surfsense_local/backend/modules/llm/connections/router.py index 97cf76b7d..d92a14a9e 100644 --- a/surfsense_local/backend/modules/llm/connections/router.py +++ b/surfsense_local/backend/modules/llm/connections/router.py @@ -23,6 +23,11 @@ from modules.llm.providers.openai_compatible import ( OpenAICompatibleChatProvider, OpenAICompatibleImageProvider, ) +from modules.llm.providers.openai_compatible.speech import ( + NonRetryableSpeechError, + RemoteSpeech, +) +from modules.llm.providers.protocols import SpokenTurn from modules.llm.providers.types import Message from modules.llm.schemas import ( ChatTestRead, @@ -36,6 +41,7 @@ from modules.llm.selectable import selectable_for router = APIRouter(prefix="/connections") DEFAULT_IMAGE_TEST_PROMPT = "A simple blue circle centered on a plain white background." +DEFAULT_SPEECH_TEST_TEXT = "Hello. This is how your podcasts will sound." DEFAULT_CHAT_TEST_PROMPT = "Reply with one short sentence confirming you can answer." # Enough that a thinking model reaches its answer, and still little enough that # testing cannot run a bill up. A plain answer never approaches this, because the @@ -290,3 +296,31 @@ async def test_connection_image( media_type=image.media_type, headers={"Cache-Control": "no-store"}, ) + + +@router.post("/{connection_id}/speech-test") +async def test_connection_speech( + connection_id: int, payload: ModelTestWrite, session: SessionDep +) -> Response: + """One line in the typed voice, or the server's default, so an audio model + can be heard before it is chosen.""" + connection = await transact(session, allowed_connection, connection_id) + model = _requested_model(payload) + speech = RemoteSpeech( + model, + base_url=connection.base_url, + api_key=connection.api_key, + ) + # Empty leaves the voice to the server's default, where it has one. + voice = (payload.voice or "").strip() + line = SpokenTurn(voice, payload.prompt or DEFAULT_SPEECH_TEST_TEXT) + try: + # Only audio.cpp reads the language; a server infers it from the text. + audio = await speech.synthesize([line], "") + except NonRetryableSpeechError as error: + raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(error)) from error + return Response( + content=audio.content, + media_type=audio.media_type, + headers={"Cache-Control": "no-store"}, + ) diff --git a/surfsense_local/backend/modules/llm/connections/service.py b/surfsense_local/backend/modules/llm/connections/service.py index bcc19a5b5..8744f8e48 100644 --- a/surfsense_local/backend/modules/llm/connections/service.py +++ b/surfsense_local/backend/modules/llm/connections/service.py @@ -25,6 +25,8 @@ _DECLARED_AS = { "image": ModelType.IMAGE_GEN, "video": ModelType.VIDEO_GEN, "audio": ModelType.AUDIO_GEN, + # OpenRouter's word for text-to-speech output. + "speech": ModelType.AUDIO_GEN, } @@ -114,10 +116,13 @@ def _classify( # The same answer the remote catalog gives, from the one rule in lookup.py. unusable = lookup.unusable_reason(name, catalog_provider) if modalities: + # `audio` and `speech` name one type; a model declaring both has it once. declared = tuple( - model_type - for modality, model_type in _DECLARED_AS.items() - if modality in modalities + dict.fromkeys( + model_type + for modality, model_type in _DECLARED_AS.items() + if modality in modalities + ) ) return DiscoveredModel(name, declared, "declared", unusable) catalogued = lookup.classify(name, provider=catalog_provider) @@ -138,9 +143,9 @@ async def _request_entries( base_url: str, api_key: str | None, *, - image_only: bool = False, + every_modality: bool = False, ) -> dict[str, set[str]]: - params = {"output_modalities": "image"} if image_only else None + params = {"output_modalities": "all"} if every_modality else None async with httpx.AsyncClient( timeout=DISCOVERY_TIMEOUT, headers=key_headers(base_url, api_key), @@ -158,7 +163,7 @@ async def probe_connection(base_url: str, api_key: str | None) -> list[Discovere async def discover_models(connection: ProviderConnection) -> list[DiscoveredModel]: baseline, optional = await asyncio.gather( _request_entries(connection.base_url, connection.api_key), - _optional_image_entries(connection.base_url, connection.api_key), + _optional_other_entries(connection.base_url, connection.api_key), ) merged = {name: set(modalities) for name, modalities in baseline.items()} for name, modalities in optional.items(): @@ -171,17 +176,17 @@ async def discover_models(connection: ProviderConnection) -> list[DiscoveredMode ) -async def _optional_image_entries( +async def _optional_other_entries( base_url: str, api_key: str | None ) -> dict[str, set[str]]: - """Ask again for image models, because a default listing may not hold them. + """Ask again for every modality, because a default listing may not hold them. - OpenRouter serves 444 models from /models and 54 from the image-filtered - query, 43 of which the first call never returns at all. This is enumeration, - not classification. An endpoint that ignores the parameter answers with the + OpenRouter's /models holds 464 models; `output_modalities=all` holds 635, + among them its 46 image and 21 speech models. This is enumeration, not + classification. An endpoint that ignores the parameter answers with the same set, so the union is a no-op and no provider has to be recognised. """ try: - return await _request_entries(base_url, api_key, image_only=True) + return await _request_entries(base_url, api_key, every_modality=True) except (httpx.HTTPError, ValueError): return {} diff --git a/surfsense_local/backend/modules/llm/models.py b/surfsense_local/backend/modules/llm/models.py index 5b71fc84c..18bcf2a0f 100644 --- a/surfsense_local/backend/modules/llm/models.py +++ b/surfsense_local/backend/modules/llm/models.py @@ -1,7 +1,7 @@ from dataclasses import replace from datetime import datetime -from sqlalchemy import CheckConstraint, ForeignKey, String, UniqueConstraint, func +from sqlalchemy import JSON, CheckConstraint, ForeignKey, String, UniqueConstraint, func from sqlalchemy.orm import Mapped, mapped_column, relationship from modules.llm.model_type import ModelType @@ -54,6 +54,10 @@ class SelectedModel(Base): params_b: Mapped[float | None] vendor: Mapped[str | None] line: Mapped[Line | None] = mapped_column(text_enum(Line)) + # What the user set for this model that no endpoint states, keyed by the + # slice that owns each entry: `voices` belongs to `llm/voices`. Null until + # something is set, and cleared when the slot takes another model. + settings: Mapped[dict | None] = mapped_column(JSON) updated_at: Mapped[datetime] = mapped_column( server_default=func.now(), onupdate=func.now() ) diff --git a/surfsense_local/backend/modules/llm/providers/audiocpp/joined_wav.py b/surfsense_local/backend/modules/llm/providers/audiocpp/joined_wav.py index 6093445fa..7054d5d7e 100644 --- a/surfsense_local/backend/modules/llm/providers/audiocpp/joined_wav.py +++ b/surfsense_local/backend/modules/llm/providers/audiocpp/joined_wav.py @@ -10,7 +10,7 @@ from collections.abc import Sequence GAP_SECONDS = 0.35 -def joined_wav(turns: Sequence[bytes]) -> bytes: +def joined_wav(turns: Sequence[bytes], gap: float = GAP_SECONDS) -> bytes: """The turns in order, which must share one format, as one model's do.""" frames: list[bytes] = [] params = None @@ -23,7 +23,7 @@ def joined_wav(turns: Sequence[bytes]) -> bytes: raise ValueError("the turns came back in different audio formats") if index: channels, width, rate = params - frames.append(b"\x00" * round(GAP_SECONDS * rate) * width * channels) + frames.append(b"\x00" * round(gap * rate) * width * channels) frames.append(turn.readframes(turn.getnframes())) if params is None: raise ValueError("nothing was voiced") diff --git a/surfsense_local/backend/modules/llm/providers/openai_compatible/image.py b/surfsense_local/backend/modules/llm/providers/openai_compatible/image.py index caeca0346..eefb995f5 100644 --- a/surfsense_local/backend/modules/llm/providers/openai_compatible/image.py +++ b/surfsense_local/backend/modules/llm/providers/openai_compatible/image.py @@ -38,9 +38,7 @@ class OpenAICompatibleImageProvider: async def generate(self, model: str, prompt: str) -> GeneratedImage: key = (self._connection_id, self._base_url) preferred = _route_cache.get(key, STANDARD_ROUTE) - alternate = ( - EXTENSION_ROUTE if preferred == STANDARD_ROUTE else STANDARD_ROUTE - ) + alternate = EXTENSION_ROUTE if preferred == STANDARD_ROUTE else STANDARD_ROUTE try: status, payload = await self._post(preferred, model, prompt) @@ -50,9 +48,7 @@ class OpenAICompatibleImageProvider: else: route = preferred if status >= 400: - raise NonRetryableImageError( - f"image endpoint returned HTTP {status}" - ) + raise NonRetryableImageError(f"image endpoint returned HTTP {status}") image = await self._normalize(payload) except NonRetryableImageError: raise @@ -62,9 +58,7 @@ class OpenAICompatibleImageProvider: _route_cache[key] = route return image - async def _post( - self, route: str, model: str, prompt: str - ) -> tuple[int, object]: + async def _post(self, route: str, model: str, prompt: str) -> tuple[int, object]: async with ( httpx.AsyncClient(timeout=TIMEOUT, headers=self._headers()) as client, client.stream( @@ -86,9 +80,7 @@ class OpenAICompatibleImageProvider: async def _normalize(self, payload: object) -> GeneratedImage: if not isinstance(payload, dict) or not isinstance(payload.get("data"), list): raise ValueError("image endpoint returned no data list") - entry = next( - (item for item in payload["data"] if isinstance(item, dict)), None - ) + entry = next((item for item in payload["data"] if isinstance(item, dict)), None) if entry is None: raise ValueError("image endpoint returned no image") diff --git a/surfsense_local/backend/modules/llm/providers/openai_compatible/joined_replies.py b/surfsense_local/backend/modules/llm/providers/openai_compatible/joined_replies.py new file mode 100644 index 000000000..5e940bf23 --- /dev/null +++ b/surfsense_local/backend/modules/llm/providers/openai_compatible/joined_replies.py @@ -0,0 +1,93 @@ +"""One episode from each line's reply, in the format the server sent. + +The format is read from the bytes, not from what was asked for: a server can +ignore `response_format`. WAV joins frame by frame with a pause between +speakers. MP3 joins frame to frame with no pause, because a silent frame would +need the episode's encoder settings, which no reply states. +""" + +import struct +from collections.abc import Sequence + +from modules.llm.providers.audiocpp.joined_wav import joined_wav +from modules.llm.providers.protocols import SynthesizedAudio + +__all__ = ["joined_replies"] + + +def joined_replies(replies: Sequence[bytes]) -> SynthesizedAudio: + """Raises ValueError when the replies are neither WAV nor MP3, or mixed.""" + kinds = {_kind(reply) for reply in replies} + if kinds == {"wav"}: + # A line's pieces run on; only a new line gets the pause. + lines = [joined_wav(_stacked_wavs(reply), gap=0) for reply in replies] + return SynthesizedAudio(joined_wav(lines), "audio/wav") + if kinds == {"mp3"}: + return SynthesizedAudio(b"".join(map(_mp3_frames, replies)), "audio/mpeg") + raise ValueError(f"the lines came back as {', '.join(sorted(kinds))}") + + +def _kind(reply: bytes) -> str: + if reply[:4] == b"RIFF" and reply[8:12] == b"WAVE": + return "wav" + if reply[:3] == b"ID3" or _frame_length(reply, 0): + return "mp3" + return "neither WAV nor MP3" + + +def _stacked_wavs(reply: bytes) -> list[bytes]: + """Each WAV in a reply, by the size its RIFF header states. A streamed WAV + states no size, and is taken whole.""" + parts = [] + at = 0 + while reply[at : at + 4] == b"RIFF": + (size,) = struct.unpack_from(" len(reply): + parts.append(reply[at:]) + return parts + parts.append(reply[at:end]) + at = end + return parts or [reply] + + +def _mp3_frames(reply: bytes) -> bytes: + """The audio frames alone: without the ID3v2 tag, and without the Xing or + Info frame, which states one line's length for the whole file.""" + at = 0 + if reply[:3] == b"ID3": + size = _syncsafe(reply[6:10]) + at = 10 + size + (10 if reply[5] & 0x10 else 0) + length = _frame_length(reply, at) + if length and (b"Xing" in reply[at : at + 40] or b"Info" in reply[at : at + 40]): + at += length + return reply[at:] + + +def _syncsafe(four: bytes) -> int: + return (four[0] << 21) | (four[1] << 14) | (four[2] << 7) | four[3] + + +# Kbit/s by index for MPEG-1 and for MPEG-2/2.5, Layer III. +_BITRATES = { + 1: (0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320), + 2: (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160), +} +_RATES = {3: (44100, 48000, 32000), 2: (22050, 24000, 16000), 0: (11025, 12000, 8000)} + + +def _frame_length(data: bytes, at: int) -> int: + """The byte length of the Layer III frame at `at`, or 0 when none starts there.""" + if len(data) < at + 4 or data[at] != 0xFF or data[at + 1] & 0xE0 != 0xE0: + return 0 + version = (data[at + 1] >> 3) & 0x3 + layer = (data[at + 1] >> 1) & 0x3 + bitrate_index = data[at + 2] >> 4 + rate_index = (data[at + 2] >> 2) & 0x3 + if version == 1 or layer != 1 or bitrate_index in (0, 15) or rate_index == 3: + return 0 + mpeg1 = version == 3 + bitrate = _BITRATES[1 if mpeg1 else 2][bitrate_index] * 1000 + rate = _RATES[version][rate_index] + padding = (data[at + 2] >> 1) & 0x1 + return (144 if mpeg1 else 72) * bitrate // rate + padding diff --git a/surfsense_local/backend/modules/llm/providers/openai_compatible/speech.py b/surfsense_local/backend/modules/llm/providers/openai_compatible/speech.py new file mode 100644 index 000000000..1f99d33cd --- /dev/null +++ b/surfsense_local/backend/modules/llm/providers/openai_compatible/speech.py @@ -0,0 +1,113 @@ +"""Podcast voices from a server's /audio/speech, one request per turn.""" + +import logging +import wave + +import httpx + +from modules.llm.connections.key_headers import key_headers +from modules.llm.providers.openai_compatible.joined_replies import joined_replies +from modules.llm.providers.protocols import SpokenTurn, SynthesizedAudio +from shared import cancellation + +__all__ = ["NonRetryableSpeechError", "RemoteSpeech"] + +logger = logging.getLogger(__name__) + +TURN_TIMEOUT = httpx.Timeout(120.0, connect=10.0) + + +class NonRetryableSpeechError(RuntimeError): + """Voicing began, so repeating the job would bill the episode twice.""" + + +class RemoteSpeech: + def __init__( + self, + model: str, + *, + base_url: str, + api_key: str | None, + transport: httpx.AsyncBaseTransport | None = None, + ) -> None: + self._model = model + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._transport = transport + + async def check_memory(self) -> None: + """The server voices it, so nothing here has to fit.""" + + async def synthesize( + self, turns: list[SpokenTurn], language: str + ) -> SynthesizedAudio: + async with httpx.AsyncClient( + timeout=TURN_TIMEOUT, + headers=key_headers(self._base_url, self._api_key), + transport=self._transport, + ) as client: + voiced = [] + # WAV first, because only WAV takes a pause between speakers. + # OpenRouter takes only mp3 or pcm, and pcm states no sample rate. + chosen = "wav" + for index, turn in enumerate(turns, start=1): + reply = await self._voice(client, index, len(turns), turn, chosen) + if reply.is_error and index == 1: + # Asked once, on the first line: a refusal after it is + # that line's own, not the format's. + retry = await self._voice(client, index, len(turns), turn, "mp3") + if not retry.is_error: + chosen, reply = "mp3", retry + if reply.is_error: + raise NonRetryableSpeechError( + f"the server could not voice turn {index} of {len(turns)}: " + f"{_server_message(reply)}" + ) + voiced.append(reply.content) + try: + return joined_replies(voiced) + except (wave.Error, EOFError, ValueError) as error: + raise NonRetryableSpeechError( + f"the server answered, but not as WAV or MP3: {error}" + ) from error + + async def _voice( + self, + client: httpx.AsyncClient, + index: int, + total: int, + turn: SpokenTurn, + response_format: str, + ) -> httpx.Response: + # A request in flight cannot be stopped; a turn not yet sent can. + cancellation.raise_if_cancelled() + logger.info( + "speech: turn %s/%s voice=%s %s chars as %s", + index, + total, + turn.voice, + len(turn.text), + response_format, + ) + try: + body = { + "model": self._model, + "input": turn.text, + "response_format": response_format, + } + # No voice leaves it to the server's default; only a test sends none. + if turn.voice: + body["voice"] = turn.voice + return await client.post(f"{self._base_url}/audio/speech", json=body) + except httpx.HTTPError as error: + raise NonRetryableSpeechError( + f"the server could not voice turn {index} of {total}: {error}" + ) from error + + +def _server_message(reply: httpx.Response) -> str: + """OpenAI's shape, {"error": {"message": ...}}, which most servers copy.""" + try: + return str(reply.json()["error"]["message"]) + except (ValueError, KeyError, TypeError): + return f"HTTP {reply.status_code}" diff --git a/surfsense_local/backend/modules/llm/providers/openai_compatible/voice_list.py b/surfsense_local/backend/modules/llm/providers/openai_compatible/voice_list.py new file mode 100644 index 000000000..e4b96848f --- /dev/null +++ b/surfsense_local/backend/modules/llm/providers/openai_compatible/voice_list.py @@ -0,0 +1,58 @@ +"""A server's voices, asked of the server itself. + +The OpenAI API has no route that lists voices, and models.dev carries none, +so nothing else knows them. `GET /audio/voices` is Kokoro-FastAPI's; a server +without it leaves the user to type the voices. +""" + +import httpx + +from modules.llm.connections.key_headers import key_headers +from modules.llm.providers.protocols import Voice + +__all__ = ["listed_voices"] + +LIST_TIMEOUT = httpx.Timeout(10.0, connect=5.0) + +# Held for the process, never stored: a server gains voices on its own +# schedule, and the next start asks again. Only answers are held; a server +# that could not be reached is asked again. +_answers: dict[tuple[int, str], list[Voice] | None] = {} + + +def listed_voices( + connection_id: int, + base_url: str, + api_key: str | None, + *, + transport: httpx.BaseTransport | None = None, +) -> list[Voice] | None: + """The server's voices, or None when it lists none. Ids alone: no server + states a voice's gender or languages here, so none is claimed.""" + key = (connection_id, base_url) + if key in _answers: + return _answers[key] + try: + with httpx.Client( + timeout=LIST_TIMEOUT, + headers=key_headers(base_url, api_key), + transport=transport, + ) as client: + reply = client.get(f"{base_url.rstrip('/')}/audio/voices") + except httpx.HTTPError: + return None + _answers[key] = _voices(reply) + return _answers[key] + + +def _voices(reply: httpx.Response) -> list[Voice] | None: + if reply.is_error: + return None + try: + listed = reply.json()["voices"] + ids = [entry["id"] for entry in listed] + except (ValueError, KeyError, TypeError): + return None + if not ids or not all(isinstance(id, str) and id for id in ids): + return None + return [Voice(id, id, None, ()) for id in dict.fromkeys(ids)] diff --git a/surfsense_local/backend/modules/llm/providers/protocols.py b/surfsense_local/backend/modules/llm/providers/protocols.py index 00a63d032..f14324816 100644 --- a/surfsense_local/backend/modules/llm/providers/protocols.py +++ b/surfsense_local/backend/modules/llm/providers/protocols.py @@ -82,7 +82,8 @@ class ImageGenerator(Protocol): class Voice: id: str label: str - gender: Literal["female", "male"] + # None where the source does not say: OpenAI documents none for its voices. + gender: Literal["female", "male"] | None # The languages this voice speaks, as the model's entry names them: one for # a Kokoro voice, every one the model speaks for a Supertonic voice. languages: tuple[str, ...] @@ -103,9 +104,7 @@ class SynthesizedAudio: class TextToSpeech(Protocol): - """Anything that voices a script: audio.cpp on this computer today.""" - - def voices(self) -> list[Voice]: ... + """Anything that voices a script: audio.cpp here, or a server's /audio/speech.""" async def check_memory(self) -> None: """Raise, with the sentence a person reads, when voicing cannot fit.""" diff --git a/surfsense_local/backend/modules/llm/resolution.py b/surfsense_local/backend/modules/llm/resolution.py index 811b2ef7e..8e45b9146 100644 --- a/surfsense_local/backend/modules/llm/resolution.py +++ b/surfsense_local/backend/modules/llm/resolution.py @@ -14,6 +14,7 @@ from modules.llm.providers.openai_compatible import ( OpenAICompatibleChatProvider, OpenAICompatibleImageProvider, ) +from modules.llm.providers.openai_compatible.speech import RemoteSpeech from modules.llm.providers.protocols import Generator, ImageGenerator, TextToSpeech from modules.llm.providers.sdcpp import provider as sdcpp from modules.llm.providers.sdcpp.generator import LocalImageGenerator @@ -24,10 +25,6 @@ class ModelResolutionError(RuntimeError): pass -class VoiceNotLocalError(ModelResolutionError): - """The audio model is a server's, and nothing calls a remote speech endpoint.""" - - @dataclass(frozen=True) class ResolvedGeneration: selection: SelectedModel @@ -96,14 +93,41 @@ def resolve_image_generation(session: Session) -> ResolvedImageGeneration: ) +def speech_selected(session: Session) -> None: + """Raise unless the chosen audio model can be reached for, without reaching + it: the format list asks, and must not call a server to answer.""" + selected = selected_audio(session) + if selected.provider == audiocpp.PROVIDER: + local_speech(selected) + else: + stored_connection(session, selected) + + def resolve_text_to_speech(session: Session) -> TextToSpeech: + selected = selected_audio(session) + if selected.provider == audiocpp.PROVIDER: + return local_speech(selected) + return _remote_speech(selected, _connection(session, selected)) + + +def _remote_speech( + selected: SelectedModel, connection: ProviderConnection +) -> RemoteSpeech: + return RemoteSpeech( + selected.name, + base_url=connection.base_url, + api_key=connection.api_key, + ) + + +def selected_audio(session: Session) -> SelectedModel: selected = session.get(SelectedModel, ModelType.AUDIO_GEN) if selected is None: raise ModelResolutionError("no audio model selected") - if selected.provider != audiocpp.PROVIDER: - raise VoiceNotLocalError( - "podcasts are voiced by an audio model on this computer" - ) + return selected + + +def local_speech(selected: SelectedModel) -> AudioCppSpeech: engine = get_local_catalog().audiocpp installed = engine.installed_model(selected.name) if installed is None: @@ -119,10 +143,16 @@ def resolve_text_to_speech(session: Session) -> TextToSpeech: def _connection(session: Session, selected: SelectedModel) -> ProviderConnection: + """The selection's connection, once egress to its host is allowed.""" + connection = stored_connection(session, selected) + egress.require(session, egress.host_destination(connection.base_url)) + return connection + + +def stored_connection(session: Session, selected: SelectedModel) -> ProviderConnection: if selected.provider != "openai_compatible" or selected.connection_id is None: raise ModelResolutionError(f"unknown provider: {selected.provider}") connection = session.get(ProviderConnection, selected.connection_id) if connection is None: raise ModelResolutionError("selected model connection no longer exists") - egress.require(session, egress.host_destination(connection.base_url)) return connection diff --git a/surfsense_local/backend/modules/llm/router.py b/surfsense_local/backend/modules/llm/router.py index 6673cf89a..4d5113b1c 100644 --- a/surfsense_local/backend/modules/llm/router.py +++ b/surfsense_local/backend/modules/llm/router.py @@ -35,6 +35,7 @@ from modules.llm.schemas import ( ) from modules.llm.selectable import selectable_for from modules.llm.selection import choose_model, complete_onboarding +from modules.llm.voices.router import router as voices_router from shared.config import get_llm_settings router = APIRouter(prefix="/llm", tags=["llm"]) @@ -42,6 +43,7 @@ router.include_router(local_catalog_router) router.include_router(install_jobs_router) router.include_router(remote_catalog_router) router.include_router(connections_router) +router.include_router(voices_router) @router.get( diff --git a/surfsense_local/backend/modules/llm/schemas.py b/surfsense_local/backend/modules/llm/schemas.py index 9d5604616..5602eda73 100644 --- a/surfsense_local/backend/modules/llm/schemas.py +++ b/surfsense_local/backend/modules/llm/schemas.py @@ -91,6 +91,8 @@ class ModelTestWrite(BaseModel): model: str = Field(min_length=1, max_length=512) prompt: str | None = Field(default=None, max_length=2000) + # A speech test's voice; the first reviewed one when omitted, if any. + voice: str | None = Field(default=None, max_length=100) class ChatTestRead(BaseModel): diff --git a/surfsense_local/backend/modules/llm/selection.py b/surfsense_local/backend/modules/llm/selection.py index feced979f..871d73f49 100644 --- a/surfsense_local/backend/modules/llm/selection.py +++ b/surfsense_local/backend/modules/llm/selection.py @@ -110,6 +110,13 @@ def _store( if selected is None: selected = SelectedModel(model_type=model_type, name=model_name) session.add(selected) + elif (selected.provider, selected.connection_id, selected.name) != ( + provider_name, + connection_id, + model_name, + ): + # Settings are the model's own; another model starts without them. + selected.settings = None selected.provider = provider_name selected.connection_id = connection_id selected.name = model_name diff --git a/surfsense_local/backend/modules/llm/voices/__init__.py b/surfsense_local/backend/modules/llm/voices/__init__.py new file mode 100644 index 000000000..687c39eb2 --- /dev/null +++ b/surfsense_local/backend/modules/llm/voices/__init__.py @@ -0,0 +1,3 @@ +"""The voices the chosen audio model offers podcasts, from one place: the local +model's reviewed roster, a server's own list, or the ones the user added, kept +in the selection's settings.""" diff --git a/surfsense_local/backend/modules/llm/voices/roster.py b/surfsense_local/backend/modules/llm/voices/roster.py new file mode 100644 index 000000000..e7fa791b9 --- /dev/null +++ b/surfsense_local/backend/modules/llm/voices/roster.py @@ -0,0 +1,75 @@ +"""Which voices the chosen audio model offers, and where they came from.""" + +from dataclasses import dataclass +from typing import Literal + +from sqlalchemy.orm import Session + +from modules.egress import service as egress +from modules.llm.models import ProviderConnection, SelectedModel +from modules.llm.providers import audiocpp +from modules.llm.providers.openai_compatible.voice_list import listed_voices +from modules.llm.providers.protocols import Voice +from modules.llm.resolution import local_speech, selected_audio, stored_connection +from modules.llm.voices.saved import saved_voices + +__all__ = ["SpeechRoster", "VoicedBy", "server_roster", "speech_voices", "voices_page"] + +Source = Literal["local", "server", "saved"] + + +@dataclass(frozen=True) +class VoicedBy: + """The server that voices a podcast, named for the brief.""" + + server: str + model: str + + +@dataclass(frozen=True) +class SpeechRoster: + """The voices a brief may pick, and where they came from. Empty only for a + server model with none added yet.""" + + voices: list[Voice] + source: Source + voiced_by: VoicedBy | None + + +def speech_voices(session: Session) -> SpeechRoster: + selected = selected_audio(session) + if selected.provider == audiocpp.PROVIDER: + return SpeechRoster(local_speech(selected).voices(), "local", None) + connection = stored_connection(session, selected) + source, ids = server_roster(session, selected, connection) + return SpeechRoster( + # Ids alone: no server states a voice's gender or languages. + [Voice(voice, voice, None, ()) for voice in ids], + source, + VoicedBy(connection.label, selected.name), + ) + + +def server_roster( + session: Session, selected: SelectedModel, connection: ProviderConnection +) -> tuple[Literal["server", "saved"], list[str]]: + """The server's own list when it has one; otherwise the voices the user + added. The server is asked only once its host is allowed; until then the + added voices stand, since reading them reaches nothing.""" + try: + egress.require(session, egress.host_destination(connection.base_url)) + except egress.EgressDeniedError: + listed = None + else: + listed = listed_voices(connection.id, connection.base_url, connection.api_key) + if listed is not None: + return "server", [voice.id for voice in listed] + return "saved", saved_voices(selected) + + +def voices_page(connection: ProviderConnection, model: str) -> str | None: + """The page that names a model's voices, where its provider documents one: + OpenRouter's docs send readers to each model's page.""" + if connection.catalog_provider == "openrouter": + return f"https://openrouter.ai/{model}" + return None diff --git a/surfsense_local/backend/modules/llm/voices/router.py b/surfsense_local/backend/modules/llm/voices/router.py new file mode 100644 index 000000000..342bc61f2 --- /dev/null +++ b/surfsense_local/backend/modules/llm/voices/router.py @@ -0,0 +1,95 @@ +from typing import Annotated + +from fastapi import APIRouter, HTTPException, Response, status +from pydantic import BaseModel, Field, StringConstraints +from sqlalchemy.orm import Session + +from api.dependencies import SessionDep, transact +from modules.llm.connections.router import allowed_connection +from modules.llm.model_type import ModelType +from modules.llm.models import ProviderConnection, SelectedModel +from modules.llm.providers.openai_compatible.speech import ( + NonRetryableSpeechError, + RemoteSpeech, +) +from modules.llm.providers.protocols import SpokenTurn +from modules.llm.voices.roster import server_roster, voices_page +from modules.llm.voices.saved import forget, keep + +router = APIRouter(prefix="/selection/audio_gen/voices", tags=["llm"]) + +# Heard before it is kept: a voice the server refuses never reaches a podcast. +TRIAL_LINE = "Hello. This is how your podcasts will sound." + +VoiceId = Annotated[ + str, StringConstraints(strip_whitespace=True, min_length=1, max_length=100) +] + + +class VoicesRead(BaseModel): + # What a voice is played through: the connection's speech test. + connection_id: int + model: str + source: str + voices: list[str] + voices_page: str | None + + +class VoiceWrite(BaseModel): + voice: VoiceId + # The line to hear it say, in the interface's language; English otherwise. + text: str | None = Field(default=None, min_length=1, max_length=300) + + +@router.get("", response_model=VoicesRead) +def list_voices(session: SessionDep) -> VoicesRead: + selected, connection = _server_selection(session) + source, voices = server_roster(session, selected, connection) + session.commit() # egress stamps the host's last call + return VoicesRead( + connection_id=connection.id, + model=selected.name, + source=source, + voices=voices, + voices_page=voices_page(connection, selected.name), + ) + + +@router.post("", status_code=status.HTTP_201_CREATED) +async def add_voice(payload: VoiceWrite, session: SessionDep) -> Response: + """Voice one line in it, then keep it; a refusal keeps nothing.""" + selected, connection = await transact(session, _server_selection) + speech = RemoteSpeech( + selected.name, base_url=connection.base_url, api_key=connection.api_key + ) + try: + await speech.synthesize( + [SpokenTurn(payload.voice, payload.text or TRIAL_LINE)], "" + ) + except NonRetryableSpeechError as error: + raise HTTPException(status.HTTP_502_BAD_GATEWAY, str(error)) from error + await transact(session, _keep, payload.voice) + return Response(status_code=status.HTTP_201_CREATED) + + +@router.delete("", status_code=status.HTTP_204_NO_CONTENT) +def remove_voice(voice: VoiceId, session: SessionDep) -> Response: + selected, _ = _server_selection(session) + forget(session, selected, voice) + session.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +def _server_selection(session: Session) -> tuple[SelectedModel, ProviderConnection]: + """The audio slot's server model and its connection, once egress allows it.""" + selected = session.get(SelectedModel, ModelType.AUDIO_GEN) + if selected is None or selected.connection_id is None: + raise HTTPException( + status.HTTP_409_CONFLICT, "the audio model is not a server's" + ) + return selected, allowed_connection(session, selected.connection_id) + + +def _keep(session: Session, voice: str) -> None: + selected, _ = _server_selection(session) + keep(session, selected, voice) diff --git a/surfsense_local/backend/modules/llm/voices/saved.py b/surfsense_local/backend/modules/llm/voices/saved.py new file mode 100644 index 000000000..564fffde2 --- /dev/null +++ b/surfsense_local/backend/modules/llm/voices/saved.py @@ -0,0 +1,41 @@ +"""The voices the user added for the audio slot's server model, kept under +`voices` in the selection's settings. Only this module reads or writes it.""" + +from sqlalchemy.orm import Session + +from modules.llm.models import SelectedModel + +__all__ = ["forget", "keep", "saved_voices"] + +KEY = "voices" + + +def saved_voices(selected: SelectedModel) -> list[str]: + voices = (selected.settings or {}).get(KEY) + return ( + [voice for voice in voices if isinstance(voice, str)] + if isinstance(voices, list) + else [] + ) + + +def keep(session: Session, selected: SelectedModel, voice: str) -> None: + voices = saved_voices(selected) + if voice not in voices: + _write(selected, [*voices, voice]) + session.flush() + + +def forget(session: Session, selected: SelectedModel, voice: str) -> None: + _write(selected, [kept for kept in saved_voices(selected) if kept != voice]) + session.flush() + + +def _write(selected: SelectedModel, voices: list[str]) -> None: + # Reassigned, not mutated: a JSON column tracks assignment only. + settings = { + key: value for key, value in (selected.settings or {}).items() if key != KEY + } + if voices: + settings[KEY] = voices + selected.settings = settings or None diff --git a/surfsense_local/backend/tests/integration/artifacts/test_routes.py b/surfsense_local/backend/tests/integration/artifacts/test_routes.py index 855b1c214..8c21deae5 100644 --- a/surfsense_local/backend/tests/integration/artifacts/test_routes.py +++ b/surfsense_local/backend/tests/integration/artifacts/test_routes.py @@ -14,6 +14,8 @@ from modules.llm.models import ProviderConnection, SelectedModel from shared.config import get_llm_settings, get_storage_settings from shared.db import create_session_factory from shared.queue import studio_queue +from tests.integration.llm import conftest as speech_server +from tests.integration.llm.conftest import openai_server # noqa: F401 pytestmark = pytest.mark.integration @@ -124,14 +126,11 @@ async def test_podcast_is_gated_on_an_audio_model( assert podcast["requires_model_types"] == ["text_gen", "audio_gen"] -async def test_a_podcast_voices_only_on_this_computer( - client: AsyncClient, engine: Engine, workspace_id: int, choose_model: None -) -> None: - """A server's audio model can be chosen, but nothing calls a remote speech - endpoint yet, so the podcast says where the model must run.""" +def choose_server_voice(engine: Engine, base_url: str, name: str = "kokoro") -> None: + """A server's audio model as the podcast's voice.""" with create_session_factory(engine)() as session: server = ProviderConnection( - label="speech server", provider="openai_compatible", base_url="http://tts" + label="speech server", provider="openai_compatible", base_url=base_url ) session.add(server) session.flush() @@ -140,16 +139,114 @@ async def test_a_podcast_voices_only_on_this_computer( model_type=ModelType.AUDIO_GEN, provider="openai_compatible", connection_id=server.id, - name="tts-1", + name=name, ) ) session.commit() + +def podcast_job(engine: Engine, workspace_id: int, *voices: str) -> dict: + """A podcast job whose speakers take `voices`, one each, written in French.""" + speakers = [ + {"name": name, "role": "host", "voice": voice} + for name, voice in zip(("Ana", "Ben"), voices, strict=False) + ] + return { + "format": "podcast", + "document_ids": [make_ready_source(engine, workspace_id)], + "options": {"language": "fr", "speakers": speakers}, + } + + +async def test_a_server_that_lists_its_voices_fills_the_brief( + client: AsyncClient, + engine: Engine, + workspace_id: int, + choose_model: None, + openai_server: str, # noqa: F811 + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Asked of the server, as Kokoro-FastAPI answers: its ids fill the picker, + and only they voice a job. It states no language, so any script one goes.""" + monkeypatch.setattr(speech_server, "AUDIO_VOICES", ["af_heart", "am_adam"]) + choose_server_voice(engine, openai_server) + + brief_url = f"/workspaces/{workspace_id}/studio/podcast/brief" + opened = (await client.get(brief_url)).json() + assert opened["voices_source"] == "server" + assert opened["voiced_by"] == {"server": "speech server", "model": "kokoro"} + assert [v["id"] for v in opened["voices"]] == ["af_heart", "am_adam"] + assert {v["gender"] for v in opened["voices"]} == {None} + assert "fr" in opened["languages"] + assert [s["voice"] for s in opened["brief"]["speakers"]] == ["af_heart", "am_adam"] + + url = f"/workspaces/{workspace_id}/studio/jobs" + unlisted = await client.post(url, json=podcast_job(engine, workspace_id, "bf_emma")) + assert unlisted.status_code == 422 + created = await client.post( + url, json=podcast_job(engine, workspace_id, "af_heart", "am_adam") + ) + assert created.status_code == 201, created.text + + +async def test_a_server_that_lists_no_voices_takes_the_ones_added_for_it( + client: AsyncClient, + engine: Engine, + workspace_id: int, + choose_model: None, + openai_server: str, # noqa: F811 +) -> None: + """None is guessed: until voices are added in Settings the brief has none + and a job is refused; then they are its picker.""" + choose_server_voice(engine, openai_server) + brief_url = f"/workspaces/{workspace_id}/studio/podcast/brief" + url = f"/workspaces/{workspace_id}/studio/jobs" + + opened = (await client.get(brief_url)).json() + assert (opened["voices_source"], opened["voices"]) == ("saved", []) + refused = await client.post(url, json=podcast_job(engine, workspace_id, "alloy")) + assert refused.status_code == 422 + assert "Settings" in refused.json()["detail"] + + for voice in ("alloy", "echo"): + added = await client.post("/llm/selection/audio_gen/voices", json={"voice": voice}) + assert added.status_code == 201, added.text + + opened = (await client.get(brief_url)).json() + assert [v["id"] for v in opened["voices"]] == ["alloy", "echo"] + created = await client.post( + url, json=podcast_job(engine, workspace_id, "alloy", "echo") + ) + assert created.status_code == 201, created.text + + +async def test_a_server_not_yet_allowed_is_not_asked( + client: AsyncClient, engine: Engine, workspace_id: int, choose_model: None +) -> None: + """The format list reaches no server, and the brief asks one only once its + host is allowed: until then only the added voices stand.""" + choose_server_voice(engine, "https://tts.example.com/v1") + url = f"/workspaces/{workspace_id}/studio/formats" podcast = next(f for f in (await client.get(url)).json() if f["key"] == "podcast") + assert podcast["available"] is True - assert podcast["available"] is False - assert podcast["unavailable_reason"] == "Needs an audio model on this computer" + brief_url = f"/workspaces/{workspace_id}/studio/podcast/brief" + opened = (await client.get(brief_url)).json() + assert (opened["voices_source"], opened["voices"]) == ("saved", []) + + +async def test_a_model_on_this_computer_voices_from_its_roster( + client: AsyncClient, + workspace_id: int, + choose_model: None, + local_voice: None, +) -> None: + """Nothing on a server: no line saying who voices it, grouped by gender.""" + brief_url = f"/workspaces/{workspace_id}/studio/podcast/brief" + opened = (await client.get(brief_url)).json() + + assert (opened["voices_source"], opened["voiced_by"]) == ("local", None) KOKORO = ("kokoro-82m-q8_0", "Q8_0") diff --git a/surfsense_local/backend/tests/integration/llm/conftest.py b/surfsense_local/backend/tests/integration/llm/conftest.py index 388d36b3e..731e6c30c 100644 --- a/surfsense_local/backend/tests/integration/llm/conftest.py +++ b/surfsense_local/backend/tests/integration/llm/conftest.py @@ -1,5 +1,7 @@ +import io import json import threading +import wave from collections.abc import Iterator from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import parse_qs, urlsplit @@ -203,6 +205,11 @@ REMOTE_REQUESTS: list[tuple[str, str]] = [] # Set to an HTTP status to make /models refuse, as a provider rejecting the key # or rate-limiting would; the fixture resets it. MODELS_STATUS: int | None = None +# The voices GET /audio/voices lists, as Kokoro-FastAPI does; None is a 404, +# as OpenAI, Groq and OpenRouter answer. +AUDIO_VOICES: list[str] | None = None +# Voices /audio/speech refuses with a 400, as a server answers an unknown one. +REFUSED_VOICES: set[str] = set() class StubOpenAICompatible(BaseHTTPRequestHandler): @@ -210,11 +217,15 @@ class StubOpenAICompatible(BaseHTTPRequestHandler): def do_GET(self) -> None: parsed = urlsplit(self.path) - if parsed.path == "/models" and MODELS_STATUS is not None: + if parsed.path == "/audio/voices" and AUDIO_VOICES is not None: + self._json({"voices": [{"id": voice} for voice in AUDIO_VOICES]}) + elif parsed.path == "/models" and MODELS_STATUS is not None: self.send_error(MODELS_STATUS) elif parsed.path == "/models": - image_only = parse_qs(parsed.query).get("output_modalities") == ["image"] - models = REMOTE_MODELS[1:] if image_only else REMOTE_MODELS[:1] + # As OpenRouter: the default listing holds only the chat models, + # and `output_modalities=all` every model it serves. + every = parse_qs(parsed.query).get("output_modalities") == ["all"] + models = REMOTE_MODELS if every else REMOTE_MODELS[:1] self._json({"object": "list", "data": models}) else: self.send_error(404) @@ -235,6 +246,23 @@ class StubOpenAICompatible(BaseHTTPRequestHandler): ] frames.append("data: [DONE]\n\n") self._send("".join(frames).encode()) + elif ( + self.path == "/audio/speech" + and json.loads(body).get("voice") in REFUSED_VOICES + ): + self.send_response(400) + error = json.dumps({"error": {"message": "unknown voice"}}).encode() + self.send_header("Content-Length", str(len(error))) + self.end_headers() + self.wfile.write(error) + elif self.path == "/audio/speech": + out = io.BytesIO() + with wave.open(out, "wb") as clip: + clip.setnchannels(1) + clip.setsampwidth(2) + clip.setframerate(24000) + clip.writeframes(b"\x00\x00" * 2400) + self._send(out.getvalue()) elif self.path == "/images/generations": encoded = "iVBORw0KGgpmYWtl" self._json({"data": [{"b64_json": encoded, "media_type": "image/png"}]}) @@ -257,9 +285,11 @@ class StubOpenAICompatible(BaseHTTPRequestHandler): @pytest.fixture def openai_server() -> Iterator[str]: """A real OpenAI-compatible endpoint on a real port.""" - global MODELS_STATUS + global MODELS_STATUS, AUDIO_VOICES, REFUSED_VOICES REMOTE_REQUESTS.clear() MODELS_STATUS = None + AUDIO_VOICES = None + REFUSED_VOICES = set() server = ThreadingHTTPServer(("127.0.0.1", 0), StubOpenAICompatible) threading.Thread(target=server.serve_forever, daemon=True).start() url = f"http://127.0.0.1:{server.server_port}" diff --git a/surfsense_local/backend/tests/integration/llm/test_connections.py b/surfsense_local/backend/tests/integration/llm/test_connections.py index 92aa801d5..6aae33346 100644 --- a/surfsense_local/backend/tests/integration/llm/test_connections.py +++ b/surfsense_local/backend/tests/integration/llm/test_connections.py @@ -697,3 +697,51 @@ async def test_an_unreachable_provider_is_named_as_unreachable( assert listed.status_code == 502 assert listed.json()["detail"]["code"] == "provider_unreachable" + + +async def test_a_speech_model_is_heard_before_it_voices_podcasts( + client: AsyncClient, openai_server: str +) -> None: + """One short line as a clip the dialog plays: in the typed voice, or with + none, so the server speaks in its own default where it has one.""" + connection = await _connect(client, openai_server) + url = f"/llm/connections/{connection['id']}/speech-test" + + tested = await client.post(url, json={"model": "tts-1", "voice": " af_heart "}) + assert tested.status_code == 200, tested.text + assert tested.headers["content-type"] == "audio/wav" + assert tested.headers["cache-control"] == "no-store" + path, body = REMOTE_REQUESTS[-1] + assert (path, json.loads(body)["voice"]) == ("/audio/speech", "af_heart") + + unvoiced = await client.post(url, json={"model": "tts-1"}) + assert unvoiced.status_code == 200, unvoiced.text + assert "voice" not in json.loads(REMOTE_REQUESTS[-1][1]) + + +async def test_a_speech_model_only_the_full_listing_holds_fills_the_audio_slot( + client: AsyncClient, openai_server: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """OpenRouter leaves its speech models out of /models; its `speech` output + is the audio slot.""" + monkeypatch.setattr( + conftest, + "REMOTE_MODELS", + [ + *conftest.REMOTE_MODELS, + { + "id": "hexgrad/kokoro-82m", + "architecture": {"output_modalities": ["speech"]}, + }, + ], + ) + connection = await _connect(client, openai_server) + + models = { + model["name"]: model + for model in ( + await client.get(f"/llm/connections/{connection['id']}/models") + ).json() + } + + assert models["hexgrad/kokoro-82m"]["selectable_for"] == ["audio_gen"] diff --git a/surfsense_local/backend/tests/integration/llm/test_voices.py b/surfsense_local/backend/tests/integration/llm/test_voices.py new file mode 100644 index 000000000..771a27b3d --- /dev/null +++ b/surfsense_local/backend/tests/integration/llm/test_voices.py @@ -0,0 +1,162 @@ +"""The voices a server's audio model offers podcasts: listed by the server, +or added by the user to the audio selection once each has been heard.""" + +import json + +import pytest +from httpx import AsyncClient + +from . import conftest +from .conftest import REMOTE_REQUESTS + +pytestmark = pytest.mark.integration + +URL = "/llm/selection/audio_gen/voices" + + +async def _choose( + client: AsyncClient, + base_url: str, + model: str = "tts-1", + catalog_provider: str = "custom", +) -> int: + """A server's audio model in the audio slot; returns its connection id.""" + reply = await client.post( + "/llm/connections", + json={ + "label": f"Speech {model}", + "provider": "openai_compatible", + "base_url": base_url, + "api_key": "secret", + "catalog_provider": catalog_provider, + }, + ) + assert reply.status_code == 201, reply.text + connection = reply.json()["id"] + await _select(client, connection, model) + return connection + + +async def _select(client: AsyncClient, connection: int, model: str) -> None: + chosen = await client.put( + "/llm/selection/audio_gen", + json={ + "provider": "openai_compatible", + "connection_id": connection, + "name": model, + "allow_unlisted": True, + }, + ) + assert chosen.status_code == 200, chosen.text + + +async def test_a_server_that_lists_its_voices_offers_those( + client: AsyncClient, openai_server: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """Asked of the server, as Kokoro-FastAPI answers; nothing to add.""" + monkeypatch.setattr(conftest, "AUDIO_VOICES", ["af_heart", "am_adam"]) + connection = await _choose(client, openai_server, "kokoro") + + assert (await client.get(URL)).json() == { + "connection_id": connection, + "model": "kokoro", + "source": "server", + "voices": ["af_heart", "am_adam"], + "voices_page": None, + } + + +async def test_a_voice_is_kept_only_once_the_server_has_voiced_it( + client: AsyncClient, openai_server: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """A server that lists none takes the user's own: each heard before it is + kept, so a podcast never meets a voice the server refuses.""" + monkeypatch.setattr(conftest, "REFUSED_VOICES", {"nobody"}) + connection = await _choose(client, openai_server) + + assert (await client.get(URL)).json() == { + "connection_id": connection, + "model": "tts-1", + "source": "saved", + "voices": [], + "voices_page": None, + } + + refused = await client.post(URL, json={"voice": "nobody"}) + assert refused.status_code == 502 + assert "unknown voice" in refused.json()["detail"] + + added = await client.post(URL, json={"voice": " alloy "}) + assert added.status_code == 201, added.text + path, body = REMOTE_REQUESTS[-1] + assert (path, json.loads(body)["voice"]) == ("/audio/speech", "alloy") + assert (await client.post(URL, json={"voice": "alloy"})).status_code == 201 + await client.post(URL, json={"voice": "echo"}) + + assert (await client.get(URL)).json()["voices"] == ["alloy", "echo"] + + removed = await client.delete(f"{URL}?voice=alloy") + assert removed.status_code == 204 + assert (await client.get(URL)).json()["voices"] == ["echo"] + + +async def test_choosing_another_model_starts_its_voices_afresh( + client: AsyncClient, openai_server: str +) -> None: + """The voices belong to the model in the slot; another model has its own.""" + connection = await _choose(client, openai_server) + await client.post(URL, json={"voice": "alloy"}) + + await _select(client, connection, "tts-1") + assert (await client.get(URL)).json()["voices"] == ["alloy"] + await _select(client, connection, "tts-1-hd") + assert (await client.get(URL)).json()["voices"] == [] + + +async def test_openrouter_links_the_page_that_names_a_models_voices( + client: AsyncClient, openai_server: str +) -> None: + """OpenRouter lists no voices by API; its docs point to each model's page.""" + await _choose( + client, + openai_server, + "bytedance-seed/seed-audio-1-0", + catalog_provider="openrouter", + ) + + assert (await client.get(URL)).json()[ + "voices_page" + ] == "https://openrouter.ai/bytedance-seed/seed-audio-1-0" + + +async def test_a_model_on_this_computer_has_no_server_voices( + client: AsyncClient, +) -> None: + """Its voices are its reviewed roster; nothing here is for it.""" + assert (await client.get(URL)).status_code == 409 + + +async def test_a_removed_server_takes_its_voices_with_it( + client: AsyncClient, openai_server: str +) -> None: + """They go with the audio selection, which goes with its connection.""" + connection = await _choose(client, openai_server) + await client.post(URL, json={"voice": "alloy"}) + + await client.delete(f"/llm/connections/{connection}") + await _choose(client, openai_server) + + assert (await client.get(URL)).json()["voices"] == [] + + +async def test_a_voice_is_heard_in_the_line_it_is_given( + client: AsyncClient, openai_server: str +) -> None: + """The app sends the line in the interface's language, so the voice is + heard speaking it; nothing says which languages a server's voice speaks.""" + await _choose(client, openai_server) + + await client.post(URL, json={"voice": "ff_siwis", "text": "Bonjour."}) + + path, body = REMOTE_REQUESTS[-1] + assert (path, json.loads(body)["input"]) == ("/audio/speech", "Bonjour.") diff --git a/surfsense_local/backend/tests/integration/worker/test_studio.py b/surfsense_local/backend/tests/integration/worker/test_studio.py index 796b7f435..f27dd2d1a 100644 --- a/surfsense_local/backend/tests/integration/worker/test_studio.py +++ b/surfsense_local/backend/tests/integration/worker/test_studio.py @@ -14,6 +14,7 @@ from modules.llm.providers.audiocpp.memory import NotEnoughMemoryError from modules.llm.providers.audiocpp.speech import AudioCppSpeech, VoicedModel from modules.llm.providers.llamacpp import RouterClient from modules.llm.providers.openai_compatible import NonRetryableImageError +from modules.llm.providers.openai_compatible.speech import NonRetryableSpeechError from modules.llm.providers.protocols import ( GeneratedImage, SpokenTurn, @@ -530,6 +531,32 @@ def test_a_memory_refusal_is_not_retried( assert artifact.document.error_message == SHORT +class RefusingServer(ShortOfMemoryAtVoicing): + async def synthesize(self, turns: list[SpokenTurn], language: str): + raise NonRetryableSpeechError("the server could not voice turn 1 of 2: bad key") + + +def test_a_servers_voicing_refusal_is_not_retried( + session: Session, stub_model: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A retry would draft and bill the whole episode again.""" + _capture_model( + monkeypatch, + '{"title": "T", "segments": [{"title": "One"}]}', + '{"turns": [{"speaker": 1, "text": "Hi."}, {"speaker": 2, "text": "Hello."}]}', + ) + monkeypatch.setattr( + "worker.studio.job.resolve_text_to_speech", lambda _session: RefusingServer() + ) + artifact = make_artifact(session, fmt="podcast", options=BRIEF) + + run(artifact.id) # returning, not raising, is what spares a Huey retry + + session.expire_all() + assert artifact.document.status is DocumentStatus.FAILED + assert "bad key" in artifact.document.error_message + + def test_a_podcast_cancelled_while_voicing_stops_at_the_next_turn( session: Session, stub_model: None, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/surfsense_local/backend/tests/unit/llm/catalog/remote/test_classifier.py b/surfsense_local/backend/tests/unit/llm/catalog/remote/test_classifier.py index e211a9f7d..0dae313b5 100644 --- a/surfsense_local/backend/tests/unit/llm/catalog/remote/test_classifier.py +++ b/surfsense_local/backend/tests/unit/llm/catalog/remote/test_classifier.py @@ -95,6 +95,13 @@ def test_a_model_that_emits_two_media_carries_two_types() -> None: } +def test_a_model_that_only_speaks_is_an_audio_model() -> None: + """tts-1 declares text in, audio out, and no context window.""" + assert classify("tts-1", _entry(["text"], ["audio"], context=None)) == { + ModelType.AUDIO_GEN + } + + def test_video_out_is_video_generation() -> None: """Named because the type exists in the source, not because a tab wants it.""" assert classify("happyhorse-1.1-t2v", _entry(["text"], ["video"])) == { diff --git a/surfsense_local/backend/tests/unit/llm/providers/openai_compatible/__init__.py b/surfsense_local/backend/tests/unit/llm/providers/openai_compatible/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_local/backend/tests/unit/llm/providers/openai_compatible/test_speech.py b/surfsense_local/backend/tests/unit/llm/providers/openai_compatible/test_speech.py new file mode 100644 index 000000000..6a5541677 --- /dev/null +++ b/surfsense_local/backend/tests/unit/llm/providers/openai_compatible/test_speech.py @@ -0,0 +1,199 @@ +"""Podcast voices from a server's /audio/speech, against a stub of it.""" + +import asyncio +import io +import json +import wave + +import httpx +import pytest + +from modules.llm.providers.openai_compatible.speech import ( + NonRetryableSpeechError, + RemoteSpeech, +) +from modules.llm.providers.protocols import SpokenTurn + +pytestmark = pytest.mark.unit + + +def wav(frames: int, rate: int = 24000) -> bytes: + """What a server answers a WAV speech request with: 16-bit mono PCM.""" + out = io.BytesIO() + with wave.open(out, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(rate) + w.writeframes(b"\x01\x00" * frames) + return out.getvalue() + + +class StubServer: + def __init__(self, fail_on: int | None = None) -> None: + self.fail_on = fail_on + self.requests: list[httpx.Request] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if len(self.requests) == self.fail_on: + return httpx.Response( + 400, json={"error": {"message": "voice 'nobody' is not valid"}} + ) + return httpx.Response(200, content=wav(1000)) + + def bodies(self) -> list[dict]: + return [json.loads(request.content) for request in self.requests] + + +def speak(server: StubServer, turns: list[SpokenTurn]): + """Voice `turns` through the adapter against `server`.""" + speech = RemoteSpeech( + "tts-1", + base_url="https://api.example.com/v1", + api_key="secret", + transport=httpx.MockTransport(server), + ) + return asyncio.run(speech.synthesize(turns, "en")) + + +def test_each_turn_is_one_speech_request_joined_into_one_wav() -> None: + """WAV is asked for every time, because only WAV joins without an encoder.""" + server = StubServer() + + audio = speak(server, [SpokenTurn("coral", "Hi."), SpokenTurn("onyx", "Hello.")]) + + assert [str(r.url) for r in server.requests] == [ + "https://api.example.com/v1/audio/speech" + ] * 2 + assert server.requests[0].headers["authorization"] == "Bearer secret" + assert server.bodies() == [ + {"model": "tts-1", "voice": "coral", "input": "Hi.", "response_format": "wav"}, + { + "model": "tts-1", + "voice": "onyx", + "input": "Hello.", + "response_format": "wav", + }, + ] + assert audio.media_type == "audio/wav" + with wave.open(io.BytesIO(audio.content)) as joined: + # Two 1000-frame turns and the 0.35 s pause between speakers. + assert joined.getnframes() == 2000 + 8400 + + +def test_a_refused_turn_names_the_servers_reason_and_is_not_retried() -> None: + """A retry would draft and bill the episode again, and fail the same way.""" + server = StubServer(fail_on=2) + + with pytest.raises(NonRetryableSpeechError, match="voice 'nobody' is not valid"): + speak(server, [SpokenTurn("coral", "Hi."), SpokenTurn("onyx", "Hello.")]) + + +def test_voicing_on_a_server_needs_no_memory_here() -> None: + """Drafting is never refused for memory the server, not this machine, spends.""" + speech = RemoteSpeech("tts-1", base_url="http://tts", api_key=None) + + asyncio.run(speech.check_memory()) + + +def test_a_server_that_answers_in_another_format_says_so() -> None: + """Only WAV and MP3 join; Ogg back from a server that ignored the format + is refused rather than stored as an episode nothing can play whole.""" + + def ogg(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b"OggS\x00\x02fake opus") + + speech = RemoteSpeech( + "tts-1", + base_url="http://tts", + api_key=None, + transport=httpx.MockTransport(ogg), + ) + + with pytest.raises(NonRetryableSpeechError, match="not as WAV or MP3"): + asyncio.run(speech.synthesize([SpokenTurn("coral", "Hi.")], "en")) + + +def test_a_reply_of_several_wavs_back_to_back_joins_as_all_of_them() -> None: + """Some servers stream a long line as one WAV per sentence in one body.""" + + def stacked(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=wav(600) + wav(400)) + + speech = RemoteSpeech( + "tts-1", + base_url="http://tts", + api_key=None, + transport=httpx.MockTransport(stacked), + ) + audio = asyncio.run(speech.synthesize([SpokenTurn("coral", "Hi.")], "en")) + + with wave.open(io.BytesIO(audio.content)) as joined: + assert joined.getnframes() == 1000 + + +# MPEG-1 Layer III, 128 kbit/s, 44.1 kHz, no padding: 417 bytes a frame. +FRAME = b"\xff\xfb\x90\x00" + b"\x00" * 413 + + +def mp3(frames: int, *, tag: bool = True, xing: bool = True) -> bytes: + """An mp3 as a server sends one: an ID3v2 tag, a Xing frame, audio frames.""" + head = b"ID3\x04\x00\x00\x00\x00\x00\x05" + b"x" * 5 if tag else b"" + info = b"\xff\xfb\x90\x00" + b"\x00" * 32 + b"Xing" + b"\x00" * 377 if xing else b"" + return head + info + FRAME * frames + + +class MpegOnlyServer: + """OpenRouter's shape: `wav` is refused, `mp3` answered.""" + + def __init__(self) -> None: + self.formats: list[str] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + wanted = json.loads(request.content)["response_format"] + self.formats.append(wanted) + if wanted != "mp3": + return httpx.Response( + 400, json={"error": {"message": "response_format must be mp3 or pcm"}} + ) + return httpx.Response(200, content=mp3(3)) + + +def test_a_server_that_refuses_wav_voices_the_episode_as_one_mp3() -> None: + """Asked once, then mp3 for every line; each line's own tag and Xing + frame go, or a player reads the first line's length as the episode's.""" + server = MpegOnlyServer() + speech = RemoteSpeech( + "hexgrad/kokoro-82m", + base_url="http://tts", + api_key=None, + transport=httpx.MockTransport(server), + ) + + audio = asyncio.run( + speech.synthesize( + [SpokenTurn("af_heart", "Hi."), SpokenTurn("am_adam", "Hello.")], "en" + ) + ) + + assert server.formats == ["wav", "mp3", "mp3"] + assert audio.media_type == "audio/mpeg" + assert audio.content == FRAME * 6 + + +def test_a_refusal_wav_did_not_cause_is_not_asked_again_as_mp3() -> None: + """Only the first line tries mp3; a later refusal is that line's own.""" + server = StubServer(fail_on=2) + + with pytest.raises(NonRetryableSpeechError, match="turn 2 of 2"): + speak(server, [SpokenTurn("coral", "Hi."), SpokenTurn("onyx", "Hello.")]) + assert [b["response_format"] for b in server.bodies()] == ["wav", "wav"] + + +def test_a_line_with_no_voice_leaves_the_voice_to_the_server() -> None: + """Only "Test voice" sends one: the server speaks in its default, if any.""" + server = StubServer() + + speak(server, [SpokenTurn("", "Hi.")]) + + assert "voice" not in server.bodies()[0] diff --git a/surfsense_local/backend/tests/unit/llm/providers/openai_compatible/test_voice_list.py b/surfsense_local/backend/tests/unit/llm/providers/openai_compatible/test_voice_list.py new file mode 100644 index 000000000..555e3e987 --- /dev/null +++ b/surfsense_local/backend/tests/unit/llm/providers/openai_compatible/test_voice_list.py @@ -0,0 +1,83 @@ +"""A server's voices, asked of the server: nothing is kept in the repo.""" + +import httpx +import pytest + +from modules.llm.providers.openai_compatible.voice_list import listed_voices + +pytestmark = pytest.mark.unit + + +def ask(handler, connection_id: int = 1): + """The voices the server behind `handler` lists, or None.""" + return listed_voices( + connection_id, + f"http://tts-{connection_id}/v1", + "secret", + transport=httpx.MockTransport(handler), + ) + + +def test_a_server_that_lists_its_voices_fills_the_picker() -> None: + """Kokoro-FastAPI's shape. It states ids alone, so no gender or language + is given: the brief does not claim one for them.""" + seen: list[httpx.Request] = [] + + def kokoro(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response( + 200, json={"voices": [{"id": "af_heart"}, {"id": "am_adam"}]} + ) + + voices = ask(kokoro, connection_id=1) + + assert str(seen[0].url) == "http://tts-1/v1/audio/voices" + assert seen[0].headers["authorization"] == "Bearer secret" + assert [(v.id, v.label, v.gender, v.languages) for v in voices] == [ + ("af_heart", "af_heart", None, ()), + ("am_adam", "am_adam", None, ()), + ] + + +@pytest.mark.parametrize( + "reply", + [ + httpx.Response(404), + httpx.Response(200, json={"data": []}), + httpx.Response(200, json={"voices": [{"name": "no id"}]}), + httpx.Response(200, text=""), + ], +) +def test_a_server_that_cannot_list_them_leaves_the_user_to_type_them( + reply: httpx.Response, +) -> None: + """OpenAI, Groq and OpenRouter have no such route: nothing is guessed.""" + assert ask(lambda request: reply, connection_id=2) is None + + +def test_the_answer_is_asked_once_per_server() -> None: + """Held for the process and never stored, so the next start asks again.""" + calls: list[int] = [] + + def server(request: httpx.Request) -> httpx.Response: + calls.append(1) + return httpx.Response(404) + + ask(server, connection_id=3) + ask(server, connection_id=3) + + assert calls == [1] + + +def test_an_unreachable_server_is_asked_again_next_time() -> None: + """No answer is not an answer: it says nothing about the route.""" + calls: list[int] = [] + + def flaky(request: httpx.Request) -> httpx.Response: + calls.append(1) + if len(calls) == 1: + raise httpx.ConnectError("down") + return httpx.Response(200, json={"voices": [{"id": "af_heart"}]}) + + assert ask(flaky, connection_id=4) is None + assert [v.id for v in ask(flaky, connection_id=4)] == ["af_heart"] diff --git a/surfsense_local/backend/tests/unit/llm/test_openai_compatible.py b/surfsense_local/backend/tests/unit/llm/test_openai_compatible.py index 07d1d4bdd..e19340591 100644 --- a/surfsense_local/backend/tests/unit/llm/test_openai_compatible.py +++ b/surfsense_local/backend/tests/unit/llm/test_openai_compatible.py @@ -100,6 +100,34 @@ def test_types_come_from_the_endpoint_then_the_catalogue_then_nowhere() -> None: assert models[name].capability_known is False +def test_an_endpoints_own_word_for_speech_is_the_audio_slot() -> None: + """OpenRouter declares `speech` for text-to-speech and `transcription` for + speech-to-text; each output a model declares is a type, as with images.""" + models = { + model.name: model + for model in parse_models( + { + "data": [ + {"id": "omni", "output_modalities": ["text", "audio"]}, + { + "id": "hexgrad/kokoro-82m", + "architecture": {"output_modalities": ["speech"]}, + }, + { + "id": "openai/whisper-1", + "architecture": {"output_modalities": ["transcription"]}, + }, + ] + } + ) + } + + assert models["omni"].types == (ModelType.TEXT_GEN, ModelType.AUDIO_GEN) + assert models["hexgrad/kokoro-82m"].types == (ModelType.AUDIO_GEN,) + assert models["openai/whisper-1"].types == () + assert models["openai/whisper-1"].capability_known is True + + def test_delta_reads_openai_sse_and_ignores_done() -> None: """Chat streaming retains the existing OpenAI delta behavior.""" assert _delta('data: {"choices":[{"delta":{"content":"hi"}}]}') == Delta("hi") diff --git a/surfsense_local/backend/worker/studio/job.py b/surfsense_local/backend/worker/studio/job.py index 7f0706817..2cab249ff 100644 --- a/surfsense_local/backend/worker/studio/job.py +++ b/surfsense_local/backend/worker/studio/job.py @@ -10,6 +10,7 @@ from modules.documents.models import Document, DocumentStatus from modules.llm.model_type import ModelType from modules.llm.providers.audiocpp.memory import NotEnoughMemoryError from modules.llm.providers.openai_compatible import NonRetryableImageError +from modules.llm.providers.openai_compatible.speech import NonRetryableSpeechError from modules.llm.providers.protocols import TextToSpeech from modules.llm.resolution import ( ModelResolutionError, @@ -124,7 +125,10 @@ def _generate(session: Session, artifact: Artifact) -> None: document.error_message, ) # A retry would repeat minutes of drafting and fail the same way. - if isinstance(failure, NonRetryableImageError | NotEnoughMemoryError): + if isinstance( + failure, + NonRetryableImageError | NonRetryableSpeechError | NotEnoughMemoryError, + ): return raise # Huey retries; a later success clears the message. diff --git a/surfsense_local/frontend/src/components/ui/icons.tsx b/surfsense_local/frontend/src/components/ui/icons.tsx index 65952d7e4..af29e090e 100644 --- a/surfsense_local/frontend/src/components/ui/icons.tsx +++ b/surfsense_local/frontend/src/components/ui/icons.tsx @@ -50,6 +50,7 @@ import { PencilEdit02Icon as PencilEdit02IconData, Pdf01Icon as Pdf01IconData, PencilIcon as PencilIconData, + PlayIcon as PlayIconData, PlusIcon as PlusIconData, PodcastIcon as PodcastIconData, Presentation02Icon as Presentation02IconData, @@ -149,6 +150,7 @@ export const FilterIcon = createIcon(FilterIconData) export const FolderOpenIcon = createIcon(FolderOpenIconData) export const Image01Icon = createIcon(Image01IconData) export const AudioWaveformIcon = createIcon(AudioWaveformIconData) +export const PlayIcon = createIcon(PlayIconData) export const InformationCircleIcon = createIcon(InformationCircleIconData) export const LayoutGridIcon = createIcon(LayoutGridIconData) export const LicenseIcon = createIcon(LicenseIconData) diff --git a/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx b/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx index 6b5586472..16b6afffe 100644 --- a/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx +++ b/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx @@ -82,6 +82,7 @@ function WorkspaceDashboard({ onModelRequired, onModelSelected, onOpenLicense, + onOpenAudioSettings, modelsVisited, }: { workspace: Workspace @@ -94,6 +95,7 @@ function WorkspaceDashboard({ onModelRequired: () => void onModelSelected: (selection: ModelSelection) => void onOpenLicense: () => void + onOpenAudioSettings: () => void modelsVisited: number }) { const [inspect, setInspect] = useState(null) @@ -347,6 +349,7 @@ function WorkspaceDashboard({ isCreating={studio.isCreating} error={studio.error} onGenerate={studio.create} + onSetUpVoices={onOpenAudioSettings} /> } artifacts={ @@ -564,6 +567,7 @@ export function DashboardPage({ onModelRequired={() => openSettings("chat-models")} onModelSelected={onModelSelected} onOpenLicense={() => openSettings("license")} + onOpenAudioSettings={() => openSettings("audio-models")} modelsVisited={modelsVisited} /> { + const typed = voice.trim() + const body = typed + ? { model, voice: typed, prompt: text } + : { model, prompt: text } + const response = await request(`/llm/connections/${id}/speech-test`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal, + }) + return response.blob() +} + export async function testConnectionImage( id: number, model: string, diff --git a/surfsense_local/frontend/src/features/models/remote/models/server-model-picker.test.tsx b/surfsense_local/frontend/src/features/models/remote/models/server-model-picker.test.tsx index 3f58c8868..03190f968 100644 --- a/surfsense_local/frontend/src/features/models/remote/models/server-model-picker.test.tsx +++ b/surfsense_local/frontend/src/features/models/remote/models/server-model-picker.test.tsx @@ -60,6 +60,14 @@ const gatewayModels = [ capability_source: "catalog", selectable_for: ["text_gen"], }, + { + connection_id: 1, + connection_label: "Chat gateway", + name: "tts-1", + types: ["audio_gen"], + capability_source: "catalog", + selectable_for: ["audio_gen"], + }, { connection_id: 1, connection_label: "Chat gateway", @@ -257,6 +265,50 @@ describe("choosing a model from a server", () => { ).toBeTruthy() }) + it("plays a test line in the typed voice before the model takes the audio slot", async () => { + const sent: unknown[] = [] + vi.stubGlobal( + "fetch", + serving((path, init) => { + if ( + path !== "/llm/connections/1/speech-test" || + init?.method !== "POST" + ) + return null + sent.push(JSON.parse(String(init.body))) + return new Response(new Uint8Array([1, 2, 3]), { + headers: { "Content-Type": "audio/wav" }, + }) + }) + ) + vi.stubGlobal("URL", { + ...URL, + createObjectURL: vi.fn(() => "blob:test-voice"), + revokeObjectURL: vi.fn(), + }) + const user = userEvent.setup() + + render( undefined} modelType="audio_gen" />) + await user.click( + await screen.findByRole("button", { name: "Show audio models" }) + ) + await user.click(await screen.findByRole("button", { name: "Use tts-1" })) + + expect(screen.getByText(/Speech support is confirmed/)).toBeTruthy() + await user.type(screen.getByRole("textbox", { name: "Voice" }), "af_heart") + await user.click(screen.getByRole("button", { name: "Test voice" })) + const clip = await screen.findByLabelText("Test voiced by tts-1") + expect(clip.getAttribute("src")).toBe("blob:test-voice") + // The line is spoken in the interface's language. + expect(sent).toEqual([ + { + model: "tts-1", + voice: "af_heart", + prompt: "Hello. This is how your podcasts will sound.", + }, + ]) + }) + it("assigns a typed model id without the listing's check", async () => { const fetchMock = serving() vi.stubGlobal("fetch", fetchMock) @@ -407,4 +459,47 @@ describe("choosing a model from a server", () => { )[1] expect(JSON.parse(String(finalCreate[1]?.body)).allow_unverified).toBe(true) }) + + it("sets up the voices of the server's audio model inside that server", async () => { + vi.stubGlobal( + "fetch", + serving((path, init) => { + if (path === "/llm/selection/audio_gen" && !init?.method) + return Response.json({ + model_type: "audio_gen", + provider: "openai_compatible", + connection_id: 1, + name: "tts-1", + updated_at: "2026-09-10T00:00:00Z", + }) + if (path === "/llm/selection/audio_gen/voices") + return Response.json({ + connection_id: 1, + model: "tts-1", + source: "saved", + voices: [], + voices_page: null, + }) + return null + }) + ) + + render( undefined} modelType="audio_gen" />) + + const server = await screen.findByRole("region", { name: /Chat gateway/ }) + expect( + await within(server).findByRole("heading", { + name: "Voices for this model", + }) + ).toBeTruthy() + }) + + it("has no voices to set up for a server's chat model", async () => { + vi.stubGlobal("fetch", serving()) + + render( undefined} modelType="text_gen" />) + + await screen.findByRole("region", { name: /Chat gateway/ }) + expect(screen.queryByText("Voices for this model")).toBeNull() + }) }) diff --git a/surfsense_local/frontend/src/features/models/remote/models/server-models.tsx b/surfsense_local/frontend/src/features/models/remote/models/server-models.tsx index 7628f2e86..4a686849f 100644 --- a/surfsense_local/frontend/src/features/models/remote/models/server-models.tsx +++ b/surfsense_local/frontend/src/features/models/remote/models/server-models.tsx @@ -24,6 +24,7 @@ import { intl } from "@/i18n/intl" import type { ModelType } from "../../model-type" import type { ModelSelection } from "../../selection/api" import { useSelection } from "../../selection/use-selection" +import { ServerVoices } from "../voices/server-voices" import type { Connection } from "../connections/api" import { DisconnectButton } from "../connections/disconnect-button" import type { ConnectionModel } from "./api" @@ -188,6 +189,12 @@ export function ServerModels({ ) : null} + {current && modelType === "audio_gen" ? ( + // Its voices belong to this server's model, so they are set up here. +
+ +
+ ) : null} + + )} + {error ?

{error}

: null} + + ) +} + +function VoiceRow({ + voice, + read, + onRemove, +}: { + voice: string + read: ServerVoicesRead + onRemove?: () => void +}) { + const [clip, setClip] = useState(null) + const [playing, setPlaying] = useState(false) + const [error, setError] = useState(null) + + useEffect( + () => () => { + if (clip) URL.revokeObjectURL(clip) + }, + [clip] + ) + + const play = () => { + setPlaying(true) + setError(null) + testConnectionSpeech(read.connection_id, read.model, voice, trialLine()) + .then((blob) => setClip(URL.createObjectURL(blob))) + .catch((cause: unknown) => setError(messageFrom(cause))) + .finally(() => setPlaying(false)) + } + + return ( +
  • +
    + + {voice} + + + {onRemove ? ( + + ) : null} +
    + {clip ? ( +
    • + ) +} diff --git a/surfsense_local/frontend/src/features/models/remote/voices/use-server-voices.ts b/surfsense_local/frontend/src/features/models/remote/voices/use-server-voices.ts new file mode 100644 index 000000000..fe406cb78 --- /dev/null +++ b/surfsense_local/frontend/src/features/models/remote/voices/use-server-voices.ts @@ -0,0 +1,27 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" + +import { MODELS_QUERY_KEY } from "../../models-query" +import { addServerVoice, readServerVoices, removeServerVoice } from "./api" + +export const serverVoicesQueryKey = [ + ...MODELS_QUERY_KEY, + "remote", + "voices", +] as const + +/** Keyed under the models, so choosing another model asks again. */ +export function useServerVoices() { + const client = useQueryClient() + const refresh = () => + client.invalidateQueries({ queryKey: serverVoicesQueryKey }) + const voices = useQuery({ + queryKey: serverVoicesQueryKey, + queryFn: ({ signal }) => readServerVoices(signal), + }) + const add = useMutation({ mutationFn: addServerVoice, onSuccess: refresh }) + const remove = useMutation({ + mutationFn: removeServerVoice, + onSuccess: refresh, + }) + return { voices, add, remove } +} diff --git a/surfsense_local/frontend/src/features/onboarding/model-step/model-step.tsx b/surfsense_local/frontend/src/features/onboarding/model-step/model-step.tsx index 52ea14f04..a32dbb88a 100644 --- a/surfsense_local/frontend/src/features/onboarding/model-step/model-step.tsx +++ b/surfsense_local/frontend/src/features/onboarding/model-step/model-step.tsx @@ -19,7 +19,6 @@ import type { LocalBuild, LocalRow } from "@/features/models/local/chat/api" import { useInstall } from "@/features/models/local/installs/use-install" import { ConnectionDialog } from "@/features/models/remote/connections/connection-dialog" import { useConnections } from "@/features/models/remote/connections/use-connections" -import { serversCanServe } from "@/features/models/remote/servers-can-serve" import { useSelect } from "@/features/models/selection/use-selection" import { DeleteModelDialog } from "@/features/models/your-models/delete-model-dialog" import type { YourModelRow } from "@/features/models/your-models/your-model-row" @@ -40,9 +39,8 @@ const COPY: Record< OnboardingSlot, { title: () => string - // `servers`: whether this slot offers servers (`serversCanServe`). - description: (servers: boolean) => string - noLocal: (servers: boolean) => string + description: () => string + noLocal: () => string /** Hugging Face search, for llama.cpp's GGUF models only. */ searchable: boolean } @@ -53,24 +51,18 @@ const COPY: Record< id: "onboarding_chat_step_title", defaultMessage: "Choose a text generation model", }), - description: (servers) => - intl.formatMessage( - { - id: "onboarding_chat_step_body", - defaultMessage: - "Answers you in chat. {servers, select, yes {Run one on this computer so your chats stay private, or use one from a server.} other {Run one on this computer so your chats stay private.}}", - }, - { servers: servers ? "yes" : "no" } - ), - noLocal: (servers) => - intl.formatMessage( - { - id: "onboarding_chat_step_no_local_empty", - defaultMessage: - "No tested model can run on this computer.{servers, select, yes { Use a server instead.} other {}}", - }, - { servers: servers ? "yes" : "no" } - ), + description: () => + intl.formatMessage({ + id: "onboarding_chat_step_body", + defaultMessage: + "Answers you in chat. Run one on this computer so your chats stay private, or use one from a server.", + }), + noLocal: () => + intl.formatMessage({ + id: "onboarding_chat_step_no_local_empty", + defaultMessage: + "No tested model can run on this computer. Use a server instead.", + }), searchable: true, }, image_gen: { @@ -79,24 +71,18 @@ const COPY: Record< id: "onboarding_image_step_title", defaultMessage: "Choose an image generation model", }), - description: (servers) => - intl.formatMessage( - { - id: "onboarding_image_step_body", - defaultMessage: - "Creates images for you. {servers, select, yes {Run one on this computer, or use one from a server.} other {Run one on this computer.}}", - }, - { servers: servers ? "yes" : "no" } - ), - noLocal: (servers) => - intl.formatMessage( - { - id: "onboarding_image_step_no_local_empty", - defaultMessage: - "Image models cannot run on this computer.{servers, select, yes { Use a server instead.} other {}}", - }, - { servers: servers ? "yes" : "no" } - ), + description: () => + intl.formatMessage({ + id: "onboarding_image_step_body", + defaultMessage: + "Creates images for you. Run one on this computer, or use one from a server.", + }), + noLocal: () => + intl.formatMessage({ + id: "onboarding_image_step_no_local_empty", + defaultMessage: + "Image models cannot run on this computer. Use a server instead.", + }), // sd.cpp has no search: its models are the few the catalog ships. searchable: false, }, @@ -106,24 +92,18 @@ const COPY: Record< id: "onboarding_image_edit_step_title", defaultMessage: "Choose an image editing model", }), - description: (servers) => - intl.formatMessage( - { - id: "onboarding_image_edit_step_body", - defaultMessage: - "Edits images for you. {servers, select, yes {Run one on this computer, or use one from a server.} other {Run one on this computer.}}", - }, - { servers: servers ? "yes" : "no" } - ), - noLocal: (servers) => - intl.formatMessage( - { - id: "onboarding_image_edit_step_no_local_empty", - defaultMessage: - "Image editing models cannot run on this computer.{servers, select, yes { Use a server instead.} other {}}", - }, - { servers: servers ? "yes" : "no" } - ), + description: () => + intl.formatMessage({ + id: "onboarding_image_edit_step_body", + defaultMessage: + "Edits images for you. Run one on this computer, or use one from a server.", + }), + noLocal: () => + intl.formatMessage({ + id: "onboarding_image_edit_step_no_local_empty", + defaultMessage: + "Image editing models cannot run on this computer. Use a server instead.", + }), // Nor has it for editing: the same few models. searchable: false, }, @@ -133,24 +113,18 @@ const COPY: Record< id: "onboarding_video_step_title", defaultMessage: "Choose a video generation model", }), - description: (servers) => - intl.formatMessage( - { - id: "onboarding_video_step_body", - defaultMessage: - "Makes short video clips. {servers, select, yes {Run one on this computer, or use one from a server.} other {Run one on this computer.}}", - }, - { servers: servers ? "yes" : "no" } - ), - noLocal: (servers) => - intl.formatMessage( - { - id: "onboarding_video_step_no_local_empty", - defaultMessage: - "Video models cannot run on this computer.{servers, select, yes { Use a server instead.} other {}}", - }, - { servers: servers ? "yes" : "no" } - ), + description: () => + intl.formatMessage({ + id: "onboarding_video_step_body", + defaultMessage: + "Makes short video clips. Run one on this computer, or use one from a server.", + }), + noLocal: () => + intl.formatMessage({ + id: "onboarding_video_step_no_local_empty", + defaultMessage: + "Video models cannot run on this computer. Use a server instead.", + }), // Nor for video. searchable: false, }, @@ -160,24 +134,18 @@ const COPY: Record< id: "onboarding_audio_step_title", defaultMessage: "Choose an audio model", }), - description: (servers) => - intl.formatMessage( - { - id: "onboarding_audio_step_body", - defaultMessage: - "Creates podcasts for you. {servers, select, yes {Run one on this computer, or use one from a server.} other {Run one on this computer.}}", - }, - { servers: servers ? "yes" : "no" } - ), - noLocal: (servers) => - intl.formatMessage( - { - id: "onboarding_audio_step_no_local_empty", - defaultMessage: - "Audio models cannot run on this computer.{servers, select, yes { Use a server instead.} other {}}", - }, - { servers: servers ? "yes" : "no" } - ), + description: () => + intl.formatMessage({ + id: "onboarding_audio_step_body", + defaultMessage: + "Creates podcasts for you. Run one on this computer, or use one from a server.", + }), + noLocal: () => + intl.formatMessage({ + id: "onboarding_audio_step_no_local_empty", + defaultMessage: + "Audio models cannot run on this computer. Use a server instead.", + }), // Nor has audio.cpp. searchable: false, }, @@ -218,7 +186,6 @@ export function ModelStep({ const select = useSelect(modelType) const remove = slotDeletes[modelType]() const connections = useConnections() - const servers = serversCanServe(modelType) const [onServer, setOnServer] = useState(false) // With nothing connected yet there is no server page to show: Connect // opens the dialog here, and a server saved from it opens on its models. @@ -354,7 +321,7 @@ export function ModelStep({ /> ) : (

    - {copy.noLocal(servers)} + {copy.noLocal()}

    )} {copy.searchable ? ( @@ -367,19 +334,13 @@ export function ModelStep({ ) : null} - {servers ? ( - <> - - - connections.data?.length - ? setOnServer(true) - : setConnecting(true) - } - /> - - ) : null} + + + connections.data?.length ? setOnServer(true) : setConnecting(true) + } + /> ) } @@ -391,7 +352,7 @@ export function ModelStep({

    {copy.title()}

    - {copy.description(servers)} + {copy.description()} {onSkip ? ( diff --git a/surfsense_local/frontend/src/features/onboarding/onboarding-page.test.tsx b/surfsense_local/frontend/src/features/onboarding/onboarding-page.test.tsx index aebce6d62..7f773393b 100644 --- a/surfsense_local/frontend/src/features/onboarding/onboarding-page.test.tsx +++ b/surfsense_local/frontend/src/features/onboarding/onboarding-page.test.tsx @@ -831,15 +831,18 @@ describe("onboarding", () => { ).toBe(true) }) - it("offers no server on the audio step, which only runs models here", async () => { + it("offers a server on the audio step, as on every other", async () => { vi.stubGlobal("fetch", backend({ selections: { ...chatChosen } })) const user = userEvent.setup() render( undefined} />) await toAudioStep(user) - await screen.findByText(/Audio models cannot run on this computer/) - expect(screen.queryByRole("region", { name: "Use a server" })).toBeNull() - expect(screen.queryByText(/server/i)).toBeNull() + expect( + await screen.findByText( + "Audio models cannot run on this computer. Use a server instead." + ) + ).toBeTruthy() + expect(screen.getByRole("region", { name: "Use a server" })).toBeTruthy() }) it("says when audio models cannot run here", async () => { diff --git a/surfsense_local/frontend/src/features/settings/models/audio-models-settings.test.tsx b/surfsense_local/frontend/src/features/settings/models/audio-models-settings.test.tsx index fc5072792..99d3e32e3 100644 --- a/surfsense_local/frontend/src/features/settings/models/audio-models-settings.test.tsx +++ b/surfsense_local/frontend/src/features/settings/models/audio-models-settings.test.tsx @@ -124,7 +124,7 @@ function serving( } describe("audio model settings", () => { - it("says audio models cannot run where no audio runtime shipped", async () => { + it("offers a server where no audio runtime shipped", async () => { // No audio.cpp staged: the catalog carries chat rows and no audio rows. vi.stubGlobal("fetch", serving([chatRow])) const user = userEvent.setup() @@ -136,6 +136,7 @@ describe("audio model settings", () => { expect( screen.getByRole("heading", { name: "Add an audio model" }) ).toBeTruthy() + expect(screen.getByRole("button", { name: "Connect" })).toBeTruthy() expect( await screen.findByText("Audio models cannot run on this computer") ).toBeTruthy() diff --git a/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx b/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx index 02148800e..be953c48e 100644 --- a/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx +++ b/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx @@ -16,7 +16,6 @@ import type { Connection } from "@/features/models/remote/connections/api" import { ConnectionDialog } from "@/features/models/remote/connections/connection-dialog" import { useConnections } from "@/features/models/remote/connections/use-connections" import { ServerModelPicker } from "@/features/models/remote/models/server-model-picker" -import { serversCanServe } from "@/features/models/remote/servers-can-serve" import type { ModelSelection } from "@/features/models/selection/api" import { InUseSummary } from "@/features/models/your-models/in-use-summary" import { LocalModelsGroup } from "@/features/models/your-models/local-models-group" @@ -59,7 +58,6 @@ export function ModelSlotSettings({ onSelected?: (selection: ModelSelection) => void onChatCleared?: () => void }) { - const servers = serversCanServe(modelType) const connections = useConnections() const [page, setPage] = useState("list") // Edited in a dialog over the list; saving leaves the list as it was. @@ -85,29 +83,22 @@ export function ModelSlotSettings({ }, { slot } )} - description={ - servers - ? intl.formatMessage({ - id: "settings_models_add_page_body", - defaultMessage: - "Run one on this computer, or use one from a server you already run.", - }) - : undefined - } + description={intl.formatMessage({ + id: "settings_models_add_page_body", + defaultMessage: + "Run one on this computer, or use one from a server you already run.", + })} back={back} scrollable="all" > - + ) } const empty = !models.isPending && - (!servers || connections.data?.length === 0) && + connections.data?.length === 0 && models.local.length === 0 && !pending && models.inUse === null @@ -146,20 +137,18 @@ export function ModelSlotSettings({ { slot } )} - {servers ? ( - - {models.canDownload - ? intl.formatMessage({ - id: "settings_models_empty_download_body", - defaultMessage: - "Download one to run on this computer, or use one from a server you already run.", - }) - : intl.formatMessage({ - id: "settings_models_empty_server_body", - defaultMessage: "Use one from a server you already run.", - })} - - ) : null} + + {models.canDownload + ? intl.formatMessage({ + id: "settings_models_empty_download_body", + defaultMessage: + "Download one to run on this computer, or use one from a server you already run.", + }) + : intl.formatMessage({ + id: "settings_models_empty_server_body", + defaultMessage: "Use one from a server you already run.", + })} + {add} @@ -180,18 +169,16 @@ export function ModelSlotSettings({ /> ) : null} - {servers ? ( - { - setEditing(connection) - setEditOpen(true) - }} - onSelected={onSelected} - onChatCleared={onChatCleared} - /> - ) : null} + { + setEditing(connection) + setEditOpen(true) + }} + onSelected={onSelected} + onChatCleared={onChatCleared} + /> )} { +): Promise { return requestJson(`/workspaces/${workspaceId}/studio/podcast/brief`, { signal, }) diff --git a/surfsense_local/frontend/src/features/studio/podcast-brief-form.test.tsx b/surfsense_local/frontend/src/features/studio/podcast-brief-form.test.tsx index 503ecda03..b8a0ff509 100644 --- a/surfsense_local/frontend/src/features/studio/podcast-brief-form.test.tsx +++ b/surfsense_local/frontend/src/features/studio/podcast-brief-form.test.tsx @@ -1,5 +1,5 @@ import { useState } from "react" -import { afterEach, describe, expect, it } from "vitest" +import { afterEach, describe, expect, it, vi } from "vitest" import { cleanup, render, screen, within } from "@testing-library/react" import userEvent from "@testing-library/user-event" @@ -28,9 +28,24 @@ function Harness({ initial = brief }: { initial?: PodcastBrief }) { return } +/** What the server sends beside a roster: the languages its voices speak. */ +const spokenBy = (roster: Voice[]) => [ + ...new Set(roster.flatMap((voice) => voice.languages)), +] + function Controlled({ initial }: { initial: PodcastBrief }) { const [value, setValue] = useState(initial) - return + return ( + undefined} + onChange={setValue} + /> + ) } afterEach(cleanup) @@ -165,6 +180,10 @@ describe("podcast brief form", () => { undefined} onChange={setValue} /> ) @@ -177,4 +196,145 @@ describe("podcast brief form", () => { expect(value(within(rows[0]).getByLabelText("Voice"))).toBe("M1") expect(value(within(rows[1]).getByLabelText("Voice"))).toBe("F1") }) + + it("lists voices whose gender nobody documents in one list", () => { + // OpenAI names its voices and nothing else about them. + const openai: Voice[] = [ + { id: "alloy", label: "Alloy", gender: null, languages: ["en"] }, + { id: "ash", label: "Ash", gender: null, languages: ["en"] }, + ] + render( + undefined} + onChange={() => undefined} + /> + ) + + const voice = screen.getByLabelText("Voice") + expect(within(voice).queryAllByRole("group")).toEqual([]) + expect( + within(voice) + .getAllByRole("option") + .map((option) => option.textContent) + ).toEqual(["Alloy", "Ash"]) + }) + + it("offers a server's listed voices in every language, as they state none", async () => { + // Kokoro-FastAPI lists ids alone. + const listed: Voice[] = [ + { id: "af_heart", label: "af_heart", gender: null, languages: [] }, + { id: "am_adam", label: "am_adam", gender: null, languages: [] }, + ] + const user = userEvent.setup() + function Listed() { + const [value, setValue] = useState({ + ...brief, + language: "en", + speakers: [{ name: "Host", role: "host", voice: "af_heart" }], + }) + return ( + undefined} + onChange={setValue} + /> + ) + } + render() + + await user.selectOptions(screen.getByLabelText("Language"), "fr") + + const voice = screen.getByLabelText("Voice") + expect(value(voice)).toBe("af_heart") + expect( + within(voice) + .getAllByRole("option") + .map((option) => option.textContent) + ).toEqual(["af_heart", "am_adam"]) + expect( + screen + .getByRole("button", { name: "Add speaker" }) + .hasAttribute("disabled") + ).toBe(false) + }) + + it("says a server voices it, and reads like the local form otherwise", () => { + render( + undefined} + onChange={() => undefined} + /> + ) + + expect( + screen.getByText( + "Voiced by seed-audio-1-0 on OpenRouter. Each line is billed by the provider." + ) + ).toBeTruthy() + expect(screen.getByLabelText("Language")).toBeTruthy() + expect(screen.getByText("Speakers")).toBeTruthy() + expect(screen.queryByText("Script language")).toBeNull() + expect(screen.queryByText("Your voices for this model")).toBeNull() + expect(screen.queryByText("Listed by the server")).toBeNull() + }) + + it("asks for voices before anything else when the server model has none", async () => { + const user = userEvent.setup() + const setUp = vi.fn() + render( + undefined} + /> + ) + + expect( + screen.getByText( + "This model needs voices before it can voice a podcast. The server doesn’t list them, so add the ones you want." + ) + ).toBeTruthy() + expect(screen.queryByRole("group", { name: /Speaker \d/ })).toBeNull() + await user.click(screen.getByRole("button", { name: "Set up voices" })) + expect(setUp).toHaveBeenCalledOnce() + }) + + it("says nothing about a server for a model on this computer", () => { + render() + + expect(screen.queryByText(/Voiced by/)).toBeNull() + expect(screen.getByLabelText("Language")).toBeTruthy() + }) }) diff --git a/surfsense_local/frontend/src/features/studio/podcast-brief-form.tsx b/surfsense_local/frontend/src/features/studio/podcast-brief-form.tsx index a81312dc8..ac2f05274 100644 --- a/surfsense_local/frontend/src/features/studio/podcast-brief-form.tsx +++ b/surfsense_local/frontend/src/features/studio/podcast-brief-form.tsx @@ -15,12 +15,14 @@ import { type PodcastBrief, type PodcastSpeaker, type Voice, + type VoicedBy, + type VoicesSource, } from "./api" type PodcastStyle = PodcastBrief["style"] type PodcastDuration = PodcastBrief["duration"] type PodcastRole = PodcastSpeaker["role"] -type VoiceGender = Voice["gender"] +type VoiceGender = NonNullable const STYLE_LABELS: Record string> = { conversational: () => @@ -132,25 +134,34 @@ const title = (word: string) => word[0].toUpperCase() + word.slice(1) export function PodcastBriefForm({ brief, voices, + languages: codes, + voicesSource, + voicedBy, + onSetUpVoices, onChange, }: { brief: PodcastBrief voices: Voice[] + languages: string[] + voicesSource: VoicesSource + /** The server that voices it; null for a model on this computer. */ + voicedBy: VoicedBy | null + /** Opens where a server model's voices are added. */ + onSetUpVoices: () => void onChange: (brief: PodcastBrief) => void }) { const id = useId() // By the name shown, in the interface language: the model's order is its own. - const languages = [...new Set(voices.flatMap((voice) => voice.languages))] + const languages = codes .map((code) => ({ code, name: intl.formatDisplayName(code, { type: "language" }) ?? code, })) .sort((a, b) => a.name.localeCompare(b.name, intl.locale)) - const spoken = voices.filter((voice) => - voice.languages.includes(brief.language) - ) + const spoken = voices.filter((voice) => speaks(voice, brief.language)) const taken = new Set(brief.speakers.map((speaker) => speaker.voice)) const free = spoken.find((voice) => !taken.has(voice.id)) + const canAdd = free !== undefined const update = (patch: Partial) => onChange({ ...brief, ...patch }) @@ -158,7 +169,7 @@ export function PodcastBriefForm({ // A speaker keeps a voice that speaks the new language, as a Supertonic // voice speaks all of them; the rest take that language's free voices. const changeLanguage = (language: string) => { - const next = voices.filter((voice) => voice.languages.includes(language)) + const next = voices.filter((voice) => speaks(voice, language)) const kept = new Set( brief.speakers .map((speaker) => speaker.voice) @@ -185,7 +196,7 @@ export function PodcastBriefForm({ }) const addSpeaker = () => { - if (!free) return + if (!canAdd) return const slot = brief.speakers.length const role = ROLE_BY_SLOT[slot] ?? "guest" update({ @@ -194,7 +205,7 @@ export function PodcastBriefForm({ { name: title(role === "cohost" ? "co-host" : role), role, - voice: free.id, + voice: free?.id ?? "", }, ], }) @@ -203,8 +214,34 @@ export function PodcastBriefForm({ const removeSpeaker = (index: number) => update({ speakers: brief.speakers.filter((_, at) => at !== index) }) + const server = voicedBy ? : null + + if (voicesSource === "saved" && voices.length === 0) { + return ( +
    + {server} +
    +

    + {intl.formatMessage({ + id: "studio_podcast_brief_no_voices_body", + defaultMessage: + "This model needs voices before it can voice a podcast. The server doesn’t list them, so add the ones you want.", + })} +

    + +
    +
    + ) + } + return (
    + {server}
    = MAX_SPEAKERS} + disabled={!canAdd || brief.speakers.length >= MAX_SPEAKERS} onClick={addSpeaker} > @@ -357,19 +394,27 @@ export function PodcastBriefForm({ return group.length ? ( {group.map((voice) => ( - + /> ))} ) : null })} + {/* A provider that never states a gender gets one plain list. */} + {spoken + .filter((voice) => voice.gender === null) + .map((voice) => ( + + ))} {brief.speakers.length > 1 ? (