From 531c1f681021de2956531b14f77e8aa4dfc5204a Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 14 Oct 2026 15:04:38 -0700 Subject: [PATCH] fix(local): keep a ChatGPT plan out of speech settings, and docs that match v1 After dev's one-click ChatGPT sign-in, the local-only speech section must still offer no remote model; a test now proves it with a plan that claims the slot, and fails without the localOnly gate. The docs drop two stale Known gaps (no speech screen; no transcription test), name the settings nav as it reads, count migrations to 0030, mark steps 2 to 7 done in the proposal, and note the curated recogniser's type in the catalog proposal. --- docs/architecture/local-models/catalog.md | 3 +-- docs/architecture/overview.md | 2 +- docs/architecture/packaging.md | 2 +- docs/proposals/local-speech-to-text.md | 12 +++++----- docs/proposals/model-catalog.md | 2 +- .../models/add-model/add-model-options.tsx | 2 +- .../models/speech-models-settings.test.tsx | 24 +++++++++++++++++++ 7 files changed, 35 insertions(+), 12 deletions(-) diff --git a/docs/architecture/local-models/catalog.md b/docs/architecture/local-models/catalog.md index 60a0bb811..87a392606 100644 --- a/docs/architecture/local-models/catalog.md +++ b/docs/architecture/local-models/catalog.md @@ -687,7 +687,7 @@ on search and repo reads, and the job's generic error during an install. ## On the screen -Settings has one section per model type, **Chat**, **Image**, **Image editing**, **Video**, **Audio** and **Speech recognition** in the nav, each headed **Text generation models**, **Image generation models**, **Image editing models**, **Video generation models**, **Audio generation models** or **Speech recognition models** on its own page. +Settings has one section per model type, **Text gen**, **Image gen**, **Image edit**, **Video**, **Audio** and **Speech recognition** in the nav, each headed **Text generation models**, **Image generation models**, **Image editing models**, **Video generation models**, **Audio generation models** or **Speech recognition models** on its own page. Each names the model in use at the top, then groups the slot's models by source: **This computer**, every build on disk, curated or not, with Use (when its type can fill the slot) or In use and Delete after confirmation; then one group per @@ -813,7 +813,6 @@ and the screen in `download-chat-models.test.tsx`, `install-view.test.tsx`, `ins ## Known gaps - Only the three voice defaults and Nemotron's `Q8_0` are validated; `validated` is empty on every other build. -- No screen offers the speech recogniser yet: the Audio section drops rows without `voicing`, and Settings has no speech recognition section ([proposal](../../proposals/local-speech-to-text.md#catalog-selection-and-settings)). - Qwen3.8 27B is a hybrid model (`qwen35`): the fit estimate charges a KV cache on all 65 layers where 16 keep one, about four times too much, and does not size its recurrent state ([runtime](runtime.md)). - Qwen3.8 27B commits no `sampling`: its repo has no `params` file, so it runs on llama-server's defaults rather than the model card's. - `template.system_role` and llama.cpp's `run.args` are committed but unread: chat asks the loaded template for its system role, and the router ignores per-model load arguments. sd.cpp's `image` defaults and `run.args` reach sd-server as launch flags. `template.tools` and `template.reasoning` reach a row's support, which the screen does not show. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 7f8da1b2e..12bebf9ac 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -114,7 +114,7 @@ The rules that keep the processes out of each other's way: - [`backend/modules/`](../../surfsense_local/backend/modules/): one folder per feature, holding whichever of its models, schemas, router and service it needs: `workspaces`, `documents`, `chunks`, `chat`, `artifacts` (Studio), `events`, `llm` (runtimes, catalog, connections, selection), `egress`, `agent`, `plugins`, `license`, `migration` (import) and `health`. `shared.db.import_models()` imports every model at startup, because relationships name their targets as strings. - [`backend/worker/`](../../surfsense_local/backend/worker/): `consumer.py` drains one queue per process, `jobs.py` holds status transitions and cancellation, `notify.py` the change notice, and `ingestion/` and `studio/` the two pipelines. - [`backend/shared/`](../../surfsense_local/backend/shared/): configuration, the engine and its pragmas, `upgrade_to_head()`, the two Huey queues, `retrieve()`, and the secret that encrypts stored keys. -- [`backend/alembic/`](../../surfsense_local/backend/alembic/): revisions `0001` to `0028`, all hand-written. `env.py` has no `target_metadata`, so autogenerate cannot run by accident. +- [`backend/alembic/`](../../surfsense_local/backend/alembic/): revisions `0001` to `0030`, all hand-written. `env.py` has no `target_metadata`, so autogenerate cannot run by accident. - [`frontend/src/features/`](../../surfsense_local/frontend/src/features/): `chat`, `agent`, `sources`, `studio`, `workspaces`, `dashboard`, `models`, `onboarding`, `settings`, `egress`, `plugins`, `license`, `migration`, `updates` and `feedback`. - [`electron/src/main/`](../../surfsense_local/electron/src/main/): `index.ts` (the window, IPC and the paths it hands to boot), `sidecars/` (boot, the supervisor, one spec per sidecar, and the image-model, preset, audio and agent watchers), `session-log/` (this run's output, for issue reports), `permissions/` (who may use the microphone), `microphone/` (the operating system's switch), `secret.ts`, `updater.ts` and `document-files.ts`. diff --git a/docs/architecture/packaging.md b/docs/architecture/packaging.md index 40444bc68..2f9a25d90 100644 --- a/docs/architecture/packaging.md +++ b/docs/architecture/packaging.md @@ -164,7 +164,7 @@ Seven run in CI: `test_license_key.py` inside the release workflow, `test_frozen ## Known gaps - No issue on audio.cpp asks for archives that meet the app's floors yet, so `build-audiocpp.yml` has no end date. -- `nemotron_asr` is checked only for a loader: no packaging test transcribes with it yet, so a staged server that cannot transcribe would still ship. +- On macOS `nemotron_asr` is checked only for a loader: the `macos-archive` job stages upstream's archive and transcribes nothing, so a macOS server that cannot transcribe would still ship. - No packaging test checks that the frozen API carries the agent's `SKILL.md`; `api.spec` collects it, and it was checked by hand. - [`desktop-tests.yml`](../../.github/workflows/desktop-tests.yml) runs on pull requests and by hand (`workflow_dispatch`), never on a push to `dev`, and only on Linux, so a break that lands on `dev`, or shows only on Windows or macOS, shows first in a release build ([#2226](https://github.com/MODSetter/SurfSense/issues/2226)). - The packaged `llama-server`, `audiocpp_server` and `sd-server` are started only on the Linux runner ([#1986](https://github.com/MODSetter/SurfSense/issues/1986)). diff --git a/docs/proposals/local-speech-to-text.md b/docs/proposals/local-speech-to-text.md index 37215b184..dec9aa937 100644 --- a/docs/proposals/local-speech-to-text.md +++ b/docs/proposals/local-speech-to-text.md @@ -382,12 +382,12 @@ The runtime licence policy keeps GPL and LGPL code out of the app's own binaries Each step ships alone and leaves the app working. Steps 1 to 7 are v1. 1. **The measurement, no app change. Done** on 14 Oct 2026 ([Measurements](#measurements)): Nemotron 3.5 and Qwen3-ASR-0.6B on audio.cpp v0.8.2 built with the app's recipe flags. Nemotron took half the memory and a third of the time, so it is the model. -2. **The runtime, staged and invisible.** `FAMILIES`, the Linux `objdump` and Windows per-CPU gates, the macOS archive check, the sidecar spec, its watcher with crash restart, and the API's URL. Nothing writes `transcription.json` yet, so nothing starts. Adds ADR 0056, or the next number free then (0040 to 0046 are proposed by the file-agent work): "Speech recognition runs on a second audio.cpp server that only the API calls". Updates [`packaging.md`](../architecture/packaging.md). -3. **Catalog, selection and Settings.** The type, the revision, the slot, the manifest block and entry, the classifier, `transcription.json`, the rows and the Settings section. Users can install and choose a speech model; nothing reads it yet, which [selection](../architecture/local-models/selection.md) allows. Updates [`data-model.md`](../architecture/data-model.md), [`selection.md`](../architecture/local-models/selection.md) and [`catalog.md`](../architecture/local-models/catalog.md), and adds glossary rows for "speech recognition", "dictation" and "transcript", apart from the "speech" row. -4. **The API slice,** `modules/llm/transcription/` (router, internal router, gate, sessions, admission, selected model, languages, audio checks, cleanup, Silero) and the provider client behind the `Transcriber` protocol, with the hold rule and `fetch_silero_vad.py`. Tested against a fake server, and against the staged server by `test_audiocpp_transcription.py`; no screen changes. Extends the ADR with the gate, and updates [`admission.md`](../architecture/local-models/admission.md) and [`studio.md`](../architecture/studio.md). -5. **Audio files as sources,** with `miniaudio`, yielding, progress, cancel, awaiting rows and the set-up popover they open, and the search caveat in the same release, because without yielding one recording blocks every upload and note. Updates [`documents.md`](../architecture/documents.md) and `data-model.md`. -6. **The microphone in Electron,** and a notarized macOS build that shows the prompt. Updates `packaging.md`. -7. **Dictation,** phrase by phrase. Updates [`chat.md`](../architecture/chat.md). Folds v1 into `architecture/`, moves steps 8 to 11 into a new `proposals/speech-recognition-next.md`, and deletes this file. +2. **The runtime, staged and invisible. Done.** `FAMILIES`, the Linux `objdump` and Windows per-CPU gates, the macOS archive check, the sidecar spec, its watcher with crash restart, and the API's URL. Nothing writes `transcription.json` yet, so nothing starts. Adds [ADR 0056](../adr/0056-speech-recognition-runs-on-a-second-audiocpp-server.md), "Speech recognition runs on a second audio.cpp server that only the API calls". Updates [`packaging.md`](../architecture/packaging.md). +3. **Catalog, selection and Settings. Done.** The type, the revision, the slot, the manifest block and entry, the classifier, `transcription.json`, the rows and the Settings section. Users can install and choose a speech model; nothing reads it yet, which [selection](../architecture/local-models/selection.md) allows. Updates [`data-model.md`](../architecture/data-model.md), [`selection.md`](../architecture/local-models/selection.md) and [`catalog.md`](../architecture/local-models/catalog.md), and adds glossary rows for "speech recognition", "dictation" and "transcript", apart from the "speech" row. +4. **The API slice, done:** `modules/llm/transcription/` (router, internal router, gate, sessions, admission, selected model, languages, audio checks, cleanup, Silero) and the provider client behind the `Transcriber` protocol, with the hold rule and `fetch_silero_vad.py`. Tested against a fake server, and against the staged server by `test_audiocpp_transcription.py`; no screen changes. Extends the ADR with the gate, and updates [`admission.md`](../architecture/local-models/admission.md) and [`studio.md`](../architecture/studio.md). +5. **Audio files as sources, done:** with `miniaudio`, yielding, progress, cancel, awaiting rows and the set-up popover they open, and the search caveat in the same release, because without yielding one recording blocks every upload and note. Updates [`documents.md`](../architecture/documents.md) and `data-model.md`. +6. **The microphone in Electron, done.** A notarized macOS build that shows the prompt is still owed ([packaging](../architecture/packaging.md), Known gaps). Updates `packaging.md`. +7. **Dictation, done:** phrase by phrase. Updates [`chat.md`](../architecture/chat.md). Folding v1 into `architecture/`, moving steps 8 to 11 into `proposals/speech-recognition-next.md` and deleting this file wait for the measurements under [Open questions](#open-questions). 8. **Next: light and broad languages.** The pin moved to v0.9.1 or later on its own, with the TTS peaks and the voicing gate re-run, then OWSM v4 base and small after a FLEURS eval of ar, hi, bn, ur, ta, te, sw, uk, he and fa, listing only the languages that pass. 9. **Next: more formats.** `scripts/ffmpeg-decode/` builds a pinned FFmpeg in CI on `ubuntu-22.04`, Windows and macOS arm64 with `--disable-everything --disable-network --disable-autodetect`, only the needed demuxers, decoders and parsers, `aresample` and a `pcm_s16le` WAV muxer, gated on `-buildconf` showing no `--enable-gpl`, `--enable-nonfree` or `--enable-lib` option, on fixtures in every allowed format and on the `objdump` floor. The worker runs it as the decode child, with a timeout. `.opus` joins; `.m4a` and `.aac` join once the AAC question is settled; `.webm` joins only for a file whose only stream is audio, until the maintainer answers the video question. Evaluated beside it: decoding in the renderer through Electron's own FFmpeg for files added from the window, streamed in chunks to 16 kHz mono WAV. 10. **Next: listen and seek.** `GET /documents/{id}/original` commits before its `FileResponse`, since every transaction is `BEGIN IMMEDIATE` and a player would hold the write lock, as Studio's file route does. An audio viewer joins the [source preview](source-preview.md)'s registry, with the transcript beside it, and a citation of a recording opens it at the time from `segments.json`. diff --git a/docs/proposals/model-catalog.md b/docs/proposals/model-catalog.md index b5cb69f97..65e0d205d 100644 --- a/docs/proposals/model-catalog.md +++ b/docs/proposals/model-catalog.md @@ -294,7 +294,7 @@ The snapshot this replaced dropped the provider and keyed by model id alone. Tha - Diffusion architectures (`sd1`, `sdxl`, `sd3`, `flux`, `flux2`, `qwen_image`, `z_image`, `lumina2` and the rest) are `IMAGE_GEN`. They were refused because the old catalog was text only; the app ships sd.cpp. - Video architectures (`wan`, `ltxv`, `hyvid`, `cosmos`) are `VIDEO_GEN`. - Text-to-speech models are `AUDIO_GEN`. -- Embedders, rerankers, speech recognisers, labellers, OCR, draft heads and projectors are known-none: no type describes what they produce. +- Embedders, rerankers, speech recognisers, labellers, OCR, draft heads and projectors are known-none: no type describes what they produce. The exception is a curated speech recogniser's family, which is `SPEECH_TO_TEXT` (`nemotron_asr`, [catalog](../architecture/local-models/catalog.md)). - Any other architecture is `TEXT_GEN`. A denylist ages the right way: an unknown architecture is usually a chat model released last week. Evidence rules: diff --git a/surfsense_local/frontend/src/features/models/add-model/add-model-options.tsx b/surfsense_local/frontend/src/features/models/add-model/add-model-options.tsx index 892524fc5..6097df42b 100644 --- a/surfsense_local/frontend/src/features/models/add-model/add-model-options.tsx +++ b/surfsense_local/frontend/src/features/models/add-model/add-model-options.tsx @@ -21,7 +21,7 @@ export function AddModelOptions({ }: { /** The slot's catalog for this computer. */ download: ReactNode - /** The slot being filled, so the connect dialog offers only what fills it. */ + /** The slot being filled, so a ChatGPT plan is offered only where one fills it. */ modelType: ModelType /** Absent where the slot offers no server. */ onConnected?: (connection: Connection) => void diff --git a/surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx b/surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx index eca87e963..7c58a810d 100644 --- a/surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx +++ b/surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx @@ -109,6 +109,30 @@ describe("speech recognition settings", () => { expect(screen.queryByText(/Kokoro/)).toBeNull() }) + it("offers no ChatGPT plan either, even one that says it serves the slot", async () => { + vi.stubGlobal( + "fetch", + serving([], (path) => + path === "/llm/connections/chatgpt" + ? Response.json({ serves: ["text_gen", "speech_to_text"], hosts: [] }) + : null + ) + ) + const user = userEvent.setup() + renderSettings() + + await user.click(await screen.findByRole("button", { name: "Add model" })) + expect( + await screen.findByText( + "Speech recognition models cannot run on this computer" + ) + ).toBeTruthy() + expect( + screen.queryByRole("region", { name: "Use your ChatGPT plan" }) + ).toBeNull() + expect(screen.queryByRole("region", { name: "Use a server" })).toBeNull() + }) + it("says the models could not load, not that this computer cannot run one", async () => { vi.stubGlobal( "fetch",