mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-11-17 22:08:05 +00:00
The embedding model is now chosen in the last onboarding step and fixed for the whole library. Skipping keeps bge-small, which ships with the app and can't be deleted. Changing the model later is not built. - An embedding_indexes row records the model's spec and its vector table, and each document records which index embedded it. Migration 0021 stamps existing libraries with bge-small. Routes that queue embedding work return 409 embedding_not_chosen until a model is chosen. - Ingest, Studio and search embed through the active index; the old worker embedding module is removed. Workers wait for the schema, and documents interrupted when the app closed are marked failed. - Curated embedders (bge-small, granite-97m, granite-311m) live in the local manifest, run by a new onnxruntime engine. EMBEDDING is a catalog type, never a selection slot. - Hugging Face search finds ONNX embedders. Each download is checked: a probe sets its width, and a ten-question search check must pass. Small tokenizer.json files that Hugging Face lists without a checksum are hashed at the pinned commit. Results use the GGUF search's shapes. - Onboarding renders the embedding step with the same component and shared list as the other steps; only its choice hook differs. Use marks the choice and Finish sends it. Bundled models have no Delete. - Settings has a read-only Embedding section. - ADRs 0036 and 0037, a new embedding architecture doc, and updates to the docs it touches. New strings are translated into every language.
38 lines
1.1 KiB
Python
38 lines
1.1 KiB
Python
from dataclasses import dataclass
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modules.embedding.models import EmbeddingIndex, IndexState
|
|
from modules.embedding.spec import EmbedderSpec
|
|
|
|
|
|
class EmbeddingNotChosenError(Exception):
|
|
"""Nothing may be embedded before onboarding fixes the model."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__("no embedding model has been chosen yet")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ActiveIndex:
|
|
id: int
|
|
spec: EmbedderSpec
|
|
vector_table: str
|
|
|
|
|
|
def active_index(session: Session) -> ActiveIndex | None:
|
|
"""The index search reads and ingest writes. None before onboarding chooses."""
|
|
row = session.scalars(
|
|
select(EmbeddingIndex).where(EmbeddingIndex.state == IndexState.ACTIVE)
|
|
).one_or_none()
|
|
if row is None:
|
|
return None
|
|
return ActiveIndex(row.id, EmbedderSpec.model_validate(row.spec), row.vector_table)
|
|
|
|
|
|
def require_active_index(session: Session) -> ActiveIndex:
|
|
index = active_index(session)
|
|
if index is None:
|
|
raise EmbeddingNotChosenError
|
|
return index
|