mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-10-19 15:44:27 +00:00
84c7ffad18
The embedding model is now chosen in the last onboarding step and fixed for the whole library. Skipping keeps bge-small, which ships with the app and can't be deleted. Changing the model later is not built. - An embedding_indexes row records the model's spec and its vector table, and each document records which index embedded it. Migration 0021 stamps existing libraries with bge-small. Routes that queue embedding work return 409 embedding_not_chosen until a model is chosen. - Ingest, Studio and search embed through the active index; the old worker embedding module is removed. Workers wait for the schema, and documents interrupted when the app closed are marked failed. - Curated embedders (bge-small, granite-97m, granite-311m) live in the local manifest, run by a new onnxruntime engine. EMBEDDING is a catalog type, never a selection slot. - Hugging Face search finds ONNX embedders. Each download is checked: a probe sets its width, and a ten-question search check must pass. Small tokenizer.json files that Hugging Face lists without a checksum are hashed at the pinned commit. Results use the GGUF search's shapes. - Onboarding renders the embedding step with the same component and shared list as the other steps; only its choice hook differs. Use marks the choice and Finish sends it. Bundled models have no Delete. - Settings has a read-only Embedding section. - ADRs 0036 and 0037, a new embedding architecture doc, and updates to the docs it touches. New strings are translated into every language.
53 lines
1.8 KiB
Python
53 lines
1.8 KiB
Python
"""Fixing the library's embedder, once, when onboarding finishes.
|
|
|
|
The one place outside migrations that emits DDL: the vector table's width is the
|
|
chosen model's, which no migration can know. Safe only while the table is empty,
|
|
which is what the refusals guarantee.
|
|
"""
|
|
|
|
from sqlalchemy import exists, select, text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modules.chunks.models import Chunk
|
|
from modules.embedding.active import active_index
|
|
from modules.embedding.bundled import BGE
|
|
from modules.embedding.models import EmbeddingIndex, IndexState
|
|
from modules.embedding.spec import EmbedderSpec
|
|
|
|
# The table migration 0001 created, which the chunk delete trigger names.
|
|
FIRST_TABLE = "chunk_vectors"
|
|
|
|
|
|
class IndexAlreadyBuiltError(Exception):
|
|
"""The library already has vectors from some model, or a locked choice."""
|
|
|
|
|
|
def lock_index(session: Session, spec: EmbedderSpec) -> EmbeddingIndex:
|
|
if session.scalar(
|
|
select(exists().where(EmbeddingIndex.state == IndexState.ACTIVE))
|
|
):
|
|
raise IndexAlreadyBuiltError("an embedding model is already chosen")
|
|
if session.scalar(select(exists().select_from(Chunk))):
|
|
raise IndexAlreadyBuiltError("the library already holds embedded passages")
|
|
|
|
session.execute(text(f"DROP TABLE IF EXISTS {FIRST_TABLE}"))
|
|
session.execute(
|
|
text(
|
|
f"CREATE VIRTUAL TABLE {FIRST_TABLE} "
|
|
f"USING vec0(embedding float[{int(spec.dimension)}])"
|
|
)
|
|
)
|
|
index = EmbeddingIndex(
|
|
spec=spec.model_dump(mode="json"),
|
|
vector_table=FIRST_TABLE,
|
|
state=IndexState.ACTIVE,
|
|
)
|
|
session.add(index)
|
|
session.flush()
|
|
return index
|
|
|
|
|
|
def lock_default_if_unchosen(session: Session) -> None:
|
|
"""Finishing onboarding with no embedder chosen means the bundled one."""
|
|
if active_index(session) is None:
|
|
lock_index(session, BGE)
|