mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-10-19 07:34:58 +00:00
84c7ffad18
The embedding model is now chosen in the last onboarding step and fixed for the whole library. Skipping keeps bge-small, which ships with the app and can't be deleted. Changing the model later is not built. - An embedding_indexes row records the model's spec and its vector table, and each document records which index embedded it. Migration 0021 stamps existing libraries with bge-small. Routes that queue embedding work return 409 embedding_not_chosen until a model is chosen. - Ingest, Studio and search embed through the active index; the old worker embedding module is removed. Workers wait for the schema, and documents interrupted when the app closed are marked failed. - Curated embedders (bge-small, granite-97m, granite-311m) live in the local manifest, run by a new onnxruntime engine. EMBEDDING is a catalog type, never a selection slot. - Hugging Face search finds ONNX embedders. Each download is checked: a probe sets its width, and a ten-question search check must pass. Small tokenizer.json files that Hugging Face lists without a checksum are hashed at the pinned commit. Results use the GGUF search's shapes. - Onboarding renders the embedding step with the same component and shared list as the other steps; only its choice hook differs. Use marks the choice and Finish sends it. Bundled models have no Delete. - Settings has a read-only Embedding section. - ADRs 0036 and 0037, a new embedding architecture doc, and updates to the docs it touches. New strings are translated into every language.
86 lines
2.8 KiB
Python
86 lines
2.8 KiB
Python
"""One embedding model's manifest entry from its repo at a commit.
|
|
|
|
Pure, like the other engines' assemblies: every rule is a unit test over
|
|
recorded input.
|
|
"""
|
|
|
|
from typing import Any
|
|
|
|
from local_manifest.onnxruntime.entry import EmbeddingEntry
|
|
from local_manifest.recorded import RepoAtRevision
|
|
from local_manifest.unreadable import UnreadableBuildError
|
|
from modules.llm.catalog.local.build import Build, BuildFile, FileRole
|
|
|
|
|
|
def pinned_builds(entry: EmbeddingEntry, snapshot: RepoAtRevision) -> list[Build]:
|
|
"""Each build's weights with the repo's tokenizer, every file hashed."""
|
|
listed = {f.path: f for f in snapshot.listing}
|
|
builds = []
|
|
for label, weights in entry.builds:
|
|
files = []
|
|
for role, path in (
|
|
(FileRole.WEIGHTS, weights),
|
|
(FileRole.TOKENIZER, entry.tokenizer),
|
|
):
|
|
found = listed.get(path)
|
|
if found is None or not found.sha256 or found.size_bytes <= 0:
|
|
raise UnreadableBuildError(
|
|
f"{snapshot.repo}: {path} has no hash or size"
|
|
)
|
|
files.append(
|
|
BuildFile(
|
|
role,
|
|
path,
|
|
found.size_bytes,
|
|
found.sha256,
|
|
snapshot.repo,
|
|
snapshot.revision,
|
|
)
|
|
)
|
|
builds.append(Build(label, tuple(files)))
|
|
return builds
|
|
|
|
|
|
def entry_for(
|
|
entry: EmbeddingEntry,
|
|
snapshot: RepoAtRevision,
|
|
builds: list[Build],
|
|
architecture: str,
|
|
validated: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""The written entry. `architecture` is the repo config's `model_type`."""
|
|
validated = validated or {}
|
|
return {
|
|
"id": entry.id,
|
|
"name": entry.name,
|
|
"family": entry.family,
|
|
"publisher": entry.publisher,
|
|
"description": entry.description,
|
|
"license": entry.license,
|
|
"source_repo": entry.source_repo,
|
|
"aliases": list(entry.aliases),
|
|
"evidence": {
|
|
"architecture": architecture,
|
|
"pipeline_tag": snapshot.pipeline_tag,
|
|
},
|
|
"embedding": dict(entry.embedding),
|
|
"builds": [
|
|
{
|
|
"quantization": build.quantization,
|
|
"files": [
|
|
{
|
|
"role": file.role.value,
|
|
"repo": file.repo,
|
|
"upstream_repo": entry.upstream_repo,
|
|
"revision": file.revision,
|
|
"path": file.path,
|
|
"size_bytes": file.size_bytes,
|
|
"sha256": file.sha256,
|
|
}
|
|
for file in build.files
|
|
],
|
|
"validated": {"onnxruntime": validated.get(build.weights.path)},
|
|
}
|
|
for build in builds
|
|
],
|
|
}
|