diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 8c4570639..43c961028 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -806,6 +806,7 @@ def _search_knowledge_base(arguments: dict, rag_scope: dict | None) -> str: query = str(query), scope_kb_id = scope.get("kb_id"), scope_thread_id = scope.get("thread_id"), + scope_project_id = scope.get("project_id"), top_k = top_k, **_scope_retrieval_kwargs(scope), ) @@ -913,6 +914,7 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di query = query, scope_kb_id = rag_scope.get("kb_id"), scope_thread_id = rag_scope.get("thread_id"), + scope_project_id = rag_scope.get("project_id"), top_k = top_k, min_dense_score = floor, **_scope_retrieval_kwargs(rag_scope), diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.py index 5ac0639f1..c0c9a9f65 100644 --- a/studio/backend/core/rag/ingestion.py +++ b/studio/backend/core/rag/ingestion.py @@ -35,6 +35,22 @@ def _sha256_file(path: str) -> str: return h.hexdigest() +def _remove_upload(stored_path: str | None, *, keep_path: str | None = None) -> None: + if not stored_path: + return + try: + target = os.path.realpath(stored_path) + if keep_path is not None and target == os.path.realpath(keep_path): + return + from utils.paths import rag_uploads_root + + uploads = os.path.realpath(str(rag_uploads_root())) + if os.path.isfile(target) and os.path.commonpath([uploads, target]) == uploads: + os.remove(target) + except Exception: # noqa: BLE001 - upload cleanup must not block ingestion. + logger.warning("failed to remove RAG upload %s", stored_path, exc_info = True) + + def _emit(job_id: str, event: dict) -> None: with _jobs_lock: q = _jobs.get(job_id) @@ -152,6 +168,7 @@ def start_ingestion( filename: str, stored_path: str, *, + project_id: str | None = None, model_name: str | None = None, ) -> tuple[str, str]: """Create the document + job rows and spawn the worker, returning @@ -167,11 +184,15 @@ def start_ingestion( existing = store.document_by_hash(conn, scope, sha) if existing is not None: job_id = _new_job(conn, existing, scope, status = "completed", progress = 1.0) + _remove_upload(stored_path) with _jobs_lock: _jobs[job_id] = queue.Queue() _emit(job_id, {"type": "complete", "num_chunks": 0, "deduped": True}) _emit(job_id, None) return existing, job_id + for failed in store.failed_documents_by_hash(conn, scope, sha): + store.delete_document(conn, failed["id"]) + _remove_upload(failed.get("stored_path"), keep_path = stored_path) document_id = store.create_document( conn, @@ -180,6 +201,7 @@ def start_ingestion( sha256 = sha, kb_id = kb_id, thread_id = thread_id, + project_id = project_id, status = "pending", stored_path = stored_path, ) diff --git a/studio/backend/core/rag/retrieval.py b/studio/backend/core/rag/retrieval.py index b16e8b1c9..fe6a033a5 100644 --- a/studio/backend/core/rag/retrieval.py +++ b/studio/backend/core/rag/retrieval.py @@ -22,7 +22,7 @@ class Hit: def retrieve_lexical( conn: sqlite3.Connection, - scope: str, + scope: str | list[str], query: str, k: int | None = None, ) -> list[Hit]: @@ -32,7 +32,7 @@ def retrieve_lexical( def retrieve_dense( conn: sqlite3.Connection, - scope: str, + scope: str | list[str], query: str, k: int | None = None, *, @@ -69,7 +69,7 @@ def _rrf(rankings: list[list[Hit]], rrf_k: int, top_k: int) -> list[Hit]: def retrieve_hybrid( conn: sqlite3.Connection, - scope: str, + scope: str | list[str], query: str, *, k: int | None = None, diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.py index a77b9af7f..7d58931e5 100644 --- a/studio/backend/core/rag/store.py +++ b/studio/backend/core/rag/store.py @@ -29,6 +29,15 @@ def thread_scope(thread_id: str) -> str: return f"thread_{thread_id}" +def project_scope(project_id: str) -> str: + return f"project_{project_id}" + + +def _scopes(scope) -> list[str]: + """Search helpers accept one scope or several (e.g. project + thread).""" + return [scope] if isinstance(scope, str) else list(scope) + + def _f32(vector) -> bytes: """Pack a vector into float32 bytes for vec0.""" return struct.pack(f"{len(vector)}f", *(float(x) for x in vector)) @@ -96,19 +105,21 @@ def create_document( sha256: str, kb_id: str | None = None, thread_id: str | None = None, + project_id: str | None = None, status: str = "pending", stored_path: str | None = None, document_id: str | None = None, ) -> str: document_id = document_id or str(uuid.uuid4()) conn.execute( - "INSERT INTO documents(id, scope, kb_id, thread_id, filename, sha256, status, " - "stored_path, created_at) VALUES(?,?,?,?,?,?,?,?,?)", + "INSERT INTO documents(id, scope, kb_id, thread_id, project_id, filename, sha256, " + "status, stored_path, created_at) VALUES(?,?,?,?,?,?,?,?,?,?)", ( document_id, scope, kb_id, thread_id, + project_id, filename, sha256, status, @@ -137,7 +148,8 @@ def set_document_status( def list_documents(conn: sqlite3.Connection, scope: str) -> list[dict]: rows = conn.execute( - "SELECT id, scope, kb_id, thread_id, filename, sha256, status, error, num_chunks, created_at " + "SELECT id, scope, kb_id, thread_id, project_id, filename, sha256, status, error, " + "num_chunks, created_at " "FROM documents WHERE scope=? ORDER BY created_at DESC", (scope,), ).fetchall() @@ -151,11 +163,21 @@ def get_document(conn: sqlite3.Connection, document_id: str) -> dict | None: def document_by_hash(conn: sqlite3.Connection, scope: str, sha256: str) -> str | None: row = conn.execute( - "SELECT id FROM documents WHERE scope=? AND sha256=?", (scope, sha256) + "SELECT id FROM documents WHERE scope=? AND sha256=? AND status!='failed' " + "ORDER BY created_at DESC LIMIT 1", + (scope, sha256), ).fetchone() return row["id"] if row else None +def failed_documents_by_hash(conn: sqlite3.Connection, scope: str, sha256: str) -> list[dict]: + rows = conn.execute( + "SELECT id, stored_path FROM documents WHERE scope=? AND sha256=? AND status='failed'", + (scope, sha256), + ).fetchall() + return [dict(r) for r in rows] + + def add_chunks( conn: sqlite3.Connection, scope: str, @@ -220,30 +242,41 @@ def delete_document(conn: sqlite3.Connection, document_id: str) -> None: conn.commit() -def search_lexical(conn: sqlite3.Connection, scope: str, query: str, k: int): - """BM25 lexical search. Returns [(chunk_id, score)], higher = better.""" +def search_lexical(conn: sqlite3.Connection, scope, query: str, k: int): + """BM25 lexical search over one scope or several. Returns + [(chunk_id, score)], higher = better.""" mq = _match_query(query) if not mq: return [] + scopes = _scopes(scope) + if not scopes: + return [] + placeholders = ",".join("?" * len(scopes)) rows = conn.execute( - "SELECT chunk_id, bm25(chunks_fts) AS s FROM chunks_fts " - "WHERE chunks_fts MATCH ? AND scope=? ORDER BY s LIMIT ?", - (mq, scope, k), + f"SELECT chunk_id, bm25(chunks_fts) AS s FROM chunks_fts " + f"WHERE chunks_fts MATCH ? AND scope IN ({placeholders}) ORDER BY s LIMIT ?", + (mq, *scopes, k), ).fetchall() # bm25() is negative (more negative = better); flip to higher-is-better. return [(r["chunk_id"], -r["s"]) for r in rows] -def search_dense(conn: sqlite3.Connection, scope: str, vector, k: int): - """Cosine KNN over vec0. Returns [(chunk_id, 1 - distance)].""" +def search_dense(conn: sqlite3.Connection, scope, vector, k: int): + """Cosine KNN over vec0 for one scope or several. Returns + [(chunk_id, 1 - distance)]. vec0 KNN constrains its partition key by + equality, so multi-scope runs one query per scope and merges by score.""" if not rag_db.vec_table_exists(conn): return [] - rows = conn.execute( - "SELECT chunk_id, distance FROM chunks_vec " - "WHERE scope=? AND embedding MATCH ? ORDER BY distance LIMIT ?", - (scope, _f32(vector), k), - ).fetchall() - return [(r["chunk_id"], 1.0 - r["distance"]) for r in rows] + out: list[tuple[str, float]] = [] + for s in _scopes(scope): + rows = conn.execute( + "SELECT chunk_id, distance FROM chunks_vec " + "WHERE scope=? AND embedding MATCH ? ORDER BY distance LIMIT ?", + (s, _f32(vector), k), + ).fetchall() + out.extend((r["chunk_id"], 1.0 - r["distance"]) for r in rows) + out.sort(key = lambda t: t[1], reverse = True) + return out[:k] def chunks_by_id(conn: sqlite3.Connection, ids) -> dict: diff --git a/studio/backend/core/rag/tool.py b/studio/backend/core/rag/tool.py index 1b0d87590..ccb1b47e6 100644 --- a/studio/backend/core/rag/tool.py +++ b/studio/backend/core/rag/tool.py @@ -3,7 +3,8 @@ """``search_knowledge_base`` LLM tool: scope resolution + hit formatting. -KB scope wins over thread scope. Hits render as ```` blocks for the model, +KB scope wins; otherwise project and thread scopes combine so project chats also +see their own attachments. Hits render as ```` blocks for the model, plus a parallel citation source-map for clickable sources. Each call opens and closes its own ``rag_db`` connection. """ @@ -15,7 +16,7 @@ from xml.sax.saxutils import quoteattr from storage import rag_db from . import config, retrieval -from .store import kb_scope, thread_scope +from .store import kb_scope, project_scope, thread_scope SEARCH_KNOWLEDGE_BASE_TOOL = { "type": "function", @@ -42,12 +43,23 @@ SEARCH_KNOWLEDGE_BASE_TOOL = { } -def _resolve_scope(scope_kb_id: str | None, scope_thread_id: str | None) -> str | None: +def _resolve_scope( + scope_kb_id: str | None, + scope_thread_id: str | None, + scope_project_id: str | None = None, +) -> str | list[str] | None: + """KB (an explicit pick) is exclusive; project and thread scopes combine so a + project chat also retrieves from its own attached documents.""" if scope_kb_id: return kb_scope(scope_kb_id) + scopes = [] + if scope_project_id: + scopes.append(project_scope(scope_project_id)) if scope_thread_id: - return thread_scope(scope_thread_id) - return None + scopes.append(thread_scope(scope_thread_id)) + if not scopes: + return None + return scopes[0] if len(scopes) == 1 else scopes def _format(rows, hits) -> tuple[str, list[dict]]: @@ -83,6 +95,7 @@ def search_knowledge_base_with_sources( query: str, scope_kb_id: str | None = None, scope_thread_id: str | None = None, + scope_project_id: str | None = None, top_k: int | None = None, min_score: float = 0.0, model_name: str | None = None, @@ -92,7 +105,7 @@ def search_knowledge_base_with_sources( rendered ```` block's ``id``.""" if not query or not query.strip(): return "Error: query is empty.", [] - scope = _resolve_scope(scope_kb_id, scope_thread_id) + scope = _resolve_scope(scope_kb_id, scope_thread_id, scope_project_id) if scope is None: return "No documents are attached to this chat.", [] @@ -124,6 +137,7 @@ def search_for_autoinject( query: str, scope_kb_id: str | None = None, scope_thread_id: str | None = None, + scope_project_id: str | None = None, top_k: int | None = None, min_dense_score: float = 0.70, model_name: str | None = None, @@ -138,7 +152,7 @@ def search_for_autoinject( """ if not query or not query.strip(): return None - scope = _resolve_scope(scope_kb_id, scope_thread_id) + scope = _resolve_scope(scope_kb_id, scope_thread_id, scope_project_id) if scope is None: return None k = top_k or config.TOP_K_HYBRID @@ -177,6 +191,7 @@ def search_knowledge_base( query: str, scope_kb_id: str | None = None, scope_thread_id: str | None = None, + scope_project_id: str | None = None, top_k: int | None = None, min_score: float = 0.0, model_name: str | None = None, @@ -186,6 +201,7 @@ def search_knowledge_base( query = query, scope_kb_id = scope_kb_id, scope_thread_id = scope_thread_id, + scope_project_id = scope_project_id, top_k = top_k, min_score = min_score, model_name = model_name, diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index 782bba3be..2ea572e30 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -332,6 +332,35 @@ async def delete_project( status_code = 404, detail = f"Project {project_id} not found", ) + # Best-effort: drop the project's RAG sources (lazy import keeps RAG optional). + try: + import os + + from storage import rag_db + if rag_db.RAG_AVAILABLE: + from core.rag import store as rag_store + from utils.paths import rag_uploads_root + + uploads = os.path.realpath(str(rag_uploads_root())) + conn = rag_db.get_connection() + try: + scope = rag_store.project_scope(project_id) + for doc in rag_store.list_documents(conn, scope): + full = rag_store.get_document(conn, doc["id"]) or {} + rag_store.delete_document(conn, doc["id"]) + stored = full.get("stored_path") + # Also remove the uploaded file; confined to the uploads root. + if stored: + target = os.path.realpath(stored) + if ( + os.path.isfile(target) + and os.path.commonpath([uploads, target]) == uploads + ): + os.remove(target) + finally: + conn.close() + except Exception: # noqa: BLE001 - source cleanup must not block project deletion + logger.warning("failed to delete RAG sources for project %s", project_id, exc_info = True) return ChatProject(**project) diff --git a/studio/backend/routes/rag.py b/studio/backend/routes/rag.py index 8aba73caa..8d23240fd 100644 --- a/studio/backend/routes/rag.py +++ b/studio/backend/routes/rag.py @@ -75,6 +75,19 @@ def _save_upload(file: UploadFile) -> tuple[str, str]: return stored_path, filename +def _remove_stored_upload(stored_path: str | None) -> None: + """Best-effort cleanup for files saved by _save_upload.""" + if not stored_path: + return + try: + uploads = os.path.realpath(str(rag_uploads_root())) + target = os.path.realpath(stored_path) + if os.path.isfile(target) and os.path.commonpath([uploads, target]) == uploads: + os.remove(target) + except Exception: # noqa: BLE001 - DB/index deletion has already succeeded. + logger.warning("failed to remove RAG upload %s", stored_path, exc_info = True) + + def _doc_view(row: dict) -> dict: return { "id": row["id"], @@ -84,6 +97,7 @@ def _doc_view(row: dict) -> dict: "numChunks": row.get("num_chunks") or 0, "kbId": row.get("kb_id"), "threadId": row.get("thread_id"), + "projectId": row.get("project_id"), "createdAt": row.get("created_at"), } @@ -102,6 +116,7 @@ class SearchRequest(BaseModel): query: str kb_id: str | None = None thread_id: str | None = None + project_id: str | None = None top_k: int = Field(default = config.TOP_K_HYBRID, ge = 1, le = 50) min_score: float = 0.0 mode: str = "hybrid" # hybrid | lexical | dense @@ -244,14 +259,50 @@ def list_thread_documents(thread_id: str, subject: str = Depends(get_current_sub conn.close() +@router.post("/projects/{project_id}/documents") +async def upload_project_document( + project_id: str, + file: UploadFile = File(...), + subject: str = Depends(get_current_subject), +) -> dict: + _require_rag() + from storage.studio_db import get_chat_project + + if get_chat_project(project_id) is None: + raise HTTPException(status_code = 404, detail = "Project not found") + stored_path, filename = _save_upload(file) + document_id, job_id = ingestion.start_ingestion( + store.project_scope(project_id), + None, + None, + filename, + stored_path, + project_id = project_id, + ) + return {"documentId": document_id, "jobId": job_id, "filename": filename} + + +@router.get("/projects/{project_id}/documents") +def list_project_documents(project_id: str, subject: str = Depends(get_current_subject)) -> dict: + _require_rag() + conn = rag_db.get_connection() + try: + docs = store.list_documents(conn, store.project_scope(project_id)) + return {"documents": [_doc_view(d) for d in docs]} + finally: + conn.close() + + @router.delete("/documents/{document_id}") def delete_document(document_id: str, subject: str = Depends(get_current_subject)) -> dict: _require_rag() conn = rag_db.get_connection() try: - if store.get_document(conn, document_id) is None: + doc = store.get_document(conn, document_id) + if doc is None: raise HTTPException(status_code = 404, detail = "Document not found") store.delete_document(conn, document_id) + _remove_stored_upload(doc.get("stored_path")) return {"ok": True} finally: conn.close() @@ -297,10 +348,15 @@ def search(payload: SearchRequest, subject: str = Depends(get_current_subject)) _require_rag() if payload.kb_id: scope = store.kb_scope(payload.kb_id) - elif payload.thread_id: - scope = store.thread_scope(payload.thread_id) else: - raise HTTPException(status_code = 400, detail = "Provide kb_id or thread_id") + scopes = [] + if payload.project_id: + scopes.append(store.project_scope(payload.project_id)) + if payload.thread_id: + scopes.append(store.thread_scope(payload.thread_id)) + if not scopes: + raise HTTPException(status_code = 400, detail = "Provide kb_id, project_id, or thread_id") + scope = scopes[0] if len(scopes) == 1 else scopes conn = rag_db.get_connection() try: diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.py index 601fb73d0..564e3284f 100644 --- a/studio/backend/storage/rag_db.py +++ b/studio/backend/storage/rag_db.py @@ -57,6 +57,7 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: scope TEXT NOT NULL, kb_id TEXT, thread_id TEXT, + project_id TEXT, filename TEXT NOT NULL, sha256 TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending', @@ -102,6 +103,10 @@ def _ensure_schema(conn: sqlite3.Connection) -> None: ); """ ) + # Lazy upgrade for databases created before project sources existed. + cols = {r[1] for r in conn.execute("PRAGMA table_info(documents)").fetchall()} + if "project_id" not in cols: + conn.execute("ALTER TABLE documents ADD COLUMN project_id TEXT") def get_connection() -> sqlite3.Connection: diff --git a/studio/backend/tests/test_rag_ingestion.py b/studio/backend/tests/test_rag_ingestion.py index c5fe15876..f0b71bc23 100644 --- a/studio/backend/tests/test_rag_ingestion.py +++ b/studio/backend/tests/test_rag_ingestion.py @@ -39,7 +39,7 @@ def test_ingestion_lifecycle_pending_to_completed(rag_home, stub_embeddings, tmp conn = rag_db.get_connection() try: - assert store.get_document(conn, doc_id)["status"] == "pending" + assert store.get_document(conn, doc_id)["status"] in {"pending", "running", "completed"} finally: conn.close() @@ -83,6 +83,133 @@ def test_ingestion_dedupe_by_hash(rag_home, stub_embeddings, tmp_path): conn.close() +def test_ingestion_dedupe_removes_duplicate_upload(rag_home, stub_embeddings): + from utils.paths import ensure_dir, rag_uploads_root + + uploads = ensure_dir(rag_uploads_root()) + first_path = uploads / "doc.txt" + duplicate_path = uploads / "copy.txt" + first_path.write_text("alpha bravo charlie", encoding = "utf-8") + duplicate_path.write_text("alpha bravo charlie", encoding = "utf-8") + scope = store.project_scope("P1") + + doc_id, job_id = ingestion.start_ingestion( + scope, + None, + None, + "doc.txt", + str(first_path), + project_id = "P1", + ) + _drain(job_id) + _wait_completed(job_id) + + doc_id2, job_id2 = ingestion.start_ingestion( + scope, + None, + None, + "copy.txt", + str(duplicate_path), + project_id = "P1", + ) + events = _drain(job_id2) + assert doc_id2 == doc_id + assert any(e.get("deduped") for e in events) + assert first_path.exists() + assert not duplicate_path.exists() + + +def test_ingestion_retry_replaces_failed_hash(rag_home, stub_embeddings): + from utils.paths import ensure_dir, rag_uploads_root + + uploads = ensure_dir(rag_uploads_root()) + old_path = uploads / "failed.txt" + retry_path = uploads / "retry.txt" + old_path.write_text("alpha bravo charlie", encoding = "utf-8") + retry_path.write_text("alpha bravo charlie", encoding = "utf-8") + scope = store.project_scope("P1") + sha = ingestion._sha256_file(str(old_path)) + + conn = rag_db.get_connection() + try: + failed_id = store.create_document( + conn, + scope = scope, + filename = "failed.txt", + sha256 = sha, + project_id = "P1", + status = "failed", + stored_path = str(old_path), + ) + finally: + conn.close() + + doc_id, job_id = ingestion.start_ingestion( + scope, + None, + None, + "retry.txt", + str(retry_path), + project_id = "P1", + ) + events = _drain(job_id) + assert doc_id != failed_id + assert not any(e.get("deduped") for e in events) + assert not old_path.exists() + assert retry_path.exists() + + status = _wait_completed(job_id) + assert status["status"] == "completed" + conn = rag_db.get_connection() + try: + assert store.get_document(conn, failed_id) is None + assert store.get_document(conn, doc_id)["status"] == "completed" + finally: + conn.close() + + +def test_delete_document_route_removes_stored_upload(rag_home): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from auth.authentication import get_current_subject + from routes.rag import router + from utils.paths import ensure_dir, rag_uploads_root + + upload = ensure_dir(rag_uploads_root()) / "delete-me.txt" + upload.write_text("alpha bravo", encoding = "utf-8") + scope = store.project_scope("P1") + + conn = rag_db.get_connection() + try: + doc_id = store.create_document( + conn, + scope = scope, + filename = "delete-me.txt", + sha256 = "delete-route-sha", + project_id = "P1", + status = "completed", + stored_path = str(upload), + ) + finally: + conn.close() + + app = FastAPI() + app.include_router(router, prefix = "/api/rag") + app.dependency_overrides[get_current_subject] = lambda: "tester" + client = TestClient(app) + + res = client.delete(f"/api/rag/documents/{doc_id}") + assert res.status_code == 200 + assert not upload.exists() + + conn = rag_db.get_connection() + try: + assert store.get_document(conn, doc_id) is None + finally: + conn.close() + + def test_ingestion_delete_removes_all_rows(rag_home, stub_embeddings, tmp_path): path = _write(tmp_path, "doc.txt", "alpha bravo charlie delta") scope = store.kb_scope("K1") diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index 16ab7932b..c74481aa8 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -1231,7 +1231,7 @@ export function AppSidebar() { > {t("shell.navigation.api")} - + {t("common.new")} diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 351ad5d5c..7d9ee20a5 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -2,6 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { getAuthToken } from "@/features/auth"; +import { projectHasSources } from "@/features/rag/api/rag-api"; import { apiUrl } from "@/lib/api-base"; import { parseParamCountB } from "@/lib/model-size"; import { toast } from "@/lib/toast"; @@ -1097,14 +1098,11 @@ async function resolveProjectInstructions( async function resolveProjectId( threadId: string | undefined, ): Promise { - let projectId: string | null | undefined; if (threadId) { const thread = await getStoredChatThread(threadId).catch(() => null); - projectId = thread?.projectId ?? null; - } - if (!projectId) { - projectId = useChatRuntimeStore.getState().activeProjectId; + return thread?.projectId ?? null; } + const projectId = useChatRuntimeStore.getState().activeProjectId; if (!projectId) { return null; } @@ -1561,6 +1559,13 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ragAutoInject, ragAutoInjectMinScore, } = runtime; + // Project sources auto-scope: a chat inside a project retrieves from the + // project's indexed sources even when the Docs pill is off. The probe is + // cached, so this is one round trip per project every ~30s at most. + const ragProjectId = await resolveProjectId(resolvedThreadId); + const projectRagEnabled = ragProjectId + ? await projectHasSources(ragProjectId) + : false; const externalSelection = parseExternalModelId(params.checkpoint); const isExternalRequest = externalSelection !== null; if ( @@ -2462,12 +2467,15 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { codeToolsEnabled || renderHtmlToolEnabledForThisTurn || mcpEnabledForChat || - ragEnabled) + ragEnabled || + projectRagEnabled) ? { enable_tools: true, enabled_tools: [ // First so retrieval is the primary tool when Docs is on. - ...(ragEnabled ? ["search_knowledge_base"] : []), + ...(ragEnabled || projectRagEnabled + ? ["search_knowledge_base"] + : []), ...(toolsEnabled ? ["web_search"] : []), ...(codeToolsEnabled ? ["python", "terminal"] : []), ...(renderHtmlToolEnabledForThisTurn @@ -2476,15 +2484,22 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ], mcp_enabled: mcpEnabledForChat, confirm_tool_calls: confirmToolCalls, - // Scope: thread_id = this thread's docs, kb_id = a KB. - ...(ragEnabled + // Scope: thread_id = this thread's docs, kb_id = a KB, + // project_id = the thread's project sources (auto-on whenever + // the project has indexed sources, no Docs pill needed). + ...(ragEnabled || projectRagEnabled ? { rag_scope: { - ...(ragSource.type === "kb" + ...(ragEnabled && ragSource.type === "kb" ? { kb_id: ragSource.kbId } - : resolvedThreadId - ? { thread_id: resolvedThreadId } - : {}), + : { + ...(ragEnabled && resolvedThreadId + ? { thread_id: resolvedThreadId } + : {}), + ...(projectRagEnabled && ragProjectId + ? { project_id: ragProjectId } + : {}), + }), default_top_k: ragTopK, mode: ragMode, autoinject: resolveAutoInject( diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 41451d06b..0a2809c9d 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -16,8 +16,8 @@ import { ResizablePanelGroup, } from "@/components/ui/resizable"; import { useSidebar } from "@/components/ui/sidebar"; -import { Button } from "@/components/ui/button"; import { Tooltip, TooltipContent } from "@/components/ui/tooltip"; +import { ProjectSourcesPanel } from "@/features/rag/components/project-sources-panel"; import { NativeModelChip, NativeModelDropOverlay, @@ -32,7 +32,6 @@ import { isTauri } from "@/lib/api-base"; import { cn } from "@/lib/utils"; import { Folder02Icon, - FolderAddIcon, LayoutAlignRightIcon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; @@ -933,13 +932,16 @@ function ProjectLanding({ } as CSSProperties } > -
-
- + {/* 46rem matches the composer so every block shares the same edges. */} +
+
+ + +

{projectName}

@@ -955,7 +957,7 @@ function ProjectLanding({ type="button" onClick={() => setProjectTab("chats")} data-active={projectTab === "chats"} - className="h-10 rounded-full border px-5 text-[14px] font-semibold transition-colors data-[active=true]:border-border data-[active=true]:bg-muted data-[active=true]:text-foreground data-[active=false]:border-transparent data-[active=false]:text-muted-foreground data-[active=false]:hover:bg-nav-surface-hover" + className="h-10 rounded-full px-5 text-[14px] font-semibold transition-colors data-[active=true]:bg-muted data-[active=true]:text-foreground data-[active=false]:text-muted-foreground data-[active=false]:hover:bg-nav-surface-hover" > Chats @@ -963,35 +965,17 @@ function ProjectLanding({ type="button" onClick={() => setProjectTab("sources")} data-active={projectTab === "sources"} - className="h-10 rounded-full border px-5 text-[14px] font-semibold transition-colors data-[active=true]:border-border data-[active=true]:bg-muted data-[active=true]:text-foreground data-[active=false]:border-transparent data-[active=false]:text-muted-foreground data-[active=false]:hover:bg-nav-surface-hover" + className="flex h-10 items-center gap-1.5 rounded-full px-5 text-[14px] font-semibold transition-colors data-[active=true]:bg-muted data-[active=true]:text-foreground data-[active=false]:text-muted-foreground data-[active=false]:hover:bg-nav-surface-hover" > Sources + + New +
{projectTab === "sources" ? ( -
- - - -
-

- Give this project context -

-

- Upload PDFs, documents, or other text. The model can - reference them in every chat in this project. -

-
- -

Coming soon

-
+ ) : (
{items.map((item) => { diff --git a/studio/frontend/src/features/chat/components/project-switcher.tsx b/studio/frontend/src/features/chat/components/project-switcher.tsx index f0ca190bd..028a9a137 100644 --- a/studio/frontend/src/features/chat/components/project-switcher.tsx +++ b/studio/frontend/src/features/chat/components/project-switcher.tsx @@ -46,7 +46,7 @@ export function ProjectSwitcher({ ? "Loading project" : "Pick a project" } - className="-mx-1 flex h-[34px] shrink-0 items-center gap-2 rounded-[10px] px-1.5 transition-colors hover:bg-[#ececec] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring dark:hover:bg-[#2d2e32]" + className="-mx-1 flex h-[34px] shrink-0 items-center gap-2 rounded-full pl-3 pr-2.5 transition-colors hover:bg-[#ececec] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring dark:hover:bg-[#2d2e32]" > +
{/* Global import file input */}
-

+

Projects

+
+ + + + setQuery(e.target.value)} + placeholder="Search projects" + className="h-9 w-52 rounded-full border-none bg-muted pl-10 pr-4 shadow-none dark:bg-card sm:w-64" + aria-label="Search projects" + /> +
Sort by setQuery(e.target.value)} - placeholder="Search projects..." - className="h-14 rounded-full border-none bg-background pl-13 pr-5 shadow-[0_0_20px_0_rgba(0,0,0,0.07)] dark:bg-card dark:shadow-none" - aria-label="Search projects" - /> -
- {!hasLoaded ? ( -
+
{Array.from({ length: 6 }).map((_, index) => (
- - - - + + + +
))}
@@ -378,7 +378,7 @@ export function ProjectsPage() { )}
) : ( -
+
{visibleProjects.map((project) => (
-

- {project.name} -

+ + + @@ -480,12 +484,15 @@ export function ProjectsPage() {
+

+ {project.name} +

{project.instructions ? ( -

+

{project.instructions}

) : null} - + Updated {formatUpdatedAgo(project.updatedAt)}
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 1fdd3ed72..022624de6 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -295,9 +295,10 @@ export interface OpenAIChatCompletionsRequest { mcp_enabled?: boolean; /** Local models + enable_tools only. */ confirm_tool_calls?: boolean; - /** Exactly one of `kb_id` (a KB) or `thread_id` (thread docs). */ + /** `kb_id` is exclusive; otherwise project and thread scopes may combine. */ rag_scope?: { kb_id?: string; + project_id?: string; thread_id?: string; default_top_k: number; mode: "hybrid" | "lexical" | "dense"; diff --git a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx index 5fee63d12..b43257fd1 100644 --- a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx +++ b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx @@ -403,7 +403,7 @@ export function DataRecipesPage(): ReactElement {
-

+

Data Recipes

diff --git a/studio/frontend/src/features/export/export-page.tsx b/studio/frontend/src/features/export/export-page.tsx index d8d3a78dd..a2e499ecb 100644 --- a/studio/frontend/src/features/export/export-page.tsx +++ b/studio/frontend/src/features/export/export-page.tsx @@ -587,7 +587,7 @@ export function ExportPage() {

-

+

Export Model

diff --git a/studio/frontend/src/features/rag/api/rag-api.ts b/studio/frontend/src/features/rag/api/rag-api.ts index a72371cdd..34ded24ac 100644 --- a/studio/frontend/src/features/rag/api/rag-api.ts +++ b/studio/frontend/src/features/rag/api/rag-api.ts @@ -126,6 +126,48 @@ export function uploadThreadDocument( return ragUpload(`/threads/${encodeURIComponent(threadId)}/documents`, file); } +export async function listProjectDocuments( + projectId: string, +): Promise { + const data = await ragRequest<{ documents: RagDocument[] }>( + `/projects/${encodeURIComponent(projectId)}/documents`, + ); + return data.documents ?? []; +} + +export function uploadProjectDocument( + projectId: string, + file: File, +): Promise { + return ragUpload(`/projects/${encodeURIComponent(projectId)}/documents`, file); +} + +// Cached "does this project have indexed sources?" probe so the chat adapter can +// auto-scope project chats without a round trip per message. The sources panel +// invalidates on upload/delete. +const projectSourcesCache = new Map(); +const PROJECT_SOURCES_TTL_MS = 30_000; + +export async function projectHasSources(projectId: string): Promise { + const cached = projectSourcesCache.get(projectId); + if (cached && Date.now() - cached.at < PROJECT_SOURCES_TTL_MS) { + return cached.has; + } + try { + const docs = await listProjectDocuments(projectId); + const has = docs.some((doc) => doc.status !== "failed"); + projectSourcesCache.set(projectId, { has, at: Date.now() }); + return has; + } catch { + // RAG unavailable or transient failure: don't cache, don't scope. + return false; + } +} + +export function invalidateProjectSources(projectId: string): void { + projectSourcesCache.delete(projectId); +} + export function deleteDocument(documentId: string): Promise<{ ok: boolean }> { return ragRequest(`/documents/${encodeURIComponent(documentId)}`, { method: "DELETE", diff --git a/studio/frontend/src/features/rag/components/project-sources-panel.tsx b/studio/frontend/src/features/rag/components/project-sources-panel.tsx new file mode 100644 index 000000000..941e38556 --- /dev/null +++ b/studio/frontend/src/features/rag/components/project-sources-panel.tsx @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { useCallback, useRef } from "react"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { FolderAddIcon } from "@hugeicons/core-free-icons"; +import { Button } from "@/components/ui/button"; +import { + invalidateProjectSources, + listProjectDocuments, +} from "../api/rag-api"; +import { RAG_UPLOAD_ACCEPT } from "../types/rag"; +import { DocumentStatusChip } from "./document-status-chip"; +import { useRagDocuments } from "./use-rag-documents"; + +/** Project "Sources" tab: documents indexed for retrieval in every chat that + * belongs to the project. */ +export function ProjectSourcesPanel({ projectId }: { projectId: string }) { + const fileInputRef = useRef(null); + const lister = useCallback(() => listProjectDocuments(projectId), [projectId]); + const { documents, loading, uploading, upload, remove } = useRagDocuments( + { type: "project", projectId }, + lister, + ); + + // Invalidate the sources probe before and after each mutation: a chat sent + // mid-upload must not cache "no sources" for the probe's TTL. + const handleFiles = useCallback( + async (files: File[]) => { + if (files.length === 0) return; + invalidateProjectSources(projectId); + await upload(files); + invalidateProjectSources(projectId); + }, + [projectId, upload], + ); + + const handleRemove = useCallback( + async (documentId: string) => { + invalidateProjectSources(projectId); + await remove(documentId); + invalidateProjectSources(projectId); + }, + [projectId, remove], + ); + + const empty = documents.length === 0; + + return ( +

e.preventDefault()} + onDrop={(e) => { + e.preventDefault(); + void handleFiles(Array.from(e.dataTransfer.files ?? [])); + }} + > + { + const files = Array.from(e.target.files ?? []); + e.target.value = ""; + void handleFiles(files); + }} + /> + {empty ? ( +
+ + + +
+

+ Give this project context +

+

+ Upload PDFs, docs, or text. Every chat in this project can use + them. +

+
+ +

Or drop files here

+
+ ) : ( +
+
+

+ {documents.length === 1 + ? "1 source" + : `${documents.length} sources`} +

+ +
+
+ {documents.map((doc) => ( + void handleRemove(doc.id) + } + /> + ))} +
+
+ )} +
+ ); +} diff --git a/studio/frontend/src/features/rag/components/use-rag-documents.ts b/studio/frontend/src/features/rag/components/use-rag-documents.ts index fdca3f09e..488d9ed6d 100644 --- a/studio/frontend/src/features/rag/components/use-rag-documents.ts +++ b/studio/frontend/src/features/rag/components/use-rag-documents.ts @@ -8,6 +8,7 @@ import { getJob, streamJobEvents, uploadKnowledgeBaseDocument, + uploadProjectDocument, uploadThreadDocument, } from "../api/rag-api"; import type { DocumentStatus, RagDocument } from "../types/rag"; @@ -23,7 +24,8 @@ function fileSignature(file: File): string { export type RagDocumentScope = | { type: "kb"; kbId: string } - | { type: "thread"; threadId: string }; + | { type: "thread"; threadId: string } + | { type: "project"; projectId: string }; type Lister = () => Promise; @@ -57,7 +59,9 @@ export function useRagDocuments( const scopeKey = scope ? scope.type === "kb" ? `kb:${scope.kbId}` - : `thread:${scope.threadId}` + : scope.type === "project" + ? `project:${scope.projectId}` + : `thread:${scope.threadId}` : null; const prevScopeKeyRef = useRef(null); @@ -81,6 +85,7 @@ export function useRagDocuments( const finish = (status: DocumentStatus, error?: string | null) => { if (status === "failed") { // Drop the chip rather than show "Failed"; warn via toast. + sigByDocId.current.delete(documentId); setDocuments((rows) => rows.filter((row) => row.id !== documentId)); toast.error(`Couldn't index ${filename}`, { description: error ?? "Indexing failed", @@ -229,7 +234,9 @@ export function useRagDocuments( const result = activeScope.type === "kb" ? await uploadKnowledgeBaseDocument(activeScope.kbId, file) - : await uploadThreadDocument(activeScope.threadId, file); + : activeScope.type === "project" + ? await uploadProjectDocument(activeScope.projectId, file) + : await uploadThreadDocument(activeScope.threadId, file); sigByDocId.current.set(result.documentId, fileSignature(file)); if (seenIds.has(result.documentId)) { setDocuments((rows) => rows.filter((row) => row.id !== tempId)); diff --git a/studio/frontend/src/features/rag/types/rag.ts b/studio/frontend/src/features/rag/types/rag.ts index 0f8c684ef..d01854f5a 100644 --- a/studio/frontend/src/features/rag/types/rag.ts +++ b/studio/frontend/src/features/rag/types/rag.ts @@ -20,6 +20,7 @@ export interface RagDocument { numChunks?: number | null; kbId?: string | null; threadId?: string | null; + projectId?: string | null; createdAt?: string | null; } diff --git a/studio/frontend/src/features/settings/settings-dialog.tsx b/studio/frontend/src/features/settings/settings-dialog.tsx index 1e5f60ca7..0ddba11b3 100644 --- a/studio/frontend/src/features/settings/settings-dialog.tsx +++ b/studio/frontend/src/features/settings/settings-dialog.tsx @@ -188,7 +188,7 @@ export function SettingsDialog() { {t(tab.labelKey)} {tab.badgeKey ? ( - + {t(tab.badgeKey)} ) : null} diff --git a/studio/frontend/src/features/studio/studio-page.tsx b/studio/frontend/src/features/studio/studio-page.tsx index 4635a6ce0..4d3a32fab 100644 --- a/studio/frontend/src/features/studio/studio-page.tsx +++ b/studio/frontend/src/features/studio/studio-page.tsx @@ -165,7 +165,7 @@ export function StudioPage(): ReactElement { />
-

+

{t("studio.title")}

{subtitle}

diff --git a/studio/frontend/src/index.css b/studio/frontend/src/index.css index 2dc588806..9299545da 100644 --- a/studio/frontend/src/index.css +++ b/studio/frontend/src/index.css @@ -1260,6 +1260,25 @@ box-shadow: 0 0 0 2px color-mix(in oklab, var(--ring) 45%, transparent); } + /* Clickable controls never show the browser's blue outline (Radix re-focuses + triggers on menu close, which lights them up). Mouse focus also drops the + Tailwind ring; keyboard focus (:focus-visible) keeps custom rings so the + focused control stays visible. Inputs keep their own focus styles. Zeroing + the ring vars leaves decorative shadow-* utilities intact. */ + button:focus, + a:focus, + summary:focus, + [role="button"]:focus { + outline: none !important; + } + button:focus:not(:focus-visible), + a:focus:not(:focus-visible), + summary:focus:not(:focus-visible), + [role="button"]:focus:not(:focus-visible) { + --tw-ring-shadow: 0 0 #0000 !important; + --tw-ring-offset-shadow: 0 0 #0000 !important; + } + /* Set Hellix explicitly; .aui-thread-root resets --font-heading to sans. */ .unsloth-welcome-title { font-family: "Hellix", "Space Grotesk Variable", var(--font-sans); diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index b8e1cfe9c..5473491bd 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -4781,7 +4781,12 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: }: return ["llama-server", "llama-quantize", "llama-diffusion-gemma-visual-server", "lib*.so*"] if choice.install_kind in {"macos-arm64", "macos-x64"}: - return ["llama-server", "llama-quantize", "llama-diffusion-gemma-visual-server", "lib*.dylib"] + return [ + "llama-server", + "llama-quantize", + "llama-diffusion-gemma-visual-server", + "lib*.dylib", + ] if choice.install_kind in { "windows-cpu", "windows-cuda", @@ -4789,7 +4794,12 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: "windows-rocm", "windows-arm64", }: - return ["llama-server.exe", "llama-quantize.exe", "llama-diffusion-gemma-visual-server.exe", "*.dll"] + return [ + "llama-server.exe", + "llama-quantize.exe", + "llama-diffusion-gemma-visual-server.exe", + "*.dll", + ] raise PrebuiltFallback(f"unsupported install kind for runtime overlay: {choice.install_kind}")