diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c87c8d2a2f..f69946927d 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -3906,6 +3906,10 @@ class LlamaCppBackend: self._audio_probed: bool = False # Audio INPUT capability (distinct from _is_audio, which is TTS output). self._has_audio_input: bool = False + # Video INPUT capability, from llama-server's /props modalities. True + # only when the mmproj, the build and ffmpeg all line up, none of which + # the GGUF alone can tell us. + self._has_video_input: bool = False self._mmproj_has_audio: bool = False # clip.has_audio_encoder, set at load # clip.has_vision_encoder, set at load; True keeps an undeclared projector capable. self._mmproj_accepts_image: bool = True @@ -18397,6 +18401,9 @@ class LlamaCppBackend: # reported context_length matches reality. (Querying /props # before the spawn above always failed; the seeded value was the # requested/native length.) + # Clear first, or a swap into a model without video inherits + # the previous server's answer. + self._has_video_input = False self._reconcile_effective_ctx_with_server() if self._kv_cache_context_total is not None: self._n_ubatch = min( @@ -19294,6 +19301,7 @@ class LlamaCppBackend: self._audio_type = None self._audio_probed = False self._has_audio_input = False + self._has_video_input = False self._mmproj_has_audio = False self._mmproj_accepts_image = True self._port = None @@ -20576,22 +20584,42 @@ class LlamaCppBackend: flags.extend(["--fit-target", str(int(_target))]) return flags + def _query_server_props(self) -> Optional[dict]: + """llama-server's ``/props``, or None when it cannot be read.""" + url = f"{self.base_url}/props" + try: + # /props is not one of llama-server's public endpoints, so under + # UNSLOTH_DIRECT_STREAM=1 (which launches the child with --api-key) + # an unauthenticated read 401s: the context readback silently keeps + # the requested -c, and video reads as unsupported on a model that + # supports it. None when there is no child key, which is httpx's + # default and what every other call site here relies on. + resp = httpx.get(url, headers = self._auth_headers, timeout = 5.0, trust_env = False) + if resp.status_code != 200: + return None + props = resp.json() + return props if isinstance(props, dict) else None + except Exception: + return None + def _query_server_n_ctx(self) -> Optional[int]: """Per-slot context llama-server actually allocated, from ``/props``. The memory-fit step or ``--parallel`` slot split can leave this below the requested ``-c``; requests are validated against this value. + + Records the declared modalities on the way past: video input depends on + build flags and ffmpeg, neither visible from the GGUF. """ - url = f"{self.base_url}/props" - try: - resp = httpx.get(url, timeout = 5.0, trust_env = False) - if resp.status_code != 200: - return None - settings = resp.json().get("default_generation_settings") or {} - n_ctx = settings.get("n_ctx") - return int(n_ctx) if n_ctx else None - except Exception: + props = self._query_server_props() + if props is None: return None + modalities = props.get("modalities") + if isinstance(modalities, dict): + self._has_video_input = bool(modalities.get("video")) + settings = props.get("default_generation_settings") or {} + n_ctx = settings.get("n_ctx") + return int(n_ctx) if n_ctx else None def _reconcile_effective_ctx_with_server(self) -> None: """Adopt the server's real ``n_ctx`` when it is below Unsloth's value. diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 925142bcd0..5e15e1e943 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -683,6 +683,13 @@ class _InferenceRuntimeFields(BaseModel): is_audio: bool = Field(False, description = "Whether model is a TTS audio model") audio_type: Optional[str] = Field(None, description = "Audio codec type: snac, csm, bicodec, dac") has_audio_input: bool = Field(False, description = "Whether model accepts audio input (ASR)") + has_video_input: bool = Field( + False, + description = ( + "Whether llama-server accepts video input for this model, from its /props " + "modalities. False unless the mmproj, the build and ffmpeg all support it." + ), + ) requires_trust_remote_code: bool = Field( False, description = "Whether the model defaults require trust_remote_code to be enabled for loading.", @@ -1488,6 +1495,13 @@ class ChatCompletionRequest(BaseModel): None, description = "[x-unsloth] Base64-encoded audio (wav/mp3/ogg/flac/m4a) for audio-input models", ) + video_base64: Optional[str] = Field( + None, + description = ( + "[x-unsloth] Base64-encoded video (mp4/mov/webm/mkv/avi) for video-input " + "models. GGUF only: llama-server samples frames with ffmpeg." + ), + ) use_adapter: Optional[Union[bool, str]] = Field( None, description = ( diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py index e1952e444a..e38c9a59f9 100644 --- a/studio/backend/routes/data_recipe/seed.py +++ b/studio/backend/routes/data_recipe/seed.py @@ -442,41 +442,109 @@ def _get_block_total_size(block_dir: Path) -> int: return total -@router.post("/seed/upload-unstructured-file") -async def upload_unstructured_file( - file: UploadFile = FastAPIFile(...), block_id: str = Form(...) -) -> UnstructuredFileUploadResponse: - _validate_safe_id(block_id, "block_id") - - original_filename = file.filename or "upload" - ext = Path(original_filename).suffix.lower() - if ext not in UNSTRUCTURED_ALLOWED_EXTS: - raise HTTPException( - 400, - f"Unsupported file type: {ext}. Allowed: {', '.join(sorted(UNSTRUCTURED_ALLOWED_EXTS))}", - ) - - content = await file.read() - size_bytes = len(content) - - if size_bytes == 0: - raise HTTPException(400, "Empty file not allowed") - +def _require_within_budget(size_bytes: int, budget: int) -> None: + """413 on the tighter of the per-file cap and what the block has left.""" if size_bytes > UNSTRUCTURED_RECIPE_UPLOAD_MAX_BYTES: raise HTTPException( 413, f"File too large ({size_bytes} bytes). Maximum is {UNSTRUCTURED_RECIPE_UPLOAD_MAX_LABEL}.", ) - - block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id - ensure_dir(block_dir) - current_total = _get_block_total_size(block_dir) - if current_total + size_bytes > UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES: + if size_bytes > budget: raise HTTPException( 413, f"Total upload limit ({UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_LABEL}) exceeded", ) + +def _require_unstructured_ext(filename: str) -> str: + """Reject an unsupported type before any bytes are read.""" + ext = Path(filename).suffix.lower() + if ext not in UNSTRUCTURED_ALLOWED_EXTS: + raise HTTPException( + 400, + f"Unsupported file type: {ext}. Allowed: {', '.join(sorted(UNSTRUCTURED_ALLOWED_EXTS))}", + ) + return ext + + +def _read_native_drop(lease: str, budget: int) -> tuple[str, bytes]: + """Read a desktop drop; returns (filename, content). + + The webview never names a path directly: Rust signs what the OS handed it, + and this re-verifies and re-stats that grant before reading a byte. Same + contract as the RAG route's ``_save_native_path_upload``. + + ``budget`` is what is still allowed for this block. The path is a local file + of any size, so it is refused on its stat rather than after a multi-gigabyte + read, and the read itself stops one byte past the budget in case the file + grew between the two. + """ + from utils.native_path_leases import NativePathLeaseError, verify_native_path_lease + + try: + grant = verify_native_path_lease( + lease, + operation = "attach", + expected_kind = "attachment", + expected_path_type = "file", + allowed_suffixes = sorted(UNSTRUCTURED_ALLOWED_EXTS), + ) + except NativePathLeaseError as exc: + raise HTTPException(400, str(exc)) from exc + + _require_unstructured_ext(grant.canonical_path.name) + try: + size_bytes = grant.canonical_path.stat().st_size + except OSError as exc: + raise HTTPException(400, "Dropped file could not be read.") from exc + _require_within_budget(size_bytes, budget) + + try: + with grant.canonical_path.open("rb") as source: + content = source.read(budget + 1) + except OSError as exc: + raise HTTPException(400, "Dropped file could not be read.") from exc + _require_within_budget(len(content), budget) + return grant.canonical_path.name, content + + +@router.post("/seed/upload-unstructured-file") +async def upload_unstructured_file( + file: UploadFile | None = FastAPIFile(None), + block_id: str = Form(...), + native_path_lease: str | None = Form(None, alias = "nativePathLease"), +) -> UnstructuredFileUploadResponse: + _validate_safe_id(block_id, "block_id") + + block_dir = UNSTRUCTURED_UPLOAD_ROOT / block_id + # Reads 0 for a block with no directory yet, so this does not create one for + # an upload that is about to be refused. + budget = UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES - _get_block_total_size(block_dir) + + # Desktop drops arrive as a signed path, not multipart bytes: Tauri hands + # the webview a path, never a File (#9036). isinstance, not a truth test: + # called outside FastAPI, an unfilled param is still a truthy Form marker. + lease = native_path_lease if isinstance(native_path_lease, str) else None + if lease: + original_filename, content = _read_native_drop(lease, budget) + elif file is not None and hasattr(file, "read"): + original_filename = file.filename or "upload" + # Before the read, as it was: a rejected 500 MB upload must not be + # pulled into memory first. + _require_unstructured_ext(original_filename) + content = await file.read() + else: + raise HTTPException(400, "No file was provided.") + + ext = Path(original_filename).suffix.lower() + size_bytes = len(content) + + if size_bytes == 0: + raise HTTPException(400, "Empty file not allowed") + + _require_within_budget(size_bytes, budget) + ensure_dir(block_dir) + file_id = uuid4().hex raw_path = block_dir / f"{file_id}{ext}" raw_path.write_bytes(content) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 43fbad522a..de7c4155ec 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -5,6 +5,7 @@ Inference API routes for model loading and text generation. """ +import math import os import sys import time @@ -5525,6 +5526,7 @@ async def _maybe_auto_switch_model( *, require_vision: bool = False, require_image: bool = True, + modality_label: str = "image or audio", ) -> None: """Load a downloaded local GGUF named by an OpenAI request when auto-switch is on. @@ -5535,7 +5537,8 @@ async def _maybe_auto_switch_model( to a text-only target before it runs, so an image request can't evict the resident vision model only to 400 afterwards; ``require_image`` is what makes that rejection modality-aware, since an audio request needs the projector but - not a vision tower. + not a vision tower. ``modality_label`` names the inputs actually attached, so + the rejection does not report a modality the request never carried. """ from utils.openai_auto_switch_settings import ( get_openai_auto_switch_enabled, @@ -5693,7 +5696,7 @@ async def _maybe_auto_switch_model( raise HTTPException( status_code = 400, detail = openai_error_body( - "The requested model does not support the image or audio input in this request.", + f"The requested model does not support the {modality_label} input in this request.", status = 400, code = "invalid_value", param = "model", @@ -11689,6 +11692,9 @@ def _decode_audio_base64(b64: str) -> "np.ndarray": # can expand to a far larger PCM array than the encoded-size cap implies. _MAX_AUDIO_RAW_BYTES = STT_AUDIO_RAW_MAX_BYTES _MAX_AUDIO_B64_CHARS = STT_AUDIO_B64_MAX_CHARS +# The composer's 64 MB cap as padded base64: 4 chars per 3 bytes, rounded up. +# Flooring instead refused a file of exactly the size the composer allows. +_MAX_VIDEO_B64_CHARS = 4 * math.ceil((64 * 1024 * 1024) / 3) _MAX_AUDIO_SECONDS = 30 * 60 _WAV_HEADER_BYTES = 44 _MIN_TRANSCODE_AUDIO_SAMPLE_RATE = 8000 @@ -11834,6 +11840,41 @@ def _prepare_audio_for_llama(b64: str) -> tuple[str, str]: return base64.b64encode(_mono_f32_to_wav_bytes(arr, sr)).decode("ascii"), "wav" +def _video_b64_rejection(video_b64: str) -> tuple[str, Optional[tuple[int, str]]]: + """The clip's base64 without its data URI header, plus why it is refused. + + The header is not payload, so counting it would refuse a clip of exactly the + size the composer allows. Returned rather than raised so the pre-switch and + post-load checks share one rule while raising their own way. + """ + if video_b64.startswith("data:"): + video_b64 = video_b64.split(",", 1)[1] if "," in video_b64 else "" + if not video_b64: + return "", (400, "Could not read the provided video file.") + if len(video_b64) > _MAX_VIDEO_B64_CHARS: + return video_b64, (413, "Video file is too large (max 64 MB).") + return video_b64, None + + +def _inject_video_part(messages: list[dict], video_b64: str) -> None: + """Append an input_video part to the last user message, in place. + + llama-server samples the clip into frames itself (ffmpeg via mtmd), so the + container is forwarded untouched. Rides the message list like image_url and + input_audio, so it flows through the plain and tool-calling paths alike. + Ref: llama.cpp tools/server/server-common.cpp, `type == "input_video"`. + """ + part = {"type": "input_video", "input_video": {"data": video_b64}} + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content") + if isinstance(content, list): + content.append(part) + else: + msg["content"] = [{"type": "text", "text": content or ""}, part] + return + + def _inject_audio_part(messages: list[dict], audio_b64: str, audio_format: str) -> None: """Append an input_audio part to the last user message, in place. @@ -13196,6 +13237,13 @@ async def openai_chat_completions( untrack_current_request(request.scope) if _wants_multiple_choices(payload): _raise_unsupported_n("external provider chat completions") + # input_video is llama.cpp's own part type, so the proxy has nowhere to + # put the clip. Say so rather than answering as if there were no video. + if payload.video_base64: + raise HTTPException( + status_code = 400, + detail = "Video input is only supported on a local GGUF model with video support.", + ) return await _proxy_to_external_provider(payload, request, current_subject) # Reject a malformed function tool here: it would otherwise reach @@ -13233,6 +13281,7 @@ async def openai_chat_completions( _pre_parsed = None _needs_vision = False _needs_image = False + _modality_label = "image or audio" if _automatic_model_load_may_run(): _pre_parsed = _extract_content_parts(payload.messages) if not _pre_parsed[1]: @@ -13334,7 +13383,30 @@ async def openai_chat_completions( # audio-only request asks for the projector alone, since an audio model's # projector carries no vision tower. _needs_image = bool(_pre_parsed[2]) or _request_has_image(payload) - _needs_vision = _needs_image or bool(payload.audio_base64) + # Video rides that projector too. Its own /props gate can only run after + # the load, so this at least keeps a text-only target from evicting a + # working model to serve a clip it could never take. + _needs_vision = _needs_image or bool(payload.audio_base64) or bool(payload.video_base64) + # Name what is actually attached, so the refusal does not report a + # modality the request never carried. + _modality_label = ( + " or ".join( + name + for name, present in ( + ("image", _needs_image), + ("audio", bool(payload.audio_base64)), + ("video", bool(payload.video_base64)), + ) + if present + ) + or _modality_label + ) + # Size is knowable now and the switch is not cheap: refuse an oversized + # clip before it costs a model load. + if payload.video_base64: + _, _video_rejection = _video_b64_rejection(payload.video_base64) + if _video_rejection is not None: + raise HTTPException(status_code = _video_rejection[0], detail = _video_rejection[1]) await _maybe_auto_switch_model( _switch_model_for_payload(payload), @@ -13342,6 +13414,7 @@ async def openai_chat_completions( current_subject, require_vision = _needs_vision, require_image = _needs_image, + modality_label = _modality_label, ) llama_backend = get_llama_cpp_backend() @@ -13664,6 +13737,15 @@ async def openai_chat_completions( ), ) + # Injection lives in the GGUF branch below, since input_video is llama.cpp's + # own part type. Without this a transformers model answers as if the clip + # were never attached. + if payload.video_base64 and not using_gguf: + raise _reject( + 400, + "Video input is only supported on a local GGUF model with video support.", + ) + # Apply per-model recommended sampling (and any operator UNSLOTH_SAMPLING_* pin) to the # fields the client omitted, so agents and API clients get the model's tuned defaults # unless they set the field explicitly. Placed after external-provider routing (which @@ -13736,6 +13818,14 @@ async def openai_chat_completions( 400, "Audio input is not supported together with guided decoding or client-supplied tools yet.", ) + if payload.video_base64: + # Same shape: _build_openai_passthrough_body forwards an explicit + # field list, so the clip would be dropped and the model would + # answer without it. + raise _reject( + 400, + "Video input is not supported together with guided decoding or client-supplied tools yet.", + ) # Preserve the vision guard from the non-passthrough path below: # text-only tool-capable GGUFs should return a clear 400 here rather @@ -13819,6 +13909,20 @@ async def openai_chat_completions( logger.warning("Audio decode failed: %s", e, exc_info = True) raise _reject(400, "Could not decode the provided audio file.") + # Forwarded whole: llama-server owns the frame sampling, and takes the + # clip only when /props reports modalities.video. + video_b64 = None + if payload.video_base64: + if not getattr(llama_backend, "_has_video_input", False): + raise _reject( + 400, + "Video provided but the current GGUF model cannot take video input. " + "It needs an mmproj with video support, and ffmpeg/ffprobe installed.", + ) + video_b64, video_rejection = _video_b64_rejection(payload.video_base64) + if video_rejection is not None: + raise _reject(*video_rejection) + gguf_messages, _ = await _openai_messages_for_gguf_chat_async( payload, llama_backend.is_vision, @@ -13827,6 +13931,8 @@ async def openai_chat_completions( image_b64 = None if audio_b64: _inject_audio_part(gguf_messages, audio_b64, audio_format) + if video_b64: + _inject_video_part(gguf_messages, video_b64) cancel_event = threading.Event() @@ -19506,6 +19612,12 @@ async def chat_count_tokens( status_code = 503, detail = "Cannot count tokens for messages containing audio.", ) + # And video, whose frames llama-server samples at completion time. + if getattr(payload, "video_base64", None): + raise HTTPException( + status_code = 503, + detail = "Cannot count tokens for messages containing video.", + ) llama_backend = get_llama_cpp_backend() if not llama_backend.is_loaded: diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.py index bcd82cb57e..f603c47419 100644 --- a/studio/backend/tests/test_data_recipe_seed.py +++ b/studio/backend/tests/test_data_recipe_seed.py @@ -3,9 +3,12 @@ import asyncio import importlib.util +import sys from pathlib import Path +from types import SimpleNamespace import pytest +from fastapi import HTTPException def _seed_route_source() -> str: @@ -223,6 +226,70 @@ def test_total_upload_quota_is_scoped_per_block(monkeypatch, tmp_path): assert other.status == "ok" +# A desktop drop names a local file of any size, so the cap has to be enforced +# on its stat. Reading first let a multi-gigabyte drop into backend memory +# before the 413 (#9036). +def test_an_oversized_native_drop_is_refused_before_it_is_read(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + huge = tmp_path / "corpus.txt" + huge.write_bytes(b"x" * 64) + + reads: list[str] = [] + real_open = Path.open + + def tracking_open(self, *args, **kwargs): + reads.append(self.name) + return real_open(self, *args, **kwargs) + + monkeypatch.setattr(Path, "open", tracking_open) + monkeypatch.setattr(seed_route, "UNSTRUCTURED_RECIPE_UPLOAD_MAX_BYTES", 32) + monkeypatch.setattr( + seed_route, + "verify_native_path_lease", + lambda *a, **k: SimpleNamespace(canonical_path = huge), + raising = False, + ) + monkeypatch.setitem( + sys.modules, + "utils.native_path_leases", + SimpleNamespace( + NativePathLeaseError = RuntimeError, + verify_native_path_lease = lambda *a, **k: SimpleNamespace(canonical_path = huge), + ), + ) + + with pytest.raises(HTTPException) as excinfo: + asyncio.run( + seed_route.upload_unstructured_file(None, "block", native_path_lease = "signed-lease") + ) + assert excinfo.value.status_code == 413 + assert reads == [], "the file was opened before the size check" + + +# The block's remaining budget bounds the read too, so a file that grew between +# the stat and the read cannot slip past it. +def test_a_native_drop_over_the_block_budget_is_refused(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + dropped = tmp_path / "notes.txt" + dropped.write_bytes(b"y" * 64) + + monkeypatch.setattr(seed_route, "UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES", 16) + monkeypatch.setitem( + sys.modules, + "utils.native_path_leases", + SimpleNamespace( + NativePathLeaseError = RuntimeError, + verify_native_path_lease = lambda *a, **k: SimpleNamespace(canonical_path = dropped), + ), + ) + + with pytest.raises(HTTPException) as excinfo: + asyncio.run( + seed_route.upload_unstructured_file(None, "block", native_path_lease = "signed-lease") + ) + assert excinfo.value.status_code == 413 + + class _BlockPlugin: """Meta path finder making the optional seed plugin look uninstalled.""" diff --git a/studio/backend/tests/test_llama_cpp_props_readback.py b/studio/backend/tests/test_llama_cpp_props_readback.py index 1dc8bae8c2..f5831e5983 100644 --- a/studio/backend/tests/test_llama_cpp_props_readback.py +++ b/studio/backend/tests/test_llama_cpp_props_readback.py @@ -99,9 +99,16 @@ class _FakeResponse: return self._body -def _make_backend(effective_ctx = 98304, port = 51234): +def _make_backend( + effective_ctx = 98304, + port = 51234, + api_key = None, +): inst = LlamaCppBackend.__new__(LlamaCppBackend) inst._port = port + # __init__ always sets this; __new__ skips it, and the readback reads it via + # _auth_headers to authenticate against a --api-key child server. + inst._api_key = api_key inst._effective_context_length = effective_ctx inst._context_length = 262144 inst._effective_parallel_slots = 1 @@ -118,12 +125,17 @@ def _stub_props( ): def fake_get( url, + headers = None, timeout = None, trust_env = None, ): assert url.endswith("/props") assert trust_env is False + # /props sits behind llama-server's api-key middleware, so a direct-stream + # child must be addressed with the bearer token; without one the header + # stays absent rather than becoming a bogus "Bearer None". + assert headers is None or headers == {"Authorization": "Bearer test-key"} if exc is not None: raise exc return _FakeResponse(status_code, body) diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index b4be650d56..17b3efd048 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -3163,8 +3163,13 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch): *, require_vision = False, require_image = True, + modality_label = "image or audio", ): - captured.update(require_vision = require_vision, require_image = require_image) + captured.update( + require_vision = require_vision, + require_image = require_image, + modality_label = modality_label, + ) raise _Reached() monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True) @@ -3172,7 +3177,12 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch): payload = _chat_request(model = "org/B-GGUF", audio_base64 = "AAAA") with pytest.raises(_Reached): asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester")) - assert captured == {"require_vision": True, "require_image": False} + # The label follows what is attached, not the union the hook guards. + assert captured == { + "require_vision": True, + "require_image": False, + "modality_label": "audio", + } # An image in the same request does need the vision tower. img = ImageContentPart(type = "image_url", image_url = ImageUrl(url = "data:image/png;base64,AAAA")) @@ -3183,7 +3193,11 @@ def test_chat_audio_input_guards_target_before_switch(monkeypatch): ) with pytest.raises(_Reached): asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester")) - assert captured == {"require_vision": True, "require_image": True} + assert captured == { + "require_vision": True, + "require_image": True, + "modality_label": "image or audio", + } def test_completions_rejects_object_prompt_before_switch(monkeypatch): @@ -3451,7 +3465,7 @@ def test_chat_validates_confirm_and_modality_before_switch(): assert "require_vision" in src hook = inspect.getsource(inference_route._maybe_auto_switch_model) assert hook.index("require_vision") < hook.index("_load_model_impl") - assert "does not support the image or audio input" in hook + assert "does not support the {modality_label} input" in hook def test_messages_have_image_helper(): @@ -7986,3 +8000,86 @@ def test_normalize_keeps_an_explicit_empty_list_only_when_asked(monkeypatch): assert settings.normalize_model_override( {"llama_extra_args": ["--top-k", "40"]}, keep_empty_extra_args = keep ) == {"llama_extra_args": ["--top-k", "40"]} + + +def _wire_refusing_switch(monkeypatch): + backend = _FakeBackend("org/A-GGUF") + rec = _LoadRecorder(backend) + _wire( + monkeypatch, + enabled = True, + resolves_to = ("/local/B.gguf", "Q8_0", "org/B-GGUF"), + backend = backend, + recorder = rec, + ) + monkeypatch.setattr(inference_route, "_target_is_vision", lambda _p, _v = None, _i = True: False) + return rec + + +def _refusal_detail(monkeypatch, **kwargs) -> str: + rec = _wire_refusing_switch(monkeypatch) + with pytest.raises(HTTPException) as exc: + asyncio.run( + inference_route._maybe_auto_switch_model( + "org/B-GGUF", object(), "t", require_vision = True, **kwargs + ) + ) + assert rec.calls == [] + return json.dumps(exc.value.detail) + + +def test_the_switch_refusal_names_the_modality_the_request_carried(monkeypatch): + # Video joins require_vision, so without a label the user who attached a clip + # is told the model lacks "image or audio" support and never sees "video". + detail = _refusal_detail(monkeypatch, modality_label = "video") + assert "video input" in detail + assert "image or audio" not in detail + + +def test_the_refusal_lists_every_attached_modality(monkeypatch): + detail = _refusal_detail(monkeypatch, modality_label = "image or video") + assert "image or video input" in detail + + +def test_the_refusal_wording_is_unchanged_for_callers_that_pass_no_label(monkeypatch): + # The image-only callers (/messages, /responses) keep their existing text. + detail = _refusal_detail(monkeypatch) + assert "image or audio input" in detail + + +def test_a_video_request_labels_the_switch_refusal_video(monkeypatch): + """End to end through the handler: video joins require_vision, so the label + has to follow or the user who attached a clip is told about image or audio.""" + + class _Reached(Exception): + pass + + captured = {} + + async def _capture( + model, + request, + subject, + *, + require_vision = False, + require_image = True, + modality_label = "image or audio", + ): + captured.update( + require_vision = require_vision, + require_image = require_image, + modality_label = modality_label, + ) + raise _Reached() + + monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True) + monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _capture) + payload = _chat_request(model = "org/B-GGUF", video_base64 = "AAAA") + with pytest.raises(_Reached): + asyncio.run(inference_route.openai_chat_completions(payload, object(), "tester")) + # No vision tower: a video projector need not carry one, same as audio. + assert captured == { + "require_vision": True, + "require_image": False, + "modality_label": "video", + } diff --git a/studio/backend/tests/test_video_attachment_part.py b/studio/backend/tests/test_video_attachment_part.py new file mode 100644 index 0000000000..eb7fd921cc --- /dev/null +++ b/studio/backend/tests/test_video_attachment_part.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Video attachments ride the message list as llama-server's `input_video` part. + +llama.cpp takes video through its OpenAI-compatible chat endpoint as +``{"type": "input_video", "input_video": {"data": ...}}`` (tools/server/ +server-common.cpp), refusing it unless the projector, the build and ffmpeg all +line up -- which it reports at ``/props`` under ``modalities.video``. These tests +pin the wire shape and that capability read, since neither is visible from the +GGUF alone. +""" + +from __future__ import annotations + +import base64 +from pathlib import Path + +import pytest + +pytest.importorskip("torch") + +from routes.inference import _inject_video_part # noqa: E402 + + +def test_a_video_part_is_appended_to_the_last_user_message(): + messages = [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": [{"type": "text", "text": "what happens here?"}]}, + ] + _inject_video_part(messages, "AAAA") + assert messages[1]["content"][-1] == {"type": "input_video", "input_video": {"data": "AAAA"}} + # The system message is untouched. + assert messages[0]["content"] == "be brief" + + +def test_a_string_content_turn_is_promoted_to_parts(): + messages = [{"role": "user", "content": "describe the clip"}] + _inject_video_part(messages, "BBBB") + assert messages[0]["content"] == [ + {"type": "text", "text": "describe the clip"}, + {"type": "input_video", "input_video": {"data": "BBBB"}}, + ] + + +def test_only_the_newest_user_turn_carries_the_clip(): + messages = [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "second"}, + ] + _inject_video_part(messages, "CCCC") + assert messages[0]["content"] == "first" + assert messages[2]["content"][-1]["type"] == "input_video" + + +def test_a_turn_with_no_user_message_is_left_alone(): + messages = [{"role": "assistant", "content": "hello"}] + _inject_video_part(messages, "DDDD") + assert messages == [{"role": "assistant", "content": "hello"}] + + +def test_video_capability_is_read_from_the_server_props(): + """Only llama-server knows: the mmproj, MTMD_VIDEO and ffmpeg all have a vote.""" + from core.inference.llama_cpp import LlamaCppBackend + + backend = LlamaCppBackend.__new__(LlamaCppBackend) + backend._has_video_input = False + backend._query_server_props = lambda: { + "default_generation_settings": {"n_ctx": 4096}, + "modalities": {"vision": True, "video": True, "audio": False}, + } + assert backend._query_server_n_ctx() == 4096 + assert backend._has_video_input is True + + +def test_a_server_without_video_leaves_the_capability_off(): + from core.inference.llama_cpp import LlamaCppBackend + + backend = LlamaCppBackend.__new__(LlamaCppBackend) + backend._has_video_input = True + backend._query_server_props = lambda: { + "default_generation_settings": {"n_ctx": 2048}, + "modalities": {"vision": True, "video": False, "audio": False}, + } + backend._query_server_n_ctx() + assert backend._has_video_input is False + + +def test_an_unreadable_props_does_not_claim_video(): + from core.inference.llama_cpp import LlamaCppBackend + + backend = LlamaCppBackend.__new__(LlamaCppBackend) + backend._has_video_input = False + backend._query_server_props = lambda: None + assert backend._query_server_n_ctx() is None + assert backend._has_video_input is False + + +def test_the_cap_admits_a_clip_of_exactly_the_composer_limit(): + """Flooring the 4/3 inflation refused a file of exactly the allowed size.""" + import math + + from routes.inference import _MAX_VIDEO_B64_CHARS + + limit_bytes = 64 * 1024 * 1024 + # Padded base64 is 4 characters per 3 bytes, rounded up. + assert len(base64.b64encode(b"x" * 3001)) == 4 * math.ceil(3001 / 3) + assert 4 * math.ceil(limit_bytes / 3) <= _MAX_VIDEO_B64_CHARS + assert 4 * math.ceil((limit_bytes + 1024) / 3) > _MAX_VIDEO_B64_CHARS + + +def _inference_source() -> str: + return (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) + + +def test_video_is_refused_on_the_tool_passthrough_path(): + """That branch forwards an explicit field list and returns before the + injection below, so the clip would be dropped and the model would answer + without it. The audio path already refuses; video has to match.""" + source = _inference_source() + start = source.index("if using_gguf and _takes_tool_passthrough(payload, llama_backend):") + branch = source[start : start + 2500] + assert "payload.audio_base64" in branch + assert "payload.video_base64" in branch + assert "Video input is not supported together with guided decoding" in branch + + +def test_the_size_check_runs_before_the_automatic_switch(): + """A cheap length check must not cost a model load first: an oversized clip + would otherwise evict a working model and 413 only afterwards.""" + source = _inference_source() + # Anchor inside the chat-completions handler; other routes switch too. + handler = source.index("_needs_image = bool(_pre_parsed[2])") + guard = source.index("_video_b64_rejection(payload.video_base64)", handler) + switch = source.index("await _maybe_auto_switch_model(", handler) + assert guard < switch + + +def test_video_joins_the_projector_requirement_before_switching(): + """Video rides the same companion mmproj as vision, so a text-only target + cannot serve it either. Audio already votes here.""" + source = _inference_source() + start = source.index("_needs_image = bool(_pre_parsed[2])") + block = source[start : start + 400] + assert "payload.audio_base64" in block + assert "payload.video_base64" in block + + +def test_an_external_provider_refuses_video_rather_than_ignoring_it(): + """input_video is llama.cpp's own part type, so the proxy has nowhere to put + the clip and returns before any video handling below.""" + source = _inference_source() + start = source.index("if payload.provider_id or payload.provider_type:") + branch = source[start : source.index("_proxy_to_external_provider(payload", start)] + assert "payload.video_base64" in branch + assert "Video input is only supported on a local GGUF model" in branch + + +def test_a_non_gguf_model_refuses_video_rather_than_ignoring_it(): + """Injection lives in the GGUF branch, so a transformers model would answer + as if nothing were attached.""" + source = _inference_source() + assert "if payload.video_base64 and not using_gguf:" in source + + +def test_token_counting_refuses_video_like_image_and_audio(): + """The completion injects the clip; this route cannot, so counting here + would silently undercount the turn.""" + source = _inference_source() + start = source.index("Cannot count tokens for messages containing images.") + block = source[start : start + 700] + assert "Cannot count tokens for messages containing audio." in block + assert "Cannot count tokens for messages containing video." in block + + +def test_both_video_checks_share_one_rule(): + """Two size checks that drift let the pre-switch one pass what the post-load + one refuses, which is the model load this was meant to avoid.""" + source = _inference_source() + assert source.count("_video_b64_rejection(payload.video_base64)") == 2 diff --git a/studio/backend/tests/test_video_pr9057_simulation.py b/studio/backend/tests/test_video_pr9057_simulation.py new file mode 100644 index 0000000000..74f58c0dab --- /dev/null +++ b/studio/backend/tests/test_video_pr9057_simulation.py @@ -0,0 +1,443 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""PR 9057 review simulation: every axis a video attachment can travel. + +Not part of the PR. Written during review to answer "does this break anything, +and does the fix actually work", covering: the common no-video-capability model, +a swap from a capable model to a non-capable one, the external-provider and +non-GGUF passthroughs, an oversized clip, a data-URI wrapper, an llama.cpp build +too old to declare modalities at all, and the shape of the runtime fields old +clients read. +""" + +from __future__ import annotations + +import base64 +import math + +import pytest + +pytest.importorskip("torch") + +from core.inference.llama_cpp import LlamaCppBackend # noqa: E402 +from models.inference import ( # noqa: E402 + ChatCompletionRequest, + InferenceStatusResponse, + _InferenceRuntimeFields, +) +from routes.inference import ( # noqa: E402 + _MAX_VIDEO_B64_CHARS, + _inject_video_part, + _video_b64_rejection, +) + +LIMIT = 64 * 1024 * 1024 + + +def _props_backend(props): + b = LlamaCppBackend.__new__(LlamaCppBackend) + b._has_video_input = False + b._query_server_props = lambda: props + return b + + +# -------------------------------------------------------------------------- +# A. the base64 ceiling, measured against the real encoder +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", [0, 1, 2, 3, 4, 5, 6, 1023, 1024, 3000, 3001, 3002]) +def test_the_ceiling_formula_matches_what_base64_actually_produces(n): + assert len(base64.b64encode(b"x" * n)) == 4 * math.ceil(n / 3) + + +def test_a_clip_of_exactly_the_composer_limit_is_admitted(): + # 67108864 = 3*22369621 + 1, so the last byte costs a full padded quad. + assert _MAX_VIDEO_B64_CHARS == 89478488 + assert 4 * math.ceil(LIMIT / 3) == 89478488 + exact = "A" * 89478488 + assert _video_b64_rejection(exact)[1] is None + + +def test_the_old_floor_expression_would_have_refused_it(): + # What the review flagged: floor(64MiB * 4 / 3) == 89478485, three characters + # short, so the largest file the composer offers 413s. + floored = (LIMIT * 4) // 3 + assert floored == 89478485 + assert floored < 4 * math.ceil(LIMIT / 3) + + +def test_one_character_over_the_ceiling_is_refused_413(): + assert _video_b64_rejection("A" * (_MAX_VIDEO_B64_CHARS + 1))[1] == ( + 413, + "Video file is too large (max 64 MB).", + ) + + +def test_the_data_uri_header_is_not_counted_against_the_cap(): + # A composer that sends a data URI must not lose bytes to its own header. + payload = "A" * _MAX_VIDEO_B64_CHARS + stripped, rejection = _video_b64_rejection(f"data:video/mp4;base64,{payload}") + assert rejection is None + assert stripped == payload + + +@pytest.mark.parametrize( + "mime", + ["video/mp4", "video/quicktime", "video/webm", "video/x-matroska", "video/x-msvideo"], +) +def test_every_container_the_composer_accepts_survives_the_data_uri_strip(mime): + stripped, rejection = _video_b64_rejection(f"data:{mime};base64,QUJD") + assert rejection is None and stripped == "QUJD" + + +@pytest.mark.parametrize("bad", ["", "data:", "data:video/mp4;base64", "data:,"]) +def test_an_unreadable_payload_is_a_400_not_a_crash(bad): + stripped, rejection = _video_b64_rejection(bad) + assert rejection == (400, "Could not read the provided video file.") + + +def test_a_bare_payload_with_no_header_is_passed_through_untouched(): + assert _video_b64_rejection("QUJD") == ("QUJD", None) + + +# -------------------------------------------------------------------------- +# B. capability read: old builds, odd payloads, and swaps +# -------------------------------------------------------------------------- + + +def test_a_build_too_old_to_declare_modalities_reports_no_video(): + """The key backwards-compat case: llama.cpp only grew `modalities` in /props + recently, and every older build simply omits the key.""" + b = _props_backend({"default_generation_settings": {"n_ctx": 4096}}) + assert b._query_server_n_ctx() == 4096 + assert b._has_video_input is False + + +@pytest.mark.parametrize( + "props", + [ + {"modalities": None}, + {"modalities": []}, + {"modalities": "vision"}, + {"modalities": {"vision": True}}, + {"modalities": {"video": None}}, + {"modalities": {"video": 0}}, + {"modalities": {"video": "false"}}, # a non-empty string is truthy: see below + {}, + ], +) +def test_a_malformed_modalities_block_never_crashes_the_context_readback(props): + b = _props_backend({**props, "default_generation_settings": {"n_ctx": 2048}}) + assert b._query_server_n_ctx() == 2048 + assert isinstance(b._has_video_input, bool) + + +def test_only_a_real_json_true_turns_the_capability_on(): + for value, expected in ((True, True), (False, False), (None, False), (0, False)): + b = _props_backend({"modalities": {"video": value}}) + b._query_server_n_ctx() + assert b._has_video_input is expected, value + + +def test_a_swap_to_a_model_without_video_does_not_inherit_the_old_answer(): + """A stale True here would offer video on a model that cannot take it, and + llama-server would refuse the completion after the upload.""" + b = _props_backend( + {"modalities": {"video": True}, "default_generation_settings": {"n_ctx": 8192}} + ) + b._query_server_n_ctx() + assert b._has_video_input is True + b._query_server_props = lambda: { + "modalities": {"video": False}, + "default_generation_settings": {"n_ctx": 8192}, + } + b._query_server_n_ctx() + assert b._has_video_input is False + + +def test_an_unreachable_props_leaves_the_capability_off_rather_than_guessing(): + b = _props_backend(None) + b._has_video_input = True + assert b._query_server_n_ctx() is None + # Nothing clears it here, which is why the load path clears it explicitly: + assert "self._has_video_input = False" in _llama_cpp_source() + + +def _llama_cpp_source() -> str: + from pathlib import Path + return ( + Path(__file__).resolve().parent.parent / "core" / "inference" / "llama_cpp.py" + ).read_text(encoding = "utf-8") + + +def test_the_load_path_and_the_unload_path_both_clear_the_capability(): + src = _llama_cpp_source() + assert src.count("self._has_video_input = False") == 2 + + +class _Resp: + def __init__(self, status_code, payload): + self.status_code = status_code + self._payload = payload + + def json(self): + if isinstance(self._payload, Exception): + raise self._payload + return self._payload + + +def _stub_props_http( + monkeypatch, + resp, + record = None, +): + """Replace httpx.get inside the backend module and capture the call.""" + import core.inference.llama_cpp as llama_mod + + def _get(url, **kwargs): + if record is not None: + record.update(url = url, **kwargs) + if isinstance(resp, Exception): + raise resp + return resp + + monkeypatch.setattr(llama_mod.httpx, "get", _get) + + +def _live_backend(api_key = None): + b = LlamaCppBackend.__new__(LlamaCppBackend) + b._has_video_input = False + b._api_key = api_key + b._port = 9999 + b._host = "127.0.0.1" + return b + + +@pytest.mark.parametrize("junk", [[1, 2], "props", 7, None, 3.5]) +def test_a_props_body_that_is_not_an_object_is_rejected_not_crashed(monkeypatch, junk): + """A proxy or a future build could answer with a list; the readback must + degrade to "cannot tell", not raise into the load path.""" + b = _live_backend() + _stub_props_http(monkeypatch, _Resp(200, junk)) + assert b._query_server_props() is None + assert b._query_server_n_ctx() is None + assert b._has_video_input is False + + +@pytest.mark.parametrize("status", [401, 403, 404, 500, 503]) +def test_a_non_200_props_never_claims_video(monkeypatch, status): + b = _live_backend() + _stub_props_http(monkeypatch, _Resp(status, {"modalities": {"video": True}})) + assert b._query_server_props() is None + assert b._has_video_input is False + + +def test_an_undecodable_props_body_is_swallowed(monkeypatch): + b = _live_backend() + _stub_props_http(monkeypatch, _Resp(200, ValueError("not json"))) + assert b._query_server_props() is None + + +def test_a_dead_server_is_swallowed(monkeypatch): + b = _live_backend() + _stub_props_http(monkeypatch, OSError("connection refused")) + assert b._query_server_props() is None + + +def test_the_props_request_carries_the_child_api_key_when_direct_stream_set_one(monkeypatch): + """llama-server's api-key middleware protects /props (it is not in the + public_endpoints set), so an unauthenticated read 401s and the capability + silently reads False under UNSLOTH_DIRECT_STREAM=1.""" + record: dict = {} + b = _live_backend(api_key = "secret-token") + _stub_props_http(monkeypatch, _Resp(200, {"modalities": {"video": True}}), record = record) + b._query_server_props() + assert record.get("headers") == {"Authorization": "Bearer secret-token"} + + +def test_the_props_request_sends_no_auth_header_when_there_is_no_child_key(monkeypatch): + record: dict = {} + b = _live_backend(api_key = None) + _stub_props_http(monkeypatch, _Resp(200, {}), record = record) + b._query_server_props() + assert record.get("headers") is None + + +# -------------------------------------------------------------------------- +# C. the wire shape old and new clients see +# -------------------------------------------------------------------------- + + +def test_the_runtime_field_is_declared_and_defaults_off_for_every_non_gguf_model(): + """A transformers or MLX model never sets it, so the composer must read + False and refuse video rather than offering it.""" + assert "has_video_input" in _InferenceRuntimeFields.model_fields + assert InferenceStatusResponse().has_video_input is False + + +def test_the_generic_runtime_mapper_actually_picks_the_capability_up(): + """`_llama_runtime_fields` maps a response field to `_` on the backend. + If that mapping missed, `has_video_input` would be hardcoded False on the + wire and the whole feature would be unreachable from the UI.""" + from routes.inference import _llama_runtime_fields + + class _Stub: + pass + + backend = _Stub() + for name in _InferenceRuntimeFields.model_fields: + setattr(backend, f"_{name}", None) + backend._has_video_input = True + backend._has_audio_input = False + for extra in ( + "requested_spec_mode", + "requested_parallel_slots", + "effective_parallel_slots", + "requested_extra_args", + "is_diffusion", + ): + setattr(backend, extra, None) + fields = _llama_runtime_fields(backend) + assert fields["has_video_input"] is True + assert fields["has_audio_input"] is False + + +def test_an_old_client_that_sends_no_video_field_is_unaffected(): + req = ChatCompletionRequest(messages = [{"role": "user", "content": "hi"}]) + assert req.video_base64 is None + + +def test_an_old_backend_would_ignore_the_new_field_rather_than_422(): + """`extra: allow`, so a newer desktop app talking to an older backend loses + the clip silently instead of breaking every message. Worth knowing; not + something this PR can fix from the new side.""" + assert ChatCompletionRequest.model_config["extra"] == "allow" + + +def test_the_field_round_trips_through_json_unchanged(): + payload = "data:video/mp4;base64,QUJD" + req = ChatCompletionRequest.model_validate( + {"messages": [{"role": "user", "content": "hi"}], "video_base64": payload} + ) + assert req.video_base64 == payload + assert req.model_dump()["video_base64"] == payload + + +# -------------------------------------------------------------------------- +# D. injection, on every message shape a real session produces +# -------------------------------------------------------------------------- + + +def test_an_empty_message_list_is_a_no_op(): + messages: list[dict] = [] + _inject_video_part(messages, "AAAA") + assert messages == [] + + +def test_a_user_turn_with_a_none_content_is_promoted_without_losing_the_clip(): + messages = [{"role": "user", "content": None}] + _inject_video_part(messages, "AAAA") + assert messages[0]["content"] == [ + {"type": "text", "text": ""}, + {"type": "input_video", "input_video": {"data": "AAAA"}}, + ] + + +def test_the_clip_lands_beside_an_image_rather_than_replacing_it(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "compare these"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA"}}, + ], + } + ] + _inject_video_part(messages, "VVVV") + types = [p["type"] for p in messages[0]["content"]] + assert types == ["text", "image_url", "input_video"] + + +def test_a_tool_turn_after_the_last_user_turn_does_not_steal_the_clip(): + messages = [ + {"role": "user", "content": "watch this"}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "1"}]}, + {"role": "tool", "tool_call_id": "1", "content": "42"}, + ] + _inject_video_part(messages, "VVVV") + assert messages[0]["content"][-1]["type"] == "input_video" + assert messages[2]["content"] == "42" + + +def test_a_long_multi_turn_thread_only_carries_one_clip(): + messages = [] + for i in range(20): + messages.append({"role": "user", "content": f"q{i}"}) + messages.append({"role": "assistant", "content": f"a{i}"}) + messages.append({"role": "user", "content": "last"}) + _inject_video_part(messages, "VVVV") + injected = [ + m + for m in messages + if isinstance(m["content"], list) and any(p["type"] == "input_video" for p in m["content"]) + ] + assert len(injected) == 1 + assert injected[0]["content"][0]["text"] == "last" + + +def test_the_part_shape_is_exactly_what_llama_server_parses(): + messages = [{"role": "user", "content": "x"}] + _inject_video_part(messages, "PAYLOAD") + part = messages[0]["content"][-1] + assert set(part) == {"type", "input_video"} + assert part["type"] == "input_video" + assert set(part["input_video"]) == {"data"} + assert part["input_video"]["data"] == "PAYLOAD" + + +# -------------------------------------------------------------------------- +# E. the refusal paths, read off the handler +# -------------------------------------------------------------------------- + + +def _routes_source() -> str: + from pathlib import Path + return (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) + + +@pytest.mark.parametrize( + "needle", + [ + # external provider (OpenAI / Anthropic / any proxied backend) + 'raise HTTPException(\n status_code = 400,\n detail = "Video input is only supported on a local GGUF model with video support.",', + # local non-GGUF (transformers, MLX) + "if payload.video_base64 and not using_gguf:", + # GGUF that cannot take video + 'if not getattr(llama_backend, "_has_video_input", False):', + # tool / guided-decoding passthrough + '"Video input is not supported together with guided decoding or client-supplied tools yet."', + # token counting + '"Cannot count tokens for messages containing video."', + ], +) +def test_every_path_that_cannot_serve_a_clip_refuses_out_loud(needle): + assert needle in _routes_source() + + +def test_the_size_check_is_paid_before_the_model_switch_not_after(): + src = _routes_source() + handler = src.index("_needs_image = bool(_pre_parsed[2])") + assert src.index("_video_b64_rejection(payload.video_base64)", handler) < src.index( + "await _maybe_auto_switch_model(", handler + ) + + +def test_the_external_provider_refusal_precedes_the_proxy_call(): + src = _routes_source() + start = src.index("if payload.provider_id or payload.provider_type:") + branch = src[start : src.index("_proxy_to_external_provider(payload", start)] + assert "payload.video_base64" in branch diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index 267467a980..1bb7109f0c 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -2442,6 +2442,17 @@ const Composer: FC<{ ); const [materializingDroppedAudio, setMaterializingDroppedAudio] = useState(false); + const hasPendingVideoAttachments = useNativeIntentStore((s) => + Boolean( + nativeAttachmentTargetKey && + (s.pendingVideoAttachments[nativeAttachmentTargetKey]?.length ?? 0) > 0, + ), + ); + const registeringVideoDrops = useNativeIntentStore( + (s) => s.registeringVideoDrops > 0, + ); + const [materializingDroppedVideo, setMaterializingDroppedVideo] = + useState(false); // A parked send must not fire on a failed drop: the user is owed the toast and // their text, not a send of the text alone. Assigned below, once the callback exists. const cancelQueuedSendRef = useRef<(() => void) | null>(null); @@ -2468,6 +2479,16 @@ const Composer: FC<{ seenAudioDropFailuresRef.current = audioDropFailures; cancelQueuedSendRef.current?.(); }, [audioDropFailures]); + const videoDropFailures = useNativeIntentStore( + (s) => (nativeAttachmentTargetKey ? s.videoDropFailures[nativeAttachmentTargetKey] : 0) ?? 0, + ); + const seenVideoDropFailuresRef = useRef(videoDropFailures); + // Cancel the parked send before `endVideoDropRegistration` reopens the gate. + useEffect(() => { + if (seenVideoDropFailuresRef.current === videoDropFailures) return; + seenVideoDropFailuresRef.current = videoDropFailures; + cancelQueuedSendRef.current?.(); + }, [videoDropFailures]); // Registering and reading a dropped clip is async, so hold the send gate: // the composer sees nothing until `addAttachment` lands. useEffect(() => { @@ -2585,6 +2606,123 @@ const Composer: FC<{ }; }, [nativeAttachmentTargetKey, aui]); + // Same drain as audio, one queue over: one clip per message, and the send + // gate has to hold across the read either way. + useEffect(() => { + if (!nativeAttachmentTargetKey) { + return; + } + const targetKey = nativeAttachmentTargetKey; + const identityAtSetup = composerIdentityRef.current; + useNativeIntentStore + .getState() + .claimVideoAttachments(identityAtSetup, targetKey); + let disposed = false; + let draining = false; + + // A re-key follows the same composer; a thread switch parks the clip back. + const stillThisComposer = () => + composerIdentityRef.current === identityAtSetup; + // A remount hides the new key, so tag the batch; the next instance claims it. + const requeue = (intents: NativeIntent[]) => { + const key = stillThisComposer() + ? (nativeAttachmentTargetKeyRef.current ?? targetKey) + : targetKey; + const store = useNativeIntentStore.getState(); + store.addVideoAttachments(key, intents); + store.noteVideoDropOwner(key, identityAtSetup); + }; + + const drainPendingVideo = async () => { + if (disposed || draining) return; + draining = true; + setMaterializingDroppedVideo(true); + try { + while (!disposed) { + const intents = useNativeIntentStore + .getState() + .takeVideoAttachments(targetKey); + if (intents.length === 0) break; + for (const [index, intent] of intents.entries()) { + if (disposed) { + requeue(intents.slice(index)); + return; + } + let file: File; + try { + file = await nativeAttachmentIntentToFile(intent); + } catch (error) { + toast.error("Could not attach dropped video", { + description: + error instanceof Error ? error.message : String(error), + }); + // Do not let a send parked on this clip go out as bare text. + if (stillThisComposer()) cancelQueuedSendRef.current?.(); + continue; + } + // The read is async; a chat switch in that window must not steal the clip. + if ( + disposed || + nativeAttachmentTargetKeyRef.current !== targetKey + ) { + requeue(intents.slice(index)); + return; + } + try { + await aui.composer().addAttachment(file); + } catch { + // Chat-wide, not per file (no video mmproj, no ffmpeg, too large, + // already attached), and every adapter path toasted: stop quietly. + if (stillThisComposer()) cancelQueuedSendRef.current?.(); + return; + } + } + } + } finally { + draining = false; + // A drain for a target already left must not touch the flag; cleanup + // cleared it, and the live target may have set it again. + if (!disposed) { + // The early returns requeue mid-batch, and a drop can land while + // `draining` gated the subscription. + const pending = + useNativeIntentStore.getState().pendingVideoAttachments[targetKey] + ?.length ?? 0; + // Only the instance still owning this composer re-drains; otherwise + // the batch stays parked rather than looping here forever. + if (pending > 0 && stillThisComposer()) { + void drainPendingVideo(); + } else { + setMaterializingDroppedVideo(false); + } + } + } + }; + + const unsubscribe = useNativeIntentStore.subscribe((state) => { + // A predecessor's requeue can land after setup, so keep watching. + const orphaned = Object.entries(state.videoDropOwners).some( + ([key, owner]) => owner === identityAtSetup && key !== targetKey, + ); + if (orphaned) { + useNativeIntentStore + .getState() + .claimVideoAttachments(identityAtSetup, targetKey); + return; + } + if ((state.pendingVideoAttachments[targetKey]?.length ?? 0) > 0) { + void drainPendingVideo(); + } + }); + void drainPendingVideo(); + + return () => { + disposed = true; + setMaterializingDroppedVideo(false); + unsubscribe(); + }; + }, [nativeAttachmentTargetKey, aui]); + useEffect(() => { if (!nativeAttachmentTargetKey) { return; @@ -2726,6 +2864,10 @@ const Composer: FC<{ registeringAudioDrops || hasPendingAudioAttachments || materializingDroppedAudio; + const hasMaterializingVideoAttachments = + registeringVideoDrops || + hasPendingVideoAttachments || + materializingDroppedVideo; const threadIsRunning = useAuiState(({ thread }) => thread.isRunning); const threadListItemId = useAuiState( ({ threadListItem }) => threadListItem.id, @@ -2765,6 +2907,7 @@ const Composer: FC<{ !hasPendingAttachments && !hasMaterializingImageAttachments && !hasMaterializingAudioAttachments && + !hasMaterializingVideoAttachments && !disabled && !overlay; const canQueueCurrentPrompt = @@ -3598,7 +3741,14 @@ const Composer: FC<{ cancelQueuedSendRef.current = cancelQueuedSend; const enqueueSend = useCallback( - (waitingOn: "indexing" | "images" | "audio" | "settings" = "indexing") => { + ( + waitingOn: + | "indexing" + | "images" + | "audio" + | "video" + | "settings" = "indexing", + ) => { if (pendingSendRef.current) return; pendingSendRef.current = true; setPendingSend(true); @@ -3607,9 +3757,11 @@ const Composer: FC<{ ? "Waiting for dropped images" : waitingOn === "audio" ? "Waiting for dropped audio" - : waitingOn === "settings" - ? "Loading this chat's settings" - : "Waiting for documents to finish indexing"; + : waitingOn === "video" + ? "Waiting for dropped video" + : waitingOn === "settings" + ? "Loading this chat's settings" + : "Waiting for documents to finish indexing"; waitToastRef.current = toast(title, { description: "Your message will send automatically once they are ready.", duration: Infinity, @@ -3625,19 +3777,30 @@ const Composer: FC<{ if ( disabled || overlay || - (!hasMaterializingImageAttachments && !hasMaterializingAudioAttachments) || + (!hasMaterializingImageAttachments && + !hasMaterializingAudioAttachments && + !hasMaterializingVideoAttachments) || !hasSendableContent || isComposingRef.current || hasPendingAttachments ) { return; } - enqueueSend(hasMaterializingImageAttachments ? "images" : "audio"); + // Name what is actually being waited on, or a parked video drop reports + // itself as audio. + enqueueSend( + hasMaterializingImageAttachments + ? "images" + : hasMaterializingAudioAttachments + ? "audio" + : "video", + ); }, [ disabled, overlay, hasMaterializingImageAttachments, hasMaterializingAudioAttachments, + hasMaterializingVideoAttachments, hasSendableContent, hasPendingAttachments, isComposingRef, @@ -3650,9 +3813,11 @@ const Composer: FC<{ isComposingRef.current || hasPendingAttachments || hasMaterializingImageAttachments || - hasMaterializingAudioAttachments, + hasMaterializingAudioAttachments || + hasMaterializingVideoAttachments, [ hasMaterializingAudioAttachments, + hasMaterializingVideoAttachments, hasMaterializingImageAttachments, hasPendingAttachments, hasSendableContent, @@ -3756,7 +3921,8 @@ const Composer: FC<{ indexingActive || threadScopedSettingsPending || hasMaterializingImageAttachments || - hasMaterializingAudioAttachments + hasMaterializingAudioAttachments || + hasMaterializingVideoAttachments ) { return; } @@ -3793,6 +3959,7 @@ const Composer: FC<{ threadScopedSettingsPending, hasMaterializingImageAttachments, hasMaterializingAudioAttachments, + hasMaterializingVideoAttachments, aui, canQueueCurrentPrompt, canQueuePastedTextPrompt, @@ -3854,7 +4021,8 @@ const Composer: FC<{ uploading: hasPendingAttachments || hasMaterializingImageAttachments || - hasMaterializingAudioAttachments, + hasMaterializingAudioAttachments || + hasMaterializingVideoAttachments, researchActive: isResearchActive, runActive: threadIsRunning || promptQueueActive, queueDisabled: Boolean(disableQueue), diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index e0b16c4f0f..ad2456f24a 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1501,6 +1501,43 @@ export function findLatestUserAudioBase64( return pendingAudio ?? undefined; } +function extractVideoPartBase64( + part: { type: string } | null | undefined, +): string | undefined { + if (!part || part.type !== "file") return undefined; + const filePart = part as unknown as { data?: string; mimeType?: string }; + if (!filePart.data || !/^video\//i.test(filePart.mimeType ?? "")) return undefined; + return filePart.data.startsWith("data:") + ? filePart.data.split(",")[1] + : filePart.data; +} + +/** Base64 of the clip on the newest user turn. Only the newest counts, like + * audio: replaying an older one would re-sample it into frames and spend the + * context of every text follow-up. */ +export function findLatestUserVideoBase64( + messages: RunMessages, +): string | undefined { + for (let i = messages.length - 1; i >= 0; i -= 1) { + const message = messages[i]; + if (!message || message.role !== "user") continue; + for (const part of message.content ?? []) { + const base64 = extractVideoPartBase64(part); + if (base64) return base64; + } + if ("attachments" in message) { + for (const attachment of message.attachments ?? []) { + for (const part of attachment.content ?? []) { + const base64 = extractVideoPartBase64(part); + if (base64) return base64; + } + } + } + break; + } + return undefined; +} + // The Canvas instructions createOpenAIStreamAdapter appends, named so the recount prices the same // text the request carries. export const CANVAS_TOOL_INSTRUCTION = @@ -2088,6 +2125,7 @@ function queuedResolvedModelFromStore( isAudio: activeModel.isAudio, audioType: activeModel.audioType, hasAudioInput: activeModel.hasAudioInput, + hasVideoInput: activeModel.hasVideoInput, } : null, }; @@ -3539,6 +3577,7 @@ async function resolveQueuedEmptyLocalModel( isAudio: status.is_audio ?? false, audioType: status.audio_type ?? null, hasAudioInput: status.has_audio_input ?? false, + hasVideoInput: status.has_video_input ?? false, }, }, }; @@ -4486,6 +4525,7 @@ export function createOpenAIStreamAdapter( survivingMessages, !queuedRunSettings && !continuation, ); + const videoBase64 = findLatestUserVideoBase64(survivingMessages); const hasOutboundImage = Boolean(imageBase64); // Keep render_html local-only and mirror the backend image-turn gate. @@ -5445,6 +5485,7 @@ export function createOpenAIStreamAdapter( presence_penalty: params.presencePenalty, image_base64: imageBase64, audio_base64: audioBase64, + video_base64: videoBase64, cancel_id: cancelId, ...(sandboxSessionId ? { session_id: sandboxSessionId } : {}), ...(resolvedThreadId ? { thread_id: resolvedThreadId } : {}), diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 1ac6283cc6..dd22e5a49f 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -2770,8 +2770,21 @@ export function ChatPage({ }, [artifactViewKey], ); + const handleNativeVideoDrop = useCallback( + (intents: NativeIntent[]) => { + useNativeIntentStore.getState().addVideoAttachments(artifactViewKey, intents); + }, + [artifactViewKey], + ); const nativeModelDropState = useNativeModelDrop({ - enabled: active && view.mode === "single", + // Compare used to disable this outright, so a drop there vanished with no + // overlay and no message (#9036). Keep listening and refuse out loud. The + // refusal covers models too: nothing may load behind a compare view. + enabled: active, + dropsUnsupportedReason: + view.mode === "single" + ? undefined + : "Dropped files need a single chat. Open one, then drop it there.", attachmentScope, attachmentTargetKey: artifactViewKey, nativePathLeasesSupported, @@ -2781,6 +2794,7 @@ export function ChatPage({ onAttach: handleNativeAttachmentDrop, onAttachImages: handleNativeImageDrop, onAttachAudio: handleNativeAudioDrop, + onAttachVideo: handleNativeVideoDrop, }); const handleCheckpointChange = useCallback( diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index c0eec72ce2..ad11075123 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -221,6 +221,7 @@ function describeModel(model: { is_mlx?: boolean; is_audio?: boolean; has_audio_input?: boolean; + has_video_input?: boolean; }): string | undefined { const tags: string[] = []; if (model.is_gguf) tags.push("GGUF"); @@ -251,6 +252,7 @@ function toChatModelSummary(model: { is_audio?: boolean; audio_type?: string | null; has_audio_input?: boolean; + has_video_input?: boolean; }): ChatModelSummary { return { id: model.id, @@ -263,6 +265,7 @@ function toChatModelSummary(model: { isAudio: Boolean(model.is_audio), audioType: model.audio_type ?? null, hasAudioInput: Boolean(model.has_audio_input), + hasVideoInput: Boolean(model.has_video_input), }; } @@ -281,6 +284,7 @@ export function syncModelCapabilities( is_audio?: boolean; audio_type?: string | null; has_audio_input?: boolean; + has_video_input?: boolean; }, ): void { const store = useChatRuntimeStore.getState(); @@ -291,6 +295,9 @@ export function syncModelCapabilities( isAudio: Boolean(resp.is_audio), audioType: resp.audio_type ?? null, hasAudioInput: Boolean(resp.has_audio_input), + // /api/models/list omits this for the active GGUF row, so without it the + // video adapter reads false after every load and status hydration. + hasVideoInput: Boolean(resp.has_video_input), }; const idx = models.findIndex((m) => m.id === modelId); if (idx === -1) { diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts index 6883dbab3c..4b4a152f5e 100644 --- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts +++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts @@ -99,6 +99,7 @@ function ensureActiveModelInStoreList( isAudio: status.is_audio ?? false, audioType: status.audio_type ?? null, hasAudioInput: status.has_audio_input ?? false, + hasVideoInput: status.has_video_input ?? false, }; const existing = store.models.find((model) => model.id === checkpointId); if (existing) { diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx index f7733e2e64..d9425f6984 100644 --- a/studio/frontend/src/features/chat/runtime-provider.tsx +++ b/studio/frontend/src/features/chat/runtime-provider.tsx @@ -62,6 +62,7 @@ import { readOpenDocumentAttachmentContent, } from "./open-document"; import { AudioAttachmentAdapter } from "./audio-attachment-adapter"; +import { VideoAttachmentAdapter } from "./video-attachment-adapter"; import { awaitThreadScopedSettingsWrite, beginThreadScopedPairing, @@ -1543,6 +1544,9 @@ function useStudioRuntimeAdapters( new CompositeAttachmentAdapter([ new VisionImageAdapter(), new AudioAttachmentAdapter(), + // Before the document adapters: a composite takes the first match, + // and .mkv/.mov must not fall through to them. + new VideoAttachmentAdapter(), new TextAttachmentAdapter(), new HtmlAttachmentAdapter(), new PDFAttachmentAdapter(), diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index 4a40d1da19..d901281b18 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -38,6 +38,7 @@ import { getAudioSizeError, } from "@/lib/audio-utils"; import { isTauri } from "@/lib/api-base"; +import { isVideoFile } from "@/lib/video-utils"; import { isDownloadCancelled } from "@/lib/native-files"; import { isMultimodalResponse } from "./types/api"; import { getImageInputUnavailableReason } from "./utils/image-input-support"; @@ -919,6 +920,7 @@ export function SharedComposer({ const next: PendingImage[] = []; let droppedImageForUnavailable = false; let audioSizeError: string | null = null; + let videoUnsupported = false; for (let i = 0; i < files.length; i++) { const file = files[i]; if (!file) continue; @@ -935,6 +937,12 @@ export function SharedComposer({ }); continue; } + // video_base64 targets the single loaded GGUF, so at most one side of + // a compare could answer. Say that rather than drop the file. + if (isVideoFile(file)) { + videoUnsupported = true; + continue; + } // Handle image files if (!file.type.match(/^image\/(jpeg|png|webp|gif)$/i)) continue; if (file.size > MAX_IMAGE_SIZE) continue; @@ -950,6 +958,11 @@ export function SharedComposer({ if (audioSizeError) { toast.error(audioSizeError); } + if (videoUnsupported) { + toast.error("Video can't be attached in compare mode", { + description: "Open a single chat with a video-capable model instead.", + }); + } setPendingImages((prev) => [...prev, ...next]); }, [setPendingAudioStore, attachUnavailableReason], @@ -1667,6 +1680,7 @@ export function SharedComposer({ isAudio: Boolean(resp.is_audio), audioType: resp.audio_type ?? null, hasAudioInput: Boolean(resp.has_audio_input), + hasVideoInput: Boolean(resp.has_video_input), }; if (idx === -1) { store.setModels([ diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 190107b821..dc851c3824 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -20,6 +20,7 @@ export interface BackendModelDetails { is_audio?: boolean; audio_type?: string | null; has_audio_input?: boolean; + has_video_input?: boolean; } export interface ListModelsResponse { @@ -217,6 +218,7 @@ export interface LoadModelResponse { is_audio?: boolean; audio_type?: string | null; has_audio_input?: boolean; + has_video_input?: boolean; inference?: { temperature?: number; top_p?: number; @@ -305,6 +307,7 @@ export interface InferenceStatusResponse { is_audio?: boolean; audio_type?: string | null; has_audio_input?: boolean; + has_video_input?: boolean; loading: string[]; loaded: string[]; inference?: { @@ -560,6 +563,7 @@ export interface OpenAIChatCompletionsRequest { presence_penalty?: number; image_base64?: string; audio_base64?: string; + video_base64?: string; use_adapter?: boolean | string | null; enable_thinking?: boolean | null; reasoning_effort?: diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 76f67e370c..be1c0d3e04 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -56,6 +56,9 @@ export interface ChatModelSummary { isAudio?: boolean; audioType?: string | null; hasAudioInput?: boolean; + /** llama-server takes video for this model: mmproj video support, a build + * with video enabled, and ffmpeg installed. */ + hasVideoInput?: boolean; } export interface ChatLoraSummary { diff --git a/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts b/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts index af401a603e..f7cf56873f 100644 --- a/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts +++ b/studio/frontend/src/features/chat/utils/queued-model-capabilities.ts @@ -2,7 +2,12 @@ import type { ChatModelSummary } from "../types/runtime"; export type QueuedModelCapabilities = Pick< ChatModelSummary, - "isVision" | "isGguf" | "isAudio" | "audioType" | "hasAudioInput" + | "isVision" + | "isGguf" + | "isAudio" + | "audioType" + | "hasAudioInput" + | "hasVideoInput" >; /** diff --git a/studio/frontend/src/features/chat/utils/refresh-context-usage.ts b/studio/frontend/src/features/chat/utils/refresh-context-usage.ts index 0ad05d74a6..bda4a3b60d 100644 --- a/studio/frontend/src/features/chat/utils/refresh-context-usage.ts +++ b/studio/frontend/src/features/chat/utils/refresh-context-usage.ts @@ -7,6 +7,7 @@ import { buildLocalTokenCountReasoning, buildOutboundMessagesForTokenCount, findLatestUserAudioBase64, + findLatestUserVideoBase64, messagesContainImage, } from "../api/chat-adapter"; import { countChatInputTokens } from "../api/chat-api"; @@ -275,6 +276,15 @@ export async function refreshContextUsage( // no audio branch, so counting would price a text-only prompt. Decline as images do. if (findLatestUserAudioBase64(runMessages)) return; + // Same for video, and more so: the real request replays the clip as + // video_base64 and llama-server expands it into frames, while + // toOpenAIMessages has no video branch -- so counting would price a + // text-only prompt and the bar would show room the window does not have. + // /chat/count_tokens 503s on video for the same reason. Declining here also + // keeps up to 85 MB of base64 out of branchSignature's JSON.stringify, + // which is the synchronous main-thread cost the image bail above exists for. + if (findLatestUserVideoBase64(runMessages)) return; + if (fromLiveBranch) { countedBranch = branchSignature(runMessages); } else { diff --git a/studio/frontend/src/features/chat/video-attachment-adapter.ts b/studio/frontend/src/features/chat/video-attachment-adapter.ts new file mode 100644 index 0000000000..3ee92d9fc1 --- /dev/null +++ b/studio/frontend/src/features/chat/video-attachment-adapter.ts @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { fileToBase64 } from "@/lib/audio-utils"; +import { + VIDEO_ACCEPT, + getVideoSizeError, + videoMimeForFile, +} from "@/lib/video-utils"; +import type { + Attachment, + AttachmentAdapter, + CompleteAttachment, + PendingAttachment, +} from "@assistant-ui/react"; +import { toast } from "sonner"; +import { externalModelLabel } from "./lib/external-model-label"; +import { useChatRuntimeStore } from "./stores/chat-runtime-store"; + +// crypto.randomUUID is undefined in non-secure contexts (HTTP over a LAN IP). +function newAttachmentId(): string { + if (typeof globalThis.crypto?.randomUUID === "function") { + return globalThis.crypto.randomUUID(); + } + return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; +} + +/** Video shares the "Add photos & files" picker, like audio. llama-server + * samples the clip into frames with ffmpeg, so the container is forwarded + * untouched. */ +export class VideoAttachmentAdapter implements AttachmentAdapter { + accept = VIDEO_ACCEPT; + private readonly attachmentIds = new Set(); + + async add({ file }: { file: File }): Promise { + const state = useChatRuntimeStore.getState(); + const checkpoint = state.params.checkpoint; + const activeModel = state.models.find((m) => m.id === checkpoint); + const modelLoaded = !!checkpoint && !state.modelLoading; + let unavailableReason: string | null = null; + if (!modelLoaded) { + // Mirror the image and audio gates: a failed load reads differently from + // no model picked. + unavailableReason = state.lastModelLoadError + ? "The last model failed to load. Check the server logs, then load a model before adding video." + : "Load a model before adding video."; + } else if (!activeModel?.hasVideoInput) { + const label = + activeModel?.name || + externalModelLabel(checkpoint) || + checkpoint || + "Current model"; + // Three causes land here and the server does not say which, so name all + // three: /props reports video only when all of them line up. + unavailableReason = `${label} cannot accept video. Video needs a GGUF model whose mmproj supports video, a llama.cpp build with video enabled, and ffmpeg installed on this machine.`; + } + if (unavailableReason) { + toast.error(unavailableReason); + throw new Error(unavailableReason); + } + const sizeReason = getVideoSizeError(file.size); + if (sizeReason) { + toast.error(sizeReason); + throw new Error(sizeReason); + } + // One clip per message: a second would blow the context in frames alone. + if (this.attachmentIds.size > 0) { + const duplicateReason = "Only one video can be attached per message."; + toast.error(duplicateReason); + throw new Error(duplicateReason); + } + + const id = newAttachmentId(); + this.attachmentIds.add(id); + return { + id, + type: "file", + name: file.name, + contentType: videoMimeForFile(file), + file, + status: { type: "requires-action", reason: "composer-send" }, + }; + } + + async send(attachment: PendingAttachment): Promise { + try { + const data = await fileToBase64(attachment.file); + return { + id: attachment.id, + type: "file", + name: attachment.name, + contentType: attachment.contentType, + content: [ + { + type: "file", + filename: attachment.name, + data, + // Normalised at pick time: the extractor keys off this, and a + // browser that answered "" or application/octet-stream for an mkv + // would otherwise cost the clip silently. + mimeType: attachment.contentType || "video/mp4", + }, + ], + status: { type: "complete" }, + }; + } finally { + this.attachmentIds.delete(attachment.id); + } + } + + remove(attachment: Attachment): Promise { + this.attachmentIds.delete(attachment.id); + return Promise.resolve(); + } +} diff --git a/studio/frontend/src/features/native-intents/components/native-model-drop-overlay.tsx b/studio/frontend/src/features/native-intents/components/native-model-drop-overlay.tsx index 2192d55fb4..4aa3c9b960 100644 --- a/studio/frontend/src/features/native-intents/components/native-model-drop-overlay.tsx +++ b/studio/frontend/src/features/native-intents/components/native-model-drop-overlay.tsx @@ -6,13 +6,14 @@ function overlayCopy(state: NativeModelDropState): { title: string; description: if (state.status === "invalid") { return { title: "Can't use these files", - description: "Drop a .gguf model, or documents to chat with.", + description: + state.reason ?? "Drop a .gguf model, or documents to chat with.", }; } if (state.status === "attach") { - // Only documents are indexed; images and audio ride the next message. + // Only documents are indexed; images, audio and video ride the next message. const description = - state.kind === "images" || state.kind === "audio" + state.kind === "images" || state.kind === "audio" || state.kind === "video" ? "Attached to your next message." : state.kind === "mixed" ? "Documents indexed, attachments sent with your next message." @@ -22,7 +23,9 @@ function overlayCopy(state: NativeModelDropState): { title: string; description: ? "image" : state.kind === "audio" ? "audio file" - : "file"; + : state.kind === "video" + ? "video" + : "file"; return { title: state.count === 1 diff --git a/studio/frontend/src/features/native-intents/drop-paths.ts b/studio/frontend/src/features/native-intents/drop-paths.ts index 564c185965..0b0b9a0442 100644 --- a/studio/frontend/src/features/native-intents/drop-paths.ts +++ b/studio/frontend/src/features/native-intents/drop-paths.ts @@ -15,8 +15,15 @@ export const CHAT_AUDIO_DROP_ACCEPT = ".wav,.mp3,.m4a,.ogg,.oga,.flac"; const AUDIO_EXTS = CHAT_AUDIO_DROP_ACCEPT.split(",").map((ext) => ext.trim().toLowerCase()); +/** Chat video attachments; keep in sync with `native_path_policy.rs` + * `VIDEO_ATTACHMENT_EXTS`. llama-server decodes with ffmpeg, so this is what + * ffmpeg reads, not what the webview can play. */ +export const CHAT_VIDEO_DROP_ACCEPT = ".mp4,.mov,.webm,.mkv,.avi"; + +const VIDEO_EXTS = CHAT_VIDEO_DROP_ACCEPT.split(",").map((ext) => ext.trim().toLowerCase()); + /** What the window actually takes, for the rejection toast and the overlay. */ -export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${CHAT_IMAGE_DROP_ACCEPT}, one of ${CHAT_AUDIO_DROP_ACCEPT}, or a single .gguf model.`; +export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${CHAT_IMAGE_DROP_ACCEPT}, one of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`; function hasExt(path: string, ext: string): boolean { return path.toLowerCase().endsWith(ext); @@ -28,7 +35,14 @@ export type NativeDropClass = | { kind: "docs"; paths: string[] } | { kind: "images"; paths: string[] } | { kind: "audio"; paths: string[] } - | { kind: "attach"; docs: string[]; images: string[]; audio: string[] } + | { kind: "video"; paths: string[] } + | { + kind: "attach"; + docs: string[]; + images: string[]; + audio: string[]; + video: string[]; + } | { kind: "unsupported" }; /** What a native drag payload is, before any of it is registered with Rust. */ @@ -48,18 +62,40 @@ export function classifyDropPaths(paths: string[]): NativeDropClass { const audio = paths.filter((path) => AUDIO_EXTS.some((ext) => hasExt(path, ext)), ); - if (docs.length + images.length + audio.length !== paths.length) { + const video = paths.filter((path) => + VIDEO_EXTS.some((ext) => hasExt(path, ext)), + ); + if ( + docs.length + images.length + audio.length + video.length !== + paths.length + ) { return { kind: "unsupported" }; } // The audio adapter takes one clip per message; a larger batch never attaches. if (audio.length > 1) { return { kind: "unsupported" }; } - if (docs.length === 0 && images.length === 0 && audio.length === 0) { + // Same for video: one clip expands into a run of frames, so a batch would + // blow the context before reaching the model. + if (video.length > 1) { + return { kind: "unsupported" }; + } + if ( + docs.length === 0 && + images.length === 0 && + audio.length === 0 && + video.length === 0 + ) { return { kind: "none" }; } - if (images.length === 0 && audio.length === 0) return { kind: "docs", paths: docs }; - if (docs.length === 0 && audio.length === 0) return { kind: "images", paths: images }; - if (docs.length === 0 && images.length === 0) return { kind: "audio", paths: audio }; - return { kind: "attach", docs, images, audio }; + const kinds = [docs, images, audio, video].filter( + (group) => group.length > 0, + ); + if (kinds.length === 1) { + if (docs.length > 0) return { kind: "docs", paths: docs }; + if (images.length > 0) return { kind: "images", paths: images }; + if (audio.length > 0) return { kind: "audio", paths: audio }; + return { kind: "video", paths: video }; + } + return { kind: "attach", docs, images, audio, video }; } diff --git a/studio/frontend/src/features/native-intents/index.ts b/studio/frontend/src/features/native-intents/index.ts index 9695a3be88..026fce3d9b 100644 --- a/studio/frontend/src/features/native-intents/index.ts +++ b/studio/frontend/src/features/native-intents/index.ts @@ -14,7 +14,10 @@ export { } from "./api"; export type { NativeDocumentFolderSelection } from "./api"; export { nativeDropTargetAt } from "./native-drop-targets"; +export { nativeAttachmentIntentToFile } from "./native-attachment-file"; export { useNativeDropTarget } from "./use-native-drop-target"; +export { useNativeFileDrop } from "./use-native-file-drop"; +export type { NativeFileDrop, NativeFileDropOptions } from "./use-native-file-drop"; export { NativeAttachmentTargetContext, useNativeAttachmentTargetKey, diff --git a/studio/frontend/src/features/native-intents/store.ts b/studio/frontend/src/features/native-intents/store.ts index 12cfe28f40..4662dfc520 100644 --- a/studio/frontend/src/features/native-intents/store.ts +++ b/studio/frontend/src/features/native-intents/store.ts @@ -13,6 +13,7 @@ interface NativeIntentState { pendingAttachments: PendingNativeAttachments; pendingImageAttachments: PendingNativeAttachments; pendingAudioAttachments: PendingNativeAttachments; + pendingVideoAttachments: PendingNativeAttachments; // Image drops registering with Rust, before they have a queue to sit in. Not // keyed: until the intents land there is no settled target, and the OS drop // went to the window, which has one composer to send from. @@ -20,33 +21,45 @@ interface NativeIntentState { // Same for audio: cover the register-and-read window or a fast submit // goes out without the clip. registeringAudioDrops: number; + // Same for video: one clip is a long read, and a submit in that window would + // go out without it. + registeringVideoDrops: number; // Bumped, per chat, when a drop fails before it reaches a queue. The composer // watches its own key so a failure elsewhere cannot cancel its parked send. imageDropFailures: Record; audioDropFailures: Record; + videoDropFailures: Record; // Owner of a queued image batch, by composer identity. A remount means the // outgoing instance cannot hand the batch over itself, so it leaves a note. imageDropOwners: Record; // Same for audio: a new chat re-keys mid-read, so the clip needs a note // to follow the composer. audioDropOwners: Record; + videoDropOwners: Record; addIntent: (intent: NativeIntent) => void; addAttachments: (targetKey: string, intents: NativeIntent[]) => void; addImageAttachments: (targetKey: string, intents: NativeIntent[]) => void; addAudioAttachments: (targetKey: string, intents: NativeIntent[]) => void; + addVideoAttachments: (targetKey: string, intents: NativeIntent[]) => void; takeAttachments: (targetKey: string) => NativeIntent[]; takeImageAttachments: (targetKey: string) => NativeIntent[]; takeAudioAttachments: (targetKey: string) => NativeIntent[]; + takeVideoAttachments: (targetKey: string) => NativeIntent[]; beginImageDropRegistration: () => void; endImageDropRegistration: () => void; beginAudioDropRegistration: () => void; endAudioDropRegistration: () => void; + beginVideoDropRegistration: () => void; + endVideoDropRegistration: () => void; failImageDropRegistration: (targetKey: string) => void; failAudioDropRegistration: (targetKey: string) => void; + failVideoDropRegistration: (targetKey: string) => void; noteImageDropOwner: (targetKey: string, identity: string) => void; claimImageAttachments: (identity: string, targetKey: string) => void; noteAudioDropOwner: (targetKey: string, identity: string) => void; claimAudioAttachments: (identity: string, targetKey: string) => void; + noteVideoDropOwner: (targetKey: string, identity: string) => void; + claimVideoAttachments: (identity: string, targetKey: string) => void; clearModelIntent: (intentId?: string) => void; } @@ -55,12 +68,16 @@ export const useNativeIntentStore = create((set, get) => ({ pendingAttachments: {}, pendingImageAttachments: {}, pendingAudioAttachments: {}, + pendingVideoAttachments: {}, registeringImageDrops: 0, registeringAudioDrops: 0, + registeringVideoDrops: 0, imageDropFailures: {}, audioDropFailures: {}, + videoDropFailures: {}, imageDropOwners: {}, audioDropOwners: {}, + videoDropOwners: {}, addAttachments: (targetKey, intents) => { const current = get().pendingAttachments; const pendingAttachments = enqueueNativeAttachments( @@ -219,6 +236,74 @@ export const useNativeIntentStore = create((set, get) => ({ for (const key of stale) delete nextOwners[key]; set({ pendingAudioAttachments, audioDropOwners: nextOwners }); }, + addVideoAttachments: (targetKey, intents) => { + const current = get().pendingVideoAttachments; + const pendingVideoAttachments = enqueueNativeAttachments( + current, + targetKey, + intents, + ); + if (pendingVideoAttachments !== current) { + set({ pendingVideoAttachments }); + } + }, + takeVideoAttachments: (targetKey) => { + const current = get().pendingVideoAttachments; + const [queued, pendingVideoAttachments] = dequeueNativeAttachments( + current, + targetKey, + ); + if (pendingVideoAttachments !== current) { + set({ pendingVideoAttachments }); + } + return queued; + }, + beginVideoDropRegistration: () => { + set({ registeringVideoDrops: get().registeringVideoDrops + 1 }); + }, + endVideoDropRegistration: () => { + set({ registeringVideoDrops: Math.max(0, get().registeringVideoDrops - 1) }); + }, + failVideoDropRegistration: (targetKey) => { + const current = get().videoDropFailures; + set({ + videoDropFailures: { + ...current, + [targetKey]: (current[targetKey] ?? 0) + 1, + }, + }); + }, + noteVideoDropOwner: (targetKey, identity) => { + if (!identity) return; + set({ videoDropOwners: { ...get().videoDropOwners, [targetKey]: identity } }); + }, + claimVideoAttachments: (identity, targetKey) => { + if (!identity) return; + const owners = get().videoDropOwners; + const stale = Object.keys(owners).filter( + (key) => owners[key] === identity && key !== targetKey, + ); + if (stale.length === 0) return; + const queues = get().pendingVideoAttachments; + let pendingVideoAttachments = queues; + for (const key of stale) { + const queued = queues[key] ?? []; + if (queued.length > 0) { + pendingVideoAttachments = enqueueNativeAttachments( + pendingVideoAttachments, + targetKey, + queued, + ); + } + if (key in pendingVideoAttachments) { + pendingVideoAttachments = { ...pendingVideoAttachments }; + delete pendingVideoAttachments[key]; + } + } + const nextOwners = { ...owners }; + for (const key of stale) delete nextOwners[key]; + set({ pendingVideoAttachments, videoDropOwners: nextOwners }); + }, addIntent: (intent) => { if (intent.kind !== "model") { return; diff --git a/studio/frontend/src/features/native-intents/use-native-drop.ts b/studio/frontend/src/features/native-intents/use-native-drop.ts index ddc3756c9a..591154756e 100644 --- a/studio/frontend/src/features/native-intents/use-native-drop.ts +++ b/studio/frontend/src/features/native-intents/use-native-drop.ts @@ -10,14 +10,23 @@ import type { NativeIntent } from "./types"; export type NativeModelDropState = | { status: "idle" } | { status: "valid"; action: "load" | "replace" | "chip" } - | { status: "attach"; count: number; kind: "docs" | "images" | "audio" | "mixed" } - | { status: "invalid" }; + | { + status: "attach"; + count: number; + kind: "docs" | "images" | "audio" | "video" | "mixed"; + } + // `reason` explains a refusal the file types alone do not. Absent means the + // files themselves are the problem. + | { status: "invalid"; reason?: string }; interface NativeModelDropOptions { enabled?: boolean; attachmentScope?: string; // Where a drop on this window belongs, for reporting a failure back to it. attachmentTargetKey?: string; + /** Set when this view takes no drops at all. Refuses every droppable payload + * with this sentence instead of swallowing it, and loads nothing. */ + dropsUnsupportedReason?: string; nativePathLeasesSupported: boolean; hasActiveModel: boolean; isModelLoading: boolean; @@ -25,6 +34,7 @@ interface NativeModelDropOptions { onAttach?: (intents: NativeIntent[]) => Promise | void; onAttachImages?: (intents: NativeIntent[]) => Promise | void; onAttachAudio?: (intents: NativeIntent[]) => Promise | void; + onAttachVideo?: (intents: NativeIntent[]) => Promise | void; } function canAttachDocs(options: NativeModelDropOptions): boolean { @@ -39,6 +49,10 @@ function canAttachAudio(options: NativeModelDropOptions): boolean { return Boolean(options.onAttachAudio); } +function canAttachVideo(options: NativeModelDropOptions): boolean { + return Boolean(options.onAttachVideo); +} + function canAutoLoadModel(options: NativeModelDropOptions): boolean { return ( options.nativePathLeasesSupported && @@ -51,22 +65,39 @@ function attachmentCount(dropped: ReturnType): number if ( dropped.kind === "docs" || dropped.kind === "images" || - dropped.kind === "audio" + dropped.kind === "audio" || + dropped.kind === "video" ) { return dropped.paths.length; } if (dropped.kind === "attach") { - return dropped.docs.length + dropped.images.length + dropped.audio.length; + return ( + dropped.docs.length + + dropped.images.length + + dropped.audio.length + + dropped.video.length + ); } return 0; } +/** Anything this handler would otherwise act on. "none" and "unsupported" + * already have their own answers. */ +function isActionableKind( + dropped: ReturnType, +): boolean { + return dropped.kind !== "none" && dropped.kind !== "unsupported"; +} + function dropStateForPaths( paths: string[], options: NativeModelDropOptions, ): NativeModelDropState { const dropped = classifyDropPaths(paths); if (dropped.kind === "none") return { status: "idle" }; + if (options.dropsUnsupportedReason && isActionableKind(dropped)) { + return { status: "invalid", reason: options.dropsUnsupportedReason }; + } if (dropped.kind === "docs") { return canAttachDocs(options) ? { status: "attach", count: dropped.paths.length, kind: "docs" } @@ -82,12 +113,18 @@ function dropStateForPaths( ? { status: "attach", count: dropped.paths.length, kind: "audio" } : { status: "invalid" }; } + if (dropped.kind === "video") { + return canAttachVideo(options) + ? { status: "attach", count: dropped.paths.length, kind: "video" } + : { status: "invalid" }; + } if (dropped.kind === "attach") { const docsSupported = dropped.docs.length === 0 || canAttachDocs(options); const imagesSupported = dropped.images.length === 0 || canAttachImages(options); const audioSupported = dropped.audio.length === 0 || canAttachAudio(options); - return docsSupported && imagesSupported && audioSupported + const videoSupported = dropped.video.length === 0 || canAttachVideo(options); + return docsSupported && imagesSupported && audioSupported && videoSupported ? { status: "attach", count: attachmentCount(dropped), kind: "mixed" } : { status: "invalid" }; } @@ -105,9 +142,11 @@ interface RegisteredDrop { docs: NativeIntent[]; images: NativeIntent[]; audio: NativeIntent[]; + video: NativeIntent[]; docsFailed: number; imagesFailed: number; audioFailed: number; + videoFailed: number; error?: Error; } @@ -136,7 +175,7 @@ async function registerEach(paths: string[]) { async function registerDroppedAttachments( dropped: Extract< ReturnType, - { kind: "docs" | "images" | "audio" | "attach" } + { kind: "docs" | "images" | "audio" | "video" | "attach" } >, ): Promise { const docPaths = @@ -157,19 +196,28 @@ async function registerDroppedAttachments( : dropped.kind === "attach" ? dropped.audio : []; - const [docs, images, audio] = await Promise.all([ + const videoPaths = + dropped.kind === "video" + ? dropped.paths + : dropped.kind === "attach" + ? dropped.video + : []; + const [docs, images, audio, video] = await Promise.all([ registerEach(docPaths), registerEach(imagePaths), registerEach(audioPaths), + registerEach(videoPaths), ]); return { docs: docs.intents, images: images.intents, audio: audio.intents, + video: video.intents, docsFailed: docs.failed, imagesFailed: images.failed, audioFailed: audio.failed, - error: docs.error ?? images.error ?? audio.error, + videoFailed: video.failed, + error: docs.error ?? images.error ?? audio.error ?? video.error, }; } @@ -181,6 +229,8 @@ function sameDropState( if (a.status === "valid" && b.status === "valid") return a.action === b.action; if (a.status === "attach" && b.status === "attach") return a.count === b.count && a.kind === b.kind; + if (a.status === "invalid" && b.status === "invalid") + return a.reason === b.reason; return true; } @@ -234,10 +284,17 @@ export function useNativeModelDrop(options: NativeModelDropOptions): NativeModel toast.error(SUPPORTED_DROP_HINT); return; } + // Before the model branch too: this view loads nothing, so a dropped + // GGUF must not replace the active model behind it. + if (currentOptions.dropsUnsupportedReason && isActionableKind(dropped)) { + toast.error(currentOptions.dropsUnsupportedReason); + return; + } if ( dropped.kind === "docs" || dropped.kind === "images" || dropped.kind === "audio" || + dropped.kind === "video" || dropped.kind === "attach" ) { const needsDocs = @@ -249,6 +306,9 @@ export function useNativeModelDrop(options: NativeModelDropOptions): NativeModel const needsAudio = dropped.kind === "audio" || (dropped.kind === "attach" && dropped.audio.length > 0); + const needsVideo = + dropped.kind === "video" || + (dropped.kind === "attach" && dropped.video.length > 0); if (needsDocs && !canAttachDocs(currentOptions)) { toast.error("Attaching files needs the desktop backend", { description: "Retry once Studio has finished starting up.", @@ -267,12 +327,19 @@ export function useNativeModelDrop(options: NativeModelDropOptions): NativeModel }); return; } + if (needsVideo && !canAttachVideo(currentOptions)) { + toast.error("Attaching video is unavailable right now", { + description: "Retry once this chat is ready for attachments.", + }); + return; + } // Hold the send gate across registration too. Between the drop and the // intents reaching the queue there is nothing for the composer to see, // so an Enter in that window would send the text without the image. const store = useNativeIntentStore.getState(); if (needsImages) store.beginImageDropRegistration(); if (needsAudio) store.beginAudioDropRegistration(); + if (needsVideo) store.beginVideoDropRegistration(); try { const registered = await registerDroppedAttachments(dropped); const latestOptions = optionsRef.current; @@ -296,19 +363,27 @@ export function useNativeModelDrop(options: NativeModelDropOptions): NativeModel if (registered.audioFailed > 0 && failureKey) { store.failAudioDropRegistration(failureKey); } - // A failed document cancels a send parked behind the image or audio - // gate too, or the draft goes out with only what survived. + if (registered.videoFailed > 0 && failureKey) { + store.failVideoDropRegistration(failureKey); + } + // A failed document cancels a send parked behind the image, audio or + // video gate too, or the draft goes out with only what survived. if (registered.docsFailed > 0 && failureKey) { if (needsImages) store.failImageDropRegistration(failureKey); if (needsAudio) store.failAudioDropRegistration(failureKey); + if (needsVideo) store.failVideoDropRegistration(failureKey); } if (registered.audio.length > 0) { await attachOptions.onAttachAudio?.(registered.audio); } + if (registered.video.length > 0) { + await attachOptions.onAttachVideo?.(registered.video); + } if ( registered.docsFailed + registered.imagesFailed + - registered.audioFailed > + registered.audioFailed + + registered.videoFailed > 0 ) { toast.error("Could not attach dropped files", { @@ -323,12 +398,16 @@ export function useNativeModelDrop(options: NativeModelDropOptions): NativeModel if (needsAudio && failureKey) { store.failAudioDropRegistration(failureKey); } + if (needsVideo && failureKey) { + store.failVideoDropRegistration(failureKey); + } toast.error("Could not attach dropped files", { description: error instanceof Error ? error.message : String(error), }); } finally { if (needsImages) store.endImageDropRegistration(); if (needsAudio) store.endAudioDropRegistration(); + if (needsVideo) store.endVideoDropRegistration(); } return; } diff --git a/studio/frontend/src/features/native-intents/use-native-file-drop.ts b/studio/frontend/src/features/native-intents/use-native-file-drop.ts new file mode 100644 index 0000000000..7150f93768 --- /dev/null +++ b/studio/frontend/src/features/native-intents/use-native-file-drop.ts @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { isTauri } from "@/lib/api-base"; +import { toast } from "@/lib/toast"; +import type React from "react"; +import { useCallback, useRef, useState } from "react"; +import { registerNativeAttachmentPath } from "./api"; +import { nativeAttachmentIntentToFile } from "./native-attachment-file"; +import type { NativeIntent } from "./types"; +import { useNativeDropTarget } from "./use-native-drop-target"; + +const PATH_SEPARATOR_RE = /[\\/]/; + +/** File name at the end of an OS path, either separator. */ +function nativeFileName(path: string): string { + const segments = path.split(PATH_SEPARATOR_RE); + return segments[segments.length - 1] || path; +} + +/** Extensions from an `accept` list (".pdf,.md"), lowercased with their dot. */ +function acceptedExts(accept: string | undefined): string[] { + if (!accept) return []; + return accept + .split(",") + .map((entry) => entry.trim().toLowerCase()) + .filter((entry) => entry.startsWith(".")); +} + +function hasAcceptedExt(name: string, exts: string[]): boolean { + if (exts.length === 0) { + return true; + } + const lower = name.toLowerCase(); + return exts.some((ext) => lower.endsWith(ext)); +} + +/** Nothing in the payload was droppable here. */ +function toastNothingAccepted( + names: string[], + accept: string | undefined, +): void { + // A folder is one extension-less name, and the native side takes files only. + const looksLikeFolder = names.some((name) => !name.includes(".")); + if (looksLikeFolder) { + toast.error("Folders can't be dropped here", { + description: "Drop the files inside it, or use the picker button.", + }); + return; + } + toast.error( + names.length === 1 + ? "That file type can't be dropped here" + : "Those file types can't be dropped here", + accept ? { description: `Accepts ${accept}.` } : undefined, + ); +} + +/** Some of the payload was droppable and the rest was not. */ +function toastPartiallySkipped(count: number): void { + if (count <= 0) { + return; + } + toast.error( + count === 1 + ? "Skipped a file this zone doesn't accept" + : `Skipped ${count} files this zone doesn't accept`, + ); +} + +function reasonText(reason: unknown): string { + return reason instanceof Error ? reason.message : String(reason); +} + +/** Register each path, then read it back if the caller wants Files. Per path, + * so one bad file does not discard the siblings that registered cleanly. */ +async function registerDroppedPaths( + paths: string[], + register: (path: string) => Promise, + asIntents: boolean, +): Promise<{ + ready: Array; + failed: number; + reason?: unknown; +}> { + const settled = await Promise.allSettled( + paths.map(async (path) => { + const intent = await register(path); + return asIntents ? intent : await nativeAttachmentIntentToFile(intent); + }), + ); + const ready = settled.flatMap((result) => + result.status === "fulfilled" ? [result.value] : [], + ); + const rejection = settled.find((result) => result.status === "rejected"); + return { + ready, + failed: settled.length - ready.length, + reason: rejection?.status === "rejected" ? rejection.reason : undefined, + }; +} + +/** Registered or read back, and did not survive it. */ +function toastReadFailures(count: number, reason: unknown): void { + toast.error( + count === 1 + ? "Couldn't read a dropped file" + : `Couldn't read ${count} dropped files`, + { description: reason === undefined ? undefined : reasonText(reason) }, + ); +} + +export interface NativeFileDropOptions { + /** Receives real Files, whether the OS or the DOM delivered them. */ + onFiles: (files: File[]) => void | Promise; + /** Take registered paths instead of Files, for zones that upload by lease. + * The native reader only serves media inline, so documents need this. */ + onNativeIntents?: (intents: NativeIntent[]) => void | Promise; + /** Extensions this zone takes, as an `accept` list (".pdf,.md"). Omitted takes anything. */ + accept?: string; + /** Refuse drops, with `disabledReason` said out loud rather than swallowed. */ + disabled?: boolean; + disabledReason?: string; + /** Single-slot pickers take only the first file of a batch. */ + multiple?: boolean; + /** Register under a policy other than the attachment one (datasets, models). */ + register?: (path: string) => Promise; +} + +export interface NativeFileDrop { + /** Attach to the element that owns the drop; claims native drops landing on it. */ + ref: (element: HTMLElement | null) => void; + dragging: boolean; + dragHandlers: { + onDragEnter: (event: React.DragEvent) => void; + onDragOver: (event: React.DragEvent) => void; + onDragLeave: (event: React.DragEvent) => void; + onDrop: (event: React.DragEvent) => void; + }; +} + +/** One drop zone that works on web and on desktop. + * + * Tauri delivers OS drops window-wide and suppresses the webview's own drop + * events, so a zone wired only to `onDrop` is dead in the desktop app (#9036). + * This claims the native drop for the element and hands back the same Files. */ +export function useNativeFileDrop( + options: NativeFileDropOptions, +): NativeFileDrop { + const [dragging, setDragging] = useState(false); + // Read through a ref so a fresh caller closure does not re-register the target. + const latest = useRef(options); + latest.current = options; + // dragenter/dragleave fire per child, so a raw boolean flickers on inner moves. + const dragDepth = useRef(0); + + const deliver = useCallback((files: File[]) => { + const current = latest.current; + if (files.length === 0) return; + void current.onFiles( + current.multiple === false ? files.slice(0, 1) : files, + ); + }, []); + + const handleNativePaths = useCallback( + async (paths: string[]) => { + const current = latest.current; + if (current.disabled) { + toast.error( + current.disabledReason ?? "This drop zone is busy right now", + ); + return; + } + const exts = acceptedExts(current.accept); + const supported = paths.filter((path) => + hasAcceptedExt(nativeFileName(path), exts), + ); + if (supported.length === 0) { + toastNothingAccepted(paths.map(nativeFileName), current.accept); + return; + } + const takeIntents = current.onNativeIntents; + const { ready, failed, reason } = await registerDroppedPaths( + current.multiple === false ? supported.slice(0, 1) : supported, + current.register ?? registerNativeAttachmentPath, + Boolean(takeIntents), + ); + if (ready.length > 0) { + if (takeIntents) { + void takeIntents(ready as NativeIntent[]); + } else { + deliver(ready as File[]); + } + } + if (failed > 0) { + toastReadFailures(failed, reason); + return; + } + toastPartiallySkipped(paths.length - supported.length); + }, + [deliver], + ); + + const ref = useNativeDropTarget({ + onDrop: (paths) => void handleNativePaths(paths), + onDragOver: (over) => setDragging(over && !latest.current.disabled), + }); + + const endDrag = useCallback(() => { + dragDepth.current = 0; + setDragging(false); + }, []); + + // Files only: preventDefault on a text drag kills editing in wrapped inputs. + const isFileDrag = (event: React.DragEvent): boolean => + Array.from(event.dataTransfer?.types ?? []).includes("Files"); + + // preventDefault runs even under Tauri and while disabled, or the webview + // navigates to the dropped file. + const dragHandlers = { + onDragEnter: (event: React.DragEvent) => { + if (!isFileDrag(event)) return; + event.preventDefault(); + if (isTauri || latest.current.disabled) return; + dragDepth.current += 1; + setDragging(true); + }, + onDragOver: (event: React.DragEvent) => { + if (!isFileDrag(event)) return; + event.preventDefault(); + if (isTauri || latest.current.disabled) return; + event.dataTransfer.dropEffect = "copy"; + }, + onDragLeave: (event: React.DragEvent) => { + if (isTauri || !isFileDrag(event)) return; + dragDepth.current = Math.max(0, dragDepth.current - 1); + if (dragDepth.current === 0) setDragging(false); + }, + onDrop: (event: React.DragEvent) => { + if (!isFileDrag(event)) { + return; + } + event.preventDefault(); + // The native target owns this on desktop; both would attach it twice. + if (isTauri) { + return; + } + endDrag(); + const current = latest.current; + if (current.disabled) { + toast.error( + current.disabledReason ?? "This drop zone is busy right now", + ); + return; + } + const exts = acceptedExts(current.accept); + const dropped = Array.from(event.dataTransfer.files ?? []); + const supported = dropped.filter((file) => + hasAcceptedExt(file.name, exts), + ); + if (dropped.length > 0 && supported.length === 0) { + toastNothingAccepted( + dropped.map((file) => file.name), + current.accept, + ); + return; + } + deliver(supported); + toastPartiallySkipped(dropped.length - supported.length); + }, + }; + + return { ref, dragging, dragHandlers }; +} diff --git a/studio/frontend/src/features/rag/components/project-source-dropzone.tsx b/studio/frontend/src/features/rag/components/project-source-dropzone.tsx index 5d4c930da1..a291a9a96e 100644 --- a/studio/frontend/src/features/rag/components/project-source-dropzone.tsx +++ b/studio/frontend/src/features/rag/components/project-source-dropzone.tsx @@ -309,7 +309,14 @@ export function ProjectSourceDropzone({ // handler, which would attach it to the chat behind the dialog. const nativeDropRef = useNativeDropTarget({ onDrop: (paths) => { - if (disabled) return; + // Claimed but refusing, so say so: returning quietly made the file + // vanish with no border and no message (#9036). + if (disabled) { + toast.error("Sources are still uploading", { + description: "Wait for them to finish, then drop again.", + }); + return; + } void addNativePaths(paths); }, onDragOver: (over) => setDragging(over && !disabled), diff --git a/studio/frontend/src/features/rag/components/project-sources-panel.tsx b/studio/frontend/src/features/rag/components/project-sources-panel.tsx index 9733af0bcc..65e90a98c2 100644 --- a/studio/frontend/src/features/rag/components/project-sources-panel.tsx +++ b/studio/frontend/src/features/rag/components/project-sources-panel.tsx @@ -2,6 +2,9 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { Button } from "@/components/ui/button"; +import { useNativeFileDrop } from "@/features/native-intents"; +import type { NativeIntent } from "@/features/native-intents"; +import { cn } from "@/lib/utils"; import { FolderAddIcon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { useCallback, useEffect, useRef } from "react"; @@ -14,7 +17,8 @@ import { import { RAG_UPLOAD_ACCEPT, isLinkedFolderManaged } from "../types/rag"; import { DocumentStatusChip } from "./document-status-chip"; import { LinkedFoldersManager } from "./linked-folders-manager"; -import { useRagDocuments } from "./use-rag-documents"; +import { fileItems, useRagDocuments } from "./use-rag-documents"; +import type { RagUploadItem } from "./use-rag-documents"; /** Project "Sources" tab: documents indexed for retrieval in every chat that * belongs to the project. */ @@ -31,16 +35,37 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) { // cannot cache "no sources" for the probe's TTL, and announce after it, which // is the half other instances and other tabs listen for. Announcing before // would refetch and resurrect the row this panel has already dropped. - const handleFiles = useCallback( - async (files: File[]) => { - if (files.length === 0) return; + const handleItems = useCallback( + async (items: RagUploadItem[]) => { + if (items.length === 0) return; invalidateProjectSources(projectId); - await upload(files); + await upload(items); announceProjectSourcesUpdated(projectId); }, [projectId, upload], ); + const handleFiles = useCallback( + (files: File[]) => handleItems(fileItems(files)), + [handleItems], + ); + + // Desktop drops arrive as paths; the upload mints a lease per file rather + // than reading a document through the webview. + const handleNativeIntents = useCallback( + (intents: NativeIntent[]) => + handleItems( + intents.map((intent) => ({ + kind: "native" as const, + token: intent.path.token, + name: intent.path.displayLabel, + sizeBytes: intent.path.sizeBytes, + modifiedMs: intent.path.modifiedMs, + })), + ), + [handleItems], + ); + const handleRemove = useCallback( async (documentId: string) => { invalidateProjectSources(projectId); @@ -68,15 +93,18 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) { const empty = documents.length === 0; + // Tauri suppresses webview drop events, so the plain `onDrop` this panel + // carried never fired on desktop: no border, file ignored (#9036). + const { ref: dropRef, dragging, dragHandlers } = useNativeFileDrop({ + onFiles: handleFiles, + onNativeIntents: handleNativeIntents, + accept: RAG_UPLOAD_ACCEPT, + disabled: uploading, + disabledReason: "Wait for the current upload to finish, then drop again.", + }); + return ( -
e.preventDefault()} - onDrop={(e) => { - e.preventDefault(); - void handleFiles(Array.from(e.dataTransfer.files ?? [])); - }} - > +
{empty ? ( -
+
Or drop files here

) : ( -
+

{documents.length === 1 diff --git a/studio/frontend/src/features/recipe-studio/api/index.ts b/studio/frontend/src/features/recipe-studio/api/index.ts index f4e8167cb3..dafe7c504a 100644 --- a/studio/frontend/src/features/recipe-studio/api/index.ts +++ b/studio/frontend/src/features/recipe-studio/api/index.ts @@ -447,13 +447,24 @@ type UnstructuredFileUploadResponse = { error?: string; }; +/** A desktop drop, redeemed server-side: Tauri hands the webview a path, never + * a File, so the bytes never cross the bridge. */ +export interface NativeUnstructuredUpload { + nativePathLease: string; + name: string; + size: number; +} + +export type UnstructuredUploadSource = File | NativeUnstructuredUpload; + export async function uploadUnstructuredFile( - file: File, + file: UnstructuredUploadSource, blockId: string, signal?: AbortSignal, ): Promise { const formData = new FormData(); - formData.append("file", file); + if (file instanceof File) formData.append("file", file); + else formData.append("nativePathLease", file.nativePathLease); formData.append("block_id", blockId); const res = await authFetch( diff --git a/studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx b/studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx index bb03c1f125..983f0607c9 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/seed/unstructured-drop-zone.tsx @@ -1,4 +1,5 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef } from "react"; +import { consumeNativePathToken, useNativeFileDrop } from "@/features/native-intents"; import { CloudUploadIcon, Cancel01Icon, @@ -17,6 +18,13 @@ import { const ACCEPTED_EXTENSIONS = [".txt", ".pdf", ".docx", ".md"]; +/** One queued upload: browser bytes, or a desktop drop the backend redeems. */ +type UploadCandidate = { + name: string; + size: number; + source: File | { token: string }; +}; + type FileEntry = { id: string; name: string; @@ -56,7 +64,6 @@ export function UnstructuredDropZone({ const filesRef = useRef(files); const blockIdRef = useRef(blockId); const mountedRef = useRef(true); - const [isDragOver, setIsDragOver] = useState(false); useEffect(() => { filesRef.current = files; @@ -68,9 +75,9 @@ export function UnstructuredDropZone({ const totalSize = files.reduce((sum, f) => sum + f.size, 0); - const handleFiles = useCallback( - async (newFiles: File[]) => { - const valid = newFiles.filter((f) => { + const uploadCandidates = useCallback( + async (candidates: UploadCandidate[]) => { + const valid = candidates.filter((f) => { if (!isValidExtension(f.name)) return false; if (f.size > UNSTRUCTURED_RECIPE_UPLOAD_MAX_BYTES) return false; return true; @@ -94,14 +101,25 @@ export function UnstructuredDropZone({ onFilesChange((prev) => [...prev, ...entries]); for (let i = 0; i < valid.length; i++) { - const file = valid[i]; + const candidate = valid[i]; const entry = entries[i]; let updatedId = ""; let updatedStatus: FileEntry["status"] = "error"; let updatedError: string | undefined; try { + // Leases are short-lived, so mint one per upload, not at drop time. + const source = + candidate.source instanceof File + ? candidate.source + : { + nativePathLease: ( + await consumeNativePathToken(candidate.source.token, "attach") + ).nativePathLease, + name: candidate.name, + size: candidate.size, + }; const result = await uploadUnstructuredFile( - file, + source, blockId, entry.abortController?.signal, ); @@ -132,6 +150,18 @@ export function UnstructuredDropZone({ [blockId, onFilesChange], ); + const handleFiles = useCallback( + (newFiles: File[]) => + uploadCandidates( + newFiles.map((file) => ({ + name: file.name, + size: file.size, + source: file, + })), + ), + [uploadCandidates], + ); + const deletedIdsRef = useRef(new Set()); const handleRemove = useCallback( (index: number) => { @@ -170,26 +200,24 @@ export function UnstructuredDropZone({ [blockId, onFilesChange], ); - const handleDrop = useCallback( - (e: React.DragEvent) => { - e.preventDefault(); - setIsDragOver(false); - if (disabled) return; - const dropped = Array.from(e.dataTransfer.files); - handleFiles(dropped); - }, - [disabled, handleFiles], - ); - - const handleDragOver = useCallback( - (e: React.DragEvent) => { - e.preventDefault(); - if (!disabled) setIsDragOver(true); - }, - [disabled], - ); - - const handleDragLeave = useCallback(() => setIsDragOver(false), []); + // Tauri suppresses webview drop events, so the plain `onDrop` this zone + // carried was dead on desktop (#9036). + const { ref: dropRef, dragging: isDragOver, dragHandlers } = useNativeFileDrop({ + onFiles: handleFiles, + // A seed corpus can run to hundreds of MB, so the backend redeems the + // signed path itself rather than routing bytes through the webview. + onNativeIntents: (intents) => + uploadCandidates( + intents.map((intent) => ({ + name: intent.path.displayLabel, + size: intent.path.sizeBytes ?? 0, + source: { token: intent.path.token }, + })), + ), + accept: ACCEPTED_EXTENSIONS.join(","), + disabled, + disabledReason: "This block is busy. Try the drop again in a moment.", + }); const handleClick = useCallback(() => { if (!disabled) inputRef.current?.click(); @@ -209,14 +237,16 @@ export function UnstructuredDropZone({ return (

(null); - const [dragging, setDragging] = useState(false); const readFile = useCallback( (file: File | undefined | null) => { @@ -52,6 +56,14 @@ export function ReferenceMediaPicker({ [kind, onChange], ); + // Tauri suppresses the webview's own drop events, so the handlers below never + // fire on the desktop app; this claims the OS drop for the button (#9036). + const { ref: dropRef, dragging, dragHandlers } = useNativeFileDrop({ + onFiles: (files) => readFile(files[0]), + accept: kind === "video" ? CHAT_VIDEO_DROP_ACCEPT : CHAT_AUDIO_DROP_ACCEPT, + multiple: false, + }); + const icon = kind === "video" ? FlimSlateIcon : MusicNote01Icon; if (value) { @@ -89,17 +101,9 @@ export function ReferenceMediaPicker({ return (