From 8a2d4fa3fd7b413cdada6659444df558b3820d8e Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 18 Aug 2026 16:12:24 +0000 Subject: [PATCH] Studio: an absent speculative mode is auto, not off, for the GPU arbiter _canonicalize_spec_mode returns None for None, empty and whitespace alike, and every consumer resolves that to auto (the module docstring says so, and load_model spells it '_canonicalize_spec_mode(...) or auto'), where a remote or local sidecar can be discovered and launched with its default GPU offload. zero_vram_chat_load grouped None with off, so a manual gpu_layers=0 load that omitted the mode was classified zero-VRAM and skipped acquire_for(CHAT). The launch-time mask disagreed: it reads the FINISHED argv, sees the drafter that auto had since added, and keeps the GPUs visible. Same load, no arbiter, real VRAM, free to land on a resident image or video pipeline and OOM both. Only the canonical off is speculation-free. The three existing tests passed the mode positionally as absent while meaning off, so they encoded the assumption being corrected; they now say off explicitly. The empty-string case moves with them: it canonicalizes to None like an omitted mode, so it was never off either. --- studio/backend/core/inference/llama_cpp.py | 13 +++++- studio/backend/tests/test_gpu_memory_mode.py | 47 +++++++++++++++----- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index f4940dfa20..52f89cb09d 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1800,8 +1800,17 @@ def zero_vram_chat_load( return False # Any speculative mode may launch a GPU drafter and only the request's own knobs are known here, so treat every selection as GPU-bearing except # "off". Canonicalize first: the UI sends the literal "off", which a bare truthiness test read as "speculation requested". - spec_mode = _canonicalize_spec_mode(speculative_type) - if needs_mmproj or spec_mode not in (None, "off"): + # ...and an ABSENT mode is not a quiet "off": _canonicalize_spec_mode returns None + # for None, "" and whitespace alike, and every consumer resolves that to "auto" + # (`_canonicalize_spec_mode(...) or "auto"` in load_model), where a remote or local + # sidecar can be discovered and launched with its default GPU offload. Grouping None + # with "off" here let an API or defaulted load skip acquire_for(CHAT) while the + # launch-time mask, which reads the FINISHED argv and so sees the drafter that was + # added, kept the GPUs visible. The two then disagreed about the same load: no + # arbiter, real VRAM, free to land on a resident image or video pipeline and OOM + # both. Only the canonical "off" is speculation-free. + spec_mode = _canonicalize_spec_mode(speculative_type) or "auto" + if needs_mmproj or spec_mode != "off": return False if LlamaCppBackend._is_vulkan_backend(): return False diff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.py index 51a6150273..28c8339ecd 100644 --- a/studio/backend/tests/test_gpu_memory_mode.py +++ b/studio/backend/tests/test_gpu_memory_mode.py @@ -1380,25 +1380,29 @@ def test_zero_vram_chat_load_only_for_a_deliberate_cpu_only_offload(not_vulkan): # Manual + gpu_layers=0 is the one shape that launches with the GPUs hidden from the child, so it is the one shape allowed # to skip the GPU arbiter. Auto (or any pinned layer count) puts the model on the GPU and must still evict a pipeline. zero = llama_cpp_module.zero_vram_chat_load - assert zero("manual", 0) is True - assert zero("auto", 0) is False - assert zero("manual", 1) is False - assert zero("manual", -1) is False + # Speculation named explicitly off: an ABSENT mode resolves to auto everywhere else + # in the module, so it is GPU-bearing and covered by its own test below. + assert zero("manual", 0, [], False, "off") is True + assert zero("auto", 0, [], False, "off") is False + assert zero("manual", 1, [], False, "off") is False + assert zero("manual", -1, [], False, "off") is False def test_zero_vram_chat_load_refuses_every_gpu_companion(not_vulkan): # The launch-time mask keeps the GPUs visible for a device pin, tensor mode, an mmproj or a drafter, so those loads DO hold # VRAM. --mmproj and --model-draft are added by the backend, so their intent arrives as flags. zero = llama_cpp_module.zero_vram_chat_load - assert zero("manual", 0, ["--device", "CUDA0"]) is False - assert zero("manual", 0, ["-dev", "CUDA0"]) is False - assert zero("manual", 0, ["--split-mode", "tensor"]) is False - assert zero("manual", 0, ["--model-draft", "/tmp/draft.gguf"]) is False - assert zero("manual", 0, [], True) is False + assert zero("manual", 0, ["--device", "CUDA0"], False, "off") is False + assert zero("manual", 0, ["-dev", "CUDA0"], False, "off") is False + assert zero("manual", 0, ["--split-mode", "tensor"], False, "off") is False + assert zero("manual", 0, ["--model-draft", "/tmp/draft.gguf"], False, "off") is False + assert zero("manual", 0, [], True, "off") is False assert zero("manual", 0, [], False, "model") is False # A CPU-pinned device and a CPU-forced drafter keep it zero-VRAM. - assert zero("manual", 0, ["--device", "none"]) is True - assert zero("manual", 0, ["--model-draft", "/tmp/d.gguf", "--spec-draft-ngl", "0"]) is True + assert zero("manual", 0, ["--device", "none"], False, "off") is True + assert zero( + "manual", 0, ["--model-draft", "/tmp/d.gguf", "--spec-draft-ngl", "0"], False, "off" + ) is True def test_zero_vram_chat_load_exempts_disabled_speculation(not_vulkan): @@ -1407,7 +1411,9 @@ def test_zero_vram_chat_load_exempts_disabled_speculation(not_vulkan): zero = llama_cpp_module.zero_vram_chat_load assert zero("manual", 0, [], False, "off") is True assert zero("manual", 0, [], False, " OFF ") is True - assert zero("manual", 0, [], False, "") is True + # Empty is NOT off: it canonicalizes to None like an omitted mode, and every + # consumer resolves that to auto, which may launch a drafter on the GPU. + assert zero("manual", 0, [], False, "") is False assert zero("manual", 0, [], False, "auto") is False assert zero("manual", 0, [], False, "mtp") is False assert zero("manual", 0, [], False, "default") is False @@ -1465,3 +1471,20 @@ def test_cmd_companion_ignores_a_projector_pinned_off_the_gpu(): # llama.cpp assigns rather than accumulates for this one, so the last flag wins. cmd = ["llama-server", "--mmproj", "p.gguf", "--no-mmproj-offload", "--mmproj-offload"] assert has(cmd, {}) is True + + +def test_zero_vram_chat_load_treats_an_absent_mode_as_auto(not_vulkan): + # _canonicalize_spec_mode returns None for None, "" and whitespace alike, and every + # consumer resolves that to "auto" (`... or "auto"` in load_model), where a remote or + # local sidecar can be discovered and launched with its default GPU offload. Reading + # an absent mode as "off" here let an API or defaulted load skip acquire_for(CHAT) + # while the launch-time mask, which reads the finished argv and so sees the drafter + # that got added, kept the GPUs visible: no arbiter, real VRAM, free to land on a + # resident image/video pipeline and OOM both. + zero = llama_cpp_module.zero_vram_chat_load + assert zero("manual", 0) is False + assert zero("manual", 0, [], False, None) is False + assert zero("manual", 0, [], False, "") is False + assert zero("manual", 0, [], False, " ") is False + # Only the canonical spelling still exempts it. + assert zero("manual", 0, [], False, "off") is True