mirror of
https://github.com/unslothai/unsloth.git
synced 2026-07-09 15:58:41 +00:00
fix(studio): keep local GGUF vision on llama-server (#5770)
* fix(studio): keep local GGUF vision on llama-server * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): lower local GGUF vision log level * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): find GGUF companions from variant dirs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Co-authored-by: imagineer99 <samleejackson0@gmail.com>
This commit is contained in:
alkinun
GitHub
parent
f033213c0b
commit
ac844b0be7
2 changed files with 156 additions and 5 deletions
|
|
@ -30,6 +30,7 @@ _loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
|
|||
sys.modules.setdefault("loggers", _loggers_stub)
|
||||
|
||||
from utils.models.model_config import (
|
||||
ModelConfig,
|
||||
is_vision_model,
|
||||
_is_vision_model_uncached,
|
||||
_vision_detection_cache,
|
||||
|
|
@ -120,6 +121,99 @@ class TestVisionCacheSubprocessPath:
|
|||
mock_raw_config.assert_called_once_with("unsloth/gemma-4-E4B-it", hf_token = None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Local GGUF capability path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLocalGgufVisionDetection:
|
||||
@patch(
|
||||
"utils.models.model_config._is_vision_model_subprocess",
|
||||
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
|
||||
)
|
||||
def test_qwen36_gguf_with_mmproj_skips_transformers(self, mock_subprocess, tmp_path):
|
||||
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
|
||||
model.write_bytes(b"")
|
||||
(tmp_path / "mmproj-F32.gguf").write_bytes(b"")
|
||||
|
||||
assert is_vision_model(str(model)) is True
|
||||
mock_subprocess.assert_not_called()
|
||||
|
||||
@patch(
|
||||
"utils.models.model_config._is_vision_model_subprocess",
|
||||
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
|
||||
)
|
||||
def test_direct_gguf_in_variant_subdir_finds_snapshot_mmproj(self, mock_subprocess, tmp_path):
|
||||
variant_dir = tmp_path / "BF16"
|
||||
variant_dir.mkdir()
|
||||
model = variant_dir / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
|
||||
model.write_bytes(b"")
|
||||
(tmp_path / "mmproj-F32.gguf").write_bytes(b"")
|
||||
|
||||
assert is_vision_model(str(model)) is True
|
||||
mock_subprocess.assert_not_called()
|
||||
|
||||
@patch(
|
||||
"utils.models.model_config._is_vision_model_subprocess",
|
||||
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
|
||||
)
|
||||
def test_qwen36_gguf_without_mmproj_skips_transformers(self, mock_subprocess, tmp_path):
|
||||
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
|
||||
model.write_bytes(b"")
|
||||
|
||||
assert is_vision_model(str(model)) is False
|
||||
mock_subprocess.assert_not_called()
|
||||
|
||||
def test_local_gguf_check_observes_mmproj_added_later(self, tmp_path):
|
||||
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
|
||||
model.write_bytes(b"")
|
||||
|
||||
assert is_vision_model(str(model)) is False
|
||||
(tmp_path / "mmproj-F32.gguf").write_bytes(b"")
|
||||
assert is_vision_model(str(model)) is True
|
||||
|
||||
@patch(
|
||||
"utils.models.model_config._is_vision_model_subprocess",
|
||||
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
|
||||
)
|
||||
def test_ui_selection_returns_local_gguf_config(self, mock_subprocess, tmp_path):
|
||||
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
|
||||
model.write_bytes(b"")
|
||||
mmproj = tmp_path / "mmproj-F32.gguf"
|
||||
mmproj.write_bytes(b"")
|
||||
|
||||
config = ModelConfig.from_ui_selection(str(model), None)
|
||||
|
||||
assert config is not None
|
||||
assert config.is_gguf is True
|
||||
assert config.is_vision is True
|
||||
assert config.gguf_mmproj_file == str(mmproj.resolve())
|
||||
mock_subprocess.assert_not_called()
|
||||
|
||||
@patch(
|
||||
"utils.models.model_config._is_vision_model_subprocess",
|
||||
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
|
||||
)
|
||||
def test_ui_selection_direct_gguf_in_variant_subdir_keeps_mmproj(
|
||||
self, mock_subprocess, tmp_path
|
||||
):
|
||||
variant_dir = tmp_path / "BF16"
|
||||
variant_dir.mkdir()
|
||||
model = variant_dir / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
|
||||
model.write_bytes(b"")
|
||||
mmproj = tmp_path / "mmproj-F32.gguf"
|
||||
mmproj.write_bytes(b"")
|
||||
|
||||
config = ModelConfig.from_ui_selection(str(model), None)
|
||||
|
||||
assert config is not None
|
||||
assert config.is_gguf is True
|
||||
assert config.is_vision is True
|
||||
assert config.gguf_mmproj_file == str(mmproj.resolve())
|
||||
mock_subprocess.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exception handling — cache the False fallback
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -721,6 +721,25 @@ def is_vision_model(model_name: str, hf_token: Optional[str] = None) -> bool:
|
|||
model_name: Model identifier (HF repo or local path)
|
||||
hf_token: Optional HF token for gated/private models
|
||||
"""
|
||||
# Local GGUF models are served by llama-server. Their multimodal
|
||||
# capability comes from a companion mmproj, not a Transformers config.
|
||||
# Do not cache this lookup: a projector may be added beside an existing
|
||||
# weight file after it was first inspected.
|
||||
if is_local_path(model_name):
|
||||
local_path = normalize_path(model_name)
|
||||
gguf_file = detect_gguf_model(local_path)
|
||||
if gguf_file:
|
||||
companion_root = _local_gguf_companion_search_root(local_path, gguf_file)
|
||||
mmproj_file = detect_mmproj_file(gguf_file, search_root = companion_root)
|
||||
is_vision = mmproj_file is not None
|
||||
logger.debug(
|
||||
"Local GGUF vision check for '%s': mmproj=%s, is_vision=%s",
|
||||
gguf_file,
|
||||
mmproj_file,
|
||||
is_vision,
|
||||
)
|
||||
return is_vision
|
||||
|
||||
# Normalize model name so different casings of the same repo share a key
|
||||
try:
|
||||
if is_local_path(model_name):
|
||||
|
|
@ -1358,6 +1377,35 @@ def _extract_quant_label(filename: str) -> str:
|
|||
return stem.split("-")[-1]
|
||||
|
||||
|
||||
def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str:
|
||||
"""Directory to scan upward from for local GGUF companion files."""
|
||||
import re
|
||||
|
||||
selected = Path(selected_path)
|
||||
gguf_path = Path(gguf_file)
|
||||
if selected.suffix.lower() != ".gguf":
|
||||
return selected_path
|
||||
|
||||
gguf_dir = gguf_path.parent
|
||||
if not gguf_dir.name:
|
||||
return str(gguf_dir)
|
||||
|
||||
quant_dir_re = (
|
||||
r"(UD-)?("
|
||||
r"MXFP[0-9]+(?:_[A-Z0-9]+)*"
|
||||
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
|
||||
r"|TQ[0-9]+_[0-9]+"
|
||||
r"|Q[0-9]+_K_[A-Z]+"
|
||||
r"|Q[0-9]+_[0-9]+"
|
||||
r"|Q[0-9]+_K"
|
||||
r"|BF16|F16|F32"
|
||||
r")"
|
||||
)
|
||||
if re.fullmatch(quant_dir_re, gguf_dir.name, re.IGNORECASE):
|
||||
return str(gguf_dir.parent)
|
||||
return str(gguf_dir)
|
||||
|
||||
|
||||
def _iter_hf_cache_snapshots(repo_id: str):
|
||||
"""Yield HF cache snapshot dirs for *repo_id*, newest first.
|
||||
|
||||
|
|
@ -2275,10 +2323,10 @@ class ModelConfig:
|
|||
except Exception as e:
|
||||
logger.debug(f"Could not read export metadata: {e}")
|
||||
|
||||
# Pass search_root=path so detect_mmproj_file walks up to the
|
||||
# snapshot root: the weight may sit in a quant subdir while
|
||||
# mmproj-*.gguf lives at the root.
|
||||
mmproj_file = detect_mmproj_file(gguf_file, search_root = path)
|
||||
# Direct file selections may point into a quant subdir while
|
||||
# mmproj-*.gguf lives at the snapshot root.
|
||||
companion_root = _local_gguf_companion_search_root(path, gguf_file)
|
||||
mmproj_file = detect_mmproj_file(gguf_file, search_root = companion_root)
|
||||
if mmproj_file:
|
||||
gguf_is_vision = True
|
||||
logger.info(f"Detected mmproj for vision: {mmproj_file}")
|
||||
|
|
@ -2286,7 +2334,7 @@ class ModelConfig:
|
|||
logger.warning(f"Base model is vision but no mmproj file found in {gguf_dir}")
|
||||
|
||||
# Separate MTP drafter sibling (Gemma 4), mirroring mmproj.
|
||||
mtp_file = detect_mtp_file(gguf_file, search_root = path)
|
||||
mtp_file = detect_mtp_file(gguf_file, search_root = companion_root)
|
||||
if mtp_file:
|
||||
logger.info(f"Detected MTP drafter: {mtp_file}")
|
||||
|
||||
|
|
@ -2476,6 +2524,15 @@ class ModelConfig:
|
|||
identifier = resolved_identifier
|
||||
path = resolved_identifier
|
||||
|
||||
# Keep existing local GGUF selections on the llama-server path. This
|
||||
# constructor is still used by older inference helpers and must not
|
||||
# describe a .gguf weight file as loadable by FastVisionModel.
|
||||
if is_local and not is_lora and detect_gguf_model(path):
|
||||
gguf_config = cls.from_identifier(path, hf_token = hf_token)
|
||||
if gguf_config is not None:
|
||||
gguf_config.display_name = display_name
|
||||
return gguf_config
|
||||
|
||||
# --- Base Model and Vision Detection ---
|
||||
base_model = None
|
||||
is_vision = False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue