mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-29 10:13:31 +00:00
mtmd: support dots3-note vision+audio (#27524)
* text: conversion * init impl * mtmd: conversion * impl mtmd cpp * Update gguf-py/gguf/tensor_mapping.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> --------- Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
Xuan-Son Nguyen
GitHub
parent
3a653fea93
commit
54ee5ee643
15 changed files with 535 additions and 11 deletions
|
|
@ -282,6 +282,8 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
|||
"CogVLMForCausalLM": "cogvlm",
|
||||
"DeepseekOCR2ForCausalLM": "deepseek",
|
||||
"DeepseekOCRForCausalLM": "deepseek",
|
||||
"Dots3NoteForCausalLM": "dots3",
|
||||
"Dots3NoteForConditionalGeneration": "dots3",
|
||||
"DotsOCRForCausalLM": "dotsocr",
|
||||
"Exaone4_5_ForConditionalGeneration": "exaone",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@ from __future__ import annotations
|
|||
import math
|
||||
import re
|
||||
|
||||
from typing import TYPE_CHECKING, Callable, Iterable
|
||||
import torch
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Callable, Iterable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .base import MmprojModel, ModelBase, gguf
|
||||
|
||||
from .deepseek import DeepseekV2Model
|
||||
|
||||
|
|
@ -193,3 +195,129 @@ class Dots3NoteModel(DeepseekV2Model):
|
|||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("Dots3NoteForCausalLM", "Dots3NoteForConditionalGeneration")
|
||||
class Dots3NoteMmprojModel(MmprojModel):
|
||||
has_vision_encoder = True
|
||||
has_audio_encoder = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
assert self.hparams_vision is not None
|
||||
assert self.hparams_audio is not None
|
||||
|
||||
# preprocessor_config.json nests the image params under vision_config
|
||||
self.preprocessor_config = {**self.preprocessor_config, **self.preprocessor_config.get("vision_config", {})}
|
||||
|
||||
vis = self.hparams_vision
|
||||
# in this config, hidden_size is the adapter output width; embed_dim is the tower width
|
||||
vis["hidden_size"] = vis["embed_dim"]
|
||||
vis["image_size"] = 0 # dynamic resolution
|
||||
self.pyramid = [max(0, n) for n in vis["pyramid_num_routed"]]
|
||||
|
||||
if vis.get("adapter_type") != "patch_merger" or not vis.get("pre_pixel_shuffle"):
|
||||
raise ValueError("dots3-note vision conversion requires adapter_type=patch_merger and pre_pixel_shuffle")
|
||||
if vis.get("router_scoring_func", "sigmoid") != "sigmoid" or vis.get("router_scale", 1.0) != 1.0:
|
||||
raise ValueError("dots3-note vision conversion only supports sigmoid routing with router_scale=1.0")
|
||||
if vis.get("temporal_patch_size", 1) != 1 or vis.get("use_bias") or not vis.get("use_qk_norm"):
|
||||
raise ValueError("unsupported dots3-note vision config variant")
|
||||
|
||||
aud = self.hparams_audio
|
||||
if not aud.get("use_conv2d_stem") or not aud.get("use_rope") or not aud.get("use_rms_norm") or aud.get("use_causal"):
|
||||
raise ValueError("unsupported dots3-note audio config variant")
|
||||
if aud["whisper_config"].get("activation_function") != "swiglu":
|
||||
raise ValueError("dots3-note audio conversion requires the swiglu activation")
|
||||
if aud.get("merge_factor", 1) != 1 or aud.get("chunk_seconds") != 60:
|
||||
raise ValueError("unsupported dots3-note audio chunking config")
|
||||
# the graph hard-codes these rope parameters
|
||||
rope = aud.get("rope_parameters", {})
|
||||
if rope.get("partial_rotary_factor") != 0.5 or rope.get("rope_theta") != 10000.0:
|
||||
raise ValueError("unsupported dots3-note audio rope config")
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
cfg = self.global_config.get("audio_config")
|
||||
if cfg is not None:
|
||||
# aliases so MmprojModel.find_aparam() / n_block_keys can resolve them
|
||||
whisper = cfg["whisper_config"]
|
||||
cfg["hidden_size"] = whisper["d_model"]
|
||||
cfg["intermediate_size"] = whisper["encoder_ffn_dim"]
|
||||
cfg["num_attention_heads"] = whisper["encoder_attention_heads"]
|
||||
cfg["num_hidden_layers"] = whisper["encoder_layers"]
|
||||
return cfg
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
assert self.hparams_vision is not None
|
||||
assert self.hparams_audio is not None
|
||||
|
||||
self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.DOTS3NOTE_V)
|
||||
self.gguf_writer.add_vision_use_silu(True)
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["rms_norm_eps"])
|
||||
self.gguf_writer.add_vision_spatial_merge_size(self.hparams_vision["spatial_merge_size"])
|
||||
self.gguf_writer.add_vision_min_pixels(self.preprocessor_config["min_pixels"])
|
||||
self.gguf_writer.add_vision_max_pixels(self.preprocessor_config["max_pixels"])
|
||||
# pyramid MoE: per-block routed expert count, 0 = dense block
|
||||
self.gguf_writer.add_vision_expert_count_per_layer(self.pyramid)
|
||||
self.gguf_writer.add_vision_expert_used_count(int(self.hparams_vision["capacity_factor"]))
|
||||
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.DOTS3NOTE_A)
|
||||
self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["whisper_config"]["num_mel_bins"])
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-6) # Dots3NoteAudioRMSNorm default
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(("vision_encoder.", "audio_encoder.")):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
_vis_experts: dict[int, dict[str, Tensor]] | None = None
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# router params have no .weight suffix in the checkpoint, but gguf tools expect one
|
||||
if name.endswith((".gate_weight", ".router_bias")):
|
||||
name += ".weight"
|
||||
|
||||
# audio fc1 fuses gate and up for swiglu; split it
|
||||
if ".speech_encoder.layers." in name and ".fc1." in name:
|
||||
gate, up = data_torch.chunk(2, dim=0)
|
||||
yield from super().modify_tensors(gate, name.replace(".fc1.", ".fc1_gate."), bid)
|
||||
yield from super().modify_tensors(up, name.replace(".fc1.", ".fc1_up."), bid)
|
||||
return
|
||||
|
||||
# vision MoE: stack per-expert weights into a single 3D tensor per block
|
||||
if ".mlp.experts." in name:
|
||||
assert bid is not None
|
||||
n_expert = self.pyramid[bid]
|
||||
if self._vis_experts is None:
|
||||
self._vis_experts = {}
|
||||
buf = self._vis_experts.setdefault(bid, {})
|
||||
buf[name] = data_torch
|
||||
|
||||
if len(buf) >= n_expert * 3:
|
||||
for w_name in ("fc1", "fc2", "fc3"):
|
||||
datas: list[Tensor] = []
|
||||
for xid in range(n_expert):
|
||||
ename = f"vision_encoder.blocks.{bid}.mlp.experts.{xid}.{w_name}.weight"
|
||||
datas.append(buf.pop(ename))
|
||||
merged = torch.stack(datas, dim=0)
|
||||
yield from super().modify_tensors(merged, f"vision_encoder.blocks.{bid}.mlp.experts.{w_name}.weight", bid)
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
def prepare_tensors(self):
|
||||
super().prepare_tensors()
|
||||
if self._vis_experts is not None:
|
||||
leftover = [k for d in self._vis_experts.values() for k in d.keys()]
|
||||
if leftover:
|
||||
raise ValueError(f"unprocessed vision experts: {leftover}")
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# FP32 routing is load-bearing for the vision MoE (near-tied expert scores)
|
||||
if ".ffn_gate_inp." in new_name or ".exp_probs_b." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
if ".conv2d" in new_name or "a.conv_out" in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
|
|
|||
|
|
@ -364,6 +364,8 @@ class Keys:
|
|||
IMAGE_MEAN = "clip.vision.image_mean"
|
||||
IMAGE_STD = "clip.vision.image_std"
|
||||
SPATIAL_MERGE_SIZE = "clip.vision.spatial_merge_size"
|
||||
EXPERT_COUNT_PER_LAYER = "clip.vision.expert_count_per_layer" # dots3note pyramid MoE, 0 = dense layer
|
||||
EXPERT_USED_COUNT = "clip.vision.expert_used_count"
|
||||
USE_GELU = "clip.use_gelu"
|
||||
USE_SILU = "clip.use_silu"
|
||||
N_WA_PATTERN = "clip.vision.n_wa_pattern" # used by qwen2.5vl
|
||||
|
|
@ -874,6 +876,11 @@ class MODEL_TENSOR(IntEnum):
|
|||
V_ENC_FFN_UP = auto()
|
||||
V_ENC_FFN_GATE = auto()
|
||||
V_ENC_FFN_DOWN = auto()
|
||||
V_ENC_FFN_GATE_INP = auto() # dots3note vision MoE router
|
||||
V_ENC_FFN_GATE_EXPS = auto()
|
||||
V_ENC_FFN_UP_EXPS = auto()
|
||||
V_ENC_FFN_DOWN_EXPS = auto()
|
||||
V_ENC_FFN_EXP_PROBS_B = auto()
|
||||
V_ENC_ATTN_POST_NORM = auto() # gemma4
|
||||
V_ENC_FFN_POST_NORM = auto()
|
||||
V_LAYER_SCALE_1 = auto()
|
||||
|
|
@ -1591,6 +1598,11 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
|||
MODEL_TENSOR.V_ENC_FFN_UP: "v.blk.{bid}.ffn_up",
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE: "v.blk.{bid}.ffn_gate",
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN: "v.blk.{bid}.ffn_down",
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE_INP: "v.blk.{bid}.ffn_gate_inp",
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE_EXPS: "v.blk.{bid}.ffn_gate_exps",
|
||||
MODEL_TENSOR.V_ENC_FFN_UP_EXPS: "v.blk.{bid}.ffn_up_exps",
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS: "v.blk.{bid}.ffn_down_exps",
|
||||
MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B: "v.blk.{bid}.exp_probs_b",
|
||||
MODEL_TENSOR.V_ENC_ATTN_POST_NORM: "v.blk.{bid}.attn_post_norm",
|
||||
MODEL_TENSOR.V_ENC_FFN_POST_NORM: "v.blk.{bid}.ffn_post_norm",
|
||||
MODEL_TENSOR.V_LAYER_SCALE_1: "v.blk.{bid}.ls1",
|
||||
|
|
@ -1913,6 +1925,11 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
|||
MODEL_TENSOR.V_ENC_FFN_UP,
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE,
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN,
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE_INP,
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE_EXPS,
|
||||
MODEL_TENSOR.V_ENC_FFN_UP_EXPS,
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS,
|
||||
MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B,
|
||||
MODEL_TENSOR.V_ENC_ATTN_POST_NORM,
|
||||
MODEL_TENSOR.V_ENC_FFN_POST_NORM,
|
||||
MODEL_TENSOR.V_LAYER_SCALE_1,
|
||||
|
|
@ -5497,6 +5514,8 @@ class VisionProjectorType:
|
|||
COGVLM = "cogvlm"
|
||||
JANUS_PRO = "janus_pro"
|
||||
DOTSOCR = "dots_ocr"
|
||||
DOTS3NOTE_V = "dots3note_v"
|
||||
DOTS3NOTE_A = "dots3note_a" # audio
|
||||
DEEPSEEKOCR = "deepseekocr"
|
||||
DEEPSEEKOCR2 = "deepseekocr2"
|
||||
LFM2A = "lfm2a" # audio
|
||||
|
|
|
|||
|
|
@ -1327,6 +1327,12 @@ class GGUFWriter:
|
|||
def add_vision_spatial_merge_size(self, value: int) -> None:
|
||||
self.add_uint32(Keys.ClipVision.SPATIAL_MERGE_SIZE, value)
|
||||
|
||||
def add_vision_expert_count_per_layer(self, value: Sequence[int]) -> None:
|
||||
self.add_array(Keys.ClipVision.EXPERT_COUNT_PER_LAYER, value)
|
||||
|
||||
def add_vision_expert_used_count(self, value: int) -> None:
|
||||
self.add_uint32(Keys.ClipVision.EXPERT_USED_COUNT, value)
|
||||
|
||||
def add_vision_use_gelu(self, value: bool) -> None:
|
||||
self.add_bool(Keys.ClipVision.USE_GELU, value)
|
||||
|
||||
|
|
|
|||
|
|
@ -1454,6 +1454,7 @@ class TensorNameMap:
|
|||
"mlp_AR.linear_{bid}", # PaddleOCR-VL
|
||||
"merger.mlp.{bid}",
|
||||
"vision_tower.merger.mlp.{bid}", # dots.ocr
|
||||
"vision_encoder.adapter.mlp.{bid}", # dots3note
|
||||
"vit.perceive.proj.{bid}", # HunyuanVL (proj.0 = conv1, proj.2 = conv2)
|
||||
),
|
||||
|
||||
|
|
@ -1504,6 +1505,7 @@ class TensorNameMap:
|
|||
"vision_model.radio_model.model.patch_generator.embedder", # Nemotron Nano v2 VL
|
||||
"model.vision_tower.patch_embedder.input_proj", # gemma4
|
||||
"vision_tower.patch_embed.patchifier.proj", # dots.ocr
|
||||
"vision_encoder.patch_embed.proj", # dots3note
|
||||
"vision_model.conv1", # Step3-VL
|
||||
"model.vision_embedder.patch_dense", # gemma4 unified
|
||||
"model.vision_tower.patch_embedder.patch_embedding", # muse-glimmer
|
||||
|
|
@ -1512,6 +1514,7 @@ class TensorNameMap:
|
|||
MODEL_TENSOR.V_ENC_EMBD_NORM: (
|
||||
"visual.post_conv_layernorm", # glm4v
|
||||
"vision_tower.patch_embed.patchifier.norm", # dots.ocr
|
||||
"vision_encoder.patch_embed.norm", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: (
|
||||
|
|
@ -1551,6 +1554,7 @@ class TensorNameMap:
|
|||
MODEL_TENSOR.V_ENC_ATTN_QKV: (
|
||||
"visual.blocks.{bid}.attn.qkv", # qwen3vl
|
||||
"vision_tower.blocks.{bid}.attn.qkv", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.attn.qkv", # dots3note
|
||||
"model.vision.transformer.layers.{bid}.attention.query_key_value", # cogvlm
|
||||
"model.vision_model.transformer.layers.{bid}.self_attn.qkv_proj", # Deepseek-OCR CLIP
|
||||
"vision_tower.encoder.blocks.{bid}.wqkv", # Kimi-K2.5
|
||||
|
|
@ -1579,6 +1583,7 @@ class TensorNameMap:
|
|||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
|
||||
"vision_encoder.blocks.{bid}.attn.q_norm", # dots3note
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.attn.q_norm", # InternVL
|
||||
"model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1
|
||||
"visual.blocks.{bid}.attn.q_norm", # GLM-OCR
|
||||
|
|
@ -1606,6 +1611,7 @@ class TensorNameMap:
|
|||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
|
||||
"vision_encoder.blocks.{bid}.attn.k_norm", # dots3note
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.attn.k_norm", # InternVL
|
||||
"model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1
|
||||
"visual.blocks.{bid}.attn.k_norm", # GLM-OCR
|
||||
|
|
@ -1651,6 +1657,7 @@ class TensorNameMap:
|
|||
"siglip2.vision_model.encoder.layers.{bid}.layer_norm1",
|
||||
"vision_model.radio_model.model.blocks.{bid}.norm1", # Nemotron Nano v2 VL
|
||||
"vision_tower.blocks.{bid}.norm1", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.norm_1", # dots3note
|
||||
"vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL
|
||||
"model.qwen2_model.model.model.layers.{bid}.input_layernorm", # Deepseek-OCR-2 qwen2
|
||||
"model.vision_tower.layers.{bid}.norm1", # muse-glimmer
|
||||
|
|
@ -1678,6 +1685,7 @@ class TensorNameMap:
|
|||
"model.qwen2_model.model.model.layers.{bid}.self_attn.o_proj", # Deepseek-OCR-2 qwen2
|
||||
"vision_model.model.layers.{bid}.self_attn.o_proj.linear", # gemma4
|
||||
"vision_tower.blocks.{bid}.attn.proj", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.attn.proj", # dots3note
|
||||
"vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL
|
||||
"model.vision_tower.layers.{bid}.attn.proj", # muse-glimmer
|
||||
),
|
||||
|
|
@ -1706,12 +1714,14 @@ class TensorNameMap:
|
|||
"vision_model.radio_model.model.blocks.{bid}.norm2", # Nemotron Nano v2 VL
|
||||
"vision_model.model.layers.{bid}.pre_feedforward_layernorm", # gemma4
|
||||
"vision_tower.blocks.{bid}.norm2", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.norm_2", # dots3note
|
||||
"vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL
|
||||
"model.qwen2_model.model.model.layers.{bid}.post_attention_layernorm", # Deepseek-OCR-2 qwen2
|
||||
"model.vision_tower.layers.{bid}.norm2", # muse-glimmer
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_UP: (
|
||||
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",
|
||||
"model.vision_tower.encoder.layers.{bid}.mlp.fc1", # minicpmv4_6
|
||||
|
|
@ -1737,6 +1747,7 @@ class TensorNameMap:
|
|||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE: (
|
||||
"vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
|
||||
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
|
||||
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
|
||||
"visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
|
||||
|
|
@ -1745,6 +1756,7 @@ class TensorNameMap:
|
|||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN: (
|
||||
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",
|
||||
"model.vision_tower.encoder.layers.{bid}.mlp.fc2", # minicpmv4_6
|
||||
|
|
@ -1769,6 +1781,29 @@ class TensorNameMap:
|
|||
"model.vision_tower.layers.{bid}.mlp.fc2", # muse-glimmer
|
||||
),
|
||||
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE_INP: (
|
||||
"vision_encoder.blocks.{bid}.mlp.gate_weight", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B: (
|
||||
"vision_encoder.blocks.{bid}.mlp.router_bias", # dots3note
|
||||
),
|
||||
|
||||
# note: expert weights are stacked into a single 3D tensor in conversion code,
|
||||
# which emits the pseudo-names below
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE_EXPS: (
|
||||
"vision_encoder.blocks.{bid}.mlp.experts.fc1", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_UP_EXPS: (
|
||||
"vision_encoder.blocks.{bid}.mlp.experts.fc3", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS: (
|
||||
"vision_encoder.blocks.{bid}.mlp.experts.fc2", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_POST_NORM: (
|
||||
"vision_model.model.layers.{bid}.post_attention_layernorm", # gemma4
|
||||
),
|
||||
|
|
@ -1800,6 +1835,7 @@ class TensorNameMap:
|
|||
"vision_model.layernorm_pre", # llama4
|
||||
"model.vision_model.pre_layrnorm", # Deepseek-OCR CLIP
|
||||
"vision_tower.patch_embed.patchifier.norm", # dots.ocr
|
||||
"vision_encoder.patch_embed.norm", # dots3note
|
||||
"vision_model.ln_pre", # Step3-VL
|
||||
"model.vision_tower.ln_pre", # muse-glimmer
|
||||
),
|
||||
|
|
@ -1821,6 +1857,7 @@ class TensorNameMap:
|
|||
MODEL_TENSOR.V_MM_POST_NORM: (
|
||||
"visual.merger.post_projection_norm", # glm4v
|
||||
"vision_tower.post_trunk_norm", # dots.ocr
|
||||
"vision_encoder.post_trunk_norm", # dots3note
|
||||
"vit.perceive.after_rms", # HunyuanVL
|
||||
),
|
||||
|
||||
|
|
@ -1838,6 +1875,7 @@ class TensorNameMap:
|
|||
"mlp_AR.pre_norm", # PaddleOCR-VL
|
||||
"merger.ln_q",
|
||||
"vision_tower.merger.ln_q", # dots.ocr
|
||||
"vision_encoder.adapter.ln_q", # dots3note
|
||||
"model.merger.mlp.0.pre_norm", # minicpmv4_6
|
||||
),
|
||||
|
||||
|
|
@ -2173,10 +2211,12 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_ENC_CONV2D: (
|
||||
"audio_tower.conv2d{bid}", # qwen3omni
|
||||
"audio_encoder.dots_encoder.speech_encoder.conv2d{bid}", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_CONV_OUT: (
|
||||
"audio_tower.conv_out", # qwen3omni
|
||||
"audio_encoder.dots_encoder.speech_encoder.conv_out", # dots3note
|
||||
"speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation
|
||||
),
|
||||
|
||||
|
|
@ -2184,12 +2224,14 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_POST_NORM: (
|
||||
"audio_tower.layer_norm", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note
|
||||
"audio_tower.ln_post", # qwen2omni
|
||||
"encoder.layer_norm", # mimo-audio-tokenizer
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_ATTN_Q: (
|
||||
"audio_tower.layers.{bid}.self_attn.q_proj", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.q_proj", # dots3note
|
||||
"conformer.layers.{bid}.self_attn.linear_q", # lfm2
|
||||
"conformer.layers.{bid}.attention.attn.q_proj", # gemma3n
|
||||
"conformer.layers.{bid}.self_attn.q_proj", # gemma4
|
||||
|
|
@ -2200,6 +2242,7 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_ENC_ATTN_K: (
|
||||
"audio_tower.layers.{bid}.self_attn.k_proj", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.k_proj", # dots3note
|
||||
"conformer.layers.{bid}.self_attn.linear_k", # lfm2
|
||||
"conformer.layers.{bid}.attention.attn.k_proj", # gemma3n
|
||||
"conformer.layers.{bid}.self_attn.k_proj", # gemma4
|
||||
|
|
@ -2210,6 +2253,7 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_ENC_ATTN_V: (
|
||||
"audio_tower.layers.{bid}.self_attn.v_proj", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.v_proj", # dots3note
|
||||
"conformer.layers.{bid}.self_attn.linear_v", # lfm2
|
||||
"conformer.layers.{bid}.attention.attn.v_proj", # gemma3n
|
||||
"conformer.layers.{bid}.self_attn.v_proj", # gemma4
|
||||
|
|
@ -2241,6 +2285,7 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_ENC_INPUT_NORM: (
|
||||
"audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn_layer_norm", # dots3note
|
||||
"conformer.layers.{bid}.norm_self_att", # lfm2
|
||||
"conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
|
||||
"sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet
|
||||
|
|
@ -2250,6 +2295,7 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_ENC_OUTPUT: (
|
||||
"audio_tower.layers.{bid}.self_attn.out_proj", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.out_proj", # dots3note
|
||||
"conformer.layers.{bid}.self_attn.linear_out", # lfm2
|
||||
"conformer.layers.{bid}.attention.post", # gemma3n
|
||||
"conformer.layers.{bid}.self_attn.post", # gemma4
|
||||
|
|
@ -2260,6 +2306,7 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_ENC_OUTPUT_NORM: (
|
||||
"audio_tower.layers.{bid}.final_layer_norm", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.final_layer_norm", # dots3note
|
||||
"conformer.layers.{bid}.norm_out", # lfm2
|
||||
"conformer.layers.{bid}.attention.post_norm", # gemma3n
|
||||
"sound_encoder.encoder.layers.{bid}.norm_out", # parakeet
|
||||
|
|
@ -2285,6 +2332,7 @@ class TensorNameMap:
|
|||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_FFN_UP: (
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_up", # dots3note (split from fc1 in conversion code)
|
||||
"audio_tower.layers.{bid}.fc1", # ultravox
|
||||
"conformer.layers.{bid}.feed_forward1.linear1", # lfm2
|
||||
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
|
||||
|
|
@ -2294,9 +2342,12 @@ class TensorNameMap:
|
|||
"encoder.layers.{bid}.fc1", # mimo-audio-tokenizer
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_FFN_GATE: (),
|
||||
MODEL_TENSOR.A_ENC_FFN_GATE: (
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_gate", # dots3note (split from fc1 in conversion code)
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_FFN_DOWN: (
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc2", # dots3note
|
||||
"audio_tower.layers.{bid}.fc2", # ultravox
|
||||
"conformer.layers.{bid}.feed_forward1.linear2", # lfm2
|
||||
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
|
||||
|
|
@ -2380,6 +2431,7 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_MMPROJ: (
|
||||
"audio.multi_modal_projector.linear_{bid}", # ultravox, meralion
|
||||
"audio_encoder.audio_adapter.proj.{bid}", # dots3note (proj.1, proj.3)
|
||||
"audio_adapter.model.{bid}", # lfm2
|
||||
"audio_tower.proj{bid}", # qwen3omni
|
||||
"sound_projection.linear{bid}", # parakeet (linear1, linear2)
|
||||
|
|
@ -2394,6 +2446,7 @@ class TensorNameMap:
|
|||
|
||||
MODEL_TENSOR.A_MM_NORM_PRE: (
|
||||
"audio.multi_modal_projector.ln_pre", # ultravox
|
||||
"audio_encoder.audio_adapter.proj.0", # dots3note
|
||||
"sound_projection.norm", # parakeet
|
||||
),
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ add_library(mtmd
|
|||
models/models.h
|
||||
models/cogvlm.cpp
|
||||
models/conformer.cpp
|
||||
models/dots3note.cpp
|
||||
models/dotsocr.cpp
|
||||
models/exaone4_5.cpp
|
||||
models/gemma4a.cpp
|
||||
|
|
|
|||
|
|
@ -120,6 +120,12 @@ struct clip_graph {
|
|||
ffn_op_type type_op,
|
||||
int il) const;
|
||||
|
||||
ggml_tensor * build_moe_ffn(
|
||||
ggml_tensor * cur,
|
||||
const clip_layer & layer,
|
||||
ffn_op_type type_op,
|
||||
int il) const;
|
||||
|
||||
ggml_tensor * build_attn(
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@
|
|||
#define KEY_SAM_N_HEAD "clip.vision.sam.head_count"
|
||||
#define KEY_SAM_N_BLOCK "clip.vision.sam.block_count"
|
||||
#define KEY_SAM_N_EMBD "clip.vision.sam.embedding_length"
|
||||
#define KEY_VISION_N_EXPERT_USED "clip.vision.expert_used_count"
|
||||
// audio-specific
|
||||
#define KEY_AUDIO_PROJ_TYPE "clip.audio.projector_type" // for models with mixed modalities
|
||||
#define KEY_A_NUM_MEL_BINS "clip.audio.num_mel_bins"
|
||||
|
|
@ -119,7 +120,11 @@
|
|||
#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s"
|
||||
#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s"
|
||||
#define TN_FFN_UP "%s.blk.%d.ffn_up.%s"
|
||||
#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s"
|
||||
#define TN_FFN_GATE_INP "%s.blk.%d.ffn_gate_inp.%s" // MoE router (dots3note)
|
||||
#define TN_FFN_GATE_EXPS "%s.blk.%d.ffn_gate_exps.%s"
|
||||
#define TN_FFN_UP_EXPS "%s.blk.%d.ffn_up_exps.%s"
|
||||
#define TN_FFN_DOWN_EXPS "%s.blk.%d.ffn_down_exps.%s"
|
||||
#define TN_FFN_EXP_PROBS_B "%s.blk.%d.exp_probs_b.%s"
|
||||
#define TN_LN_1 "%s.blk.%d.ln1.%s" // layer norm
|
||||
#define TN_LN_2 "%s.blk.%d.ln2.%s" // layer norm
|
||||
#define TN_LS_1 "%s.blk.%d.ls1.%s" // layer scale
|
||||
|
|
@ -471,6 +476,8 @@ enum projector_type {
|
|||
PROJECTOR_TYPE_COGVLM,
|
||||
PROJECTOR_TYPE_JANUS_PRO,
|
||||
PROJECTOR_TYPE_DOTS_OCR,
|
||||
PROJECTOR_TYPE_DOTS3NOTE_V,
|
||||
PROJECTOR_TYPE_DOTS3NOTE_A,
|
||||
PROJECTOR_TYPE_DEEPSEEKOCR,
|
||||
PROJECTOR_TYPE_DEEPSEEKOCR2,
|
||||
PROJECTOR_TYPE_LFM2A,
|
||||
|
|
@ -533,6 +540,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
|||
{ PROJECTOR_TYPE_COGVLM, "cogvlm"},
|
||||
{ PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
|
||||
{ PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"},
|
||||
{ PROJECTOR_TYPE_DOTS3NOTE_V, "dots3note_v"},
|
||||
{ PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"},
|
||||
{ PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"},
|
||||
{ PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},
|
||||
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ struct clip_hparams {
|
|||
|
||||
float eps = 1e-6;
|
||||
float rope_theta = 0.0;
|
||||
int32_t n_expert_used = 0;
|
||||
std::vector<int32_t> feature_layers;
|
||||
int32_t attn_window_size = 0;
|
||||
int32_t n_wa_pattern = 0;
|
||||
|
|
@ -259,6 +260,13 @@ struct clip_layer {
|
|||
ggml_tensor * ff_down_w = nullptr;
|
||||
ggml_tensor * ff_down_b = nullptr;
|
||||
|
||||
// MoE FFN (dots3note vision pyramid blocks)
|
||||
ggml_tensor * ff_gate_inp_w = nullptr;
|
||||
ggml_tensor * ff_gate_exps_w = nullptr;
|
||||
ggml_tensor * ff_up_exps_w = nullptr;
|
||||
ggml_tensor * ff_down_exps_w = nullptr;
|
||||
ggml_tensor * ff_exp_probs_b = nullptr;
|
||||
|
||||
// layernorm 2 (or pre-FFN norm)
|
||||
ggml_tensor * ln_2_w = nullptr;
|
||||
ggml_tensor * ln_2_b = nullptr;
|
||||
|
|
|
|||
|
|
@ -514,11 +514,13 @@ ggml_tensor * clip_graph::build_vit(
|
|||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
ffn_t, il);
|
||||
cur = layer.ff_gate_exps_w
|
||||
? build_moe_ffn(cur, layer, ffn_t, il)
|
||||
: build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
ffn_t, il);
|
||||
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
|
|
@ -699,6 +701,50 @@ ggml_tensor * clip_graph::build_ffn(
|
|||
return cur;
|
||||
}
|
||||
|
||||
// MoE FFN with sigmoid router and normalized top-k weights (dots3note vision)
|
||||
// the router runs in fp32; exp_probs_b only affects expert selection, not the weights
|
||||
ggml_tensor * clip_graph::build_moe_ffn(ggml_tensor * cur, const clip_layer & layer, ffn_op_type type_op, int il) const {
|
||||
const int64_t n_tokens = cur->ne[1];
|
||||
const int64_t n_expert = layer.ff_gate_exps_w->ne[2];
|
||||
const int64_t n_expert_used = std::min((int64_t) hparams.n_expert_used, n_expert);
|
||||
GGML_ASSERT(n_expert_used > 0);
|
||||
GGML_ASSERT(type_op == FFN_SILU);
|
||||
|
||||
ggml_tensor * probs = ggml_sigmoid(ctx0, build_mm(layer.ff_gate_inp_w, cur)); // [n_expert, n_tokens]
|
||||
cb(probs, "ffn_moe_probs", il);
|
||||
|
||||
ggml_tensor * sel = layer.ff_exp_probs_b
|
||||
? ggml_add(ctx0, probs, layer.ff_exp_probs_b)
|
||||
: probs;
|
||||
ggml_tensor * selected = ggml_top_k(ctx0, sel, n_expert_used); // [n_expert_used, n_tokens]
|
||||
|
||||
ggml_tensor * weights = ggml_get_rows(ctx0,
|
||||
ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens), selected);
|
||||
weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens);
|
||||
weights = ggml_div(ctx0, weights, ggml_sum_rows(ctx0, weights));
|
||||
weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);
|
||||
cb(weights, "ffn_moe_weights", il);
|
||||
|
||||
cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, n_tokens);
|
||||
ggml_tensor * gate = ggml_mul_mat_id(ctx0, layer.ff_gate_exps_w, cur, selected); // [n_ff, n_expert_used, n_tokens]
|
||||
ggml_tensor * up = ggml_mul_mat_id(ctx0, layer.ff_up_exps_w, cur, selected);
|
||||
cur = ggml_mul(ctx0, ggml_silu(ctx0, gate), up);
|
||||
cur = ggml_mul_mat_id(ctx0, layer.ff_down_exps_w, cur, selected); // [n_embd, n_expert_used, n_tokens]
|
||||
cur = ggml_mul(ctx0, cur, weights);
|
||||
|
||||
// sum over the selected experts
|
||||
ggml_tensor * out = nullptr;
|
||||
for (int64_t i = 0; i < n_expert_used; i++) {
|
||||
ggml_tensor * v = ggml_view_2d(ctx0, cur, cur->ne[0], n_tokens, cur->nb[2], i * cur->nb[1]);
|
||||
out = out ? ggml_add(ctx0, out, v) : v;
|
||||
}
|
||||
if (n_expert_used == 1) {
|
||||
out = ggml_cont(ctx0, out);
|
||||
}
|
||||
cb(out, "ffn_moe_out", il);
|
||||
return out;
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph::build_attn(
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
|
|
@ -933,9 +979,14 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
|
|||
builder = std::make_unique<clip_graph_pixtral>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V: // same ViT + merger; pyramid MoE is handled by build_vit
|
||||
{
|
||||
builder = std::make_unique<clip_graph_dotsocr>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_A:
|
||||
{
|
||||
builder = std::make_unique<clip_graph_dots3note_a>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_QWEN2VL:
|
||||
case PROJECTOR_TYPE_QWEN25VL:
|
||||
{
|
||||
|
|
@ -1510,6 +1561,25 @@ struct clip_model_loader {
|
|||
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
|
||||
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V:
|
||||
{
|
||||
hparams.rope_theta = 10000.0f;
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
|
||||
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
|
||||
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
|
||||
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
|
||||
get_u32(KEY_VISION_N_EXPERT_USED, hparams.n_expert_used);
|
||||
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_A:
|
||||
{
|
||||
hparams.rope_theta = 10000.0f;
|
||||
hparams.audio_chunk_len = 60; // in seconds
|
||||
hparams.audio_sample_rate = 16000;
|
||||
hparams.audio_n_fft = 400;
|
||||
hparams.audio_window_len = 400;
|
||||
hparams.audio_hop_len = 160;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_KIMIVL:
|
||||
{
|
||||
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
|
||||
|
|
@ -2190,12 +2260,20 @@ struct clip_model_loader {
|
|||
layer.ln_1_b = get_tensor(string_format(TN_LN_1, prefix, il, "bias"), false);
|
||||
layer.ln_2_b = get_tensor(string_format(TN_LN_2, prefix, il, "bias"), false);
|
||||
|
||||
// MoE ffn (dots3note vision pyramid blocks); replaces the dense ffn when present
|
||||
layer.ff_gate_inp_w = get_tensor(string_format(TN_FFN_GATE_INP, prefix, il, "weight"), false);
|
||||
layer.ff_gate_exps_w = get_tensor(string_format(TN_FFN_GATE_EXPS, prefix, il, "weight"), false);
|
||||
layer.ff_up_exps_w = get_tensor(string_format(TN_FFN_UP_EXPS, prefix, il, "weight"), false);
|
||||
layer.ff_down_exps_w = get_tensor(string_format(TN_FFN_DOWN_EXPS, prefix, il, "weight"), false);
|
||||
layer.ff_exp_probs_b = get_tensor(string_format(TN_FFN_EXP_PROBS_B, prefix, il, "weight"), false);
|
||||
const bool is_moe = layer.ff_gate_exps_w != nullptr;
|
||||
|
||||
// ffn
|
||||
layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"));
|
||||
layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, prefix, il, "weight"), !is_moe);
|
||||
layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, prefix, il, "bias"), false);
|
||||
layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, prefix, il, "weight"), false);
|
||||
layer.ff_gate_b = get_tensor(string_format(TN_FFN_GATE, prefix, il, "bias"), false);
|
||||
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"));
|
||||
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"), !is_moe);
|
||||
layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias"), false);
|
||||
|
||||
// mimovl per-head attention sink bias
|
||||
|
|
@ -2677,6 +2755,7 @@ struct clip_model_loader {
|
|||
model.mm_patch_merger_w = get_tensor(string_format(TN_MM_PATCH_MERGER, "weight"), false);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V:
|
||||
{
|
||||
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
|
||||
model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
|
||||
|
|
@ -2687,6 +2766,23 @@ struct clip_model_loader {
|
|||
// post_trunk_norm: applied after all ViT blocks, before the merger
|
||||
model.post_ln_w = get_tensor(string_format(TN_MM_POST_NORM, "weight"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_A:
|
||||
{
|
||||
model.conv2d_1_w = get_tensor(string_format(TN_CONV2D, 1, "weight"));
|
||||
model.conv2d_1_b = get_tensor(string_format(TN_CONV2D, 1, "bias"));
|
||||
model.conv2d_2_w = get_tensor(string_format(TN_CONV2D, 2, "weight"));
|
||||
model.conv2d_2_b = get_tensor(string_format(TN_CONV2D, 2, "bias"));
|
||||
model.conv2d_3_w = get_tensor(string_format(TN_CONV2D, 3, "weight"));
|
||||
model.conv2d_3_b = get_tensor(string_format(TN_CONV2D, 3, "bias"));
|
||||
model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); // no bias
|
||||
// adapter: LayerNorm -> Linear -> GELU -> Linear
|
||||
model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"));
|
||||
model.mm_norm_pre_b = get_tensor(string_format(TN_MM_NORM_PRE, "bias"));
|
||||
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
|
||||
model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
|
||||
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
|
||||
model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
|
||||
} break;
|
||||
case PROJECTOR_TYPE_ULTRAVOX:
|
||||
{
|
||||
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
|
||||
|
|
@ -4075,12 +4171,18 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
|
|||
} break;
|
||||
case PROJECTOR_TYPE_PADDLEOCR:
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V:
|
||||
{
|
||||
// dynamic size
|
||||
int n_merge = ctx->model.hparams.n_merge;
|
||||
int stride = n_merge * n_merge;
|
||||
n_patches = CLIP_ALIGN(n_patches, stride) / stride;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_A:
|
||||
{
|
||||
// 3x stride-2 conv2d over mel frames
|
||||
n_patches = (img->nx() + 7) / 8;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PIXTRAL:
|
||||
case PROJECTOR_TYPE_LIGHTONOCR:
|
||||
{
|
||||
|
|
@ -4727,6 +4829,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
|
|||
set_input_i32("minimax_pos_w", pos_w);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V:
|
||||
{
|
||||
const int pw = image_size_width / patch_size;
|
||||
const int ph = image_size_height / patch_size;
|
||||
|
|
@ -5217,6 +5320,16 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
|
|||
}
|
||||
set_input_i32("pos_w", pos_data);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_A:
|
||||
{
|
||||
GGML_ASSERT(imgs.entries.size() == 1);
|
||||
const int n_pos = (imgs.entries.front().nx() + 7) / 8; // 3x stride-2 conv2d
|
||||
std::vector<int32_t> positions(n_pos);
|
||||
for (int i = 0; i < n_pos; i++) {
|
||||
positions[i] = i;
|
||||
}
|
||||
set_input_i32("positions", positions);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_GEMMA4A:
|
||||
{
|
||||
GGML_ASSERT(imgs.entries.size() == 1);
|
||||
|
|
@ -5713,6 +5826,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
|
|||
case PROJECTOR_TYPE_PIXTRAL:
|
||||
case PROJECTOR_TYPE_LIGHTONOCR:
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_A:
|
||||
return ctx->model.mm_2_w->ne[1];
|
||||
case PROJECTOR_TYPE_MLP_NORM:
|
||||
return ctx->model.mm_3_b->ne[0];
|
||||
|
|
|
|||
61
tools/mtmd/models/dots3note.cpp
Normal file
61
tools/mtmd/models/dots3note.cpp
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_dots3note_a::build() {
|
||||
// inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded
|
||||
// the reference impl zero-masks conv inputs beyond the valid length at each stage;
|
||||
// running on exactly the valid frames with the convs' zero padding is equivalent
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
GGML_ASSERT(inp->type == GGML_TYPE_F32);
|
||||
|
||||
// 3x conv2d (k=3, s=2, p=1) + gelu
|
||||
{
|
||||
auto conv_block = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
|
||||
x = ggml_conv_2d(ctx0, w, x, 2, 2, 1, 1, 1, 1);
|
||||
x = ggml_add(ctx0, x, ggml_reshape_4d(ctx0, b, 1, 1, x->ne[2], 1));
|
||||
return ggml_gelu_erf(ctx0, x);
|
||||
};
|
||||
|
||||
inp = conv_block(inp, model.conv2d_1_w, model.conv2d_1_b);
|
||||
inp = conv_block(inp, model.conv2d_2_w, model.conv2d_2_b);
|
||||
inp = conv_block(inp, model.conv2d_3_w, model.conv2d_3_b);
|
||||
// inp: [OW=n_frames/8, OH=n_mel/8, OC=480, 1]
|
||||
cb(inp, "after_conv_stem", -1);
|
||||
}
|
||||
|
||||
// [OW, OH, OC, 1] -> [OH*OC, OW], feature index f + OH*c (matches the reference permute+reshape)
|
||||
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
|
||||
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0] * inp->ne[1], inp->ne[2]);
|
||||
|
||||
// project to d_model (no bias)
|
||||
inp = ggml_mul_mat(ctx0, model.conv_out_w, inp);
|
||||
cb(inp, "after_conv_out", -1);
|
||||
|
||||
const int64_t n_pos = inp->ne[1];
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
// partial rotary: first half of each head, NEOX style
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2,
|
||||
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
};
|
||||
|
||||
ggml_tensor * cur = build_vit(inp, n_pos,
|
||||
NORM_TYPE_RMS, hparams.ffn_op,
|
||||
nullptr, add_pos);
|
||||
cb(cur, "after_transformer", -1);
|
||||
|
||||
// adapter: LayerNorm -> Linear -> GELU -> Linear
|
||||
cur = build_norm(cur, model.mm_norm_pre_w, model.mm_norm_pre_b, NORM_TYPE_NORMAL, 1e-5, -1);
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF, -1);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
|
|
@ -119,6 +119,11 @@ struct clip_graph_dotsocr : clip_graph {
|
|||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_dots3note_a : clip_graph {
|
||||
clip_graph_dots3note_a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_cogvlm : clip_graph {
|
||||
clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
|
|
|||
|
|
@ -723,6 +723,100 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa
|
|||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio_preprocessor_dots3note
|
||||
//
|
||||
// Matches Dots3NoteFeatureExtractor: the waveform is split into 60s chunks and each chunk gets
|
||||
// its own whisper-style log-mel (center=True, log10 + (max-8)/4). Only sample_length//hop frames
|
||||
// per chunk are valid; the reference masks everything beyond them, so we emit exactly that many.
|
||||
//
|
||||
|
||||
void mtmd_audio_preprocessor_dots3note::initialize() {
|
||||
cache.fill_sin_cos_table(hparams.audio_n_fft);
|
||||
cache.fill_hann_window(hparams.audio_window_len, true);
|
||||
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
|
||||
}
|
||||
|
||||
bool mtmd_audio_preprocessor_dots3note::preprocess(const float * samples,
|
||||
size_t n_samples,
|
||||
std::vector<mtmd_audio_mel> & output) {
|
||||
if (n_samples == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
GGML_ASSERT(!cache.sin_vals.empty());
|
||||
GGML_ASSERT(!cache.cos_vals.empty());
|
||||
GGML_ASSERT(!cache.filters.data.empty());
|
||||
|
||||
const int pad = hparams.audio_n_fft / 2; // center=True padding
|
||||
const int hop = hparams.audio_hop_len;
|
||||
const size_t chunk_samples = (size_t) hparams.audio_chunk_len * hparams.audio_sample_rate;
|
||||
|
||||
for (size_t start = 0; start < n_samples; start += chunk_samples) {
|
||||
const size_t n_chunk = std::min(chunk_samples, n_samples - start);
|
||||
const float * chunk = samples + start;
|
||||
|
||||
const int64_t n_valid = n_chunk / hop;
|
||||
if (n_valid == 0) {
|
||||
continue; // sub-hop tail, contributes no frames
|
||||
}
|
||||
|
||||
// reflect-pad the start; the reference zero-pads partial chunks to 60s before the STFT,
|
||||
// so a partial chunk sees zeros past its end while a full chunk reflects its own tail
|
||||
std::vector<float> padded(n_chunk + 2 * pad, 0.0f);
|
||||
for (int i = 0; i < pad; i++) {
|
||||
int src = pad - i;
|
||||
padded[i] = (src < (int) n_chunk) ? chunk[src] : 0.0f;
|
||||
}
|
||||
std::copy(chunk, chunk + n_chunk, padded.begin() + pad);
|
||||
if (n_chunk == chunk_samples) {
|
||||
for (int i = 0; i < pad; i++) {
|
||||
int src = (int) n_chunk - 2 - i;
|
||||
padded[n_chunk + pad + i] = (src >= 0) ? chunk[src] : 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
filter_params params;
|
||||
params.n_mel = hparams.n_mel_bins;
|
||||
params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
|
||||
params.hann_window_size = hparams.audio_window_len;
|
||||
params.hop_length = hop;
|
||||
params.sample_rate = hparams.audio_sample_rate;
|
||||
params.no_padding = true; // padding already applied above
|
||||
params.use_natural_log = false;
|
||||
|
||||
mtmd_audio_mel mel_full;
|
||||
if (!log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, mel_full)) {
|
||||
return false;
|
||||
}
|
||||
GGML_ASSERT(mel_full.n_len >= n_valid);
|
||||
|
||||
// per-chunk whisper-style normalization, then keep only the valid frames
|
||||
mtmd_audio_mel out;
|
||||
out.n_mel = mel_full.n_mel;
|
||||
out.n_len = n_valid;
|
||||
out.n_len_org = n_valid;
|
||||
out.data.resize((size_t) out.n_mel * (size_t) out.n_len);
|
||||
|
||||
double mmax = -1e20;
|
||||
for (int64_t m = 0; m < out.n_mel; m++) {
|
||||
for (int64_t t = 0; t < n_valid; t++) {
|
||||
mmax = std::max(mmax, (double) mel_full.data[(size_t) m * mel_full.n_len + t]);
|
||||
}
|
||||
}
|
||||
mmax -= 8.0;
|
||||
for (int64_t m = 0; m < out.n_mel; m++) {
|
||||
for (int64_t t = 0; t < n_valid; t++) {
|
||||
const double v = std::max((double) mel_full.data[(size_t) m * mel_full.n_len + t], mmax);
|
||||
out.data[(size_t) m * n_valid + t] = (float) ((v + 4.0) / 4.0);
|
||||
}
|
||||
}
|
||||
|
||||
output.push_back(std::move(out));
|
||||
}
|
||||
return !output.empty();
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio_preprocessor_mimo_audio
|
||||
//
|
||||
|
|
|
|||
|
|
@ -111,6 +111,15 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
|
|||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
|
|
|
|||
|
|
@ -825,6 +825,7 @@ struct mtmd_context {
|
|||
image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS_OCR:
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_V:
|
||||
{
|
||||
// <|img|> ... (image embeddings) ... <|endofimg|>
|
||||
img_beg = "<|img|>";
|
||||
|
|
@ -976,6 +977,13 @@ struct mtmd_context {
|
|||
aud_end = "<audio|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4ua>(ctx_a);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_DOTS3NOTE_A:
|
||||
{
|
||||
// <|audio_comp_start|> ... (embeddings) ... <|audio_comp_end|>
|
||||
aud_beg = "<|audio_comp_start|>";
|
||||
aud_end = "<|audio_comp_end|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_dots3note>(ctx_a);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MIMO_AUDIO:
|
||||
{
|
||||
aud_beg = "<|mimo_audio_start|>";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue