mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-29 18:24:29 +00:00
model : GraniteSWAForCausalLM / GraniteMoeSWAForCausalLM (#25505)
* feat(convert): Add conversion for GraniteSWAForCausalLM Branch: GraniteSWAForCausalLM AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(llama): Add granite_swa support Branch: GraniteSWAForCausalLM AI-usage: full (Bob, OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(conversion): Add conversion infra for rope_pattern array NOTE: There is other work also targeting this, so this may be removed depending on merge order. Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix(conversion): Fix SWA pattern logic and support for non-rope layers Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat(conversion): Add support for GraniteMoeSWA Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Add llama_hparams::has_rope and arch constants NOTE: This shadows the work done for Granite Speech https://github.com/ggml-org/llama.cpp/pull/25107 Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Add support for per-layer rope determination Branch: GraniteSWAForCausalLM AI-usage: full (Bob) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * style: Fix failing flake8 for extra newlines Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * test: Write out SLIDING_WINDOW_PATTERN in llama-model-saver Branch: GraniteSWAForCausalLM AI-usage: full (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix(convert): Fix missing registration for GraniteMoeSWAForCausalLM Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Load MoE params as optional Branch: GraniteSWAForCausalLM AI-usage: draft (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Handle MoE params in conversion branch: GraniteSWAForCausalLM AI-usage: full (OpenCode + Qwen3.6-35b) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * style: Remove unnecessary newline AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Remove unnecessary tensor additions to GRANITE architecture Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Correctly handle naming for ffn gate inp Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Always default hparams.rope_pattern to 1s This isn't strictly necessary, but it will allow other models to rely on hparams.has_rope(il) without needting to prepopulate. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Move to has_rope for all granite model architectures Now that we have a proper hparam for this, it's better to use it and not require a hacky fallback in the hparam method itself. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: No hacky rope_finetuned fallback in has_rope Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Fully remove rope hparam filling in granitemoe There are no granitemoe models that use NoPE (it's not actually used in the layer building below), so this was just dead code. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Save out rope_pattern in model-saver Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Set hparams.rope_finetuned for round trip Since the value is _read_ from rope_finetuned, we need to persist it when the model is saved with the saver. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Code review cleanup Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * refactor: Keep gate/up fused for MoE path Branch: GraniteSWAForCausalLM AI-usage: full (Claude + Sonnet 5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Skip GRANITE_SWA in model saver https://github.com/ggml-org/llama.cpp/pull/25505#discussion_r3773175651 Keeping is_swa_impl in the saver can break other models. Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * add sliding window pattern for model in test * style: Fix indentation Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * fix: Fix \r\n Thanks Claude! Branch: GraniteSWAForCausalLM AI-usage: none Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * feat: Keep shared expert fused Branch: GraniteSWAForCausalLM AI-usage: full (Claude + Sonnet 5) Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> * style: More indentation fixes Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> --------- Signed-off-by: Gabe Goodhart <ghart@us.ibm.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
Gabe Goodhart
GitHub
parent
6cc504a2e9
commit
7221e24f57
18 changed files with 517 additions and 18 deletions
|
|
@ -109,6 +109,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"GraniteSwitchForCausalLM": "granite",
|
||||
"GraniteSpeechForConditionalGeneration": "granite",
|
||||
"GraniteSpeechPlusForConditionalGeneration": "granite",
|
||||
"GraniteSWAForCausalLM": "granite",
|
||||
"GraniteMoeSWAForCausalLM": "granite",
|
||||
"Grok1ForCausalLM": "grok",
|
||||
"GrokForCausalLM": "grok",
|
||||
"GroveMoeForCausalLM": "grovemoe",
|
||||
|
|
|
|||
|
|
@ -74,6 +74,108 @@ class GraniteModel(LlamaModel):
|
|||
return super().filter_tensors(item)
|
||||
|
||||
|
||||
@ModelBase.register("GraniteSWAForCausalLM")
|
||||
class GraniteSWAModel(GraniteModel):
|
||||
"""Conversion for IBM's GraniteSWAForCausalLM (interleaved sliding window attention)"""
|
||||
model_arch = gguf.MODEL_ARCH.GRANITE_SWA
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
if name.endswith("sinks"):
|
||||
name += ".weight"
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
"""GraniteSWA uses Granite parameters plus sliding window configuration."""
|
||||
super().set_gguf_parameters()
|
||||
|
||||
# Add sliding_window from config
|
||||
sliding_window = self.hparams.get("sliding_window", 128)
|
||||
self.gguf_writer.add_sliding_window(sliding_window)
|
||||
logger.info("gguf: (granite_swa) sliding_window = %s", sliding_window)
|
||||
|
||||
# Derive sliding_window_pattern from layer_types
|
||||
if layer_types := self.hparams.get("layer_types"):
|
||||
is_swa = [t == "sliding_attention" for t in layer_types]
|
||||
self.gguf_writer.add_sliding_window_pattern(is_swa)
|
||||
logger.info("gguf: (granite_swa) sliding_window_pattern = %d SWA layers / %d total",
|
||||
sum(is_swa), len(is_swa))
|
||||
else:
|
||||
# Fall back to period-based pattern: i % 4 != 0
|
||||
# This matches the transformers default pattern
|
||||
n_layers = self.block_count
|
||||
is_swa = [i % 4 != 0 for i in range(n_layers)]
|
||||
self.gguf_writer.add_sliding_window_pattern(is_swa)
|
||||
logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total",
|
||||
sum(is_swa), n_layers)
|
||||
|
||||
# Add rope_pattern from no_rope_layers
|
||||
if no_rope_layers := self.hparams.get("no_rope_layers"):
|
||||
# Convert 1/0 to bool (1 = use RoPE, 0 = NoPE)
|
||||
rope_pattern = [bool(x) for x in no_rope_layers]
|
||||
self.gguf_writer.add_rope_pattern(rope_pattern)
|
||||
logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total",
|
||||
sum(rope_pattern), len(rope_pattern))
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeSWAForCausalLM")
|
||||
class GraniteMoeSWAModel(GraniteSWAModel):
|
||||
"""Conversion for IBM's GraniteMoeSWAForCausalLM (unified dense + MoE with iSWA)"""
|
||||
model_arch = gguf.MODEL_ARCH.GRANITE_SWA
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
if shared_intermediate_size := self.hparams.get("shared_intermediate_size"):
|
||||
self.gguf_writer.add_expert_shared_feed_forward_length(shared_intermediate_size)
|
||||
logger.info("gguf: (granitemoewa) shared_intermediate_size = %s", shared_intermediate_size)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
"""Split merged MoE tensors (gate+up) following standard MoE pattern."""
|
||||
|
||||
# Handle expert FFN tensors (merged gate+up) - swash format: experts.gate_up_proj
|
||||
# Kept fused since inference (build_moe_ffn) supports a single gate_up_exps
|
||||
# tensor for the routed experts.
|
||||
if name.endswith("block_sparse_moe.experts.gate_up_proj"):
|
||||
ffn_dim = self.hparams["intermediate_size"]
|
||||
assert data_torch.shape[-2] == 2 * ffn_dim, f"Merged FFN tensor size must be 2 * intermediate_size, got {data_torch.shape[-2]}"
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid)
|
||||
return
|
||||
|
||||
# Handle expert FFN down projection - swash format: experts.down_proj
|
||||
if name.endswith("block_sparse_moe.experts.down_proj"):
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_EXP, bid), bid)
|
||||
return
|
||||
|
||||
# Handle expert FFN tensors (merged gate+up) - standard granite format: input_linear.weight
|
||||
# Kept fused since inference (build_moe_ffn) supports a single gate_up_exps
|
||||
# tensor for the routed experts.
|
||||
if name.endswith("block_sparse_moe.input_linear.weight"):
|
||||
ffn_dim = self.hparams["intermediate_size"]
|
||||
assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * intermediate_size"
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_UP_EXP, bid), bid)
|
||||
return
|
||||
|
||||
# Handle shared expert FFN tensors (if present) - kept fused since
|
||||
# inference (build_ffn) supports a single ffn_up_shexp tensor with
|
||||
# LLM_FFN_SWIGLU for the shared expert.
|
||||
if name.endswith("shared_mlp.input_linear.weight"):
|
||||
ffn_dim = self.hparams.get("shared_intermediate_size", self.hparams["intermediate_size"])
|
||||
assert data_torch.shape[-2] == 2 * ffn_dim, "Merged FFN tensor size must be 2 * shared_intermediate_size"
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_SHEXP, bid), bid)
|
||||
return
|
||||
|
||||
# Handle shared expert output (if present)
|
||||
if name.endswith("shared_mlp.output_linear.weight"):
|
||||
yield from ModelBase.modify_tensors(self, data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, bid), bid)
|
||||
return
|
||||
|
||||
# Pass through to parent for all other tensors (including sinks)
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeForCausalLM", "GraniteMoeSharedForCausalLM")
|
||||
@ModelBase.example("ibm-granite/granite-3.1-3b-a800m-instruct")
|
||||
class GraniteMoeModel(GraniteModel):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue