mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-21 22:35:48 +00:00
Compare commits
No commits in common. "concedo" and "v1.118.1" have entirely different histories.
924 changed files with 15059 additions and 50151 deletions
46
.github/workflows/make-release.yml
vendored
46
.github/workflows/make-release.yml
vendored
|
|
@ -1,46 +0,0 @@
|
|||
name: Make Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Dry run - validate without creating the tag'
|
||||
required: true
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
make-release:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Run release checks
|
||||
id: checks
|
||||
run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}
|
||||
env:
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
|
||||
- name: Create release tag
|
||||
if: ${{ github.event.inputs.dry_run == 'false' }}
|
||||
run: |
|
||||
VERSION="${{ steps.checks.outputs.version }}"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git tag -a "${VERSION}" -m "Release ${VERSION}"
|
||||
git push origin "${VERSION}"
|
||||
echo "Created and pushed tag ${VERSION}"
|
||||
|
||||
- name: Dry run summary
|
||||
if: ${{ github.event.inputs.dry_run == 'true' }}
|
||||
run: |
|
||||
echo "Dry run complete - all checks passed."
|
||||
echo "Would have created tag: ${{ steps.checks.outputs.version }}"
|
||||
|
|
@ -478,8 +478,6 @@ add_library(common2
|
|||
tools/mtmd/mtmd-helper.cpp
|
||||
tools/mtmd/mtmd-image.cpp
|
||||
tools/mtmd/mtmd-helper.h
|
||||
tools/mtmd/mtmd-helper-gen.cpp
|
||||
tools/mtmd/mtmd-helper-common.h
|
||||
tools/mtmd/clip.cpp
|
||||
tools/mtmd/clip.h
|
||||
src/unicode.h
|
||||
|
|
|
|||
71
Makefile
71
Makefile
|
|
@ -55,8 +55,8 @@ ifdef KCPP_SANITIZE
|
|||
CFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error
|
||||
CXXFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error
|
||||
endif
|
||||
CFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./tools/mtmd -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c11 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC
|
||||
CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./tools/mtmd -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC
|
||||
CFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c11 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC
|
||||
CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC
|
||||
|
||||
ifndef KCPP_DEBUG
|
||||
CFLAGS += -DNDEBUG -s
|
||||
|
|
@ -579,8 +579,6 @@ mtmd.o: tools/mtmd/mtmd.cpp tools/mtmd/mtmd.h
|
|||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
mtmd-helper.o: tools/mtmd/mtmd-helper.cpp tools/mtmd/mtmd-helper.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
mtmd-helper-gen.o: tools/mtmd/mtmd-helper-gen.cpp tools/mtmd/mtmd-helper-common.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
mtmd-image.o: tools/mtmd/mtmd-image.cpp tools/mtmd/mtmd-image.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
unicode-common.o: common/unicode.cpp common/unicode.h
|
||||
|
|
@ -669,7 +667,7 @@ ggml-vulkan-shaders-noext.o: ggml/src/ggml-vulkan-shaders-noext.cpp ggml/include
|
|||
$(CXX) $(CXXFLAGS) $(VKGEN_NOEXT_FORCE) $(VULKAN_FLAGS) -c $< -o $@
|
||||
|
||||
# intermediate objects
|
||||
llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-kv-cache-msa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h
|
||||
llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
llama-model.o: src/llama-model.cpp src/llama-model.h src/models/models.h ggml/include/ggml.h include/llama.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
|
|
@ -690,7 +688,7 @@ budget.o: common/reasoning-budget.cpp common/reasoning-budget.h
|
|||
chat.o: common/chat.cpp common/chat.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
|
||||
SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/detailer.cpp src/detailer.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/ip_adapter.hpp src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/detector/yolov8.h src/model/diffusion/anima.hpp src/model/diffusion/animatediff.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/hunyuan.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/lingbot_video.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mage_flow.hpp src/model/diffusion/minimax_h3.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/kcpp_sdcpp_quantized_safetensors.hpp src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/audio_vae.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/hunyuan_vae.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/mage_vae.hpp src/model/vae/minimax_h3_audio_vae.hpp src/model/vae/minimax_h3_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h
|
||||
SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/detailer.cpp src/detailer.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/ip_adapter.hpp src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/detector/yolov8.h src/model/diffusion/anima.hpp src/model/diffusion/animatediff.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/hunyuan.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/lingbot_video.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mage_flow.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/kcpp_sdcpp_quantized_safetensors.hpp src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/hunyuan_vae.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/mage_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h
|
||||
|
||||
SDCPP_MAIN_BASENAMES := examples/cli/image_metadata.cpp examples/cli/image_metadata.h examples/cli/main.cpp examples/cli/msf_gif.h examples/common/common.cpp examples/common/common.h examples/common/log.cpp examples/common/log.h examples/common/media_io.cpp examples/common/media_io.h examples/common/resource_owners.hpp src/tokenizers/vocab/clip_merges.hpp src/tokenizers/vocab/gemma2_merges.hpp src/tokenizers/vocab/gemma2_vocab.hpp src/tokenizers/vocab/gemma_merges.hpp src/tokenizers/vocab/gemma_vocab.hpp src/tokenizers/vocab/gpt_oss_merges.hpp src/tokenizers/vocab/gpt_oss_vocab.hpp src/tokenizers/vocab/mistral_merges.hpp src/tokenizers/vocab/mistral_vocab.hpp src/tokenizers/vocab/qwen_merges.hpp src/tokenizers/vocab/t5.hpp src/tokenizers/vocab/umt5.hpp src/tokenizers/vocab/vocab.cpp src/convert.cpp src/version.cpp
|
||||
|
||||
|
|
@ -766,35 +764,35 @@ clean:
|
|||
rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o
|
||||
|
||||
# useful tools
|
||||
main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
$(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib
|
||||
$(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS)
|
||||
|
||||
ggml/src/ggml-vulkan-shaders.cpp: ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
|
||||
|
|
@ -894,18 +892,15 @@ else
|
|||
@echo 'Vulkan Shaders (no extensions) Rebuilt for Linux...'
|
||||
endif
|
||||
|
||||
# common object files for all libraries
|
||||
KOBOLDCPP_COMMON_OBJS = expose.o chat.o ggml-binops.o ggml-unops.o ggml-backend.o ggml-backend-meta.o ggml-repack.o llama.o llama-model.o embeddings_default.o music_default.o tts_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o $(OBJS) $(OBJS_SDTYPE)
|
||||
|
||||
#generated libraries
|
||||
koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL)
|
||||
koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(DEFAULT_BUILD)
|
||||
|
||||
koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL)
|
||||
koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(DEFAULT_BUILD)
|
||||
|
||||
ifdef FAILSAFE_BUILD
|
||||
koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o gpttype_adapter_failsafe.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FAILSAFE)
|
||||
koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS)
|
||||
$(FAILSAFE_BUILD)
|
||||
else
|
||||
koboldcpp_failsafe:
|
||||
|
|
@ -913,7 +908,7 @@ koboldcpp_failsafe:
|
|||
endif
|
||||
|
||||
ifdef NOAVX2_BUILD
|
||||
koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o gpttype_adapter_failsafe.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLE)
|
||||
koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS)
|
||||
$(NOAVX2_BUILD)
|
||||
else
|
||||
koboldcpp_noavx2:
|
||||
|
|
@ -921,7 +916,7 @@ koboldcpp_noavx2:
|
|||
endif
|
||||
|
||||
ifdef CUBLAS_BUILD
|
||||
koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o gpttype_adapter_cublas.o whispercpp_cublas.o clip_cublas.o ggml-backend-reg_cublas.o $(CUBLAS_OBJS) $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL)
|
||||
koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS)
|
||||
$(CUBLAS_BUILD)
|
||||
else
|
||||
koboldcpp_cublas:
|
||||
|
|
@ -929,7 +924,7 @@ koboldcpp_cublas:
|
|||
endif
|
||||
|
||||
ifdef HIPBLAS_BUILD
|
||||
koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o gpttype_adapter_cublas.o whispercpp_cublas.o clip_cublas.o ggml-backend-reg_cublas.o $(HIP_OBJS) $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL)
|
||||
koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS)
|
||||
$(HIPBLAS_BUILD)
|
||||
else
|
||||
koboldcpp_hipblas:
|
||||
|
|
@ -937,12 +932,12 @@ koboldcpp_hipblas:
|
|||
endif
|
||||
|
||||
ifdef VULKAN_BUILD
|
||||
koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL)
|
||||
koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o llama.o chat.o llama-model.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(VULKAN_BUILD)
|
||||
ifdef NOAVX2_BUILD
|
||||
koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLE)
|
||||
koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS)
|
||||
$(VULKAN_BUILD)
|
||||
koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLER)
|
||||
koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS)
|
||||
$(VULKAN_BUILD)
|
||||
else
|
||||
koboldcpp_vulkan_noavx2:
|
||||
|
|
@ -960,15 +955,15 @@ koboldcpp_vulkan_failsafe:
|
|||
endif
|
||||
|
||||
# tools
|
||||
quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
# Used to cross-compile ggml-cuda for Windows ARM64 on an x64 Windows host.
|
||||
set( CMAKE_SYSTEM_NAME Windows )
|
||||
set( CMAKE_SYSTEM_PROCESSOR arm64 )
|
||||
|
||||
if ( DEFINED CUDAToolkit_ROOT )
|
||||
file( TO_CMAKE_PATH "${CUDAToolkit_ROOT}" CUDA_ROOT )
|
||||
elseif ( DEFINED ENV{CUDA_PATH} )
|
||||
file( TO_CMAKE_PATH "$ENV{CUDA_PATH}" CUDA_ROOT )
|
||||
else()
|
||||
message( FATAL_ERROR "Set CUDAToolkit_ROOT or CUDA_PATH to a Windows CUDA Toolkit with ARM64 target libraries" )
|
||||
endif()
|
||||
|
||||
if ( DEFINED ENV{VCToolsInstallDir} )
|
||||
file( TO_CMAKE_PATH "$ENV{VCToolsInstallDir}" MSVC_TOOLS_ROOT )
|
||||
set( CMAKE_CUDA_HOST_COMPILER "${MSVC_TOOLS_ROOT}/bin/Hostx64/arm64/cl.exe" CACHE FILEPATH "" )
|
||||
endif()
|
||||
|
||||
set( CMAKE_CUDA_COMPILER "${CUDA_ROOT}/bin/nvcc.exe" CACHE FILEPATH "" )
|
||||
set( CMAKE_CUDA_FLAGS_INIT "-target-dir=arm64" )
|
||||
|
||||
# FindCUDAToolkit selects lib/x64 from the host architecture on Windows.
|
||||
set( CUDA_CUDART "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" )
|
||||
set( CUDA_cudart_LIBRARY "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" )
|
||||
set( CUDA_cublas_LIBRARY "${CUDA_ROOT}/lib/arm64/cublas.lib" CACHE FILEPATH "" )
|
||||
set( CUDA_cublasLt_LIBRARY "${CUDA_ROOT}/lib/arm64/cublasLt.lib" CACHE FILEPATH "" )
|
||||
set( CUDA_cuda_driver_LIBRARY "${CUDA_ROOT}/lib/arm64/cuda.lib" CACHE FILEPATH "" )
|
||||
232
common/arg.cpp
232
common/arg.cpp
|
|
@ -28,7 +28,6 @@
|
|||
#include <algorithm>
|
||||
#include <cinttypes>
|
||||
#include <climits>
|
||||
#include <cmath>
|
||||
#include <cstdarg>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
|
|
@ -36,7 +35,6 @@
|
|||
#include <regex>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
#include <thread> // for hardware_concurrency
|
||||
#include <vector>
|
||||
|
||||
|
|
@ -63,7 +61,6 @@ static std::initializer_list<enum llama_example> mmproj_examples = {
|
|||
LLAMA_EXAMPLE_MTMD,
|
||||
LLAMA_EXAMPLE_SERVER,
|
||||
LLAMA_EXAMPLE_CLI,
|
||||
LLAMA_EXAMPLE_TTS,
|
||||
};
|
||||
|
||||
static std::string read_file(const std::string & fname) {
|
||||
|
|
@ -363,6 +360,7 @@ static bool spec_types_is_default(const common_params & params) {
|
|||
common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) {
|
||||
common_download_hf_plan plan;
|
||||
common_download_hf_plan plan_spec;
|
||||
common_download_hf_plan plan_voc;
|
||||
common_download_opts opts;
|
||||
|
||||
const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(),
|
||||
|
|
@ -377,10 +375,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
|||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_dspark = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) != params.speculative.types.end();
|
||||
|
||||
// only download mmproj if the current example is using it
|
||||
bool use_mmproj = false;
|
||||
for (const auto & ex : mmproj_examples) {
|
||||
|
|
@ -395,7 +389,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
|||
opts.download_mtp = spec_type_draft_mtp;
|
||||
opts.download_eagle3 = spec_type_draft_eagle3;
|
||||
opts.download_dflash = spec_type_draft_dflash;
|
||||
opts.download_dspark = spec_type_draft_dspark;
|
||||
opts.download_mmproj = use_mmproj && !params.no_mmproj
|
||||
&& params.mmproj.path.empty() && params.mmproj.url.empty();
|
||||
|
||||
|
|
@ -410,12 +403,15 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
|||
opts_spec.download_mtp = true;
|
||||
opts_spec.download_dflash = true;
|
||||
opts_spec.download_eagle3 = true;
|
||||
opts_spec.download_dspark = true;
|
||||
}
|
||||
plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);
|
||||
}
|
||||
|
||||
return common_models_handler{plan, plan_spec, opts};
|
||||
if (!params.vocoder.model.hf_repo.empty()) {
|
||||
plan_voc = common_download_get_hf_plan(params.vocoder.model, opts);
|
||||
}
|
||||
|
||||
return common_models_handler{plan, plan_spec, plan_voc, opts};
|
||||
}
|
||||
|
||||
bool common_models_handler_is_preset_repo(const common_models_handler & handler) {
|
||||
|
|
@ -465,6 +461,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
|
||||
auto & plan = handler.plan;
|
||||
auto & plan_spec = handler.plan_spec;
|
||||
auto & plan_voc = handler.plan_voc;
|
||||
|
||||
auto opts = handler.opts; // copy
|
||||
opts.callback = callback;
|
||||
|
|
@ -479,6 +476,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
};
|
||||
handle_url(params.model);
|
||||
handle_url(params.mmproj);
|
||||
handle_url(params.vocoder.model);
|
||||
handle_url(params.speculative.draft.mparams);
|
||||
|
||||
// optionally, if docker repo is set, resolve it
|
||||
|
|
@ -506,6 +504,14 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
task.opts = opts;
|
||||
tasks.push_back(task);
|
||||
}
|
||||
if (!params.vocoder.model.url.empty()) {
|
||||
common_download_task task;
|
||||
task.url = params.vocoder.model.url;
|
||||
task.local_path = params.vocoder.model.path;
|
||||
task.opts = opts;
|
||||
tasks.push_back(task);
|
||||
}
|
||||
|
||||
bool had_spec_url = false;
|
||||
if (!params.speculative.draft.mparams.url.empty()) {
|
||||
common_download_task task;
|
||||
|
|
@ -539,19 +545,12 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
plan_spec.mtp = {};
|
||||
plan_spec.dflash = {};
|
||||
plan_spec.eagle3 = {};
|
||||
plan_spec.dspark = {};
|
||||
}
|
||||
|
||||
// infer the speculative type from the sidecar shipped by the draft repo when none is requested
|
||||
if (spec_types_is_default(params)) {
|
||||
if (!plan_spec.mtp.local_path.empty()) {
|
||||
params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
|
||||
plan_spec.dspark = {};
|
||||
plan_spec.dflash = {};
|
||||
plan_spec.eagle3 = {};
|
||||
} else if (!plan_spec.dspark.local_path.empty()) {
|
||||
// dspark outranks dflash, its sidecar carries the extra Markov head
|
||||
params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK };
|
||||
plan_spec.dflash = {};
|
||||
plan_spec.eagle3 = {};
|
||||
} else if (!plan_spec.dflash.local_path.empty()) {
|
||||
|
|
@ -562,20 +561,10 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
}
|
||||
}
|
||||
|
||||
// infer the speculative type from the draft GGUF metadata when none is requested
|
||||
// note: reads only the first split - sharded drafts need an explicit --spec-type
|
||||
if (spec_types_is_default(params) && !params.speculative.draft.mparams.path.empty()) {
|
||||
const auto types_gguf = common_speculative_types_from_gguf(params.speculative.draft.mparams.path);
|
||||
if (!types_gguf.empty()) {
|
||||
params.speculative.types = types_gguf;
|
||||
}
|
||||
}
|
||||
|
||||
// when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model
|
||||
const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() ||
|
||||
!plan_spec.dflash.local_path.empty() ||
|
||||
!plan_spec.eagle3.local_path.empty() ||
|
||||
!plan_spec.dspark.local_path.empty();
|
||||
!plan_spec.eagle3.local_path.empty();
|
||||
if (!plan_spec.mtp.local_path.empty() && !had_spec_url) {
|
||||
tasks.emplace_back(plan_spec.mtp, opts, [&]() {
|
||||
// only use the discovered MTP head when no draft path is set yet
|
||||
|
|
@ -606,16 +595,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
}
|
||||
});
|
||||
}
|
||||
if (!plan_spec.dspark.local_path.empty() && !had_spec_url) {
|
||||
tasks.emplace_back(plan_spec.dspark, opts, [&]() {
|
||||
// only use the discovered DSpark sidecar when no draft path is set yet
|
||||
if (params.speculative.draft.mparams.path.empty()) {
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark);
|
||||
} else {
|
||||
hf_cache::finalize_file(plan_spec.dspark);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// a wired draft sidecar counts as an explicit draft for the main plan fallback below
|
||||
if (spec_sidecar_found) {
|
||||
|
|
@ -628,6 +607,11 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
had_spec_url = true;
|
||||
}
|
||||
|
||||
// handle vocoder plan (e.g. --hf-repo-v)
|
||||
if (!plan_voc.model_files.empty()) {
|
||||
add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model);
|
||||
}
|
||||
|
||||
if (!plan.model_files.empty()) {
|
||||
add_tasks(plan.model_files, plan.primary, params.model);
|
||||
}
|
||||
|
|
@ -666,16 +650,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
}
|
||||
});
|
||||
}
|
||||
if (!plan.dspark.local_path.empty() && !had_spec_url) {
|
||||
tasks.emplace_back(plan.dspark, opts, [&]() {
|
||||
// only fall back to the discovered DSpark sidecar when no draft was explicitly provided
|
||||
if (params.speculative.draft.mparams.empty()) {
|
||||
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark);
|
||||
} else {
|
||||
hf_cache::finalize_file(plan.dspark);
|
||||
}
|
||||
});
|
||||
}
|
||||
if (!plan.preset.local_path.empty()) {
|
||||
tasks.emplace_back(plan.preset, opts, [&]() {
|
||||
// if HF repo is a preset repo, we simply run server in router mode with the preset.ini file
|
||||
|
|
@ -715,61 +689,12 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
|||
// CLI argument parsing functions
|
||||
//
|
||||
|
||||
// apply config files (if present), a later file overrides an earlier one:
|
||||
// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)
|
||||
// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)
|
||||
static void common_params_apply_system_config(common_params & params, llama_example ex) {
|
||||
std::vector<std::string> paths;
|
||||
|
||||
#if defined(_WIN32)
|
||||
const std::string program_data = common_get_env("PROGRAMDATA");
|
||||
if (!program_data.empty()) {
|
||||
paths.push_back(program_data + "\\llama.cpp\\config.ini");
|
||||
}
|
||||
#else
|
||||
paths.push_back("/etc/llama.cpp/config.ini");
|
||||
#endif
|
||||
|
||||
try {
|
||||
paths.push_back(fs_get_config_directory() + "config.ini");
|
||||
} catch (const std::exception & e) {
|
||||
LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());
|
||||
}
|
||||
|
||||
std::vector<std::string> found;
|
||||
for (const auto & path : paths) {
|
||||
std::error_code ec;
|
||||
if (std::filesystem::exists(path, ec)) {
|
||||
found.push_back(path);
|
||||
}
|
||||
}
|
||||
if (found.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
common_preset_context ctx(ex);
|
||||
ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
|
||||
for (const auto & path : found) {
|
||||
LOG_INF("using config file: %s\n", path.c_str());
|
||||
common_preset global;
|
||||
common_presets presets = ctx.load_from_ini(path, global);
|
||||
global.apply_to_params(params);
|
||||
auto it = presets.find(COMMON_PRESET_DEFAULT_NAME);
|
||||
if (it != presets.end()) {
|
||||
it->second.apply_to_params(params);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {
|
||||
common_params & params = ctx_arg.params;
|
||||
|
||||
// setup log directly from params.verbosity: see tools/cli/cli.cpp
|
||||
common_log_set_verbosity_thold(params.verbosity);
|
||||
|
||||
// config file applies first, so env variables and CLI arguments override it
|
||||
common_params_apply_system_config(params, ctx_arg.ex);
|
||||
|
||||
std::unordered_map<std::string, std::pair<common_arg *, bool>> arg_to_options;
|
||||
for (auto & opt : ctx_arg.options) {
|
||||
for (const auto & arg : opt.args) {
|
||||
|
|
@ -1402,10 +1327,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
params.n_parallel = -1; // auto by default
|
||||
} else if (ex == LLAMA_EXAMPLE_TOKENIZE) {
|
||||
params.parse_special = true; // parse special tokens by default, like the old tokenize tool
|
||||
} else if (ex == LLAMA_EXAMPLE_TTS) {
|
||||
params.out_file = "output.wav";
|
||||
params.sampling.penalty_repeat = 1.05f;
|
||||
params.sampling.penalty_last_n = -1;
|
||||
}
|
||||
|
||||
params.use_color = tty_can_use_colors();
|
||||
|
|
@ -1450,7 +1371,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
{"--version"},
|
||||
"show version and build info",
|
||||
[](common_params &) {
|
||||
llama_print_build_info(llama_version());
|
||||
fprintf(stderr, "version: %d (%s)\n", llama_build_number(), llama_commit());
|
||||
fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
|
||||
exit(0);
|
||||
}
|
||||
));
|
||||
|
|
@ -2067,9 +1989,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--repeat-last-n"}, "N",
|
||||
string_format("last n tokens to consider for penalize (default: %d, 0 = disabled)", params.sampling.penalty_last_n),
|
||||
string_format("last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)", params.sampling.penalty_last_n),
|
||||
[](common_params & params, int value) {
|
||||
if (value < 0) {
|
||||
if (value < -1) {
|
||||
throw std::runtime_error(string_format("error: invalid repeat-last-n = %d\n", value));
|
||||
}
|
||||
params.sampling.penalty_last_n = value;
|
||||
|
|
@ -2081,13 +2003,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
{"--repeat-penalty"}, "N",
|
||||
string_format("penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)", (double)params.sampling.penalty_repeat),
|
||||
[](common_params & params, const std::string & value) {
|
||||
const float penalty_repeat = std::stof(value);
|
||||
if (!std::isfinite(penalty_repeat) ||
|
||||
penalty_repeat <= 0.0f ||
|
||||
!std::isfinite(1.0f/penalty_repeat)) {
|
||||
throw std::runtime_error("error: repeat-penalty must be finite and greater than 0\n");
|
||||
}
|
||||
params.sampling.penalty_repeat = penalty_repeat;
|
||||
params.sampling.penalty_repeat = std::stof(value);
|
||||
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT;
|
||||
}
|
||||
).set_sampling());
|
||||
|
|
@ -2095,22 +2011,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
{"--presence-penalty"}, "N",
|
||||
string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present),
|
||||
[](common_params & params, const std::string & value) {
|
||||
const float penalty_present = std::stof(value);
|
||||
if (!std::isfinite(penalty_present)) {
|
||||
throw std::runtime_error("error: presence-penalty must be finite\n");
|
||||
}
|
||||
params.sampling.penalty_present = penalty_present;
|
||||
params.sampling.penalty_present = std::stof(value);
|
||||
}
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--frequency-penalty"}, "N",
|
||||
string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq),
|
||||
[](common_params & params, const std::string & value) {
|
||||
const float penalty_freq = std::stof(value);
|
||||
if (!std::isfinite(penalty_freq)) {
|
||||
throw std::runtime_error("error: frequency-penalty must be finite\n");
|
||||
}
|
||||
params.sampling.penalty_freq = penalty_freq;
|
||||
params.sampling.penalty_freq = std::stof(value);
|
||||
}
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
|
|
@ -2140,9 +2048,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--dry-penalty-last-n"}, "N",
|
||||
string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable)", params.sampling.dry_penalty_last_n),
|
||||
string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable, -1 = context size)", params.sampling.dry_penalty_last_n),
|
||||
[](common_params & params, int value) {
|
||||
if (value < 0) {
|
||||
if (value < -1) {
|
||||
throw std::runtime_error(string_format("error: invalid dry-penalty-last-n = %d\n", value));
|
||||
}
|
||||
params.sampling.dry_penalty_last_n = value;
|
||||
|
|
@ -2626,7 +2534,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
params.mtmd_batch_max_tokens = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
|
||||
if (params.is_gen_docs || llama_supports_rpc()) {
|
||||
if (llama_supports_rpc()) {
|
||||
add_opt(common_arg(
|
||||
{"--rpc"}, "SERVERS",
|
||||
"comma-separated list of RPC servers (host:port)",
|
||||
|
|
@ -2664,16 +2572,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
).set_env("LLAMA_ARG_DIO"));
|
||||
add_opt(common_arg(
|
||||
{"-lm", "--load-mode"}, "MODE",
|
||||
"model loading mode (default: auto)\n"
|
||||
"- auto: mmap, unless a device does not support it\n"
|
||||
"model loading mode (default: mmap)\n"
|
||||
"- none: no special loading mode\n"
|
||||
"- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n"
|
||||
"- mlock: force system to keep model in RAM rather than swapping or compressing\n"
|
||||
"- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n"
|
||||
"- dio: use DirectIO if available\n",
|
||||
[](common_params & params, const std::string & value) {
|
||||
/**/ if (value == "auto") { params.load_mode = LLAMA_LOAD_MODE_AUTO; }
|
||||
else if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }
|
||||
/**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }
|
||||
else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }
|
||||
else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }
|
||||
else if (value == "mmap+mlock") { params.load_mode = LLAMA_LOAD_MODE_MMAP_MLOCK; }
|
||||
|
|
@ -3029,6 +2935,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
params.model.hf_file = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE"));
|
||||
add_opt(common_arg(
|
||||
{"-hfv", "-hfrv", "--hf-repo-v"}, "<user>/<model>[:quant]",
|
||||
"Hugging Face model repository for the vocoder model (default: unused)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.vocoder.model.hf_repo = value;
|
||||
}
|
||||
).set_env("LLAMA_ARG_HF_REPO_V"));
|
||||
add_opt(common_arg(
|
||||
{"-hffv", "--hf-file-v"}, "FILE",
|
||||
"Hugging Face model file for the vocoder model (default: unused)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.vocoder.model.hf_file = value;
|
||||
}
|
||||
).set_env("LLAMA_ARG_HF_FILE_V"));
|
||||
add_opt(common_arg(
|
||||
{"-hft", "--hf-token"}, "TOKEN",
|
||||
"Hugging Face access token (default: value from HF_TOKEN environment variable)",
|
||||
|
|
@ -3363,23 +3283,12 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
{"--tools"}, "TOOL1,TOOL2,...",
|
||||
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
|
||||
"specify \"all\" to enable all tools\n"
|
||||
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n"
|
||||
"available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.server_tools = parse_csv_row(value);
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS"));
|
||||
add_opt(common_arg(
|
||||
{"--tools-runtime"}, "OPTION",
|
||||
"experimental: run tools in a separate runtime environment (default: none, use host environment)\n"
|
||||
"available options:\n"
|
||||
" 'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit\n"
|
||||
" 'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit\n"
|
||||
" 'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required\n",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.server_tools_runtime = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME"));
|
||||
add_opt(common_arg(
|
||||
{"--mcp-servers-config"}, "PATH",
|
||||
"experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n"
|
||||
|
|
@ -3647,18 +3556,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING"));
|
||||
add_opt(common_arg(
|
||||
{"--reasoning-effort"}, "LEVEL",
|
||||
"reasoning effort level given to the chat template: 'default' to keep the template default,\n"
|
||||
"or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
if (value == "default") {
|
||||
params.default_template_kwargs.erase("reasoning_effort");
|
||||
} else {
|
||||
params.default_template_kwargs["reasoning_effort"] = json(value).dump();
|
||||
}
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_EFFORT"));
|
||||
add_opt(common_arg(
|
||||
{"--reasoning-budget"}, "N",
|
||||
"token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)",
|
||||
|
|
@ -4078,9 +3975,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
{"--spec-draft-n-max"}, "N",
|
||||
string_format("number of tokens to draft for speculative decoding (default: %d)", params.speculative.draft.n_max),
|
||||
[](common_params & params, int value) {
|
||||
if (value < 0) {
|
||||
throw std::invalid_argument("invalid value");
|
||||
}
|
||||
params.speculative.draft.n_max = value;
|
||||
}
|
||||
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MAX"));
|
||||
|
|
@ -4330,18 +4224,24 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
//
|
||||
|
||||
add_opt(common_arg(
|
||||
{"--tts-lang"}, "FNAME",
|
||||
"language (ISO 639-1) for audio generation\n"
|
||||
"see tts/README.md for per-model usage notes",
|
||||
{"-mv", "--model-vocoder"}, "FNAME",
|
||||
"vocoder model for audio generation (default: unused)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_lang = value;
|
||||
params.vocoder.model.path = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-use-guide-tokens"},
|
||||
"Use guide tokens to improve TTS word recall",
|
||||
[](common_params & params) {
|
||||
params.vocoder.use_guide_tokens = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-file"}, "FNAME",
|
||||
"speaker file path for audio generation",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_speaker_file = value;
|
||||
params.vocoder.speaker_file = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
|
||||
|
|
@ -4461,6 +4361,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
|||
).set_examples({LLAMA_EXAMPLE_DEBUG}));
|
||||
|
||||
// presets
|
||||
add_opt(common_arg(
|
||||
{"--tts-oute-default"},
|
||||
string_format("use default OuteTTS models (note: can download weights from the internet)"),
|
||||
[](common_params & params) {
|
||||
params.model.hf_repo = "OuteAI/OuteTTS-0.2-500M-GGUF";
|
||||
params.model.hf_file = "OuteTTS-0.2-500M-Q8_0.gguf";
|
||||
params.vocoder.model.hf_repo = "ggml-org/WavTokenizer";
|
||||
params.vocoder.model.hf_file = "WavTokenizer-Large-75-F16.gguf";
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
|
||||
add_opt(common_arg(
|
||||
{"--embd-gemma-default"},
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ void common_params_add_preset_options(std::vector<common_arg> & args);
|
|||
struct common_models_handler {
|
||||
common_download_hf_plan plan;
|
||||
common_download_hf_plan plan_spec;
|
||||
common_download_hf_plan plan_voc;
|
||||
common_download_opts opts;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ const char * llama_build_info(void) {
|
|||
return s.c_str();
|
||||
}
|
||||
|
||||
void llama_print_build_info(const char * llama_version) {
|
||||
fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
|
||||
fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
|
||||
void llama_print_build_info(void) {
|
||||
fprintf(stderr, "%s: build = %d (%s)\n", __func__, llama_build_number(), llama_commit());
|
||||
fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
#define BUILD_INFO_H
|
||||
|
||||
#define LLAMA_BUILD_NUMBER 999
|
||||
#define LLAMA_VERSION "1.0"
|
||||
#define LLAMA_COMMIT "KOBOLDCPP"
|
||||
#define LLAMA_COMPILER "KCPP"
|
||||
#define LLAMA_TARGET "KCPP"
|
||||
|
|
@ -33,7 +32,7 @@ static inline const char * llama_build_info(void) {
|
|||
return s.c_str();
|
||||
}
|
||||
|
||||
static inline void llama_print_build_info(const char *) {
|
||||
static inline void llama_print_build_info(void) {
|
||||
fprintf(stderr, "%s: build = %d (%s)\n", __func__, llama_build_number(), llama_commit());
|
||||
fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,9 +6,6 @@
|
|||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
|
||||
using ordered_json = nlohmann::ordered_json;
|
||||
|
||||
static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {
|
||||
|
|
@ -238,43 +235,6 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::stri
|
|||
return zero_or_more(choice({ p, content_chunk }));
|
||||
}
|
||||
|
||||
common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix,
|
||||
const std::vector<common_peg_parser> & parsers) {
|
||||
if (parsers.empty()) {
|
||||
return eps();
|
||||
}
|
||||
|
||||
if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) {
|
||||
return sequence(parsers);
|
||||
}
|
||||
|
||||
std::map<uint32_t, common_peg_parser> rules;
|
||||
std::function<common_peg_parser(uint32_t)> remaining_of;
|
||||
|
||||
remaining_of = [&](uint32_t remaining) -> common_peg_parser {
|
||||
if (remaining == 0) {
|
||||
return eps();
|
||||
}
|
||||
|
||||
auto cached = rules.find(remaining);
|
||||
if (cached != rules.end()) {
|
||||
return cached->second;
|
||||
}
|
||||
|
||||
auto alternatives = choice();
|
||||
for (size_t i = 0; i < parsers.size(); i++) {
|
||||
const uint32_t bit = 1u << i;
|
||||
if (remaining & bit) {
|
||||
alternatives |= parsers[i] + remaining_of(remaining & ~bit);
|
||||
}
|
||||
}
|
||||
|
||||
return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second;
|
||||
};
|
||||
|
||||
return remaining_of((1u << parsers.size()) - 1);
|
||||
}
|
||||
|
||||
std::string & common_chat_peg_mapper::args_target() {
|
||||
return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer;
|
||||
}
|
||||
|
|
@ -594,7 +554,9 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
|||
|
||||
// Full argument: name="value" or name=value
|
||||
auto arg_rule = tool_arg(
|
||||
tool_arg_open(tool_arg_name(arg_name_parser) + literal("=")) +
|
||||
tool_arg_open(eps()) +
|
||||
tool_arg_name(arg_name_parser) +
|
||||
literal("=") +
|
||||
arg_value_parser +
|
||||
tool_arg_close(eps())
|
||||
);
|
||||
|
|
|
|||
|
|
@ -55,8 +55,6 @@ class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper {
|
|||
struct content_structure;
|
||||
struct tool_call_structure;
|
||||
|
||||
constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6;
|
||||
|
||||
class common_chat_peg_builder : public common_peg_parser_builder {
|
||||
public:
|
||||
// Tag constants (from former common_chat_peg_base_builder)
|
||||
|
|
@ -107,9 +105,6 @@ class common_chat_peg_builder : public common_peg_parser_builder {
|
|||
common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); }
|
||||
|
||||
|
||||
// Matches every parser exactly once, in any order.
|
||||
common_peg_parser permute(const std::string & rule_prefix, const std::vector<common_peg_parser> & parsers);
|
||||
|
||||
// Return a parser that parses the prefix of a string, up to a given delimiter.
|
||||
common_peg_parser prefix(const std::string & s, const std::string & delimiter = {});
|
||||
|
||||
|
|
|
|||
569
common/chat.cpp
569
common/chat.cpp
|
|
@ -934,10 +934,6 @@ static std::string common_chat_template_direct_apply_impl(
|
|||
bool enabled = inp["preserve_reasoning"].get<bool>();
|
||||
jinja::caps_apply_preserve_reasoning(ctx, enabled);
|
||||
}
|
||||
if (inp.contains("reasoning_effort") && inp["reasoning_effort"].is_string() && !inp["reasoning_effort"].empty()) {
|
||||
std::string reasoning_effort = inp["reasoning_effort"].get<std::string>();
|
||||
jinja::caps_apply_reasoning_effort(ctx, reasoning_effort);
|
||||
}
|
||||
|
||||
jinja::global_from_json(ctx, inp, inputs.mark_input);
|
||||
|
||||
|
|
@ -1128,179 +1124,6 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_
|
|||
return data;
|
||||
}
|
||||
|
||||
static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
const std::string GEN_PREFIX = "<|im_start|>assistant\n";
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
|
||||
auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;
|
||||
|
||||
data.supports_thinking = supports_reasoning;
|
||||
data.preserved_tokens = {
|
||||
"<tool_call>",
|
||||
"</tool_call>",
|
||||
};
|
||||
|
||||
if (supports_reasoning) {
|
||||
data.thinking_start_tag = "<think>";
|
||||
// Support both </think> and <tool_call> as reasoning end sequences.
|
||||
// <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model
|
||||
data.thinking_end_tags = { "</think>", "<tool_call>" };
|
||||
data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });
|
||||
}
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash
|
||||
{ COMMON_CHAT_ROLE_USER, "<|im_start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PREFIX;
|
||||
if (supports_reasoning) {
|
||||
data.generation_prompt += "<think>\n" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "\n</think>\n\n";
|
||||
}
|
||||
}
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
std::vector<std::string> tool_call_starts = { "<tool_call>" };
|
||||
|
||||
// Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the
|
||||
// starting <tool_call>. The model may hallucinate a tool name, but it is preferable over
|
||||
// constraining on <function which may occur in valid content generation, e.g. #include <functional>
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const std::string name = tool.at("function").at("name");
|
||||
tool_call_starts.push_back("<function=" + name + ">");
|
||||
});
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PREFIX);
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (supports_reasoning && extract_reasoning) {
|
||||
reasoning = p.optional("<think>" + p.space() +
|
||||
p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
|
||||
(p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
|
||||
}
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
|
||||
}
|
||||
|
||||
// Tool call parser
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));
|
||||
auto arg_string = p.rule("xml-arg-string",
|
||||
p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto parameters = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(parameters);
|
||||
|
||||
std::vector<common_peg_parser> required_args;
|
||||
std::vector<common_peg_parser> optional_args;
|
||||
|
||||
foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) {
|
||||
auto rule_name = "tool-" + name + "-arg-" + param_name;
|
||||
|
||||
auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param_name)) + ">\n");
|
||||
|
||||
auto arg_value = schema_info.resolves_to_string(param_schema) ?
|
||||
arg_string :
|
||||
p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close;
|
||||
|
||||
auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));
|
||||
|
||||
(is_required ? required_args : optional_args).push_back(arg_rule);
|
||||
});
|
||||
|
||||
// Accept required arguments in any order, as Qwen does not always adhere to the
|
||||
// order provided.
|
||||
auto args = p.permute("tool-" + name + "-args", required_args);
|
||||
if (!optional_args.empty()) {
|
||||
args = args + p.zero_or_more(p.choice(optional_args));
|
||||
}
|
||||
|
||||
auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +
|
||||
p.tool_args(args) +
|
||||
p.tool_close(p.literal("</function>\n")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, func);
|
||||
});
|
||||
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
|
||||
// Qwen3-Coder models may occasionally omit the <tool_call> token.
|
||||
auto tool_call_body = tool_choice + "</tool_call>" + p.space();
|
||||
auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body);
|
||||
auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
|
||||
|
||||
auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
|
||||
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
|
||||
|
||||
return generation_prompt +
|
||||
(reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
|
||||
}
|
||||
|
||||
// Content only parser
|
||||
return generation_prompt + (reasoning << p.content(p.rest()));
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
for (const auto & start : tool_call_starts) {
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
|
@ -2134,16 +1957,23 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
|
|||
adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
|
||||
}
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "<think>";
|
||||
data.thinking_end_tags = {"</think>"};
|
||||
data.preserved_tokens = {
|
||||
"｜DSML｜",
|
||||
"<think>",
|
||||
"</think>",
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
std::optional<json> additional_context;
|
||||
if (is_v4 && has_response_format) {
|
||||
additional_context = json{ { "response_format", inputs.json_schema } };
|
||||
}
|
||||
|
||||
const std::string DSML = "｜DSML｜";
|
||||
const std::string THINK_START = "<think>";
|
||||
const std::string THINK_END = "</think>";
|
||||
|
|
@ -2155,137 +1985,25 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
|
|||
const std::string PARAM_START = "<" + DSML + "parameter";
|
||||
const std::string PARAM_END = "</" + DSML + "parameter>";
|
||||
const std::string GEN_PROMPT = "<｜Assistant｜>";
|
||||
const std::string TC_SEPARATOR = "\n\n";
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(
|
||||
tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(
|
||||
tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END, FC_START};
|
||||
data.preserved_tokens = {
|
||||
DSML,
|
||||
THINK_START,
|
||||
THINK_END,
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
if (is_v4 && msg.reasoning_content.empty()) {
|
||||
data.generation_prompt = GEN_PROMPT + THINK_END;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += msg.render_content();
|
||||
}
|
||||
} else {
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PROMPT);
|
||||
auto end = p.end();
|
||||
|
||||
// build tool call section first since we might need it in reasoning
|
||||
auto tool_choice = p.choice();
|
||||
if (has_tool_calls) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
const auto & props = params.contains("properties") ? params.at("properties") : json::object();
|
||||
|
||||
std::set<std::string> required;
|
||||
if (params.contains("required")) {
|
||||
params.at("required").get_to(required);
|
||||
}
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
std::vector<common_peg_parser> required_parsers;
|
||||
std::vector<common_peg_parser> optional_parsers;
|
||||
for (const auto & [param_name, param_schema] : props.items()) {
|
||||
bool is_required = required.find(param_name) != required.end();
|
||||
bool is_string = schema_info.resolves_to_string(param_schema);
|
||||
|
||||
auto arg = p.tool_arg(
|
||||
p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) +
|
||||
p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
|
||||
(is_string ?
|
||||
p.tool_arg_string_value(p.until(PARAM_END)) :
|
||||
p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema",
|
||||
param_schema, false))) +
|
||||
p.tool_arg_close(p.literal(PARAM_END)));
|
||||
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
|
||||
if (is_required) {
|
||||
required_parsers.push_back(named_arg);
|
||||
} else {
|
||||
optional_parsers.push_back(named_arg);
|
||||
}
|
||||
}
|
||||
|
||||
common_peg_parser args_seq = p.eps();
|
||||
for (size_t i = 0; i < required_parsers.size(); i++) {
|
||||
if (i > 0) {
|
||||
args_seq = args_seq + p.space();
|
||||
}
|
||||
args_seq = args_seq + required_parsers[i];
|
||||
}
|
||||
|
||||
if (!optional_parsers.empty()) {
|
||||
common_peg_parser any_opt = p.choice();
|
||||
for (const auto & opt : optional_parsers) {
|
||||
any_opt |= opt;
|
||||
}
|
||||
args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
|
||||
}
|
||||
|
||||
common_peg_parser invoke_body = args_seq;
|
||||
auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") +
|
||||
p.tool_name(p.literal(name)) + p.literal("\">\n")) +
|
||||
invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END)));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, func_parser);
|
||||
});
|
||||
}
|
||||
|
||||
common_peg_parser tool_calls = p.eps();
|
||||
if (inputs.parallel_tool_calls) {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice +
|
||||
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
|
||||
} else {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
|
||||
}
|
||||
auto end = p.end();
|
||||
|
||||
auto reasoning = p.eps();
|
||||
auto reasoning_with_tc = p.eps();
|
||||
auto obligatory_tool_calls = tool_calls;
|
||||
bool allow_reasoning_with_tc = false;
|
||||
|
||||
if (!require_tools) {
|
||||
tool_calls = p.optional(tool_calls);
|
||||
}
|
||||
|
||||
if (extract_reasoning && inputs.enable_thinking) {
|
||||
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
|
||||
reasoning_with_tc = THINK_START +
|
||||
p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
|
||||
p.space() + obligatory_tool_calls;
|
||||
allow_reasoning_with_tc = true;
|
||||
} else if (extract_reasoning) {
|
||||
// Thinking disabled but reasoning extraction requested: the generation prompt
|
||||
// contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that
|
||||
|
|
@ -2303,21 +2021,101 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
|
|||
return generation_prompt + reasoning + response_format + end;
|
||||
}
|
||||
|
||||
if (!has_tool_calls) {
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
|
||||
auto content_before_tools = p.negate(p.literal(THINK_START)) +
|
||||
p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) +
|
||||
p.space();
|
||||
return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :
|
||||
generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
const auto & props = params.contains("properties") ? params.at("properties") : json::object();
|
||||
|
||||
std::set<std::string> required;
|
||||
if (params.contains("required")) {
|
||||
params.at("required").get_to(required);
|
||||
}
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
std::vector<common_peg_parser> required_parsers;
|
||||
std::vector<common_peg_parser> optional_parsers;
|
||||
for (const auto & [param_name, param_schema] : props.items()) {
|
||||
bool is_required = required.find(param_name) != required.end();
|
||||
bool is_string = schema_info.resolves_to_string(param_schema);
|
||||
|
||||
auto arg = p.tool_arg(
|
||||
p.tool_arg_open(
|
||||
p.literal(PARAM_START + " name=\"") +
|
||||
p.tool_arg_name(p.literal(param_name)) +
|
||||
p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
|
||||
(is_string
|
||||
? p.tool_arg_string_value(p.until(PARAM_END))
|
||||
: p.tool_arg_json_value(p.schema(p.json(),
|
||||
"tool-" + name + "-arg-" + param_name + "-schema",
|
||||
param_schema, false))) +
|
||||
p.tool_arg_close(p.literal(PARAM_END)));
|
||||
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
|
||||
if (is_required) {
|
||||
required_parsers.push_back(named_arg);
|
||||
} else {
|
||||
optional_parsers.push_back(named_arg);
|
||||
}
|
||||
}
|
||||
|
||||
common_peg_parser args_seq = p.eps();
|
||||
for (size_t i = 0; i < required_parsers.size(); i++) {
|
||||
if (i > 0) {
|
||||
args_seq = args_seq + p.space();
|
||||
}
|
||||
args_seq = args_seq + required_parsers[i];
|
||||
}
|
||||
|
||||
if (!optional_parsers.empty()) {
|
||||
common_peg_parser any_opt = p.choice();
|
||||
for (const auto & opt : optional_parsers) {
|
||||
any_opt |= opt;
|
||||
}
|
||||
args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
|
||||
}
|
||||
|
||||
common_peg_parser invoke_body = args_seq;
|
||||
auto func_parser = p.tool(
|
||||
p.tool_open(p.literal(INVOKE_START + " name=\"") +
|
||||
p.tool_name(p.literal(name)) + p.literal("\">\n")) +
|
||||
invoke_body + p.space() +
|
||||
p.tool_close(p.literal(INVOKE_END)));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, func_parser);
|
||||
});
|
||||
|
||||
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
|
||||
common_peg_parser tool_calls = p.eps();
|
||||
if (inputs.parallel_tool_calls) {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice +
|
||||
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
|
||||
} else {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
|
||||
}
|
||||
|
||||
if (!require_tools) {
|
||||
tool_calls = p.optional(tool_calls);
|
||||
}
|
||||
|
||||
auto content_before_tools = p.content(p.until(FC_START));
|
||||
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && !require_tools;
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
|
|
@ -3111,153 +2909,6 @@ static common_chat_params common_chat_params_init_minicpm5(const common_chat_tem
|
|||
return data;
|
||||
}
|
||||
|
||||
// An assistant turn is rendered as one or more messages, each
|
||||
// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is
|
||||
// <|eom|> (more messages follow) or <|eot|> (end of turn):
|
||||
// - chain-of-thought: to=self, terminated by <|eom|>
|
||||
// - final answer: to=user, terminated by <|eot|>
|
||||
// The generation prompt is just "<|start|>assistant"; the model emits its own
|
||||
// " to=...<|message|>".
|
||||
static common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = "<|start|>assistant";
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|start|>", "<|message|>", "<|eom|>", "<|eot|>",
|
||||
// ATEM tool-call markup emitted on " to=<tool>" turns.
|
||||
"<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",
|
||||
"</atem:invoke>", "</atem:function_calls>",
|
||||
};
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|start|>tool" },
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt += data.generation_prompt;
|
||||
}
|
||||
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
// Constrained grammar whenever tools are offered.
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
|
||||
if (!extract_reasoning && !include_grammar) {
|
||||
return start + p.content(p.rest());
|
||||
}
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
} else {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
}
|
||||
auto analysis = p.ref("analysis");
|
||||
|
||||
auto recipient = p.optional(p.literal(" to=user"));
|
||||
auto final_msg = p.rule("final", recipient + p.literal("<|message|>") +
|
||||
p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto string_value = p.ac(
|
||||
p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
|
||||
"</atem:parameter>");
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
const std::string name = function.at("name");
|
||||
auto params = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
|
||||
auto args = p.eps();
|
||||
if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) {
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
auto arg_choice = p.choice();
|
||||
for (const auto & [prop_name, prop_schema] : params.at("properties").items()) {
|
||||
auto value_parser = p.eps();
|
||||
if (schema_info.resolves_to_string(prop_schema)) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false))
|
||||
+ p.tool_arg_close(p.literal("</atem:parameter>"));
|
||||
}
|
||||
|
||||
auto arg_rule = p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop_name)) + p.literal("\">")) +
|
||||
value_parser);
|
||||
|
||||
arg_choice |= arg_rule;
|
||||
}
|
||||
args = p.zero_or_more(arg_choice + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.literal(" to=") + p.until("<|message|>") +
|
||||
p.literal("<|message|><atem:function_calls>") + p.space() +
|
||||
p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())
|
||||
<< p.tool_args(args)
|
||||
<< p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + name, tool_parser);
|
||||
});
|
||||
|
||||
auto tool_calls = inputs.parallel_tool_calls
|
||||
? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))
|
||||
: p.trigger_rule("tool-call", tool_choice);
|
||||
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + analysis) + start + tool_calls;
|
||||
}
|
||||
auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
|
||||
return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls));
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + final_msg;
|
||||
});
|
||||
|
||||
data.parser = parser.save();
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
|
||||
"<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
static json common_chat_extra_context() {
|
||||
json ctx = json::object();
|
||||
std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
|
||||
|
|
@ -3286,12 +2937,6 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
|
|||
return common_chat_params_init_gpt_oss(tmpl, params);
|
||||
}
|
||||
|
||||
// Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators.
|
||||
if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Muse Glimmer\n");
|
||||
return common_chat_params_init_muse_glimmer(tmpl, params);
|
||||
}
|
||||
|
||||
// Functionary v3.2 - uses recipient-based format with >>>recipient\n{content}
|
||||
// Detection: template has ">>>all" for content and ">>>" prefix for tool calls
|
||||
if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) {
|
||||
|
|
@ -3375,14 +3020,6 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
|
|||
return common_chat_params_init_minicpm5(tmpl, params);
|
||||
}
|
||||
|
||||
// Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash
|
||||
if (src.find("<tool_call>") != std::string::npos &&
|
||||
src.find("<function=") != std::string::npos &&
|
||||
src.find("<parameter=") != std::string::npos) {
|
||||
LOG_DBG("Using specialized template: Qwen3-Coder\n");
|
||||
return common_chat_params_init_qwen3_coder(tmpl, params);
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1004,42 +1004,24 @@ bool fs_is_directory(const std::string & path) {
|
|||
return std::filesystem::exists(dir) && std::filesystem::is_directory(dir);
|
||||
}
|
||||
|
||||
std::string common_get_env(const std::string & name) {
|
||||
const char * value = std::getenv(name.c_str());
|
||||
return value == nullptr ? "" : value;
|
||||
}
|
||||
|
||||
void common_set_env(const std::string & name, const std::string & value) {
|
||||
#if defined(_WIN32)
|
||||
_putenv_s(name.c_str(), value.c_str());
|
||||
#else
|
||||
if (value.empty()) {
|
||||
unsetenv(name.c_str());
|
||||
} else {
|
||||
setenv(name.c_str(), value.c_str(), 1);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string fs_get_cache_directory() {
|
||||
std::string cache_directory = "";
|
||||
auto ensure_trailing_slash = [](std::string p) {
|
||||
// Make sure to add trailing slash
|
||||
if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
|
||||
if (p.back() != DIRECTORY_SEPARATOR) {
|
||||
p += DIRECTORY_SEPARATOR;
|
||||
}
|
||||
return p;
|
||||
};
|
||||
cache_directory = common_get_env("LLAMA_CACHE");
|
||||
if (cache_directory.empty()) {
|
||||
if (getenv("LLAMA_CACHE")) {
|
||||
cache_directory = std::getenv("LLAMA_CACHE");
|
||||
} else {
|
||||
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
|
||||
defined(__OpenBSD__) || defined(__NetBSD__)
|
||||
const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");
|
||||
const std::string home = common_get_env("HOME");
|
||||
if (!xdg_cache_home.empty()) {
|
||||
cache_directory = xdg_cache_home;
|
||||
} else if (!home.empty()) {
|
||||
cache_directory = home + "/.cache/";
|
||||
if (std::getenv("XDG_CACHE_HOME")) {
|
||||
cache_directory = std::getenv("XDG_CACHE_HOME");
|
||||
} else if (std::getenv("HOME")) {
|
||||
cache_directory = std::getenv("HOME") + std::string("/.cache/");
|
||||
} else {
|
||||
#if defined(__linux__)
|
||||
/* no $HOME is defined, fallback to getpwuid */
|
||||
|
|
@ -1054,16 +1036,9 @@ std::string fs_get_cache_directory() {
|
|||
#endif /* defined(__linux__) */
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
cache_directory = common_get_env("HOME");
|
||||
if (cache_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
}
|
||||
cache_directory += "/Library/Caches/";
|
||||
cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
|
||||
#elif defined(_WIN32)
|
||||
cache_directory = common_get_env("LOCALAPPDATA");
|
||||
if (cache_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
|
||||
}
|
||||
cache_directory = std::getenv("LOCALAPPDATA");
|
||||
#elif defined(__EMSCRIPTEN__)
|
||||
GGML_ABORT("not implemented on this platform");
|
||||
#else
|
||||
|
|
@ -1075,51 +1050,6 @@ std::string fs_get_cache_directory() {
|
|||
return ensure_trailing_slash(cache_directory);
|
||||
}
|
||||
|
||||
std::string fs_get_config_directory() {
|
||||
std::string config_directory = "";
|
||||
auto ensure_trailing_slash = [](std::string p) {
|
||||
if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
|
||||
p += DIRECTORY_SEPARATOR;
|
||||
}
|
||||
return p;
|
||||
};
|
||||
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
|
||||
defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
|
||||
const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");
|
||||
const std::string home = common_get_env("HOME");
|
||||
if (!xdg_config_home.empty()) {
|
||||
config_directory = xdg_config_home;
|
||||
} else if (!home.empty()) {
|
||||
config_directory = home + "/.config/";
|
||||
} else {
|
||||
#if defined(__linux__)
|
||||
/* no $HOME is defined, fallback to getpwuid */
|
||||
struct passwd *pw = getpwuid(getuid());
|
||||
if ((!pw) || (!pw->pw_dir)) {
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
}
|
||||
|
||||
config_directory = std::string(pw->pw_dir) + std::string("/.config/");
|
||||
#else
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
#endif
|
||||
}
|
||||
#elif defined(_WIN32)
|
||||
config_directory = common_get_env("APPDATA");
|
||||
if (config_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %APPDATA% directory");
|
||||
}
|
||||
#elif defined(__EMSCRIPTEN__)
|
||||
// caller decides what to do when there is no config directory
|
||||
throw std::runtime_error("not implemented on this platform");
|
||||
#else
|
||||
# error Unknown architecture
|
||||
#endif
|
||||
config_directory = ensure_trailing_slash(config_directory);
|
||||
config_directory += "llama.cpp";
|
||||
return ensure_trailing_slash(config_directory);
|
||||
}
|
||||
|
||||
std::string fs_get_cache_file(const std::string & filename) {
|
||||
GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
|
||||
std::string cache_directory = fs_get_cache_directory();
|
||||
|
|
@ -1281,8 +1211,6 @@ struct common_init_result::impl {
|
|||
|
||||
// note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top
|
||||
|
||||
common_threadpools threadpools;
|
||||
|
||||
llama_model_ptr model;
|
||||
llama_context_ptr context;
|
||||
|
||||
|
|
@ -1363,6 +1291,16 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
|||
params.sampling.logit_bias_eog.begin(), params.sampling.logit_bias_eog.end());
|
||||
}
|
||||
|
||||
//if (params.sampling.penalty_last_n == -1) {
|
||||
// LOG_TRC("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
|
||||
// params.sampling.penalty_last_n = llama_n_ctx(lctx);
|
||||
//}
|
||||
|
||||
//if (params.sampling.dry_penalty_last_n == -1) {
|
||||
// LOG_TRC("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
|
||||
// params.sampling.dry_penalty_last_n = llama_n_ctx(lctx);
|
||||
//}
|
||||
|
||||
// init the backend samplers as part of the context creation
|
||||
pimpl->samplers.resize(cparams.n_seq_max);
|
||||
pimpl->samplers_seq_config.resize(cparams.n_seq_max);
|
||||
|
|
@ -1384,10 +1322,6 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
|||
}
|
||||
|
||||
pimpl->context.reset(lctx);
|
||||
|
||||
set_process_priority(params.cpuparams.priority);
|
||||
|
||||
pimpl->threadpools.init(lctx, params);
|
||||
}
|
||||
|
||||
llama_model * common_init_result::model() {
|
||||
|
|
@ -1534,18 +1468,18 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
|||
common_init_result::~common_init_result() = default;
|
||||
|
||||
std::string common_get_model_endpoint() {
|
||||
std::string endpoint = common_get_env("MODEL_ENDPOINT");
|
||||
if (endpoint.empty()) {
|
||||
// the HF_ENDPOINT variable is respected for backward compatibility
|
||||
endpoint = common_get_env("HF_ENDPOINT");
|
||||
const char * model_endpoint_env = getenv("MODEL_ENDPOINT");
|
||||
// We still respect the use of environment-variable "HF_ENDPOINT" for backward-compatibility.
|
||||
const char * hf_endpoint_env = getenv("HF_ENDPOINT");
|
||||
const char * endpoint_env = model_endpoint_env ? model_endpoint_env : hf_endpoint_env;
|
||||
std::string model_endpoint = "https://huggingface.co/";
|
||||
if (endpoint_env) {
|
||||
model_endpoint = endpoint_env;
|
||||
if (model_endpoint.back() != '/') {
|
||||
model_endpoint += '/';
|
||||
}
|
||||
}
|
||||
if (endpoint.empty()) {
|
||||
return "https://huggingface.co/";
|
||||
}
|
||||
if (endpoint.back() != '/') {
|
||||
endpoint += '/';
|
||||
}
|
||||
return endpoint;
|
||||
return model_endpoint;
|
||||
}
|
||||
|
||||
char * common_get_model_or_exit(int argc, char * argv[]) {
|
||||
|
|
@ -1692,7 +1626,6 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
|
|||
mparams.progress_callback = params.load_progress_callback;
|
||||
mparams.progress_callback_user_data = params.load_progress_callback_user_data;
|
||||
mparams.no_alloc = params.no_alloc;
|
||||
mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
|
||||
|
||||
return mparams;
|
||||
}
|
||||
|
|
@ -1704,7 +1637,6 @@ struct llama_context_params common_context_params_to_llama(const common_params &
|
|||
cparams.n_seq_max = params.n_parallel;
|
||||
cparams.n_rs_seq = params.speculative.need_n_rs_seq();
|
||||
cparams.n_outputs_max = std::max(params.n_outputs_max, 0);
|
||||
cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0);
|
||||
cparams.n_batch = params.n_batch;
|
||||
cparams.n_ubatch = params.n_ubatch;
|
||||
cparams.n_threads = params.cpuparams.n_threads;
|
||||
|
|
@ -1736,10 +1668,6 @@ struct llama_context_params common_context_params_to_llama(const common_params &
|
|||
return cparams;
|
||||
}
|
||||
|
||||
//
|
||||
// Threadpool utils
|
||||
//
|
||||
|
||||
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) {
|
||||
struct ggml_threadpool_params tpp;
|
||||
|
||||
|
|
@ -1756,56 +1684,6 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo
|
|||
return tpp;
|
||||
}
|
||||
|
||||
common_threadpools::~common_threadpools() {
|
||||
if (!free_fn) {
|
||||
return;
|
||||
}
|
||||
free_fn(threadpool);
|
||||
free_fn(threadpool_batch);
|
||||
}
|
||||
|
||||
void common_threadpools::init(llama_context * ctx, const common_params & params) {
|
||||
GGML_ASSERT(!threadpool);
|
||||
GGML_ASSERT(!threadpool_batch);
|
||||
|
||||
COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads);
|
||||
|
||||
auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
|
||||
if (!cpu_dev) {
|
||||
COM_WRN("%s", "no CPU backend found\n");
|
||||
return;
|
||||
}
|
||||
auto * reg = ggml_backend_dev_backend_reg(cpu_dev);
|
||||
auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
|
||||
free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
|
||||
|
||||
struct ggml_threadpool_params tpp_batch =
|
||||
ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);
|
||||
struct ggml_threadpool_params tpp =
|
||||
ggml_threadpool_params_from_cpu_params(params.cpuparams);
|
||||
|
||||
if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
|
||||
threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
|
||||
if (!threadpool_batch) {
|
||||
COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads);
|
||||
return;
|
||||
}
|
||||
|
||||
// start the non-batch threadpool in the paused state
|
||||
tpp.paused = true;
|
||||
}
|
||||
|
||||
threadpool = ggml_threadpool_new_fn(&tpp);
|
||||
if (!threadpool) {
|
||||
COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads);
|
||||
free_fn(threadpool_batch);
|
||||
threadpool_batch = nullptr;
|
||||
return;
|
||||
}
|
||||
|
||||
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
|
||||
}
|
||||
|
||||
//
|
||||
// Batch utils
|
||||
//
|
||||
|
|
|
|||
|
|
@ -236,14 +236,14 @@ struct common_params_sampling {
|
|||
float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities
|
||||
float dynatemp_range = 0.00f; // 0.0 = disabled
|
||||
float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler
|
||||
int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty)
|
||||
int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)
|
||||
float penalty_repeat = 1.00f; // 1.0 = disabled
|
||||
float penalty_freq = 0.00f; // 0.0 = disabled
|
||||
float penalty_present = 0.00f; // 0.0 = disabled
|
||||
float dry_multiplier = 0.0f; // 0.0 = disabled; DRY repetition penalty for tokens extending repetition:
|
||||
float dry_base = 1.75f; // 0.0 = disabled; multiplier * base ^ (length of sequence before token - allowed length)
|
||||
int32_t dry_allowed_length = 2; // tokens extending repetitions beyond this receive penalty
|
||||
int32_t dry_penalty_last_n = 64; // how many tokens to scan for repetitions (0 = disable penalty)
|
||||
int32_t dry_penalty_last_n = -1; // how many tokens to scan for repetitions (0 = disable penalty, -1 = context size)
|
||||
float adaptive_target = -1.0f; // select tokens near this probability (valid range 0.0 to 1.0; negative = disabled)
|
||||
float adaptive_decay = 0.90f; // EMA decay for adaptation; history ≈ 1/(1-decay) tokens (0.0 - 0.99)
|
||||
int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
|
||||
|
|
@ -393,6 +393,14 @@ struct common_params_speculative {
|
|||
}
|
||||
};
|
||||
|
||||
struct common_params_vocoder {
|
||||
struct common_params_model model;
|
||||
|
||||
std::string speaker_file; // speaker file path
|
||||
|
||||
bool use_guide_tokens = false; // enable guide tokens to improve TTS accuracy
|
||||
};
|
||||
|
||||
struct common_params_diffusion {
|
||||
int32_t steps = 128;
|
||||
bool visual_mode = false;
|
||||
|
|
@ -448,7 +456,6 @@ struct common_params {
|
|||
int32_t n_parallel = 1; // number of parallel sequences to decode
|
||||
int32_t n_sequences = 1; // number of sequences to decode
|
||||
int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch)
|
||||
int32_t n_outputs_max_per_seq = 1; // max outputs per sequence
|
||||
int32_t grp_attn_n = 1; // group-attention factor
|
||||
int32_t grp_attn_w = 512; // group-attention width
|
||||
int32_t n_print = -1; // print token count every n tokens (-1 = disabled)
|
||||
|
|
@ -474,7 +481,7 @@ struct common_params {
|
|||
std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(), 1024 * 1024*1024);
|
||||
|
||||
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
|
||||
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model
|
||||
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP; // how to load the model
|
||||
|
||||
common_cpu_params cpuparams;
|
||||
common_cpu_params cpuparams_batch;
|
||||
|
|
@ -491,6 +498,7 @@ struct common_params {
|
|||
|
||||
struct common_params_sampling sampling;
|
||||
struct common_params_speculative speculative;
|
||||
struct common_params_vocoder vocoder;
|
||||
struct common_params_diffusion diffusion;
|
||||
|
||||
struct common_params_model model;
|
||||
|
|
@ -657,7 +665,6 @@ struct common_params {
|
|||
|
||||
// enable built-in tools
|
||||
std::vector<std::string> server_tools;
|
||||
std::string server_tools_runtime;
|
||||
|
||||
// MCP server configs (Cursor-compatible JSON)
|
||||
std::string mcp_servers_config; // path to JSON file with MCP server definitions
|
||||
|
|
@ -733,12 +740,6 @@ struct common_params {
|
|||
llama_progress_callback load_progress_callback = NULL;
|
||||
void * load_progress_callback_user_data = NULL;
|
||||
bool no_alloc = false; // Don't allocate model buffers
|
||||
|
||||
// TTS params
|
||||
std::string tts_lang = "";
|
||||
std::string tts_speaker_file = "";
|
||||
|
||||
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
|
||||
};
|
||||
|
||||
// call once at the start of a program if it uses libcommon
|
||||
|
|
@ -863,15 +864,6 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat
|
|||
|
||||
bool glob_match(const std::string & pattern, const std::string & str);
|
||||
|
||||
//
|
||||
// Environment utils
|
||||
//
|
||||
|
||||
// portable environment access, an unset variable reads as an empty string
|
||||
// and setting an empty value unsets the variable
|
||||
std::string common_get_env(const std::string & name);
|
||||
void common_set_env(const std::string & name, const std::string & value);
|
||||
|
||||
//
|
||||
// Filesystem utils
|
||||
//
|
||||
|
|
@ -882,7 +874,6 @@ bool fs_is_directory(const std::string & path);
|
|||
|
||||
std::string fs_get_cache_directory();
|
||||
std::string fs_get_cache_file(const std::string & filename);
|
||||
std::string fs_get_config_directory();
|
||||
|
||||
struct common_file_info {
|
||||
std::string path;
|
||||
|
|
@ -930,8 +921,9 @@ using common_init_result_ptr = std::unique_ptr<common_init_result>;
|
|||
|
||||
common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false);
|
||||
|
||||
struct llama_model_params common_model_params_to_llama ( common_params & params);
|
||||
struct llama_context_params common_context_params_to_llama(const common_params & params);
|
||||
struct llama_model_params common_model_params_to_llama ( common_params & params);
|
||||
struct llama_context_params common_context_params_to_llama(const common_params & params);
|
||||
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);
|
||||
|
||||
// clear LoRA adapters from context, then apply new list of adapters
|
||||
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
|
||||
|
|
@ -942,28 +934,6 @@ std::string common_get_model_endpoint();
|
|||
// for testing purposes
|
||||
char * common_get_model_or_exit(int, char*[]);
|
||||
|
||||
//
|
||||
// Threadpool utils
|
||||
//
|
||||
|
||||
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);
|
||||
|
||||
struct common_threadpools {
|
||||
common_threadpools() = default;
|
||||
~common_threadpools();
|
||||
|
||||
common_threadpools(const common_threadpools &) = delete;
|
||||
common_threadpools & operator=(const common_threadpools &) = delete;
|
||||
|
||||
void init(llama_context * ctx, const common_params & params);
|
||||
|
||||
private:
|
||||
ggml_threadpool * threadpool = nullptr;
|
||||
ggml_threadpool * threadpool_batch = nullptr;
|
||||
|
||||
decltype(ggml_threadpool_free) * free_fn = nullptr;
|
||||
};
|
||||
|
||||
//
|
||||
// Context utils
|
||||
//
|
||||
|
|
|
|||
|
|
@ -656,12 +656,6 @@ static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,
|
|||
return find_best_sibling(files, model, "dflash-", tag);
|
||||
}
|
||||
|
||||
static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files,
|
||||
const std::string & model,
|
||||
const std::string & tag = "") {
|
||||
return find_best_sibling(files, model, "dspark-", tag);
|
||||
}
|
||||
|
||||
static bool gguf_filename_is_model(const std::string & filepath) {
|
||||
if (!string_ends_with(filepath, ".gguf")) {
|
||||
return false;
|
||||
|
|
@ -676,8 +670,7 @@ static bool gguf_filename_is_model(const std::string & filepath) {
|
|||
filename.find("imatrix") == std::string::npos &&
|
||||
filename.find("mtp-") == std::string::npos &&
|
||||
filename.find("eagle3-") == std::string::npos &&
|
||||
filename.find("dflash-") == std::string::npos &&
|
||||
filename.find("dspark-") == std::string::npos;
|
||||
filename.find("dflash-") == std::string::npos;
|
||||
}
|
||||
|
||||
static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
|
||||
|
|
@ -770,7 +763,7 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
|
|||
} else {
|
||||
primary = find_best_model(all, tag);
|
||||
// a requested sidecar can resolve on its own, without a full model of the same tag
|
||||
if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) {
|
||||
if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) {
|
||||
LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
|
||||
list_available_gguf_files(all);
|
||||
return plan;
|
||||
|
|
@ -794,12 +787,9 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
|
|||
if (opts.download_eagle3) {
|
||||
plan.eagle3 = find_best_eagle3(all, primary.path, tag);
|
||||
}
|
||||
if (opts.download_dspark) {
|
||||
plan.dspark = find_best_dspark(all, primary.path, tag);
|
||||
}
|
||||
|
||||
if (primary.path.empty() &&
|
||||
plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) {
|
||||
plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) {
|
||||
LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str());
|
||||
list_available_gguf_files(all);
|
||||
}
|
||||
|
|
@ -977,8 +967,7 @@ std::vector<common_cached_model_info> common_list_cached_models() {
|
|||
split.prefix.find("mmproj") != std::string::npos ||
|
||||
split.prefix.find("mtp-") != std::string::npos ||
|
||||
split.prefix.find("eagle3-") != std::string::npos ||
|
||||
split.prefix.find("dflash-") != std::string::npos ||
|
||||
split.prefix.find("dspark-") != std::string::npos) {
|
||||
split.prefix.find("dflash-") != std::string::npos) {
|
||||
continue;
|
||||
}
|
||||
if (seen.insert(f.repo_id + ":" + split.tag).second) {
|
||||
|
|
|
|||
|
|
@ -60,7 +60,6 @@ struct common_download_opts {
|
|||
bool download_mtp = false;
|
||||
bool download_eagle3 = false;
|
||||
bool download_dflash = false;
|
||||
bool download_dspark = false;
|
||||
common_download_callback * callback = nullptr;
|
||||
};
|
||||
|
||||
|
|
@ -112,7 +111,6 @@ struct common_download_hf_plan {
|
|||
hf_cache::hf_file mtp;
|
||||
hf_cache::hf_file eagle3;
|
||||
hf_cache::hf_file dflash;
|
||||
hf_cache::hf_file dspark;
|
||||
hf_cache::hf_file preset; // if set, only this file is downloaded
|
||||
};
|
||||
common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts);
|
||||
|
|
|
|||
|
|
@ -136,10 +136,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
|
|||
devs.push_back(llama_model_get_device(model, i));
|
||||
}
|
||||
|
||||
hp_ngl = llama_model_n_layer(model);
|
||||
if (mparams->load_mtp) {
|
||||
hp_ngl += llama_model_n_layer_nextn(model);
|
||||
}
|
||||
hp_ngl = llama_model_n_layer(model) + llama_model_n_layer_nextn(model);
|
||||
hp_n_ctx_train = llama_model_n_ctx_train(model);
|
||||
hp_n_expert = llama_model_n_expert(model);
|
||||
|
||||
|
|
|
|||
|
|
@ -102,8 +102,7 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {
|
|||
const int64_t chunk_count_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT);
|
||||
const int64_t chunk_size_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE);
|
||||
|
||||
if (datasets_key != -1 && gguf_get_kv_type(ctx_gguf, datasets_key) == GGUF_TYPE_ARRAY &&
|
||||
gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) {
|
||||
if (datasets_key != -1 && gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) {
|
||||
const int64_t n = gguf_get_arr_n(ctx_gguf, datasets_key);
|
||||
imatrix.datasets.reserve(imatrix.datasets.size() + n);
|
||||
for (int64_t i = 0; i < n; ++i) {
|
||||
|
|
@ -144,13 +143,6 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {
|
|||
return false;
|
||||
}
|
||||
|
||||
if (in_sum2->type != GGML_TYPE_F32 || counts->type != GGML_TYPE_F32) {
|
||||
LOG_ERR("%s: sums and counts for %s must be F32\n", __func__, name.c_str());
|
||||
gguf_free(ctx_gguf);
|
||||
ggml_free(ctx);
|
||||
return false;
|
||||
}
|
||||
|
||||
auto & e = imatrix.entries[name];
|
||||
|
||||
const int64_t nval = ggml_nelements(in_sum2);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ namespace jinja {
|
|||
|
||||
using caps_json_fn = std::function<json()>;
|
||||
using caps_ctx_fn = std::function<void(context &)>;
|
||||
using caps_analyze_fn = std::function<void(context &, bool, value &, value &, const std::string &)>;
|
||||
using caps_analyze_fn = std::function<void(bool, value &, value &, const std::string &)>;
|
||||
|
||||
void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) {
|
||||
ctx.set_val("preserve_thinking", mk_val<value_bool>(enabled));
|
||||
|
|
@ -29,12 +29,6 @@ void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) {
|
|||
ctx.set_val("drop_thinking", mk_val<value_bool>(!enabled));
|
||||
}
|
||||
|
||||
void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort) {
|
||||
value var = mk_val<value_string>(effort); // bind to the same value for stats
|
||||
ctx.set_val("reasoning_effort", var);
|
||||
ctx.set_val("reasoning_strength", var);
|
||||
}
|
||||
|
||||
static void caps_try_execute(jinja::program & prog,
|
||||
const caps_json_fn & messages_fn,
|
||||
const caps_ctx_fn & ctx_fn,
|
||||
|
|
@ -71,7 +65,7 @@ static void caps_try_execute(jinja::program & prog,
|
|||
// ignore exceptions during capability analysis
|
||||
}
|
||||
|
||||
analyze_fn(ctx, success, messages, tools, result);
|
||||
analyze_fn(success, messages, tools, result);
|
||||
}
|
||||
|
||||
// for debugging only
|
||||
|
|
@ -96,7 +90,6 @@ std::map<std::string, bool> caps::to_map() const {
|
|||
{"supports_parallel_tool_calls", supports_parallel_tool_calls},
|
||||
{"supports_system_role", supports_system_role},
|
||||
{"supports_preserve_reasoning", supports_preserve_reasoning},
|
||||
{"supports_reasoning_effort", supports_reasoning_effort},
|
||||
{"supports_object_arguments", supports_object_arguments},
|
||||
};
|
||||
}
|
||||
|
|
@ -134,7 +127,7 @@ caps caps_get(jinja::program & prog) {
|
|||
},
|
||||
nullptr, // ctx_fn
|
||||
nullptr, // tools_fn
|
||||
[&](context &, bool success, value & messages, value &, const std::string &) {
|
||||
[&](bool success, value & messages, value &, const std::string &) {
|
||||
auto & content = messages->at(0)->at("content");
|
||||
caps_print_stats(content, "messages[0].content");
|
||||
if (has_op(content, "selectattr") || has_op(content, "array_access")) {
|
||||
|
|
@ -168,7 +161,7 @@ caps caps_get(jinja::program & prog) {
|
|||
},
|
||||
nullptr, // ctx_fn
|
||||
nullptr, // tools_fn
|
||||
[&](context &, bool, value & messages, value &, const std::string &) {
|
||||
[&](bool, value & messages, value &, const std::string &) {
|
||||
auto & content = messages->at(0)->at("content");
|
||||
caps_print_stats(content, "messages[0].content");
|
||||
if (!content->stats.used) {
|
||||
|
|
@ -244,7 +237,7 @@ caps caps_get(jinja::program & prog) {
|
|||
},
|
||||
});
|
||||
},
|
||||
[&](context &, bool success, value & messages, value & tools, const std::string &) {
|
||||
[&](bool success, value & messages, value & tools, const std::string &) {
|
||||
if (!success) {
|
||||
return; // Nothing can be inferred
|
||||
}
|
||||
|
|
@ -337,7 +330,7 @@ caps caps_get(jinja::program & prog) {
|
|||
},
|
||||
});
|
||||
},
|
||||
[&](context &, bool success, value & messages, value & tools, const std::string &) {
|
||||
[&](bool success, value & messages, value & tools, const std::string &) {
|
||||
if (!success) {
|
||||
result.supports_tool_calls = false;
|
||||
result.supports_tools = false;
|
||||
|
|
@ -439,7 +432,7 @@ caps caps_get(jinja::program & prog) {
|
|||
},
|
||||
});
|
||||
},
|
||||
[&](context &, bool success, value & messages, value &, const std::string &) {
|
||||
[&](bool success, value & messages, value &, const std::string &) {
|
||||
if (!success) {
|
||||
result.supports_parallel_tool_calls = false;
|
||||
return;
|
||||
|
|
@ -492,11 +485,10 @@ caps caps_get(jinja::program & prog) {
|
|||
});
|
||||
},
|
||||
[&](context & ctx) {
|
||||
ctx.set_val("enable_thinking", mk_val<value_bool>(true));
|
||||
caps_apply_preserve_reasoning(ctx, true);
|
||||
},
|
||||
nullptr, // tools_fn
|
||||
[&](context &, bool, value &, value &, const std::string & output) {
|
||||
[&](bool, value &, value &, const std::string & output) {
|
||||
// note: we cannot use stats here because the reasoning_content may be used for "if" condition test, but not actually outputted in the final result
|
||||
if (output.find(reasoning_placeholder) != std::string::npos) {
|
||||
result.supports_preserve_reasoning = true;
|
||||
|
|
@ -504,32 +496,6 @@ caps caps_get(jinja::program & prog) {
|
|||
}
|
||||
);
|
||||
|
||||
JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort");
|
||||
|
||||
// case: reasoning effort level
|
||||
caps_try_execute(
|
||||
prog,
|
||||
[&]() {
|
||||
// messages
|
||||
return json::array({
|
||||
{
|
||||
{"role", "user"},
|
||||
{"content", "User message"}
|
||||
},
|
||||
});
|
||||
},
|
||||
[&](context & ctx) {
|
||||
ctx.set_val("enable_thinking", mk_val<value_bool>(true));
|
||||
caps_apply_reasoning_effort(ctx, "low");
|
||||
},
|
||||
nullptr, // tools_fn
|
||||
[&](context & ctx, bool, value &, value &, const std::string &) {
|
||||
value effort = ctx.get_val("reasoning_effort");
|
||||
caps_print_stats(effort, "reasoning_effort");
|
||||
result.supports_reasoning_effort = effort->stats.used;
|
||||
}
|
||||
);
|
||||
|
||||
JJ_DEBUG("%s\n", result.to_string().c_str());
|
||||
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -16,9 +16,6 @@ struct caps {
|
|||
// supports preserve reasoning trace in the full history, not just the last assistant message
|
||||
bool supports_preserve_reasoning = false;
|
||||
|
||||
// supports reasoning effort levels
|
||||
bool supports_reasoning_effort = false;
|
||||
|
||||
// one of the 2 content capabilities must be true
|
||||
bool supports_string_content = true;
|
||||
bool supports_typed_content = false;
|
||||
|
|
@ -35,6 +32,5 @@ struct caps {
|
|||
caps caps_get(jinja::program & prog);
|
||||
|
||||
void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled);
|
||||
void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort);
|
||||
|
||||
} // namespace jinja
|
||||
|
|
|
|||
|
|
@ -266,7 +266,7 @@ value binary_expression::execute_impl(context & ctx) {
|
|||
return res;
|
||||
}
|
||||
for (int64_t i = 0; i < repeat; ++i) {
|
||||
res->val_str.append(str);
|
||||
res->val_str = res->val_str.append(str);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -763,22 +763,14 @@ struct runtime {
|
|||
gather_string_parts_recursive(val, parts);
|
||||
// join consecutive parts with the same type
|
||||
auto & p = parts->val_str.parts;
|
||||
if (p.empty()) {
|
||||
return parts;
|
||||
}
|
||||
size_t w = 0;
|
||||
for (size_t r = 1; r < p.size(); r++) {
|
||||
if (p[w].is_input == p[r].is_input) {
|
||||
p[w].val += p[r].val;
|
||||
for (size_t i = 1; i < p.size(); ) {
|
||||
if (p[i].is_input == p[i - 1].is_input) {
|
||||
p[i - 1].val += p[i].val;
|
||||
p.erase(p.begin() + i);
|
||||
} else {
|
||||
w++;
|
||||
if (w != r) {
|
||||
// the guard is needed, self-move leaves the string in an unspecified state
|
||||
p[w] = std::move(p[r]);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
p.resize(w + 1);
|
||||
return parts;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ void string::mark_input_based_on(const string & other) {
|
|||
}
|
||||
}
|
||||
|
||||
string & string::append(const string & other) {
|
||||
string string::append(const string & other) {
|
||||
for (const auto & part : other.parts) {
|
||||
parts.push_back(part);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ struct string {
|
|||
// mark this string as input if other has ALL parts as input
|
||||
void mark_input_based_on(const string & other);
|
||||
|
||||
string & append(const string & other);
|
||||
string append(const string & other);
|
||||
|
||||
// in-place transformations
|
||||
|
||||
|
|
|
|||
|
|
@ -116,8 +116,6 @@ static llama_sampler_i llama_sampler_llg_i = {
|
|||
/* .backend_accept = */ NULL,
|
||||
/* .backend_apply = */ NULL,
|
||||
/* .backend_set_input = */ NULL,
|
||||
/* .backend_reset = */ NULL,
|
||||
/* .copy_state = */ NULL,
|
||||
};
|
||||
|
||||
static size_t llama_sampler_llg_tokenize_fn(const void * user_data, const uint8_t * bytes, size_t bytes_len,
|
||||
|
|
|
|||
|
|
@ -570,34 +570,23 @@ struct parser_executor {
|
|||
}
|
||||
|
||||
static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos, const char delimiter) {
|
||||
auto save = pos;
|
||||
|
||||
++pos; // consume '\'
|
||||
if (pos >= ctx.input.size()) {
|
||||
if (!ctx.is_lenient()) {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
|
||||
}
|
||||
pos = save; // suppress unmatched '\'
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);
|
||||
}
|
||||
|
||||
char c = ctx.input[pos];
|
||||
|
||||
if (c == delimiter || c == '\\' || c == '/' || c == 'b' || c == 'f' || c == 'n' || c == 'r' || c == 't') {
|
||||
++pos;
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos);
|
||||
} else if (c == 'u') {
|
||||
return handle_unicode_escape(ctx, start, pos);
|
||||
} else {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
|
||||
}
|
||||
|
||||
if (c == 'u') {
|
||||
auto result = handle_unicode_escape(ctx, start, pos);
|
||||
if (result.need_more_input()) {
|
||||
pos = save; // suppress incomplete sequence
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
|
||||
}
|
||||
|
||||
static common_peg_parse_result handle_unicode_escape(common_peg_parse_context & ctx, size_t start, size_t & pos) {
|
||||
|
|
|
|||
|
|
@ -322,8 +322,6 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co
|
|||
preset.options[opt] = value;
|
||||
}
|
||||
LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());
|
||||
} else if (ignore_unknown_keys) {
|
||||
LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.c_str());
|
||||
} else {
|
||||
throw std::runtime_error(string_format(
|
||||
"option '%s' not recognized in preset '%s'",
|
||||
|
|
@ -365,25 +363,8 @@ struct local_model {
|
|||
std::string name;
|
||||
std::string path;
|
||||
std::string path_mmproj;
|
||||
std::string path_draft;
|
||||
};
|
||||
|
||||
// TODO @ngxson: handle "eagle3-" when it's supported by common_speculative_types_from_gguf()
|
||||
static const char * draft_prefixes[] = { "mtp-", "dspark-", "dflash-" };
|
||||
|
||||
static bool is_mmproj_file(const std::string & fname) {
|
||||
return fname.find("mmproj") != std::string::npos;
|
||||
}
|
||||
|
||||
static bool is_draft_file(const std::string & fname) {
|
||||
for (const auto & prefix : draft_prefixes) {
|
||||
if (fname.rfind(prefix, 0) == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const {
|
||||
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
|
||||
throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str()));
|
||||
|
|
@ -395,15 +376,10 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
|
|||
common_file_info model_file;
|
||||
common_file_info first_shard_file;
|
||||
common_file_info mmproj_file;
|
||||
common_file_info draft_file;
|
||||
for (const auto & file : files) {
|
||||
if (string_ends_with(file.name, ".gguf")) {
|
||||
if (is_mmproj_file(file.name)) {
|
||||
if (file.name.find("mmproj") != std::string::npos) {
|
||||
mmproj_file = file;
|
||||
} else if (is_draft_file(file.name)) {
|
||||
if (draft_file.path.empty()) {
|
||||
draft_file = file; // first sidecar found wins
|
||||
}
|
||||
} else if (file.name.find("-00001-of-") != std::string::npos) {
|
||||
first_shard_file = file;
|
||||
} else {
|
||||
|
|
@ -415,8 +391,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
|
|||
local_model model{
|
||||
/* name */ name,
|
||||
/* path */ first_shard_file.path.empty() ? model_file.path : first_shard_file.path,
|
||||
/* path_mmproj */ mmproj_file.path, // can be empty
|
||||
/* path_draft */ draft_file.path // can be empty
|
||||
/* path_mmproj */ mmproj_file.path // can be empty
|
||||
};
|
||||
if (!model.path.empty()) {
|
||||
models.push_back(model);
|
||||
|
|
@ -428,17 +403,13 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
|
|||
if (file.is_dir) {
|
||||
scan_subdir(file.path, file.name);
|
||||
} else if (string_ends_with(file.name, ".gguf")) {
|
||||
if (is_mmproj_file(file.name) || is_draft_file(file.name)) {
|
||||
continue; // companion file, cannot be loaded as a model on its own
|
||||
}
|
||||
// single file model
|
||||
std::string name = file.name;
|
||||
string_replace_all(name, ".gguf", "");
|
||||
local_model model{
|
||||
/* name */ name,
|
||||
/* path */ file.path,
|
||||
/* path_mmproj */ "",
|
||||
/* path_draft */ ""
|
||||
/* path_mmproj */ ""
|
||||
};
|
||||
models.push_back(model);
|
||||
}
|
||||
|
|
@ -453,9 +424,6 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
|
|||
if (!model.path_mmproj.empty()) {
|
||||
preset.set_option(*this, "LLAMA_ARG_MMPROJ", model.path_mmproj);
|
||||
}
|
||||
if (!model.path_draft.empty()) {
|
||||
preset.set_option(*this, "LLAMA_ARG_SPEC_DRAFT_MODEL", model.path_draft);
|
||||
}
|
||||
out[preset.name] = preset;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -59,10 +59,6 @@ struct common_preset_context {
|
|||
bool filter_allowed_keys = false;
|
||||
std::set<std::string> allowed_keys;
|
||||
|
||||
// if true, options unknown to the current example are skipped instead of being an error
|
||||
// used for config files shared by all binaries, where each binary only knows a subset of options
|
||||
bool ignore_unknown_keys = false;
|
||||
|
||||
// if only_remote_allowed is true, only accept whitelisted keys
|
||||
common_preset_context(llama_example ex);
|
||||
|
||||
|
|
|
|||
|
|
@ -217,8 +217,6 @@ static struct llama_sampler_i common_reasoning_budget_i = {
|
|||
/* .backend_accept = */ nullptr,
|
||||
/* .backend_apply = */ nullptr,
|
||||
/* .backend_set_input = */ nullptr,
|
||||
/* .backend_reset = */ nullptr,
|
||||
/* .copy_state = */ nullptr,
|
||||
};
|
||||
|
||||
static struct llama_sampler * common_reasoning_budget_clone(const struct llama_sampler * smpl) {
|
||||
|
|
|
|||
|
|
@ -184,21 +184,9 @@ std::string common_params_sampling::print() const {
|
|||
return std::string(result);
|
||||
}
|
||||
|
||||
struct common_sampler * common_sampler_init(
|
||||
const struct llama_model * model,
|
||||
struct common_params_sampling & params) {
|
||||
if (!std::isfinite(params.penalty_repeat) ||
|
||||
params.penalty_repeat <= 0.0f ||
|
||||
!std::isfinite(1.0f/params.penalty_repeat)) {
|
||||
throw std::invalid_argument("penalty_repeat must be finite and greater than 0");
|
||||
}
|
||||
if (!std::isfinite(params.penalty_freq)) {
|
||||
throw std::invalid_argument("penalty_freq must be finite");
|
||||
}
|
||||
if (!std::isfinite(params.penalty_present)) {
|
||||
throw std::invalid_argument("penalty_present must be finite");
|
||||
}
|
||||
struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params) {
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model);
|
||||
|
||||
llama_sampler_chain_params lparams = llama_sampler_chain_default_params();
|
||||
|
||||
lparams.no_perf = params.no_perf;
|
||||
|
|
@ -350,7 +338,7 @@ struct common_sampler * common_sampler_init(
|
|||
for (const auto & str : params.dry_sequence_breakers) {
|
||||
c_breakers.push_back(str.c_str());
|
||||
}
|
||||
samplers.push_back(llama_sampler_init_dry(vocab, params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size()));
|
||||
samplers.push_back(llama_sampler_init_dry(vocab, llama_model_n_ctx_train(model), params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size()));
|
||||
}
|
||||
break;
|
||||
case COMMON_SAMPLER_TYPE_TOP_K:
|
||||
|
|
@ -378,7 +366,7 @@ struct common_sampler * common_sampler_init(
|
|||
samplers.push_back(llama_sampler_init_infill(vocab));
|
||||
break;
|
||||
case COMMON_SAMPLER_TYPE_PENALTIES:
|
||||
samplers.push_back(llama_sampler_init_penalties(llama_vocab_n_tokens(vocab), params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present));
|
||||
samplers.push_back(llama_sampler_init_penalties(params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present));
|
||||
break;
|
||||
case COMMON_SAMPLER_TYPE_ADAPTIVE_P:
|
||||
// the `adaptive-p` sampler is like `dist` and `mirostat` in that it selects
|
||||
|
|
@ -518,26 +506,6 @@ struct common_sampler * common_sampler_clone(common_sampler * gsmpl) {
|
|||
};
|
||||
}
|
||||
|
||||
void common_sampler_copy(const common_sampler * src, common_sampler * dst) {
|
||||
if (!src || !dst || src == dst) {
|
||||
return;
|
||||
}
|
||||
|
||||
GGML_ASSERT((src->grmr == nullptr) == (dst->grmr == nullptr));
|
||||
GGML_ASSERT((src->rbudget == nullptr) == (dst->rbudget == nullptr));
|
||||
|
||||
llama_sampler_copy(src->grmr, dst->grmr);
|
||||
llama_sampler_copy(src->rbudget, dst->rbudget);
|
||||
llama_sampler_copy(src->chain, dst->chain);
|
||||
|
||||
dst->params = src->params;
|
||||
dst->prev = src->prev;
|
||||
dst->cur = src->cur;
|
||||
dst->cur_p = src->cur_p;
|
||||
dst->cur_p.data = src->cur_p.data ? dst->cur.data() : nullptr; // re-point to dst's buffer
|
||||
dst->t_total_us = src->t_total_us;
|
||||
}
|
||||
|
||||
void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl) {
|
||||
// TODO: measure grammar performance
|
||||
|
||||
|
|
|
|||
|
|
@ -37,9 +37,7 @@ struct common_sampler;
|
|||
// llama_sampler API overloads
|
||||
|
||||
// note: can mutate params in some cases
|
||||
struct common_sampler * common_sampler_init(
|
||||
const struct llama_model * model,
|
||||
struct common_params_sampling & params);
|
||||
struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params);
|
||||
|
||||
void common_sampler_free(struct common_sampler * gsmpl);
|
||||
|
||||
|
|
@ -47,7 +45,6 @@ void common_sampler_free(struct common_sampler * gsmpl);
|
|||
void common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool is_generated);
|
||||
void common_sampler_reset (struct common_sampler * gsmpl);
|
||||
struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl);
|
||||
void common_sampler_copy (const struct common_sampler * src, struct common_sampler * dst);
|
||||
|
||||
// arguments can be nullptr to skip printing
|
||||
void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
#include "common.h"
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpp.h"
|
||||
#include "llama.h"
|
||||
#include "log.h"
|
||||
#include "ngram-cache.cpp"
|
||||
|
|
@ -172,6 +171,12 @@ struct common_speculative_impl {
|
|||
// (optional) serialize/restore per-seq internal state (e.g. eagle3's deferred boundary).
|
||||
virtual bool get_state(llama_seq_id /*seq_id*/, std::vector<uint8_t> & /*data*/) const { return false; }
|
||||
virtual void set_state(llama_seq_id /*seq_id*/, const std::vector<uint8_t> & /*data*/) {}
|
||||
|
||||
// true if this implementation requires the target context to extract post-norm embeddings
|
||||
virtual bool need_embd() const = 0;
|
||||
|
||||
// true if this implementation requires the target context to extract pre-norm embeddings
|
||||
virtual bool need_embd_nextn() const { return false; }
|
||||
};
|
||||
|
||||
struct common_speculative_impl_draft_simple : public common_speculative_impl {
|
||||
|
|
@ -188,10 +193,6 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
|
|||
auto * ctx_dft = this->params.ctx_dft;
|
||||
auto * ctx_tgt = this->params.ctx_tgt;
|
||||
|
||||
if (!ctx_dft) {
|
||||
throw std::runtime_error("draft-simple requires a draft context");
|
||||
}
|
||||
|
||||
SPC_TRC("%s", "adding speculative implementation 'draft-simple'\n");
|
||||
SPC_TRC("- n_max=%d, n_min=%d, p_min=%f\n", this->params.n_max, this->params.n_min, this->params.p_min);
|
||||
SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n",
|
||||
|
|
@ -384,6 +385,10 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
|
|||
void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
|
||||
// noop
|
||||
}
|
||||
|
||||
bool need_embd() const override {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -902,6 +907,10 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
|
|||
pending_g_last[seq_id].resize(n_embd_dec);
|
||||
std::memcpy(pending_g_last[seq_id].data(), data.data() + sizeof(llama_pos), (size_t) n_embd_dec * sizeof(float));
|
||||
}
|
||||
|
||||
bool need_embd() const override {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// DFlash: block-diffusion drafting with a draft-side KV cache injection
|
||||
|
|
@ -913,9 +922,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
|||
|
||||
std::vector<common_sampler_ptr> smpls;
|
||||
|
||||
// backend sampler chain per seq, attached to ctx_dft
|
||||
std::vector<llama_sampler *> backend_chains;
|
||||
|
||||
int32_t n_embd_dec = 0; // draft hidden size
|
||||
int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size
|
||||
int32_t n_embd_tgt = 0; // target model hidden size
|
||||
|
|
@ -989,22 +995,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
|||
s.reset(common_sampler_init(model_dft, sparams));
|
||||
}
|
||||
|
||||
// offload draft sampling to the backend
|
||||
backend_chains.assign(n_seq, nullptr);
|
||||
if (this->params.backend_sampling) {
|
||||
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
|
||||
llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_top_k(10));
|
||||
|
||||
if (!llama_set_sampler(ctx_dft, seq_id, chain)) {
|
||||
SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id);
|
||||
llama_sampler_free(chain);
|
||||
chain = nullptr;
|
||||
}
|
||||
backend_chains[seq_id] = chain;
|
||||
}
|
||||
}
|
||||
|
||||
// turn on extraction of the target layers' input embeddings
|
||||
for (uint32_t k = 0; k < target_layer_ids_n; ++k) {
|
||||
llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
|
||||
|
|
@ -1015,18 +1005,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
|||
}
|
||||
|
||||
~common_speculative_impl_draft_dflash() override {
|
||||
auto * ctx_dft = this->params.ctx_dft;
|
||||
for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) {
|
||||
if (backend_chains[seq_id] == nullptr) {
|
||||
continue;
|
||||
}
|
||||
if (ctx_dft) {
|
||||
llama_set_sampler(ctx_dft, seq_id, nullptr);
|
||||
}
|
||||
llama_sampler_free(backend_chains[seq_id]);
|
||||
}
|
||||
backend_chains.clear();
|
||||
|
||||
llama_batch_free(batch);
|
||||
llama_batch_free(batch_inject);
|
||||
}
|
||||
|
|
@ -1054,14 +1032,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
|||
return true;
|
||||
}
|
||||
|
||||
// Target prefill may contain token IDs or multimodal embeddings. Both
|
||||
// produce the target-layer features used to seed the draft KV cache, so
|
||||
// skipping the embedding batches leaves a hole in the draft's cache and
|
||||
// the next injection fails to initialize.
|
||||
// TODO: revisit after https://github.com/ggml-org/llama.cpp/pull/24669 is merged
|
||||
const bool has_tokens = batch_in.token != nullptr;
|
||||
const bool has_embeddings = batch_in.embd != nullptr;
|
||||
if (has_tokens == has_embeddings) {
|
||||
if (batch_in.token == nullptr || batch_in.embd != nullptr) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -1269,6 +1240,10 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
|
|||
void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
|
||||
// noop
|
||||
}
|
||||
|
||||
bool need_embd() const override {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
||||
|
|
@ -1316,7 +1291,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
|||
GGML_ASSERT(ctx_tgt && ctx_dft && "MTP requires ctx_tgt and ctx_dft to be set");
|
||||
|
||||
n_embd = llama_model_n_embd_out(llama_get_model(ctx_dft));
|
||||
GGML_ASSERT(n_embd == llama_model_n_embd_out(llama_get_model(ctx_tgt)) &&
|
||||
GGML_ASSERT(n_embd == llama_model_n_embd(llama_get_model(ctx_tgt)) &&
|
||||
"MTP input row width must match the target h_nextn width");
|
||||
n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft)));
|
||||
|
||||
|
|
@ -1707,6 +1682,14 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
|
|||
const size_t row_bytes = (size_t) n_embd * sizeof(float);
|
||||
std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes);
|
||||
}
|
||||
|
||||
bool need_embd() const override {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool need_embd_nextn() const override {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
// state of self-speculation (simple implementation, not ngram-map)
|
||||
|
|
@ -1753,6 +1736,10 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl {
|
|||
void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
|
||||
// noop
|
||||
}
|
||||
|
||||
bool need_embd() const override {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
|
||||
|
|
@ -1807,6 +1794,10 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
|
|||
|
||||
common_ngram_map_accept(config[seq_id], n_accepted);
|
||||
}
|
||||
|
||||
bool need_embd() const override {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct common_speculative_impl_ngram_mod : public common_speculative_impl {
|
||||
|
|
@ -1982,6 +1973,10 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool need_embd() const override {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct common_speculative_impl_ngram_cache : public common_speculative_impl {
|
||||
|
|
@ -2121,6 +2116,10 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {
|
|||
void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override {
|
||||
// noop
|
||||
}
|
||||
|
||||
bool need_embd() const override {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct common_speculative {
|
||||
|
|
@ -2228,43 +2227,6 @@ common_speculative_type common_speculative_type_from_name(const std::string & na
|
|||
return it->second;
|
||||
}
|
||||
|
||||
std::vector<common_speculative_type> common_speculative_types_from_gguf(const std::string & path) {
|
||||
struct gguf_init_params gguf_params = {
|
||||
/* .no_alloc = */ true,
|
||||
/* .ctx = */ nullptr,
|
||||
};
|
||||
|
||||
gguf_context_ptr gguf_ctx(gguf_init_from_file(path.c_str(), gguf_params));
|
||||
if (!gguf_ctx) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
|
||||
if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
|
||||
if (arch != "dflash") {
|
||||
const uint32_t block_count = gguf_get_val_u32(gguf_ctx.get(), gguf_find_key(gguf_ctx.get(), (arch + ".block_count").c_str()));
|
||||
|
||||
if (gguf_find_tensor(gguf_ctx.get(), ("blk." + std::to_string(block_count - 1) + ".nextn.eh_proj.weight").c_str()) >= 0) {
|
||||
return { COMMON_SPECULATIVE_TYPE_DRAFT_MTP };
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
// the Markov head distinguishes draft-dspark from draft-dflash
|
||||
const auto type = gguf_find_tensor(gguf_ctx.get(), "markov_w1.weight") >= 0
|
||||
? COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK
|
||||
: COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH;
|
||||
|
||||
SPC_INF("auto-detected speculative type '%s' from the draft model metadata\n", common_speculative_type_to_str(type).c_str());
|
||||
|
||||
return { type };
|
||||
}
|
||||
|
||||
static uint32_t common_get_enabled_speculative_configs(const std::vector<common_speculative_type> & configs) {
|
||||
uint32_t result = 0;
|
||||
for (size_t i = 0; i < configs.size(); i++) {
|
||||
|
|
@ -2330,24 +2292,6 @@ common_params common_base_params_to_speculative(const common_params & params) {
|
|||
result.cache_type_k = params_spec.cache_type_k;
|
||||
result.cache_type_v = params_spec.cache_type_v;
|
||||
result.n_outputs_max = params.n_parallel;
|
||||
result.n_outputs_max_per_seq = 1;
|
||||
|
||||
// dflash/dspark decode the whole noise block in a single pass and sample every block position on the backend
|
||||
// TODO: refactor such properties to be announced by the speculative types
|
||||
// something like `struct common_speculative_type_props common_speculative_type_get_props(...);`
|
||||
const bool has_block_draft = std::any_of(
|
||||
params.speculative.types.begin(), params.speculative.types.end(),
|
||||
[](common_speculative_type t) {
|
||||
return t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
|
||||
});
|
||||
if (has_block_draft) {
|
||||
// per-seq output positions: DFlash decodes anchor + n_max masks (n_max + 1); DSpark n_max -> +1 covers both
|
||||
const int32_t per_seq = std::max(1, params_spec.n_max + 1);
|
||||
result.n_outputs_max = params.n_parallel * per_seq;
|
||||
if (params_spec.backend_sampling) {
|
||||
result.n_outputs_max_per_seq = per_seq;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
@ -2370,6 +2314,7 @@ common_speculative_init_result::common_speculative_init_result(
|
|||
const bool spec_mtp = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
|
||||
GGML_ASSERT(has_draft || spec_mtp);
|
||||
|
||||
auto mparams = common_model_params_to_llama(params);
|
||||
auto cparams = common_context_params_to_llama(params);
|
||||
|
|
@ -2432,17 +2377,6 @@ common_speculative_init_result_ptr common_speculative_init_from_params(common_pa
|
|||
return std::make_unique<common_speculative_init_result>(params, model_tgt, ctx_tgt);
|
||||
}
|
||||
|
||||
common_speculative_output_limits common_speculative_get_output_limits(
|
||||
int32_t n_batch, int32_t n_parallel, int32_t n_draft) {
|
||||
const int64_t per_seq = 1 + (int64_t) std::max(0, n_draft);
|
||||
const int64_t total = (int64_t) n_parallel * per_seq;
|
||||
|
||||
return {
|
||||
/* .total = */ (int32_t) std::min<int64_t>(n_batch, total),
|
||||
/* .per_seq = */ (int32_t) std::min<int64_t>(n_batch, per_seq),
|
||||
};
|
||||
}
|
||||
|
||||
// initialization of the speculative decoding system
|
||||
//
|
||||
common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) {
|
||||
|
|
@ -2451,28 +2385,57 @@ common_speculative * common_speculative_init(common_params_speculative & params,
|
|||
{
|
||||
uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types);
|
||||
|
||||
auto add_config_if_enabled = [&](common_speculative_type type, bool available = true) {
|
||||
if (available && (enabled_configs & (1u << type))) {
|
||||
configs.emplace_back(type, params);
|
||||
}
|
||||
};
|
||||
bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE));
|
||||
bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr;
|
||||
bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr;
|
||||
bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr;
|
||||
bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr;
|
||||
|
||||
|
||||
|
||||
bool has_ngram_cache = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_CACHE));
|
||||
bool has_ngram_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE));
|
||||
bool has_ngram_map_k = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K));
|
||||
bool has_ngram_map_k4v = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V));
|
||||
bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD));
|
||||
|
||||
// when adding a new type - update here the logic above
|
||||
static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11);
|
||||
|
||||
// this list here defines the priority of the speculators
|
||||
// the one with highest priority are listed first
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE);
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K);
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V);
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MOD);
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE);
|
||||
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE);
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params.draft.ctx_dft != nullptr);
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params.draft.ctx_dft != nullptr);
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params.draft.ctx_dft != nullptr);
|
||||
add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params.draft.ctx_dft != nullptr);
|
||||
if (has_ngram_simple) {
|
||||
// This implementation can guess a lot of tokens without any draft model.
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, params));
|
||||
}
|
||||
if (has_ngram_map_k) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, params));
|
||||
}
|
||||
if (has_ngram_map_k4v) {
|
||||
// This implementation can guess tokens with high acceptance rate but is more expensive.
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, params));
|
||||
}
|
||||
if (has_ngram_mod) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, params));
|
||||
}
|
||||
if (has_ngram_cache) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params));
|
||||
}
|
||||
if (has_draft_simple) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params));
|
||||
}
|
||||
if (has_draft_eagle3) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params));
|
||||
}
|
||||
if (has_draft_mtp) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params));
|
||||
}
|
||||
if (has_draft_dflash) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params));
|
||||
}
|
||||
if (has_draft_dspark) {
|
||||
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params));
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::unique_ptr<common_speculative_impl>> impls = {};
|
||||
|
|
@ -2607,6 +2570,34 @@ bool common_speculative_process(common_speculative * spec, const llama_batch & b
|
|||
return result;
|
||||
}
|
||||
|
||||
bool common_speculative_need_embd(common_speculative * spec) {
|
||||
if (spec == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (auto & impl : spec->impls) {
|
||||
if (impl->need_embd()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool common_speculative_need_embd_nextn(common_speculative * spec) {
|
||||
if (spec == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (auto & impl : spec->impls) {
|
||||
if (impl->need_embd_nextn()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void common_speculative_draft(common_speculative * spec) {
|
||||
if (spec == nullptr) {
|
||||
return;
|
||||
|
|
@ -2691,10 +2682,7 @@ void common_speculative_draft(common_speculative * spec) {
|
|||
void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, uint16_t n_accepted) {
|
||||
common_speculative_impl * impl = spec->impl_last[seq_id];
|
||||
|
||||
if (impl == nullptr) {
|
||||
GGML_ASSERT(n_accepted == 0);
|
||||
return;
|
||||
}
|
||||
GGML_ASSERT(impl);
|
||||
|
||||
{
|
||||
common_time_meas tm(impl->t_accept_us, !impl->gen_perf);
|
||||
|
|
|
|||
|
|
@ -14,9 +14,6 @@ const char * common_speculative_all_types_str();
|
|||
// parse user provided types
|
||||
std::vector<enum common_speculative_type> common_speculative_types_from_names(const std::vector<std::string> & names);
|
||||
|
||||
// infer the spec types from the GGUF metadata of a draft model; empty if unknown
|
||||
std::vector<enum common_speculative_type> common_speculative_types_from_gguf(const std::string & path);
|
||||
|
||||
// convert string to type
|
||||
enum common_speculative_type common_speculative_type_from_name(const std::string & name);
|
||||
|
||||
|
|
@ -28,15 +25,6 @@ int32_t common_speculative_n_max(const common_params_speculative * spec);
|
|||
|
||||
common_params common_base_params_to_speculative(const common_params & params);
|
||||
|
||||
struct common_speculative_output_limits {
|
||||
int32_t total;
|
||||
int32_t per_seq;
|
||||
};
|
||||
|
||||
// return the output limits needed for speculative decoding
|
||||
common_speculative_output_limits common_speculative_get_output_limits(
|
||||
int32_t n_batch, int32_t n_parallel, int32_t n_draft);
|
||||
|
||||
common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq);
|
||||
|
||||
void common_speculative_free(common_speculative * spec);
|
||||
|
|
@ -70,6 +58,12 @@ void common_speculative_begin(common_speculative * spec, llama_seq_id seq_id, co
|
|||
// process the batch and update the internal state of the speculative context
|
||||
bool common_speculative_process(common_speculative * spec, const llama_batch & batch);
|
||||
|
||||
// true if any implementation requires target post-norm embeddings to be extracted
|
||||
bool common_speculative_need_embd(common_speculative * spec);
|
||||
|
||||
// true if any implementation requires target nextn embeddings to be extracted
|
||||
bool common_speculative_need_embd_nextn(common_speculative * spec);
|
||||
|
||||
// generate drafts for the sequences specified with `common_speculative_get_draft_params`
|
||||
void common_speculative_draft(common_speculative * spec);
|
||||
|
||||
|
|
|
|||
|
|
@ -55,7 +55,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"DFlashDraftModel": "qwen",
|
||||
"Qwen3DSparkModel": "qwen",
|
||||
"DeepseekV4ForCausalLM": "deepseek",
|
||||
"DeepseekV4DSparkModel": "deepseek",
|
||||
"DistilBertForMaskedLM": "bert",
|
||||
"DistilBertForSequenceClassification": "bert",
|
||||
"DistilBertModel": "bert",
|
||||
|
|
@ -70,7 +69,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"Exaone4ForCausalLM": "exaone",
|
||||
"ExaoneForCausalLM": "exaone",
|
||||
"ExaoneMoEForCausalLM": "exaone",
|
||||
"ExaoneMoeForCausalLM": "exaone",
|
||||
"FalconForCausalLM": "falcon",
|
||||
"FalconH1ForCausalLM": "falcon_h1",
|
||||
"FalconMambaForCausalLM": "mamba",
|
||||
|
|
@ -103,7 +101,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"GraniteMoeForCausalLM": "granite",
|
||||
"GraniteMoeHybridForCausalLM": "granite",
|
||||
"GraniteMoeSharedForCausalLM": "granite",
|
||||
"GraniteSwitchForCausalLM": "granite",
|
||||
"GraniteSpeechForConditionalGeneration": "granite",
|
||||
"GraniteSpeechPlusForConditionalGeneration": "granite",
|
||||
"Grok1ForCausalLM": "grok",
|
||||
|
|
@ -161,8 +158,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"MiniCPM3ForCausalLM": "minicpm",
|
||||
"MiniCPMForCausalLM": "minicpm",
|
||||
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
|
||||
"MiniMaxText01ForCausalLM": "minimax",
|
||||
"MiniMaxM1ForCausalLM": "minimax",
|
||||
"MiniMaxM2ForCausalLM": "minimax",
|
||||
"MiniMaxM3SparseForCausalLM": "minimax",
|
||||
"MiniMaxM3SparseForConditionalGeneration": "minimax",
|
||||
|
|
@ -185,8 +180,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"Olmo3ForCausalLM": "olmo",
|
||||
"OlmoForCausalLM": "olmo",
|
||||
"OlmoeForCausalLM": "olmo",
|
||||
"MuseGlimmerAssistantModel": "muse_glimmer",
|
||||
"MuseGlimmerForConditionalGeneration": "muse_glimmer",
|
||||
"OpenELMForCausalLM": "openelm",
|
||||
"OrionForCausalLM": "orion",
|
||||
"PLMForCausalLM": "plm",
|
||||
|
|
@ -216,8 +209,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"Qwen3MoeForCausalLM": "qwen",
|
||||
"Qwen3NextForCausalLM": "qwen",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3_5ForCausalLM": "qwen",
|
||||
|
|
@ -303,7 +294,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
|||
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
|
||||
"Mistral3ForConditionalGeneration": "llava",
|
||||
"NemotronH_Nano_VL_V2": "nemotron",
|
||||
"MuseGlimmerForConditionalGeneration": "muse_glimmer",
|
||||
"PaddleOCRVisionModel": "ernie",
|
||||
"Phi4ForCausalLMV": "phi",
|
||||
"Qwen2AudioForConditionalGeneration": "ultravox",
|
||||
|
|
@ -313,8 +303,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
|||
"Qwen2_5_VLForConditionalGeneration": "qwenvl",
|
||||
"Qwen3ASRForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3_5ForConditionalGeneration": "qwen3vl",
|
||||
|
|
|
|||
|
|
@ -58,11 +58,6 @@ logger = logging.getLogger("hf-to-gguf")
|
|||
AnyModel = TypeVar("AnyModel", bound="type[ModelBase]")
|
||||
|
||||
|
||||
# for checkpoints that ship no config.json, we will try to provide a synthetic one
|
||||
HparamsMatcher = Callable[[Path], bool]
|
||||
HparamsLoader = Callable[[Path], dict[str, Any]]
|
||||
|
||||
|
||||
class SentencePieceTokenTypes(IntEnum):
|
||||
NORMAL = 1
|
||||
UNKNOWN = 2
|
||||
|
|
@ -82,7 +77,6 @@ class ModelBase:
|
|||
ModelType.TEXT: {},
|
||||
ModelType.MMPROJ: {},
|
||||
}
|
||||
_hparams_loaders: list[tuple[HparamsMatcher, HparamsLoader]] = []
|
||||
|
||||
dir_model: Path
|
||||
ftype: gguf.LlamaFileType
|
||||
|
|
@ -829,7 +823,7 @@ class ModelBase:
|
|||
elif any(str(v.get("quant_algo")).endswith("NVFP4") for v in quant_layers.values() if isinstance(v, dict)):
|
||||
quant_algo = "NVFP4"
|
||||
|
||||
self._is_nvfp4 = quant_algo in ("NVFP4", "W4A16_NVFP4")
|
||||
self._is_nvfp4 = quant_algo == "NVFP4"
|
||||
self._is_mxfp4 = quant_method == "mxfp4"
|
||||
|
||||
# NVFP4 weights are repacked and written directly to gguf_writer.
|
||||
|
|
@ -1046,24 +1040,6 @@ class ModelBase:
|
|||
|
||||
return part_names
|
||||
|
||||
@staticmethod
|
||||
def load_hparams_guess(dir_model: Path) -> dict[str, Any] | None:
|
||||
# some models ship no config.json, will try to guess them
|
||||
from conversion import load_all_models
|
||||
load_all_models()
|
||||
|
||||
for matcher, loader in ModelBase._hparams_loaders:
|
||||
if matcher(dir_model):
|
||||
return loader(dir_model)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def register_hparams_loader(cls, matcher: HparamsMatcher) -> Callable[[HparamsLoader], HparamsLoader]:
|
||||
def inner(loader: HparamsLoader) -> HparamsLoader:
|
||||
cls._hparams_loaders.append((matcher, loader))
|
||||
return loader
|
||||
return inner
|
||||
|
||||
@staticmethod
|
||||
def load_hparams(dir_model: Path, is_mistral_format: bool):
|
||||
if is_mistral_format:
|
||||
|
|
@ -1077,10 +1053,6 @@ class ModelBase:
|
|||
config = AutoConfig.from_pretrained(dir_model, trust_remote_code=False).to_dict()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load model config from {dir_model}: {e}")
|
||||
if not (dir_model / "config.json").is_file():
|
||||
config = ModelBase.load_hparams_guess(dir_model)
|
||||
if config is not None:
|
||||
return config
|
||||
logger.warning("Trying to load config.json instead")
|
||||
with open(dir_model / "config.json", "r", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ class ChatGLMModel(TextModel):
|
|||
|
||||
@staticmethod
|
||||
def token_bytes_to_string(b):
|
||||
from transformers.convert_slow_tokenizer import bytes_to_unicode
|
||||
from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import]
|
||||
byte_encoder = bytes_to_unicode()
|
||||
return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])
|
||||
|
||||
|
|
|
|||
|
|
@ -17,11 +17,8 @@ from .base import LazyTorchTensor, MmprojModel, ModelBase, TextModel, gguf, logg
|
|||
from .qwen import QwenModel
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekOCRForCausalLM")
|
||||
@ModelBase.register("DeepseekOCRForCausalLM", "UnlimitedOCRForCausalLM")
|
||||
class DeepseekOCRVisionModel(MmprojModel):
|
||||
# HF dynamic_preprocess() max_num, which differs per model
|
||||
preproc_max_tiles = 9
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR
|
||||
|
|
@ -46,9 +43,6 @@ class DeepseekOCRVisionModel(MmprojModel):
|
|||
# @bluebread: there's no window_size in config but just add it here anyway
|
||||
self.gguf_writer.add_vision_window_size(self.hparams.get("window_size", 14))
|
||||
|
||||
self.gguf_writer.add_vision_preproc_min_tiles(2)
|
||||
self.gguf_writer.add_vision_preproc_max_tiles(self.preproc_max_tiles)
|
||||
|
||||
# SAM configuration
|
||||
sam_hparams = hparams['sam']
|
||||
self.gguf_writer.add_vision_sam_layers_count(sam_hparams['layers'])
|
||||
|
|
@ -99,15 +93,8 @@ class DeepseekOCRVisionModel(MmprojModel):
|
|||
return super().filter_tensors((name, gen))
|
||||
|
||||
|
||||
@ModelBase.register("UnlimitedOCRForCausalLM")
|
||||
class UnlimitedOCRVisionModel(DeepseekOCRVisionModel):
|
||||
preproc_max_tiles = 32
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekOCR2ForCausalLM")
|
||||
class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
|
||||
preproc_max_tiles = 6
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR2
|
||||
|
|
@ -460,43 +447,12 @@ class DeepseekV2Model(TextModel):
|
|||
class DeepseekV32Model(DeepseekV2Model):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK32
|
||||
skip_mtp = False
|
||||
supports_mtp_export = True
|
||||
_n_main_layers: int | None = None
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.block_count = self.hparams["num_hidden_layers"]
|
||||
if not self.no_mtp:
|
||||
self.block_count += self.hparams.get("num_nextn_predict_layers", 0)
|
||||
self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None):
|
||||
type(self)._n_main_layers = self.hparams["num_hidden_layers"]
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
if (titem := super().filter_tensors(item)) is None:
|
||||
return None
|
||||
name, gen = titem
|
||||
|
||||
# DeepSeek V3.2 appends the NextN/MTP block past num_hidden_layers
|
||||
# (model.layers.61 -> blk.61 in the 62-block file).
|
||||
assert cls._n_main_layers is not None
|
||||
is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers
|
||||
|
||||
# --no-mtp: drop the appended NextN block entirely.
|
||||
if is_mtp and cls.no_mtp:
|
||||
return None
|
||||
# --mtp: keep ONLY NextN-block tensors plus the shared embeddings/
|
||||
# norm/lm_head (so the resulting GGUF carries just the draft head).
|
||||
if cls.mtp_only and not is_mtp and name not in (
|
||||
"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
|
||||
):
|
||||
return None
|
||||
|
||||
return name, gen
|
||||
|
||||
def set_vocab(self):
|
||||
from transformers import AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
|
|
@ -507,7 +463,7 @@ class DeepseekV32Model(DeepseekV2Model):
|
|||
super().set_gguf_parameters()
|
||||
|
||||
# NextN/MTP prediction layers
|
||||
if not self.no_mtp and (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
|
||||
if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
|
||||
self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
|
||||
|
||||
# DSA indexer parameters
|
||||
|
|
@ -519,10 +475,7 @@ class DeepseekV32Model(DeepseekV2Model):
|
|||
@ModelBase.register("DeepseekV4ForCausalLM")
|
||||
class DeepseekV4Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK4
|
||||
supports_mtp_export = True
|
||||
_skipped_mtp_tensors = 0
|
||||
_dsv4_main_layers: int | None = None
|
||||
_dsv4_nextn_layers: int = 0
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
type(self)._skipped_mtp_tensors = 0
|
||||
|
|
@ -533,16 +486,7 @@ class DeepseekV4Model(TextModel):
|
|||
for key, value in raw_hparams.items():
|
||||
self.hparams.setdefault(key, value)
|
||||
|
||||
# workaround for special rope_parameters (main/compress) in transformers 5.x
|
||||
if self.rope_parameters.get("full_attention", self.rope_parameters).get("rope_type") is None:
|
||||
if (rope_scaling := raw_hparams.get("rope_scaling")) is not None:
|
||||
if "rope_type" not in rope_scaling and (rope_type := rope_scaling.get("type")) is not None:
|
||||
rope_scaling["rope_type"] = rope_type
|
||||
self.rope_parameters.update(**rope_scaling)
|
||||
|
||||
self.block_count = self.hparams["num_hidden_layers"]
|
||||
if self.mtp_only:
|
||||
self.block_count += self.hparams.get("num_nextn_predict_layers", 0)
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
self._dsv4_fp8_dequantized: set[str] = set()
|
||||
|
|
@ -555,71 +499,18 @@ class DeepseekV4Model(TextModel):
|
|||
logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors)
|
||||
|
||||
# add a default chat template; if the model has a built-in template, it will be overridden later
|
||||
model_id_hint = self.remote_hf_model_id or self.dir_model.name
|
||||
is_0731 = "0731" in model_id_hint
|
||||
template_name = "deepseek-ai-DeepSeek-V4-Flash-0731.jinja" if is_0731 else "deepseek-ai-DeepSeek-V4.jinja"
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / template_name
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja"
|
||||
if template_path.is_file():
|
||||
with open(template_path, "r", encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template(f.read())
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
type(self)._dsv4_main_layers = self.hparams["num_hidden_layers"]
|
||||
type(self)._dsv4_nextn_layers = self.hparams.get("num_nextn_predict_layers", 0)
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
name, _ = item
|
||||
if name.startswith("mtp."):
|
||||
if not cls.mtp_only:
|
||||
cls._skipped_mtp_tensors += 1
|
||||
return None
|
||||
|
||||
assert cls._dsv4_main_layers is not None
|
||||
parts = name.split(".", 2)
|
||||
if len(parts) < 3 or not parts[1].isdecimal():
|
||||
raise ValueError(f"Unexpected DeepSeek-V4 MTP tensor {name!r}")
|
||||
|
||||
mtp_idx = int(parts[1])
|
||||
if mtp_idx >= cls._dsv4_nextn_layers:
|
||||
raise ValueError(f"Unexpected DeepSeek-V4 MTP layer {mtp_idx}")
|
||||
|
||||
bid = cls._dsv4_main_layers + mtp_idx
|
||||
suffix = parts[2]
|
||||
root_hc_head = {
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
}
|
||||
if suffix in root_hc_head:
|
||||
name = suffix
|
||||
elif suffix in (
|
||||
"e_proj.weight", "e_proj.scale",
|
||||
"h_proj.weight", "h_proj.scale",
|
||||
):
|
||||
name = f"layers.{bid}.nextn.{suffix}"
|
||||
elif suffix == "enorm.weight":
|
||||
name = f"layers.{bid}.nextn.enorm.weight"
|
||||
elif suffix == "hnorm.weight":
|
||||
name = f"layers.{bid}.nextn.hnorm.weight"
|
||||
elif suffix == "norm.weight":
|
||||
name = f"layers.{bid}.nextn.shared_head_norm.weight"
|
||||
else:
|
||||
name = f"layers.{bid}.{suffix}"
|
||||
return name, gen
|
||||
|
||||
if cls.mtp_only:
|
||||
keep = name in (
|
||||
"embed.weight",
|
||||
"norm.weight",
|
||||
"head.weight",
|
||||
"head.scale",
|
||||
)
|
||||
if not keep:
|
||||
return None
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
cls._skipped_mtp_tensors += 1
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
@staticmethod
|
||||
def _float8_dtypes() -> tuple[torch.dtype, ...]:
|
||||
|
|
@ -674,10 +565,6 @@ class DeepseekV4Model(TextModel):
|
|||
self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"])
|
||||
self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"])
|
||||
self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"])
|
||||
if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4:
|
||||
self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"])
|
||||
if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0:
|
||||
self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
|
||||
|
||||
def dequant_model(self):
|
||||
fp8_dtypes = self._float8_dtypes()
|
||||
|
|
@ -782,37 +669,12 @@ class DeepseekV4Model(TextModel):
|
|||
if self._dsv4_mxfp4_generated:
|
||||
return ()
|
||||
|
||||
consumed: list[str] = []
|
||||
main_layers = self.hparams["num_hidden_layers"]
|
||||
if not self.mtp_only:
|
||||
consumed.extend(self._write_hash_routing_tensors())
|
||||
elif self.hparams["num_hash_layers"] > 0:
|
||||
for bid in range(self.hparams["num_hash_layers"]):
|
||||
name = f"layers.{bid}.ffn.gate.tid2eid"
|
||||
if name in self.model_tensors:
|
||||
consumed.extend(self._write_hash_routing_tensors())
|
||||
break
|
||||
|
||||
consumed: list[str] = self._write_hash_routing_tensors()
|
||||
for bid in range(self.block_count):
|
||||
if self.mtp_only and bid < main_layers:
|
||||
continue
|
||||
consumed.extend(self._write_mxfp4_expert_tensor(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP))
|
||||
consumed.extend(self._write_mxfp4_expert_tensor(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP))
|
||||
consumed.extend(self._write_mxfp4_expert_tensor(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP))
|
||||
|
||||
for bid in range(main_layers, self.block_count):
|
||||
e_name = f"layers.{bid}.nextn.e_proj.weight"
|
||||
h_name = f"layers.{bid}.nextn.h_proj.weight"
|
||||
if e_name not in self.model_tensors and h_name not in self.model_tensors:
|
||||
continue
|
||||
if e_name not in self.model_tensors or h_name not in self.model_tensors:
|
||||
raise KeyError(f"Missing DeepSeek-V4 MTP e/h projection pair for block {bid}")
|
||||
|
||||
e_proj = LazyTorchTensor.to_eager(self.model_tensors[e_name]())
|
||||
h_proj = LazyTorchTensor.to_eager(self.model_tensors[h_name]())
|
||||
yield (f"layers.{bid}.nextn.eh_proj.weight", torch.cat((e_proj, h_proj), dim=1).contiguous())
|
||||
consumed.extend((e_name, h_name))
|
||||
|
||||
for name in consumed:
|
||||
del self.model_tensors[name]
|
||||
|
||||
|
|
@ -875,12 +737,6 @@ class DeepseekV4Model(TextModel):
|
|||
"ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"),
|
||||
"ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"),
|
||||
"ffn.shared_experts.w3.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"),
|
||||
"nextn.eh_proj.weight": (gguf.MODEL_TENSOR.NEXTN_EH_PROJ, ".weight"),
|
||||
"nextn.enorm.weight": (gguf.MODEL_TENSOR.NEXTN_ENORM, ".weight"),
|
||||
"nextn.hnorm.weight": (gguf.MODEL_TENSOR.NEXTN_HNORM, ".weight"),
|
||||
"nextn.shared_head_norm.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ".weight"),
|
||||
"nextn.embed_tokens.weight": (gguf.MODEL_TENSOR.NEXTN_EMBED_TOKENS, ".weight"),
|
||||
"nextn.shared_head_head.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, ".weight"),
|
||||
}
|
||||
|
||||
tensor_name = match.group(2)
|
||||
|
|
@ -903,12 +759,10 @@ class DeepseekV4Model(TextModel):
|
|||
return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)]
|
||||
|
||||
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
||||
del bid # unused
|
||||
del new_name, bid # unused
|
||||
|
||||
if name in self._dsv4_fp8_dequantized and n_dims >= 2:
|
||||
return gguf.GGMLQuantizationType.Q8_0
|
||||
if new_name.endswith(".nextn.eh_proj.weight"):
|
||||
return gguf.GGMLQuantizationType.Q8_0
|
||||
if name in self._dsv4_f32_tensors:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
if name in self._dsv4_bf16_tensors and n_dims >= 2:
|
||||
|
|
@ -916,122 +770,7 @@ class DeepseekV4Model(TextModel):
|
|||
|
||||
return False
|
||||
|
||||
def prepare_metadata(self, vocab_only: bool):
|
||||
from_dir = self.fname_out.is_dir()
|
||||
super().prepare_metadata(vocab_only=vocab_only)
|
||||
|
||||
if not self.mtp_only or not from_dir:
|
||||
return
|
||||
|
||||
output_type: str = self.ftype.name.partition("_")[2]
|
||||
fname_default: str = gguf.naming_convention(
|
||||
self.metadata.name, self.metadata.basename, self.metadata.finetune,
|
||||
self.metadata.version, size_label=None, output_type=output_type, model_type=None)
|
||||
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
|
||||
|
||||
def prepare_tensors(self):
|
||||
super().prepare_tensors()
|
||||
self._is_mxfp4 = True
|
||||
self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekV4DSparkModel")
|
||||
class DeepseekV4DSparkModel(DeepseekV4Model):
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
_DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
|
||||
"main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"),
|
||||
"main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"),
|
||||
"markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"),
|
||||
"markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"),
|
||||
"confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"),
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.block_count = 1 + max(
|
||||
int(match.group(1)) for name in self.model_tensors
|
||||
if (match := re.match(r"layers\.(\d+)\.", name))
|
||||
)
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
self.hparams["compress_ratios"] = [0] * self.block_count
|
||||
self.hparams["num_hash_layers"] = 0
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
if remote_hf_model_id is None:
|
||||
return super().index_tensors()
|
||||
|
||||
with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
|
||||
part_names = sorted({
|
||||
part_name for name, part_name in weight_map.items()
|
||||
if name.startswith("mtp.")
|
||||
})
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
|
||||
for part_name in part_names:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
logger.info("gguf: caching remote DSpark part '%s'", part_name)
|
||||
part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name))
|
||||
with gguf.utility.SafetensorsLocal(part_path) as model_part:
|
||||
for name in model_part:
|
||||
data = model_part[name]
|
||||
data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731
|
||||
if titem := self.filter_tensors((name, data_gen)):
|
||||
tensor_name, tensor_gen = titem
|
||||
tensors[tensor_name] = tensor_gen
|
||||
|
||||
return tensors
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if not name.startswith("mtp."):
|
||||
return None
|
||||
return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen))
|
||||
|
||||
@staticmethod
|
||||
def _rekey_mtp_tensor_name(name: str) -> str:
|
||||
match = re.match(r"mtp\.(\d+)\.(.+)$", name)
|
||||
if match is None:
|
||||
raise ValueError(f"Unexpected DSpark tensor {name!r}")
|
||||
|
||||
stage, rest = match.group(1), match.group(2)
|
||||
root_names = (
|
||||
"main_proj.scale",
|
||||
"norm.weight",
|
||||
"hc_head_fn",
|
||||
"hc_head_base",
|
||||
"hc_head_scale",
|
||||
)
|
||||
if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names:
|
||||
return rest
|
||||
return f"layers.{stage}.{rest}"
|
||||
|
||||
def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
|
||||
if name in self._DSPARK_ROOT_MAP:
|
||||
return self._DSPARK_ROOT_MAP[name]
|
||||
return super()._map_dsv4_tensor_name(name, bid)
|
||||
|
||||
def set_vocab(self):
|
||||
if self.target_model_dir is None:
|
||||
raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer")
|
||||
|
||||
original_dir = self.dir_model
|
||||
try:
|
||||
self.dir_model = self.target_model_dir
|
||||
super().set_vocab()
|
||||
finally:
|
||||
self.dir_model = original_dir
|
||||
|
||||
self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"])
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])
|
||||
self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])
|
||||
|
|
|
|||
|
|
@ -123,9 +123,7 @@ class Exaone4Model(TextModel):
|
|||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))
|
||||
|
||||
|
||||
# note: transformers >= 5.1 renamed the class to "ExaoneMoeForCausalLM" (lowercase 'e'),
|
||||
# so accept both spellings - LG AI have updated the configs of already-released models
|
||||
@ModelBase.register("ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM")
|
||||
@ModelBase.register("ExaoneMoEForCausalLM")
|
||||
class ExaoneMoEModel(Exaone4Model):
|
||||
model_arch = gguf.MODEL_ARCH.EXAONE_MOE
|
||||
|
||||
|
|
|
|||
|
|
@ -665,18 +665,7 @@ class Gemma4Model(Gemma3Model):
|
|||
swa_layers = [t == "sliding_attention" for t in self.hparams["layer_types"]]
|
||||
self.gguf_writer.add_sliding_window_pattern(swa_layers)
|
||||
|
||||
per_layer_config = self.hparams.get("per_layer_config")
|
||||
layer_types = self.hparams.get("layer_types", [])
|
||||
if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None:
|
||||
for layer_idx, layer_config in per_layer_config.items():
|
||||
layer_idx = int(layer_idx)
|
||||
if layer_idx < len(layer_types):
|
||||
if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config:
|
||||
head_dim_full = layer_config["head_dim"]
|
||||
break
|
||||
|
||||
assert head_dim_full is not None
|
||||
|
||||
head_dim_full = self.hparams["global_head_dim"]
|
||||
head_dim_swa = self.hparams["head_dim"]
|
||||
# correct the head dim for global/swa layers
|
||||
self.gguf_writer.add_key_length(head_dim_full)
|
||||
|
|
@ -696,14 +685,8 @@ class Gemma4Model(Gemma3Model):
|
|||
n_ff_arr = [n_ff if il < first_kv_shared_layer_idx else n_ff * 2 for il in range(self.block_count)]
|
||||
self.gguf_writer.add_feed_forward_length(n_ff_arr)
|
||||
|
||||
if (num_key_value_heads_full := self.hparams.get("num_global_key_value_heads")) is None and per_layer_config is not None:
|
||||
for layer_idx, layer_config in per_layer_config.items():
|
||||
layer_idx = int(layer_idx)
|
||||
if layer_idx < len(layer_types):
|
||||
if layer_types[layer_idx] == "full_attention" and "num_key_value_heads" in layer_config:
|
||||
num_key_value_heads_full = layer_config["num_key_value_heads"]
|
||||
break
|
||||
|
||||
# handle num_global_key_value_heads
|
||||
num_key_value_heads_full = self.hparams.get("num_global_key_value_heads")
|
||||
num_key_value_heads_swa = self.hparams.get("num_key_value_heads")
|
||||
if num_key_value_heads_full is not None and num_key_value_heads_swa is not None:
|
||||
value_arr = [num_key_value_heads_swa if is_swa else num_key_value_heads_full for is_swa in swa_layers]
|
||||
|
|
@ -725,19 +708,7 @@ class Gemma4Model(Gemma3Model):
|
|||
# IMPORTANT: this ROPE_FREQS tensor is ONLY used by the full_attention layers
|
||||
rope_params_full = self.hparams["rope_parameters"]["full_attention"]
|
||||
assert rope_params_full["rope_type"] == "proportional"
|
||||
|
||||
per_layer_config = self.hparams.get("per_layer_config")
|
||||
if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None:
|
||||
layer_types = self.hparams.get("layer_types", [])
|
||||
for layer_idx, layer_config in per_layer_config.items():
|
||||
layer_idx = int(layer_idx)
|
||||
if layer_idx < len(layer_types):
|
||||
if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config:
|
||||
head_dim_full = layer_config["head_dim"]
|
||||
break
|
||||
|
||||
assert head_dim_full is not None
|
||||
|
||||
head_dim_full = (self.hparams["global_head_dim"])
|
||||
partial_rotary_factor_full = rope_params_full["partial_rotary_factor"]
|
||||
n_rot_full = int(head_dim_full * partial_rotary_factor_full / 2)
|
||||
n_unrot_full = int(head_dim_full / 2) - n_rot_full
|
||||
|
|
|
|||
|
|
@ -206,70 +206,10 @@ class Glm4MoeModel(TextModel):
|
|||
@ModelBase.register("Glm4MoeLiteForCausalLM")
|
||||
class Glm4MoeLiteModel(DeepseekV2Model):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
|
||||
skip_mtp = False
|
||||
supports_mtp_export = True
|
||||
_n_main_layers: int | None = None
|
||||
|
||||
def set_vocab(self):
|
||||
return self._set_vocab_glm()
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
num_hidden_layers = self.hparams["num_hidden_layers"]
|
||||
self.num_nextn_predict_layers = self.hparams.get("num_nextn_predict_layers", 0)
|
||||
self.skip_mtp = self.no_mtp or self.num_nextn_predict_layers == 0
|
||||
|
||||
if self.skip_mtp:
|
||||
self.block_count = num_hidden_layers
|
||||
else:
|
||||
self.block_count = num_hidden_layers + self.num_nextn_predict_layers
|
||||
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
if self.skip_mtp:
|
||||
return
|
||||
|
||||
self.gguf_writer.add_nextn_predict_layers(self.num_nextn_predict_layers)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None):
|
||||
type(self)._n_main_layers = self.hparams["num_hidden_layers"]
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
if (titem := super().filter_tensors(item)) is None:
|
||||
return None
|
||||
name, gen = titem
|
||||
|
||||
if cls._n_main_layers is not None:
|
||||
match = re.match(r"model\.layers\.(\d+)\.", name)
|
||||
is_mtp = match is not None and int(match.group(1)) >= cls._n_main_layers
|
||||
if is_mtp and cls.no_mtp:
|
||||
return None
|
||||
if cls.mtp_only and not is_mtp and name not in (
|
||||
"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
|
||||
):
|
||||
return None
|
||||
|
||||
return name, gen
|
||||
|
||||
def prepare_metadata(self, vocab_only: bool):
|
||||
from_dir = self.fname_out.is_dir()
|
||||
super().prepare_metadata(vocab_only=vocab_only)
|
||||
|
||||
if not self.mtp_only or not from_dir:
|
||||
return
|
||||
|
||||
output_type: str = self.ftype.name.partition("_")[2]
|
||||
fname_default: str = gguf.naming_convention(
|
||||
self.metadata.name, self.metadata.basename, self.metadata.finetune,
|
||||
self.metadata.version, size_label=None, output_type=output_type, model_type=None)
|
||||
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
|
||||
|
||||
|
||||
@ModelBase.register("GlmMoeDsaForCausalLM")
|
||||
class GlmMoeDsaModel(DeepseekV2Model):
|
||||
|
|
|
|||
|
|
@ -123,166 +123,6 @@ class GraniteMoeModel(GraniteModel):
|
|||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("GraniteSwitchForCausalLM")
|
||||
class GraniteSwitchModel(GraniteMoeModel):
|
||||
"""Dense, all-attention Granite with N per-token embedded LoRA adapters, stacked
|
||||
over the adapter dim with a zero adapter at slot 0 (N = num_adapters + 1)."""
|
||||
model_arch = gguf.MODEL_ARCH.GRANITE_SWITCH
|
||||
|
||||
# permute q/k per-slice below (NORM-rope layout), not via the parent's auto-permute
|
||||
undo_permute = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# the weightless switch reserves one cache slot: one fewer block than num_hidden_layers
|
||||
self.block_count = self.block_count - 1
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
self._n_adapters = int(self.hparams["num_adapters"])
|
||||
self._max_lora_rank = int(self.hparams["max_lora_rank"])
|
||||
self._n_slots = self._n_adapters + 1 # +1 for the zero slot at index 0
|
||||
|
||||
n_head = int(self.hparams["num_attention_heads"])
|
||||
n_kv_head = int(self.hparams["num_key_value_heads"])
|
||||
head_dim = (
|
||||
self.hparams.get("projection_head_dim")
|
||||
or self.hparams.get("head_dim")
|
||||
or (self.hparams["hidden_size"] // n_head)
|
||||
)
|
||||
self._n_head = n_head
|
||||
self._n_kv_head = n_kv_head
|
||||
self._head_dim = int(head_dim)
|
||||
self._q_size = n_head * self._head_dim
|
||||
self._kv_size = n_kv_head * self._head_dim
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
# dense: pin expert_used_count to 0 (config carries a leftover num_experts_per_tok)
|
||||
if not self.hparams.get("num_local_experts"):
|
||||
self.gguf_writer.add_expert_used_count(0)
|
||||
|
||||
self.gguf_writer.add_adapter_count(self._n_adapters)
|
||||
self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank)
|
||||
self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"])
|
||||
self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"])
|
||||
router_gain = float(self.hparams.get("control_token_gain", 15.0))
|
||||
self.gguf_writer.add_adapter_router_gain(router_gain)
|
||||
logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain)
|
||||
|
||||
def _lora_a(self, data: Tensor) -> Tensor:
|
||||
# on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in]
|
||||
a = data.squeeze(1)
|
||||
zero = torch.zeros_like(a[:1])
|
||||
return torch.cat([zero, a], dim=0).contiguous()
|
||||
|
||||
def _lora_b(self, data: Tensor, permute_n_head: int | None = None) -> Tensor:
|
||||
# on-disk B: [n_adapters, 1, out, max_rank] -> [n_adapters+1, out, max_rank]
|
||||
b = data.squeeze(1)
|
||||
if permute_n_head is not None:
|
||||
# permute each adapter's B output rows to match the permuted q/k base
|
||||
b = torch.stack([self.permute(b[i], permute_n_head, permute_n_head) for i in range(b.shape[0])], dim=0)
|
||||
zero = torch.zeros_like(b[:1])
|
||||
return torch.cat([zero, b], dim=0).contiguous()
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
# skip the weightless switch + control-token buffers (rebuilt at load time)
|
||||
bare = name.split(".")[-1]
|
||||
if (
|
||||
name.startswith("model.switch.") or name.startswith("switch.")
|
||||
or bare in ("adapter_token_ids", "control_to_substitute_lut")
|
||||
):
|
||||
return
|
||||
|
||||
if "self_attn.qkv_proj" in name:
|
||||
if name.endswith("base_layer.weight"):
|
||||
# fused [q|k|v] rows: permute q/k row-blocks for ggml's NORM-rope layout
|
||||
q, k, v = data_torch.split([self._q_size, self._kv_size, self._kv_size], dim=0)
|
||||
q = self.permute(q, self._n_head, self._n_head)
|
||||
k = self.permute(k, self._n_kv_head, self._n_kv_head)
|
||||
fused = torch.cat([q, k, v], dim=0)
|
||||
yield (self.format_tensor_name(T.ATTN_QKV, bid), fused)
|
||||
return
|
||||
if "lora_A_slices." in name:
|
||||
slot = int(name.rsplit(".", 1)[1])
|
||||
key = {0: T.ATTN_Q, 1: T.ATTN_K, 2: T.ATTN_V}[slot]
|
||||
yield (self.format_tensor_name(key, bid, suffix=".lora_a"), self._lora_a(data_torch))
|
||||
return
|
||||
if "lora_B_slices." in name:
|
||||
slot = int(name.rsplit(".", 1)[1])
|
||||
key, ph = {
|
||||
0: (T.ATTN_Q, self._n_head),
|
||||
1: (T.ATTN_K, self._n_kv_head),
|
||||
2: (T.ATTN_V, None),
|
||||
}[slot]
|
||||
yield (self.format_tensor_name(key, bid, suffix=".lora_b"), self._lora_b(data_torch, ph))
|
||||
return
|
||||
raise ValueError(f"Unexpected qkv_proj tensor: {name}")
|
||||
|
||||
if "self_attn.o_proj" in name:
|
||||
if name.endswith("base_layer.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch)
|
||||
return
|
||||
if name.endswith("lora_A"):
|
||||
yield (self.format_tensor_name(T.ATTN_OUT, bid, suffix=".lora_a"), self._lora_a(data_torch))
|
||||
return
|
||||
if name.endswith("lora_B"):
|
||||
yield (self.format_tensor_name(T.ATTN_OUT, bid, suffix=".lora_b"), self._lora_b(data_torch))
|
||||
return
|
||||
raise ValueError(f"Unexpected o_proj tensor: {name}")
|
||||
|
||||
if "shared_mlp.input_linear" in name:
|
||||
ffn = self.hparams["shared_intermediate_size"]
|
||||
if name.endswith("base_layer.weight"):
|
||||
gate, up = data_torch.split([ffn, ffn], dim=0)
|
||||
yield (self.format_tensor_name(T.FFN_GATE, bid), gate)
|
||||
yield (self.format_tensor_name(T.FFN_UP, bid), up)
|
||||
return
|
||||
if "lora_A_slices." in name:
|
||||
slot = int(name.rsplit(".", 1)[1])
|
||||
key = {0: T.FFN_GATE, 1: T.FFN_UP}[slot]
|
||||
yield (self.format_tensor_name(key, bid, suffix=".lora_a"), self._lora_a(data_torch))
|
||||
return
|
||||
if "lora_B_slices." in name:
|
||||
slot = int(name.rsplit(".", 1)[1])
|
||||
key = {0: T.FFN_GATE, 1: T.FFN_UP}[slot]
|
||||
yield (self.format_tensor_name(key, bid, suffix=".lora_b"), self._lora_b(data_torch))
|
||||
return
|
||||
raise ValueError(f"Unexpected shared_mlp.input_linear tensor: {name}")
|
||||
|
||||
if "shared_mlp.output_linear" in name:
|
||||
if name.endswith("base_layer.weight"):
|
||||
yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch)
|
||||
return
|
||||
if name.endswith("lora_A"):
|
||||
yield (self.format_tensor_name(T.FFN_DOWN, bid, suffix=".lora_a"), self._lora_a(data_torch))
|
||||
return
|
||||
if name.endswith("lora_B"):
|
||||
yield (self.format_tensor_name(T.FFN_DOWN, bid, suffix=".lora_b"), self._lora_b(data_torch))
|
||||
return
|
||||
raise ValueError(f"Unexpected shared_mlp.output_linear tensor: {name}")
|
||||
|
||||
if bid is not None and ".layers." in name and (
|
||||
"input_layernorm" in name or "post_attention_layernorm" in name
|
||||
):
|
||||
key = T.ATTN_NORM if "input_layernorm" in name else T.FFN_NORM
|
||||
yield (self.format_tensor_name(key, bid), data_torch)
|
||||
return
|
||||
|
||||
if name in ("model.embed_tokens.weight", "embed_tokens.weight"):
|
||||
yield (self.format_tensor_name(T.TOKEN_EMBD), data_torch)
|
||||
return
|
||||
if name in ("model.norm.weight", "norm.weight"):
|
||||
yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch)
|
||||
return
|
||||
if name == "lm_head.weight":
|
||||
return # tied to token_embd
|
||||
|
||||
raise ValueError(f"graniteswitch: unhandled tensor {name!r} (bid={bid})")
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeHybridForCausalLM", "BambaForCausalLM")
|
||||
class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
|
||||
"""GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ class LlamaModel(TextModel):
|
|||
path_tekken_json = self.dir_model / "tekken.json"
|
||||
path_tokenizer_json = self.dir_model / "tokenizer.json"
|
||||
if path_tekken_json.is_file() and not path_tokenizer_json.is_file():
|
||||
return self._set_vocab_mistral()
|
||||
self._set_vocab_mistral()
|
||||
|
||||
tokenizer_config_file = self.dir_model / 'tokenizer_config.json'
|
||||
if tokenizer_config_file.is_file():
|
||||
|
|
|
|||
|
|
@ -137,15 +137,6 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel):
|
|||
class MiniCPMV4_6VisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x")
|
||||
if self.downsample_mode not in {"4x", "16x"}:
|
||||
raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}")
|
||||
if self.downsample_mode == "4x":
|
||||
self.model_tensors = {
|
||||
name: tensor for name, tensor in self.model_tensors.items()
|
||||
if ".vit_merger." not in name
|
||||
}
|
||||
|
||||
if self.hparams_vision is not None:
|
||||
# In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP
|
||||
# positional embedding bucket grid (70 x 70), while the per-slice processing
|
||||
|
|
@ -165,8 +156,8 @@ class MiniCPMV4_6VisionModel(MmprojModel):
|
|||
# (mapped to PROJECTOR_TYPE_MINICPMV4_6).
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6)
|
||||
|
||||
self.gguf_writer.add_vision_projector_scale_factor(
|
||||
2 if self.downsample_mode == "4x" else 4)
|
||||
# ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment
|
||||
self.gguf_writer.add_vision_projector_scale_factor(4)
|
||||
|
||||
# borrow wa_layer_indexes for vit_merger insertion point
|
||||
insert_layer_id = int(self.global_config.get(
|
||||
|
|
|
|||
|
|
@ -1,121 +1,13 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, Sequence, TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxText01ForCausalLM")
|
||||
@ModelBase.register("MiniMaxM1ForCausalLM")
|
||||
class MiniMaxText01Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINIMAX01
|
||||
|
||||
def _get_suppress_tokens(self) -> Sequence[int] | None:
|
||||
import json
|
||||
from transformers import AutoTokenizer
|
||||
from .base import LazyTorchTensor
|
||||
|
||||
# check added tokens embeddings in embeddings tensor for zero-valued embeddings
|
||||
# they get in the way of the token sampling process and must be suppressed
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
|
||||
tokenizer_vocab_size = tokenizer.vocab_size
|
||||
|
||||
with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
|
||||
embeddings_tensor_name = "model.embed_tokens.weight"
|
||||
embeddings_shard_name = weight_map[embeddings_tensor_name]
|
||||
with gguf.utility.SafetensorsLocal(self.dir_model / embeddings_shard_name) as model_shard:
|
||||
embeddings_data = model_shard[embeddings_tensor_name]
|
||||
|
||||
embeddings_weights_dtype = LazyTorchTensor._dtype_str_map[embeddings_data.dtype]
|
||||
embeddings_weights = torch.from_numpy(embeddings_data.mmap_bytes()).view(embeddings_weights_dtype).reshape(embeddings_data.shape)
|
||||
embeddings_vocab_size = embeddings_weights.shape[0]
|
||||
|
||||
embeddings_added_tokens = embeddings_weights[tokenizer_vocab_size:embeddings_vocab_size]
|
||||
embeddings_zero_rows = torch.all(embeddings_added_tokens == 0, dim=1)
|
||||
tokens_zero_embeddings_ids = (torch.nonzero(embeddings_zero_rows, as_tuple=False).flatten() + tokenizer_vocab_size).tolist()
|
||||
|
||||
return tokens_zero_embeddings_ids
|
||||
|
||||
def set_vocab(self) -> None:
|
||||
from pathlib import Path
|
||||
|
||||
self._set_vocab_gpt2()
|
||||
|
||||
for tmpl_file in [
|
||||
self.dir_model / "chat_template.jinja",
|
||||
Path(__file__).parent.parent / "models" / "templates" / "MiniMax-M1.jinja"
|
||||
]:
|
||||
if tmpl_file.is_file():
|
||||
self.gguf_writer.add_chat_template(tmpl_file.read_text(encoding="utf-8"))
|
||||
logger.info(f"Chat template overridden with {tmpl_file}.")
|
||||
break
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
suppress_tokens = self._get_suppress_tokens()
|
||||
if suppress_tokens:
|
||||
logger.info(f"Suppressing tokens with zero embeddings {suppress_tokens}")
|
||||
self.gguf_writer.add_suppress_tokens(suppress_tokens)
|
||||
|
||||
layernorm_full_attention_alpha = self.hparams["layernorm_full_attention_alpha"]
|
||||
layernorm_full_attention_beta = self.hparams["layernorm_full_attention_beta"]
|
||||
layernorm_linear_attention_alpha = self.hparams["layernorm_linear_attention_alpha"]
|
||||
layernorm_linear_attention_beta = self.hparams["layernorm_linear_attention_beta"]
|
||||
layernorm_mlp_alpha = self.hparams["layernorm_mlp_alpha"]
|
||||
layernorm_mlp_beta = self.hparams["layernorm_mlp_beta"]
|
||||
assert layernorm_full_attention_alpha == layernorm_linear_attention_alpha == layernorm_mlp_alpha
|
||||
assert layernorm_full_attention_beta == layernorm_linear_attention_beta == layernorm_mlp_beta == 1.0
|
||||
# we do not store the layernorm betas as they are all 1.0
|
||||
# layernorm alphas are stored as single residual_scale hparam
|
||||
self.gguf_writer.add_residual_scale(layernorm_full_attention_alpha)
|
||||
|
||||
self.gguf_writer.add_rope_dimension_count(self.hparams["rotary_dim"])
|
||||
|
||||
_experts: list[dict[str, Tensor]] | None = None
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# process the experts separately
|
||||
if name.find("block_sparse_moe.experts") != -1:
|
||||
n_experts = self.hparams["num_local_experts"]
|
||||
|
||||
assert bid is not None
|
||||
|
||||
if self._experts is None:
|
||||
self._experts = [{} for _ in range(self.block_count)]
|
||||
|
||||
self._experts[bid][name] = data_torch
|
||||
|
||||
if len(self._experts[bid]) >= n_experts * 3:
|
||||
# merge the experts into a single 3d tensor
|
||||
for wid in ["w1", "w2", "w3"]:
|
||||
datas: list[Tensor] = []
|
||||
|
||||
for xid in range(n_experts):
|
||||
ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight"
|
||||
datas.append(self._experts[bid][ename])
|
||||
del self._experts[bid][ename]
|
||||
|
||||
data_torch = torch.stack(datas, dim=0)
|
||||
|
||||
merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight"
|
||||
|
||||
new_name = self.map_tensor_name(merged_name)
|
||||
|
||||
yield from super().modify_tensors(data_torch, new_name, bid)
|
||||
return
|
||||
else:
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxM2ForCausalLM")
|
||||
|
|
|
|||
|
|
@ -1,179 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import MmprojModel, ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor":
|
||||
"""Invert transformers' `_permute_for_rope`: HF stores Q/K in rotate_half layout,
|
||||
llama.cpp consumes the interleaved (NORM) layout."""
|
||||
if tensor.ndim == 2:
|
||||
dim1, dim2 = tensor.shape
|
||||
return tensor.view(n_heads, 2, dim1 // n_heads // 2, dim2).transpose(1, 2).reshape(dim1, dim2)
|
||||
if tensor.ndim == 1:
|
||||
(dim1,) = tensor.shape
|
||||
return tensor.view(n_heads, 2, dim1 // n_heads // 2).transpose(1, 2).reshape(dim1)
|
||||
raise ValueError(f"_unpermute_for_rope: unexpected shape {tuple(tensor.shape)}")
|
||||
|
||||
|
||||
@ModelBase.register("MuseGlimmerForConditionalGeneration")
|
||||
class MuseGlimmerModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.MUSE_GLIMMER
|
||||
|
||||
def norm_shift(self, name: str) -> float:
|
||||
# All four layer norms use 1, the final norm uses 0.
|
||||
return 1.0 if name.endswith("layernorm.weight") else 0.0
|
||||
|
||||
def set_vocab(self):
|
||||
self._set_vocab_gpt2()
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
tok = AutoTokenizer.from_pretrained(self.dir_model)
|
||||
eot_id = tok.convert_tokens_to_ids("<|eot|>")
|
||||
if isinstance(eot_id, int) and eot_id >= 0:
|
||||
self.gguf_writer.add_eot_token_id(eot_id)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
hparams = self.hparams
|
||||
|
||||
self.gguf_writer.add_final_logit_softcapping(hparams["final_logit_softcapping"])
|
||||
self.gguf_writer.add_logit_scale(hparams["output_multiplier"])
|
||||
self.gguf_writer.add_sliding_window(hparams["sliding_window"])
|
||||
self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]])
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
shift = self.norm_shift(name)
|
||||
if shift != 0.0:
|
||||
data_torch = data_torch + shift
|
||||
|
||||
# Invert transformers' `_permute_for_rope` on Q/K, we keep ggml's NORM (interleaved) rope
|
||||
if ".self_attn.q_proj." in name:
|
||||
data_torch = _unpermute_for_rope(data_torch, int(self.hparams["num_attention_heads"]))
|
||||
elif ".self_attn.k_proj." in name:
|
||||
data_torch = _unpermute_for_rope(data_torch, int(self.hparams["num_key_value_heads"]))
|
||||
|
||||
# Synthesize QK-norm weights to absorb qk_scale_factor.
|
||||
# MuseGlimmer implementation: scaleless RMSNorm followed by qk_scale_factor..
|
||||
if bid is not None and name.endswith(f"model.layers.{bid}.self_attn.q_proj.weight"):
|
||||
head_dim = self.hparams["head_dim"]
|
||||
q_scale = float(self.hparams["qk_scale_factor"])
|
||||
yield (
|
||||
self.map_tensor_name(f"model.layers.{bid}.self_attn.q_norm.weight"),
|
||||
torch.full((head_dim,), q_scale, dtype=torch.float32),
|
||||
)
|
||||
yield (
|
||||
self.map_tensor_name(f"model.layers.{bid}.self_attn.k_norm.weight"),
|
||||
torch.ones((head_dim,), dtype=torch.float32),
|
||||
)
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("MuseGlimmerForConditionalGeneration")
|
||||
class MuseGlimmerVisionModel(MmprojModel):
|
||||
def get_vision_config(self) -> dict[str, Any] | None:
|
||||
c = self.global_config.get("vision_config")
|
||||
if not c:
|
||||
return None
|
||||
# MuseGlimmer actually uses dynamic size, initialize with nominal size
|
||||
image_size = c["pos_emb_height"] * c["patch_size"] * c["merge_size"]
|
||||
return {**c, "image_size": image_size}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
assert self.hparams_vision is not None
|
||||
c = self.hparams_vision # enriched vision_config from get_vision_config()
|
||||
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSE_GLIMMER)
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(float(c["layer_norm_eps"]))
|
||||
self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"]))
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
name, gen = item
|
||||
keep = ("model.vision_tower.", "model.vision_adapter.", "model.vision_projection.")
|
||||
if not any(name.startswith(k) for k in keep):
|
||||
return None
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
# 3-layer projector MLP
|
||||
_MM_MLP_MAP = {
|
||||
"model.vision_adapter.fc1": (gguf.MODEL_TENSOR.V_MMPROJ, 0),
|
||||
"model.vision_adapter.fc2": (gguf.MODEL_TENSOR.V_MMPROJ, 1),
|
||||
"model.vision_projection": (gguf.MODEL_TENSOR.V_MMPROJ, 2),
|
||||
}
|
||||
|
||||
def modify_tensors(self, data_torch, name, bid):
|
||||
assert self.hparams_vision is not None
|
||||
if ".attn.q_proj." in name or ".attn.k_proj." in name:
|
||||
n_heads = int(self.hparams_vision["num_attention_heads"])
|
||||
data_torch = _unpermute_for_rope(data_torch, n_heads)
|
||||
# Lay out the pt=2 temporal slabs of the patch embedding as a conv2d for build_inp()
|
||||
if name.endswith("patch_embedder.patch_embedding.weight"):
|
||||
n_embd = data_torch.shape[0]
|
||||
pt = int(self.hparams_vision["patch_temporal"])
|
||||
ps = int(self.hparams_vision["patch_size"])
|
||||
data_torch = data_torch.view(n_embd, pt, 3, ps, ps).sum(dim=1) # (n_embd, 3, ps, ps)
|
||||
stem, _, suffix = name.rpartition(".")
|
||||
if stem in self._MM_MLP_MAP:
|
||||
tensor_key, idx = self._MM_MLP_MAP[stem]
|
||||
yield (self.format_tensor_name(tensor_key, bid=idx, suffix="." + suffix), data_torch)
|
||||
return
|
||||
yield (self.map_tensor_name(name), data_torch)
|
||||
|
||||
|
||||
@ModelBase.register("MuseGlimmerAssistantModel")
|
||||
class MuseGlimmerAssistantModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
def set_vocab(self):
|
||||
if self.target_model_dir is None:
|
||||
raise ValueError(
|
||||
"MuseGlimmerAssistant (DFlash drafter) requires --target-model-dir pointing to the "
|
||||
"target MuseGlimmer HF directory"
|
||||
)
|
||||
|
||||
original_dir = self.dir_model
|
||||
self.dir_model = self.target_model_dir
|
||||
|
||||
from . import get_model_class
|
||||
with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f:
|
||||
target_arch = json.load(f)["architectures"][0]
|
||||
target_cls = get_model_class(target_arch)
|
||||
if target_cls is not type(self):
|
||||
target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
|
||||
else:
|
||||
super().set_vocab()
|
||||
|
||||
self.dir_model = original_dir
|
||||
|
||||
mask_token_id = self.hparams.get("mask_token_id")
|
||||
if mask_token_id is not None:
|
||||
self.gguf_writer.add_mask_token_id(int(mask_token_id))
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
h = self.hparams
|
||||
|
||||
self.gguf_writer.add_block_size(int(h["block_size"]))
|
||||
|
||||
# dflash.target_layers[k] refers to the inputs going into the ith layer, which come from the (i-1)th layer's output.
|
||||
# The transformers configuration refers to the outputs being recorded.
|
||||
self.gguf_writer.add_target_layers([int(x) + 1 for x in h["target_layer_ids"]])
|
||||
|
||||
if h.get("sliding_window") and h.get("layer_types"):
|
||||
self.gguf_writer.add_sliding_window(int(h["sliding_window"]))
|
||||
self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in h["layer_types"]])
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# DFlash defaults to NEOX (rotate_half) rope, matching transformers HF layout for Q/K, QK-norms
|
||||
# no permutation needed.
|
||||
yield (self.map_tensor_name(name), data_torch)
|
||||
|
|
@ -197,7 +197,6 @@ class NemotronHModel(GraniteHybridModel):
|
|||
"""Hybrid mamba2/attention model from NVIDIA"""
|
||||
model_arch = gguf.MODEL_ARCH.NEMOTRON_H
|
||||
is_moe: bool = False
|
||||
supports_mtp_export = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
# We have to determine the correct model architecture (MoE vs non-MoE) before
|
||||
|
|
@ -237,25 +236,6 @@ class NemotronHModel(GraniteHybridModel):
|
|||
self._ssm_layers = [i for i, val in enumerate(pattern) if val == "mamba"]
|
||||
self._mlp_layers = [i for i, val in enumerate(pattern) if val == "moe"]
|
||||
|
||||
# `--no-mtp` drops it entirely; `--mtp` exports only the MTP head
|
||||
self._mtp_bid: int | None = None
|
||||
if self.is_moe and not self.no_mtp:
|
||||
n_nextn = self.hparams.get("num_nextn_predict_layers", 0) or 0
|
||||
if n_nextn > 0:
|
||||
assert n_nextn == 1, (
|
||||
"NemotronH MTP conversion currently supports num_nextn_predict_layers == 1"
|
||||
)
|
||||
self._mtp_bid = self.block_count
|
||||
self.block_count += 1
|
||||
# The folded MTP block carries both an attention sub-layer and a
|
||||
# MoE sub-layer, so register it as both so the per-layer metadata arrays cover it
|
||||
self._attn_layers.append(self._mtp_bid)
|
||||
self._mlp_layers.append(self._mtp_bid)
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
if self.mtp_only and self._mtp_bid is None:
|
||||
raise ValueError("--mtp was requested, but this model does not contain a supported MTP head")
|
||||
|
||||
def get_attn_layers(self):
|
||||
pattern = self.hparams.get("hybrid_override_pattern") or self.hparams.get("layers_block_type")
|
||||
if pattern is None:
|
||||
|
|
@ -266,44 +246,6 @@ class NemotronHModel(GraniteHybridModel):
|
|||
|
||||
return [i for i, val in enumerate(pattern) if val == "attention"]
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if name.startswith("mtp."):
|
||||
# --no-mtp: drop the MTP head entirely
|
||||
if cls.no_mtp:
|
||||
return None
|
||||
elif cls.mtp_only:
|
||||
# --mtp: export the MTP head plus the tensors it shares with the target model
|
||||
# Include lm_head scale sidecars so NVFP4 packing sees them.
|
||||
keep = name in (
|
||||
"backbone.embeddings.weight",
|
||||
"backbone.norm_f.weight",
|
||||
"lm_head.weight",
|
||||
"lm_head.weight_scale",
|
||||
"lm_head.weight_scale_2",
|
||||
"lm_head.weight_scale_inv",
|
||||
"lm_head.input_scale",
|
||||
"lm_head.input_global_scale",
|
||||
"lm_head.weight_global_scale",
|
||||
"lm_head.weight_packed",
|
||||
)
|
||||
if not keep:
|
||||
return None
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def prepare_metadata(self, vocab_only: bool):
|
||||
from_dir = self.fname_out.is_dir()
|
||||
super().prepare_metadata(vocab_only=vocab_only)
|
||||
|
||||
if not self.mtp_only or not from_dir:
|
||||
return
|
||||
output_type: str = self.ftype.name.partition("_")[2]
|
||||
fname_default: str = gguf.naming_convention(
|
||||
self.metadata.name, self.metadata.basename, self.metadata.finetune,
|
||||
self.metadata.version, size_label=None, output_type=output_type, model_type=None)
|
||||
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
|
|
@ -342,10 +284,6 @@ class NemotronHModel(GraniteHybridModel):
|
|||
if (latent_size := self.hparams.get("moe_latent_size")) is not None:
|
||||
self.gguf_writer.add_moe_latent_size(latent_size)
|
||||
|
||||
# MTP head: number of trailing NextN blocks
|
||||
if self._mtp_bid is not None:
|
||||
self.gguf_writer.add_nextn_predict_layers(self.hparams["num_nextn_predict_layers"])
|
||||
|
||||
def set_vocab(self):
|
||||
# The NemotronH config uses pattern characters (e.g. '-') that may not
|
||||
# be supported by the installed transformers version. AutoTokenizer
|
||||
|
|
@ -412,24 +350,15 @@ class NemotronHModel(GraniteHybridModel):
|
|||
if not self.is_moe:
|
||||
self.gguf_writer.add_add_bos_token(True)
|
||||
|
||||
_MTP_SPECIAL_RENAMES = {
|
||||
"mtp.layers.0.enorm.weight": "model.layers.{bid}.enorm.weight",
|
||||
"mtp.layers.0.hnorm.weight": "model.layers.{bid}.hnorm.weight",
|
||||
"mtp.layers.0.eh_proj.weight": "model.layers.{bid}.eh_proj.weight",
|
||||
"mtp.layers.1.norm.weight": "model.layers.{bid}.post_attention_layernorm.weight",
|
||||
"mtp.layers.1.final_layernorm.weight": "model.layers.{bid}.shared_head.norm.weight",
|
||||
}
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# mtp.layers.0: NextN input fusion + attention
|
||||
# mtp.layers.1: MoE + final head norm
|
||||
if self._mtp_bid is not None and name.startswith(("mtp.layers.0.", "mtp.layers.1.")):
|
||||
suffix = name.split(".", 3)[3]
|
||||
bid = self._mtp_bid
|
||||
renamed = self._MTP_SPECIAL_RENAMES.get(name)
|
||||
name = renamed.format(bid=bid) if renamed else f"backbone.layers.{bid}.{suffix}"
|
||||
|
||||
if self.is_moe and bid is not None:
|
||||
# Skip Multi-Token Prediction (MTP) tensors. These are used for
|
||||
# for speculative decoding but we don't include them in this model
|
||||
# conversion. See https://github.com/ggml-org/llama.cpp/pull/18886
|
||||
if name.startswith("mtp."):
|
||||
logger.info(f"gguf: Skipping MTP (Speculative) layer: {name}")
|
||||
return
|
||||
|
||||
if name.endswith("mixer.gate.e_score_correction.bias"):
|
||||
yield from ModelBase.modify_tensors(self, data_torch, name, bid)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1,378 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, SentencePieceTokenTypes, TextModel, gguf, logger
|
||||
|
||||
# Pocket TTS is a CALM: the backbone conditions a flow-matching decoder that generates one
|
||||
# continuous 32-d latent per frame. There is no codebook in this model.
|
||||
# The checkpoint ships no config.json, hparams come from _load_hparams() below.
|
||||
#
|
||||
# Tricks being used to support this model via existing llama.cpp code paths:
|
||||
# - bos_before_voice and bos_emb are learned input vectors, not tokens
|
||||
# they are appended to the embedding table as extra tokens, to be looked up like any other row
|
||||
# - bos_emb lives in latent space, so input_linear is folded into it here
|
||||
# - the backbone has no lm_head, the embedding table is reused as output for the unused logits
|
||||
#
|
||||
# pipeline stage mapping:
|
||||
# mimi encoder + speaker_proj --> mapped to normal mtmd audio encoder
|
||||
# flow_lm.transformer --> mapped to normal libllama text model (autoregressive)
|
||||
# flow_lm.flow_net + out_eos --> MTMD_GEN_PROCESS_TYPE_GEN_CODE
|
||||
# mimi decoder --> MTMD_GEN_PROCESS_TYPE_GEN_WAV
|
||||
|
||||
# indices into mimi.encoder.model / mimi.decoder.model for stage i, see SEANetEncoder/SEANetDecoder
|
||||
_ENC_RES_IDX = lambda i: 1 + 3 * i # noqa: E731
|
||||
_ENC_SCALE_IDX = lambda i: 3 + 3 * i # noqa: E731
|
||||
_DEC_SCALE_IDX = lambda i: 2 + 3 * i # noqa: E731
|
||||
_DEC_RES_IDX = lambda i: 3 + 3 * i # noqa: E731
|
||||
|
||||
_N_SEANET_STAGES = 3
|
||||
_SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
def _tensor_shapes(dir_model: Path) -> dict[str, tuple[int, ...]]:
|
||||
part_names = ModelBase.get_model_part_names(dir_model, "model", ".safetensors")
|
||||
if len(part_names) != 1:
|
||||
return {}
|
||||
with gguf.utility.SafetensorsLocal(dir_model / part_names[0]) as part:
|
||||
return {name: tuple(part[name].shape) for name in part.keys()}
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(lambda dir_model: "flow_lm.bos_emb" in _tensor_shapes(dir_model))
|
||||
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected pocket-tts checkpoint, deriving hparams from tensor shapes")
|
||||
shapes = _tensor_shapes(dir_model)
|
||||
n_vocab, n_embd = shapes["flow_lm.conditioner.embed.weight"]
|
||||
n_layer = sum(1 for name in shapes if re.fullmatch(r"flow_lm\.transformer\.layers\.\d+\.norm1\.weight", name))
|
||||
n_layer_a = sum(1 for name in shapes if re.fullmatch(r"mimi\.encoder_transformer\.transformer\.layers\.\d+\.norm1\.weight", name))
|
||||
n_embd_a = shapes["mimi.encoder_transformer.transformer.layers.0.norm1.weight"][0]
|
||||
return {
|
||||
"architectures": ["PocketTTSModel"],
|
||||
"model_type": "pockettts",
|
||||
"num_hidden_layers": n_layer,
|
||||
"hidden_size": n_embd,
|
||||
"intermediate_size": shapes["flow_lm.transformer.layers.0.linear1.weight"][0],
|
||||
# the transformer is fully causal with no context limit, this only bounds the KV cache
|
||||
"max_position_embeddings": 4096,
|
||||
# not in the checkpoint, but every released variant uses head_dim 64
|
||||
"num_attention_heads": n_embd // 64,
|
||||
# extra rows for the learned input vectors, see _embd_table()
|
||||
"vocab_size": n_vocab + (2 if "flow_lm.bos_before_voice" in shapes else 1),
|
||||
"rope_theta": 10000.0,
|
||||
"layer_norm_eps": 1e-5,
|
||||
"audio_config": {
|
||||
"num_hidden_layers": n_layer_a,
|
||||
"hidden_size": n_embd_a,
|
||||
"intermediate_size": shapes["mimi.encoder_transformer.transformer.layers.0.linear1.weight"][0],
|
||||
"num_attention_heads": n_embd_a // 64,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@ModelBase.register("PocketTTSModel")
|
||||
class PocketTTSModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.POCKETTTS
|
||||
|
||||
_LAYER_TENSOR_MAP = {
|
||||
"norm1": gguf.MODEL_TENSOR.ATTN_NORM,
|
||||
"norm2": gguf.MODEL_TENSOR.FFN_NORM,
|
||||
"self_attn.out_proj": gguf.MODEL_TENSOR.ATTN_OUT,
|
||||
"linear1": gguf.MODEL_TENSOR.FFN_UP,
|
||||
"linear2": gguf.MODEL_TENSOR.FFN_DOWN,
|
||||
}
|
||||
|
||||
def set_vocab(self):
|
||||
# this is a unigram sentencepiece model, llama.cpp's SPM tokenizer cannot do
|
||||
# unigram segmentation, so use the UGM tokenizer instead
|
||||
from sentencepiece import sentencepiece_model_pb2 as model
|
||||
|
||||
proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
proto.ParseFromString(open(self.dir_model / "tokenizer.model", "rb").read())
|
||||
assert proto.trainer_spec.model_type == 1, "expected a unigram tokenizer"
|
||||
|
||||
tokens, scores, toktypes = self._create_vocab_sentencepiece()
|
||||
|
||||
# the last rows of the embedding table are not sentencepiece pieces
|
||||
extra = self._extra_tokens()
|
||||
for i, name in enumerate(extra):
|
||||
tokens[len(tokens) - len(extra) + i] = name.encode("utf-8")
|
||||
toktypes[len(tokens) - len(extra) + i] = SentencePieceTokenTypes.CONTROL
|
||||
scores[len(tokens) - len(extra) + i] = -1000.0
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("t5")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_add_space_prefix(proto.normalizer_spec.add_dummy_prefix)
|
||||
self.gguf_writer.add_remove_extra_whitespaces(proto.normalizer_spec.remove_extra_whitespaces)
|
||||
if proto.normalizer_spec.precompiled_charsmap:
|
||||
self.gguf_writer.add_precompiled_charsmap(proto.normalizer_spec.precompiled_charsmap)
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_add_eos_token(False)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if not name.startswith("flow_lm."):
|
||||
return # mimi and the flow net go to the mmproj
|
||||
|
||||
if name == "flow_lm.conditioner.embed.weight":
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), self._embd_table(data_torch))
|
||||
return
|
||||
|
||||
if name.startswith("flow_lm.out_norm."):
|
||||
suffix = "." + name.rsplit(".", 1)[1]
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT_NORM, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
if name.startswith("flow_lm.transformer.layers."):
|
||||
assert bid is not None
|
||||
key_with_suffix = name.split(f"layers.{bid}.", 1)[1]
|
||||
key, suffix = key_with_suffix.rsplit(".", 1)
|
||||
|
||||
if key == "self_attn.in_proj":
|
||||
q, k, v = data_torch.chunk(3, dim=0)
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), q)
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), k)
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v)
|
||||
return
|
||||
|
||||
tensor = self._LAYER_TENSOR_MAP.get(key)
|
||||
if tensor is not None:
|
||||
yield (self.format_tensor_name(tensor, bid, suffix="." + suffix), data_torch)
|
||||
return
|
||||
|
||||
return
|
||||
|
||||
def _extra_tokens(self) -> list[str]:
|
||||
# the conditioner's padding row, then the learned vectors appended by _embd_table().
|
||||
# bos_before_voice only exists when the pack sets insert_bos_before_voice
|
||||
names = ["<|pad|>"]
|
||||
if "flow_lm.bos_before_voice" in self.model_tensors:
|
||||
names.append("<|bos_before_voice|>")
|
||||
names.append("<|audio_bos|>")
|
||||
return names
|
||||
|
||||
def _embd_table(self, embed: Tensor) -> Tensor:
|
||||
rows = [embed]
|
||||
if "flow_lm.bos_before_voice" in self.model_tensors:
|
||||
rows.append(self.model_tensors["flow_lm.bos_before_voice"]().reshape(1, -1).to(embed.dtype))
|
||||
|
||||
# bos_emb is a latent, it only enters the backbone through input_linear
|
||||
bos_emb = self.model_tensors["flow_lm.bos_emb"]()
|
||||
input_linear = self.model_tensors["flow_lm.input_linear.weight"]()
|
||||
audio_bos = torch.nn.functional.linear(bos_emb.float(), input_linear.float()).reshape(1, -1)
|
||||
rows.append(audio_bos.to(embed.dtype))
|
||||
|
||||
return torch.cat(rows, dim=0)
|
||||
|
||||
|
||||
@ModelBase.register("PocketTTSModel")
|
||||
class PocketTTSMmprojModel(MmprojModel):
|
||||
has_audio_encoder = True
|
||||
has_vision_encoder = False
|
||||
|
||||
_MIMI_TFM_MAP = {
|
||||
"norm1": (gguf.MODEL_TENSOR.A_ENC_INPUT_NORM, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM),
|
||||
"norm2": (gguf.MODEL_TENSOR.A_ENC_OUTPUT_NORM, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM),
|
||||
"self_attn.out_proj": (gguf.MODEL_TENSOR.A_ENC_OUTPUT, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT),
|
||||
"linear1": (gguf.MODEL_TENSOR.A_ENC_FFN_UP, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP),
|
||||
"linear2": (gguf.MODEL_TENSOR.A_ENC_FFN_DOWN, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN),
|
||||
"layer_scale_1.scale": (gguf.MODEL_TENSOR.A_ENC_ATTN_SCALE, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE),
|
||||
"layer_scale_2.scale": (gguf.MODEL_TENSOR.A_ENC_FFN_SCALE_LS, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE),
|
||||
}
|
||||
_MIMI_TFM_QKV = (
|
||||
(gguf.MODEL_TENSOR.A_ENC_ATTN_Q, gguf.MODEL_TENSOR.A_ENC_ATTN_K, gguf.MODEL_TENSOR.A_ENC_ATTN_V),
|
||||
(gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V),
|
||||
)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
assert self.hparams_audio is not None
|
||||
|
||||
# voice-prompt encoder: mimi encoder + speaker_proj
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
# note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
|
||||
self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
|
||||
self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
|
||||
self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
# mimi convolves the waveform directly, it is passed around as a 1-row "mel"
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
|
||||
# generation: flow-matching decoder + mimi decoder
|
||||
# the SEANet and flow net hparams are constant across the family, clip.cpp holds them
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(self.hparams_audio["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(self.hparams_audio["num_hidden_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(self.hparams_audio["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
self.gguf_writer.add_gen_audio_model_variant(self.dir_model.name)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
del name, bid, n_dims
|
||||
# conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
|
||||
if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return False
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid # the block index of the mimi transformers is parsed here, not by the base class
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
if name in ("flow_lm.bos_emb", "flow_lm.bos_before_voice", "flow_lm.conditioner.embed.weight"):
|
||||
return # folded into the backbone embedding table
|
||||
if name.startswith("flow_lm.transformer.") or name.startswith("flow_lm.out_norm."):
|
||||
return # backbone
|
||||
|
||||
if name == "flow_lm.speaker_proj_weight":
|
||||
yield (self.format_tensor_name(T.A_ENC_SPEAKER_PROJ), data_torch)
|
||||
return
|
||||
if name == "flow_lm.input_linear.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_INPUT_LINEAR), data_torch)
|
||||
return
|
||||
if name == "flow_lm.emb_mean":
|
||||
yield (self.format_tensor_name(T.A_GEN_EMB_MEAN, suffix=""), data_torch)
|
||||
return
|
||||
if name == "flow_lm.emb_std":
|
||||
yield (self.format_tensor_name(T.A_GEN_EMB_STD, suffix=""), data_torch)
|
||||
return
|
||||
if name.startswith("flow_lm.out_eos."):
|
||||
suffix = "." + name.rsplit(".", 1)[1]
|
||||
yield (self.format_tensor_name(T.A_GEN_OUT_EOS, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
if name.startswith("flow_lm.flow_net."):
|
||||
yield from self._flow_net_tensor(name, data_torch)
|
||||
return
|
||||
|
||||
if name == "mimi.downsample.conv.conv.weight":
|
||||
yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), data_torch)
|
||||
return
|
||||
if name == "mimi.upsample.convtr.convtr.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_UPSAMPLE), data_torch)
|
||||
return
|
||||
if name == "mimi.quantizer.output_proj.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_OUT), data_torch.squeeze(-1))
|
||||
return
|
||||
|
||||
if "_transformer.transformer.layers." in name:
|
||||
yield from self._mimi_tfm_tensor(name, data_torch)
|
||||
return
|
||||
|
||||
if name.startswith("mimi.encoder.model.") or name.startswith("mimi.decoder.model."):
|
||||
yield from self._seanet_tensor(name, data_torch)
|
||||
return
|
||||
|
||||
return
|
||||
|
||||
def _flow_net_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
key = name.split("flow_lm.flow_net.", 1)[1]
|
||||
suffix = "." + key.rsplit(".", 1)[1]
|
||||
|
||||
simple = {
|
||||
"input_proj": T.A_GEN_FLOW_INPUT_PROJ,
|
||||
"cond_embed": T.A_GEN_FLOW_COND_EMBD,
|
||||
"final_layer.linear": T.A_GEN_FLOW_FINAL_PROJ,
|
||||
"final_layer.adaLN_modulation.1": T.A_GEN_FLOW_FINAL_ADA,
|
||||
}
|
||||
tensor = simple.get(key.rsplit(".", 1)[0])
|
||||
if tensor is not None:
|
||||
yield (self.format_tensor_name(tensor, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
if key.startswith("time_embed."):
|
||||
bid = int(key.split(".")[1])
|
||||
rest = key.split(f"time_embed.{bid}.", 1)[1]
|
||||
time_map = {
|
||||
"freqs": (T.A_GEN_FLOW_TIME_FREQS, ""),
|
||||
"mlp.0": (T.A_GEN_FLOW_TIME_UP, suffix),
|
||||
"mlp.2": (T.A_GEN_FLOW_TIME_DOWN, suffix),
|
||||
"mlp.3.alpha": (T.A_GEN_FLOW_TIME_NORM, ""),
|
||||
}
|
||||
entry = time_map.get(rest) or time_map.get(rest.rsplit(".", 1)[0])
|
||||
if entry is not None:
|
||||
yield (self.format_tensor_name(entry[0], bid, suffix=entry[1]), data_torch)
|
||||
return
|
||||
|
||||
if key.startswith("res_blocks."):
|
||||
bid = int(key.split(".")[1])
|
||||
rest = key.split(f"res_blocks.{bid}.", 1)[1].rsplit(".", 1)[0]
|
||||
blk_map = {
|
||||
"in_ln": T.A_GEN_FLOW_BLK_NORM,
|
||||
"mlp.0": T.A_GEN_FLOW_BLK_UP,
|
||||
"mlp.2": T.A_GEN_FLOW_BLK_DOWN,
|
||||
"adaLN_modulation.1": T.A_GEN_FLOW_BLK_ADA,
|
||||
}
|
||||
tensor = blk_map.get(rest)
|
||||
if tensor is not None:
|
||||
yield (self.format_tensor_name(tensor, bid, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
def _mimi_tfm_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]:
|
||||
is_decoder = name.startswith("mimi.decoder_transformer.")
|
||||
bid = int(name.split("_transformer.transformer.layers.", 1)[1].split(".")[0])
|
||||
key_with_suffix = name.split(f".layers.{bid}.", 1)[1]
|
||||
|
||||
if key_with_suffix == "self_attn.in_proj.weight":
|
||||
q, k, v = data_torch.chunk(3, dim=0)
|
||||
names = self._MIMI_TFM_QKV[1 if is_decoder else 0]
|
||||
for tensor, part in zip(names, (q, k, v)):
|
||||
yield (self.format_tensor_name(tensor, bid), part)
|
||||
return
|
||||
|
||||
key, suffix = key_with_suffix.rsplit(".", 1)
|
||||
entry = self._MIMI_TFM_MAP.get(key) or self._MIMI_TFM_MAP.get(key_with_suffix)
|
||||
if entry is None:
|
||||
return
|
||||
tensor = entry[1 if is_decoder else 0]
|
||||
suffix = ".weight" if key_with_suffix.endswith(".scale") else "." + suffix
|
||||
yield (self.format_tensor_name(tensor, bid, suffix=suffix), data_torch)
|
||||
|
||||
def _seanet_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
is_decoder = name.startswith("mimi.decoder.")
|
||||
idx = int(name.split(".model.", 1)[1].split(".")[0])
|
||||
suffix = "." + name.rsplit(".", 1)[1]
|
||||
|
||||
conv_in, conv_out, res1, res2, scale = (
|
||||
(T.A_GEN_WAV_SEANET_CONV_IN, T.A_GEN_WAV_SEANET_CONV_OUT, T.A_GEN_WAV_SEANET_RES_CONV1,
|
||||
T.A_GEN_WAV_SEANET_RES_CONV2, T.A_GEN_WAV_SEANET_SCALE_CONV)
|
||||
if is_decoder else
|
||||
(T.A_ENC_SEANET_CONV_IN, T.A_ENC_SEANET_CONV_OUT, T.A_ENC_SEANET_RES_CONV1,
|
||||
T.A_ENC_SEANET_RES_CONV2, T.A_ENC_SEANET_SCALE_CONV)
|
||||
)
|
||||
|
||||
if idx == 0:
|
||||
yield (self.format_tensor_name(conv_in, suffix=suffix), data_torch)
|
||||
return
|
||||
if idx == 3 * _N_SEANET_STAGES + 2:
|
||||
yield (self.format_tensor_name(conv_out, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
for stage in range(_N_SEANET_STAGES):
|
||||
res_idx = _DEC_RES_IDX(stage) if is_decoder else _ENC_RES_IDX(stage)
|
||||
scale_idx = _DEC_SCALE_IDX(stage) if is_decoder else _ENC_SCALE_IDX(stage)
|
||||
if idx == scale_idx:
|
||||
yield (self.format_tensor_name(scale, stage, suffix=suffix), data_torch)
|
||||
return
|
||||
if idx == res_idx:
|
||||
# block.1 is the dilated conv, block.3 the pointwise one (0 and 2 are ELU)
|
||||
inner = int(name.split(".block.", 1)[1].split(".")[0])
|
||||
tensor = res1 if inner == 1 else res2
|
||||
yield (self.format_tensor_name(tensor, stage, suffix=suffix), data_torch)
|
||||
return
|
||||
|
|
@ -18,7 +18,7 @@ class QwenModel(TextModel):
|
|||
|
||||
@staticmethod
|
||||
def token_bytes_to_string(b):
|
||||
from transformers.convert_slow_tokenizer import bytes_to_unicode
|
||||
from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import]
|
||||
byte_encoder = bytes_to_unicode()
|
||||
return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')])
|
||||
|
||||
|
|
@ -268,101 +268,8 @@ class Qwen3MoeModel(Qwen2MoeModel):
|
|||
super().set_vocab()
|
||||
|
||||
|
||||
class _QwenMtpMixin:
|
||||
"""Shared MTP wiring for Qwen3-Next and Qwen3.5/3.6 text variants. The HF
|
||||
config carries the MTP block under `mtp_num_hidden_layers` (computed from
|
||||
the checkpoint when absent, e.g. Qwen3-Next) and the tensors under
|
||||
`mtp.*`; we extend block_count, emit the nextn metadata key, and remap
|
||||
`mtp.*` to the standard layer-indexed nextn naming so the existing
|
||||
tensor_map handles them."""
|
||||
|
||||
supports_mtp_export = True
|
||||
hparams: dict[str, Any]
|
||||
model_arch: gguf.MODEL_ARCH
|
||||
gguf_writer: gguf.GGUFWriter
|
||||
block_count: int
|
||||
tensor_map: gguf.TensorNameMap
|
||||
no_mtp: bool
|
||||
mtp_only: bool
|
||||
_original_block_count: int | None = None
|
||||
opt_num_mtp_layers: int = 0
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.block_count = self.hparams["num_hidden_layers"]
|
||||
if not self.no_mtp:
|
||||
n_mtp = self.hparams.get("mtp_num_hidden_layers", 0)
|
||||
# Qwen-3-Next doesn't include `mtp_num_hidden_layers` in config.
|
||||
if n_mtp == 0:
|
||||
assert self.opt_num_mtp_layers != 0
|
||||
n_mtp = self.opt_num_mtp_layers
|
||||
self.block_count += n_mtp
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
hparams = {**self.hparams, **self.hparams.get("text_config", {})}
|
||||
key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None)
|
||||
type(self)._original_block_count = hparams.get(key)
|
||||
type(self).opt_num_mtp_layers = 0
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute]
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
assert cls._original_block_count is not None
|
||||
# TODO: change TextModel to super()
|
||||
if (titem := TextModel.filter_tensors(item)) is None:
|
||||
return None
|
||||
name, gen = titem
|
||||
if name.startswith("model.mtp."):
|
||||
name = name.replace("model.", "", 1)
|
||||
if name.startswith("mtp."):
|
||||
if cls.no_mtp:
|
||||
return None
|
||||
remapper = {
|
||||
"fc": "eh_proj",
|
||||
"pre_fc_norm_embedding": "enorm",
|
||||
"pre_fc_norm_hidden": "hnorm",
|
||||
"norm": "shared_head.norm",
|
||||
}
|
||||
parts = name.split(".", 3)
|
||||
if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal():
|
||||
mtp_idx = int(parts[2])
|
||||
name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}"
|
||||
cls.opt_num_mtp_layers = max(cls.opt_num_mtp_layers, mtp_idx + 1)
|
||||
elif len(parts) == 3 and parts[1] in remapper:
|
||||
name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}"
|
||||
elif cls.mtp_only:
|
||||
keep = name in (
|
||||
"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
|
||||
"embed_tokens.weight", "norm.weight",
|
||||
)
|
||||
if not keep:
|
||||
return None
|
||||
return name, gen
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters() # ty: ignore[unresolved-attribute]
|
||||
if self.no_mtp:
|
||||
return
|
||||
if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0:
|
||||
self.gguf_writer.add_nextn_predict_layers(n)
|
||||
|
||||
def prepare_metadata(self, vocab_only: bool):
|
||||
from_dir = self.fname_out.is_dir()
|
||||
super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute]
|
||||
|
||||
if not self.mtp_only or not from_dir:
|
||||
return
|
||||
|
||||
output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
fname_default: str = gguf.naming_convention(
|
||||
self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3NextForCausalLM")
|
||||
class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
|
||||
class Qwen3NextModel(Qwen2MoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3NEXT
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
|
|
@ -377,6 +284,16 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
|
|||
rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
|
||||
self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.25)))
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
if name.startswith("mtp"):
|
||||
# ignore MTP layers for now
|
||||
return None
|
||||
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.endswith(".A_log"):
|
||||
data_torch = -torch.exp(data_torch)
|
||||
|
|
@ -619,13 +536,97 @@ class _Qwen35MRopeMixin:
|
|||
self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION)
|
||||
|
||||
|
||||
class _Qwen35MtpMixin:
|
||||
"""Shared MTP wiring for Qwen3.5/3.6 text variants. The HF config carries
|
||||
the MTP block under `mtp_num_hidden_layers` and the tensors under
|
||||
`mtp.*`; we extend block_count, emit the nextn metadata key, and remap
|
||||
`mtp.*` to the standard layer-indexed nextn naming so the existing
|
||||
tensor_map handles them."""
|
||||
|
||||
supports_mtp_export = True
|
||||
hparams: dict[str, Any]
|
||||
model_arch: gguf.MODEL_ARCH
|
||||
gguf_writer: gguf.GGUFWriter
|
||||
block_count: int
|
||||
tensor_map: gguf.TensorNameMap
|
||||
no_mtp: bool
|
||||
mtp_only: bool
|
||||
_original_block_count: int | None = None
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.block_count = self.hparams["num_hidden_layers"]
|
||||
if not self.no_mtp:
|
||||
self.block_count += self.hparams.get("mtp_num_hidden_layers", 0)
|
||||
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
hparams = {**self.hparams, **self.hparams.get("text_config", {})}
|
||||
key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None)
|
||||
type(self)._original_block_count = hparams.get(key)
|
||||
return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute]
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
assert cls._original_block_count is not None
|
||||
# TODO: change TextModel to super()
|
||||
if (titem := TextModel.filter_tensors(item)) is None:
|
||||
return None
|
||||
name, gen = titem
|
||||
if name.startswith("model.mtp."):
|
||||
name = name.replace("model.", "", 1)
|
||||
if name.startswith("mtp."):
|
||||
if cls.no_mtp:
|
||||
return None
|
||||
remapper = {
|
||||
"fc": "eh_proj",
|
||||
"pre_fc_norm_embedding": "enorm",
|
||||
"pre_fc_norm_hidden": "hnorm",
|
||||
"norm": "shared_head.norm",
|
||||
}
|
||||
parts = name.split(".", 3)
|
||||
if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal():
|
||||
mtp_idx = int(parts[2])
|
||||
name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}"
|
||||
elif len(parts) == 3 and parts[1] in remapper:
|
||||
name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}"
|
||||
elif cls.mtp_only:
|
||||
keep = name in (
|
||||
"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
|
||||
"embed_tokens.weight", "norm.weight",
|
||||
)
|
||||
if not keep:
|
||||
return None
|
||||
return name, gen
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters() # ty: ignore[unresolved-attribute]
|
||||
if self.no_mtp:
|
||||
return
|
||||
if (n := self.hparams.get("mtp_num_hidden_layers", 0)) > 0:
|
||||
self.gguf_writer.add_nextn_predict_layers(n)
|
||||
|
||||
def prepare_metadata(self, vocab_only: bool):
|
||||
from_dir = self.fname_out.is_dir()
|
||||
super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute]
|
||||
|
||||
if not self.mtp_only or not from_dir:
|
||||
return
|
||||
|
||||
output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
fname_default: str = gguf.naming_convention(
|
||||
self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]
|
||||
self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM")
|
||||
class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
class Qwen3_5TextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM")
|
||||
class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
class Qwen3_5MoeTextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35MOE
|
||||
|
||||
|
||||
|
|
@ -647,13 +648,10 @@ class DFlashModel(Qwen3Model):
|
|||
# own tokenizer logic, not the Qwen default).
|
||||
from . import get_model_class
|
||||
with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f:
|
||||
target_hparams = json.load(f)
|
||||
target_arch = target_hparams["architectures"][0]
|
||||
target_arch = json.load(f)["architectures"][0]
|
||||
target_cls = get_model_class(target_arch)
|
||||
|
||||
if target_cls is not type(self):
|
||||
if target_arch == "NemotronHForCausalLM":
|
||||
setattr(self, "is_moe", "num_experts_per_tok" in target_hparams)
|
||||
target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
|
||||
else:
|
||||
super().set_vocab()
|
||||
|
|
@ -691,12 +689,6 @@ class DFlashModel(Qwen3Model):
|
|||
name = "model." + name
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name == "model.embed_tokens.weight" and not self.hparams.get("has_embed_tokens", True):
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3DSparkModel")
|
||||
class DSparkModel(DFlashModel):
|
||||
|
|
|
|||
|
|
@ -1,471 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, TextModel, gguf
|
||||
|
||||
# Tricks being used to support this model via existing llama.cpp code paths:
|
||||
# - Text projection MLP is folded into the embedding table
|
||||
# - codec_embedding is concat to the text embedding table, vocab is extended
|
||||
# example: codec_bos_id(2149) --> "<|codec_bos|>"
|
||||
# codec_eos_token_id(2150) --> "<|codec_eos_token|>"
|
||||
# codec_language_id.chinese(2055) --> "<|codec_language_chinese|>"
|
||||
# other rows --> "<|codec_0|>", "<|codec_1|>", ..., "<|codec_1023|>"
|
||||
# - output tensor codec_head is smaller than vocab, so logits will be padded at inference time
|
||||
# - suppress_tokens is used to limit the backbone to only sample either semantic or EOS (stop) token
|
||||
|
||||
# pipeline stage mapping:
|
||||
# speaker reference encoder --> mapped to normal mtmd audio encoder
|
||||
# backbone --> mapped to normal libllama text model (autoregressive)
|
||||
# code_predictor --> MTMD_GEN_PROCESS_TYPE_GEN_CODE
|
||||
# code2wav --> MTMD_GEN_PROCESS_TYPE_GEN_WAV
|
||||
|
||||
# torch activation functions used by Qwen3TTSTalkerResizeMLP (config's hidden_act)
|
||||
_ACT2FN = {
|
||||
"silu": F.silu,
|
||||
"gelu": F.gelu,
|
||||
"relu": F.relu,
|
||||
}
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3TTSForConditionalGeneration")
|
||||
class Qwen3TTSTalkerModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3TTS
|
||||
|
||||
_TEXT_PROJ_KEYS = (
|
||||
"model.text_embedding.weight",
|
||||
"text_projection.linear_fc1.weight",
|
||||
"text_projection.linear_fc1.bias",
|
||||
"text_projection.linear_fc2.weight",
|
||||
"text_projection.linear_fc2.bias",
|
||||
)
|
||||
|
||||
_text_proj_buffer: dict[str, Tensor]
|
||||
_folded_text_embed: Tensor | None
|
||||
_codec_embed: Tensor | None
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
hparams = kwargs.pop("hparams", None)
|
||||
if hparams is None:
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
raw_talker_config = dict(hparams["talker_config"])
|
||||
self._talker_config = raw_talker_config
|
||||
self.n_codec_vocab = raw_talker_config["vocab_size"]
|
||||
talker_config = dict(raw_talker_config)
|
||||
talker_config["vocab_size"] = talker_config["text_vocab_size"]
|
||||
hparams["text_config"] = talker_config
|
||||
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
|
||||
self._text_proj_buffer = {}
|
||||
self._folded_text_embed = None
|
||||
self._codec_embed = None
|
||||
|
||||
def _codec_token_names(self) -> list[str]:
|
||||
# start every row with a generic name, then override the ones with a
|
||||
# known meaning (bos/eos/language/etc, derived from the *_id fields
|
||||
# of talker_config) with a more descriptive one
|
||||
names = [f"<|codec_{i}|>" for i in range(self.n_codec_vocab)]
|
||||
for key, val in self._talker_config.items():
|
||||
if not key.endswith("_id"):
|
||||
continue
|
||||
prefix = key[:-len("_id")]
|
||||
if isinstance(val, int):
|
||||
names[val] = f"<|{prefix}|>"
|
||||
elif isinstance(val, dict):
|
||||
for subkey, subval in val.items():
|
||||
names[subval] = f"<|{prefix}_{subkey}|>"
|
||||
return names
|
||||
|
||||
def set_vocab(self):
|
||||
codec_tokens = self._codec_token_names()
|
||||
codec_toktypes = [gguf.TokenType.CONTROL] * len(codec_tokens)
|
||||
|
||||
try:
|
||||
tokens, scores, toktypes = self._create_vocab_sentencepiece()
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
tokens += [t.encode("utf-8") for t in codec_tokens]
|
||||
scores += [0.0] * len(codec_tokens)
|
||||
toktypes += codec_toktypes
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
return
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
tokens += codec_tokens
|
||||
toktypes += codec_toktypes
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(tokpre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
# make sure that the model has no chat template, so chat will be disabled
|
||||
self.gguf_writer.add_chat_template(None)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
# note: final vocab layout is [text_vocab | codec_vocab], with text_vocab is actually padded with -inf in cgraph
|
||||
# for codec_vocab, only first 2048 rows can be sampled for semantic code
|
||||
# plus codec_eos_token_id that used for signaling end of generation
|
||||
# ref: https://github.com/QwenLM/Qwen3-TTS/blob/022e286b98fbec7e1e916cb940cdf532cd9f488e/qwen_tts/core/models/modeling_qwen3_tts.py#L2059-L2063
|
||||
|
||||
vocab_size = self.hparams["vocab_size"] + self.n_codec_vocab
|
||||
codec_eos_token_id = self.hparams["vocab_size"] + self._talker_config["codec_eos_token_id"]
|
||||
self.gguf_writer.add_suppress_tokens([
|
||||
i for i in range(vocab_size - 1024, vocab_size)
|
||||
if i != codec_eos_token_id
|
||||
])
|
||||
self.gguf_writer.add_eos_token_id(codec_eos_token_id)
|
||||
self.gguf_writer.add_add_eos_token(False)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
if not name.startswith("talker.") or name.startswith("talker.code_predictor."):
|
||||
return None
|
||||
|
||||
name = name[len("talker."):]
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def _maybe_emit_token_embd(self) -> Iterable[tuple[str, Tensor]]:
|
||||
if self._folded_text_embed is None or self._codec_embed is None:
|
||||
return
|
||||
combined = torch.cat([self._folded_text_embed, self._codec_embed], dim=0)
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), combined)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# codec_embedding rows are appended after the text vocab, extending the embedding table
|
||||
if name == "model.codec_embedding.weight":
|
||||
self._codec_embed = data_torch
|
||||
yield from self._maybe_emit_token_embd()
|
||||
return
|
||||
|
||||
# codec_head is the output head for the (smaller) codec vocab; logits get padded to
|
||||
# the extended vocab size at inference time
|
||||
if name == "codec_head.weight":
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch)
|
||||
return
|
||||
|
||||
if name in self._TEXT_PROJ_KEYS:
|
||||
self._text_proj_buffer[name] = data_torch
|
||||
if len(self._text_proj_buffer) < len(self._TEXT_PROJ_KEYS):
|
||||
return
|
||||
|
||||
# fold MLP into the embedding table at conversion time, MLP won't be used at inference time anyway
|
||||
act_fn = _ACT2FN[self.hparams["hidden_act"]]
|
||||
embed = self._text_proj_buffer["model.text_embedding.weight"]
|
||||
hidden = act_fn(F.linear(embed,
|
||||
self._text_proj_buffer["text_projection.linear_fc1.weight"],
|
||||
self._text_proj_buffer["text_projection.linear_fc1.bias"]))
|
||||
folded = F.linear(hidden,
|
||||
self._text_proj_buffer["text_projection.linear_fc2.weight"],
|
||||
self._text_proj_buffer["text_projection.linear_fc2.bias"])
|
||||
self._folded_text_embed = folded
|
||||
yield from self._maybe_emit_token_embd()
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3TTSForConditionalGeneration")
|
||||
class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
# talker.code_predictor.model.layers.{bid}.<key> -> A_GEN_CODE_*
|
||||
# bypass tensor_mapping.py for now to make it simple
|
||||
_CODE_LAYER_TENSOR_MAP = {
|
||||
"input_layernorm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
|
||||
"self_attn.q_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q,
|
||||
"self_attn.q_norm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM,
|
||||
"self_attn.k_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K,
|
||||
"self_attn.k_norm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM,
|
||||
"self_attn.v_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V,
|
||||
"self_attn.o_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
|
||||
"post_attention_layernorm": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
|
||||
"mlp.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
|
||||
"mlp.up_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
|
||||
"mlp.down_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
|
||||
}
|
||||
|
||||
# note: codebook pages will be stacked to 3D
|
||||
_CODE_GEN_N_CODEBOOKS = 15
|
||||
_code_embed_buffer: dict[int, Tensor] = {}
|
||||
_code_head_buffer: dict[int, Tensor] = {}
|
||||
_wav_config_cache: dict[str, Any] | None = None
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
hparams = kwargs.pop("hparams", None)
|
||||
if hparams is None:
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]}
|
||||
# ECAPA-TDNN has a fixed 4-stage backbone, but MmprojModel.__init__ needs a n_block_keys
|
||||
hparams["speaker_encoder_config"]["n_layers"] = 4
|
||||
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
|
||||
self._wav_config_cache = None
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config.get("speaker_encoder_config")
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_SPKENC)
|
||||
|
||||
# handle speaker encoder config
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
# mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default)
|
||||
self.gguf_writer.add_audio_num_mel_bins(128)
|
||||
# 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here
|
||||
self.gguf_writer.add_audio_block_count(3)
|
||||
# ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it
|
||||
self.gguf_writer.add_audio_embedding_length(1536)
|
||||
self.gguf_writer.add_audio_head_count(1)
|
||||
self.gguf_writer.add_audio_feed_forward_length(1536)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
# handle code predictor config
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_GEN)
|
||||
code_predictor_config = self.global_config["talker_config"]["code_predictor_config"]
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(code_predictor_config["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(code_predictor_config["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(code_predictor_config["num_hidden_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(code_predictor_config["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(code_predictor_config["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"])
|
||||
# note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here
|
||||
|
||||
def _wav_decoder_config(self) -> dict[str, Any] | None:
|
||||
# code2wav has its own config.json, inside the speech_tokenizer dir
|
||||
if self._wav_config_cache is None:
|
||||
path = self.dir_model / "speech_tokenizer" / "config.json"
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
self._wav_config_cache = cfg["decoder_config"]
|
||||
return self._wav_config_cache
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
|
||||
if new_name.endswith(".weight") and (
|
||||
new_name in ("a.gen.wav.pre_conv.weight", "a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight")
|
||||
or (".up.blk." in new_name and new_name.endswith(".dwconv.weight"))
|
||||
or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
|
||||
):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
# ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
|
||||
if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
if not (
|
||||
name.startswith("speaker_encoder.")
|
||||
or name.startswith("talker.code_predictor.")
|
||||
or name == "talker.model.codec_embedding.weight"
|
||||
):
|
||||
return None
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# code2wav tensors are already named by generate_extra_tensors(), pass them through
|
||||
if name.startswith("a.gen.wav."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
# codebook-0 embedding, fed back to the talker backbone (codebooks 1-15 live in code_predictor)
|
||||
if name == "talker.model.codec_embedding.weight":
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD), data_torch)
|
||||
return
|
||||
|
||||
if name == "talker.code_predictor.model.norm.weight":
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM), data_torch)
|
||||
return
|
||||
|
||||
if name.startswith("talker.code_predictor.small_to_mtp_projection."):
|
||||
suffix = "." + name.rsplit(".", 1)[1]
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_PROJ_IN, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
if name.startswith("talker.code_predictor.model.codec_embedding."):
|
||||
idx = int(name.split("codec_embedding.")[1].split(".")[0])
|
||||
self._code_embed_buffer[idx] = data_torch
|
||||
if len(self._code_embed_buffer) < self._CODE_GEN_N_CODEBOOKS:
|
||||
return
|
||||
stacked = torch.stack([self._code_embed_buffer.pop(i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0)
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_EMBD), stacked)
|
||||
return
|
||||
|
||||
if name.startswith("talker.code_predictor.lm_head."):
|
||||
idx = int(name.split("lm_head.")[1].split(".")[0])
|
||||
self._code_head_buffer[idx] = data_torch
|
||||
if len(self._code_head_buffer) < self._CODE_GEN_N_CODEBOOKS:
|
||||
return
|
||||
stacked = torch.stack([self._code_head_buffer.pop(i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0)
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_HEAD), stacked)
|
||||
return
|
||||
|
||||
if name.startswith("talker.code_predictor.model.layers."):
|
||||
rest = name.split("model.layers.")[1] # "{bid}.<key>.weight"
|
||||
_, key_with_suffix = rest.split(".", 1) # "<key>.weight"
|
||||
key = key_with_suffix.rsplit(".", 1)[0] # "<key>"
|
||||
tensor = self._CODE_LAYER_TENSOR_MAP.get(key)
|
||||
if tensor is not None:
|
||||
yield (self.format_tensor_name(tensor, bid), data_torch)
|
||||
return
|
||||
|
||||
if "res2net_block.blocks." in name:
|
||||
assert bid is not None # the outer stage index, picked up from the tensor name automatically
|
||||
xid = int(name.split("res2net_block.blocks.")[1].split(".")[0])
|
||||
suffix = "." + name.rsplit(".", 1)[1]
|
||||
new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CONV_RES2].format(bid=bid, xid=xid) + suffix
|
||||
yield (new_name, data_torch)
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from self._generate_code2wav_tensors()
|
||||
|
||||
def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors
|
||||
from safetensors.torch import load_file
|
||||
|
||||
wav_config = self._wav_decoder_config()
|
||||
state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors")
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return state_dict[name]
|
||||
|
||||
def snake_fold(alpha: Tensor, beta: Tensor) -> tuple[Tensor, Tensor]:
|
||||
# fold SnakeBeta's exp()/reciprocal here, so the graph is only mul/sin/sqr/mul/add
|
||||
return torch.exp(alpha), 1.0 / (torch.exp(beta) + 1e-9)
|
||||
|
||||
def rvq_codebook(prefix: str, n_layers: int) -> Tensor:
|
||||
# checkpoint has EMA accumulators, so codebook[i] = embedding_sum[i] / cluster_usage[i]
|
||||
books = []
|
||||
for i in range(n_layers):
|
||||
embedding_sum = get(f"{prefix}.vq.layers.{i}._codebook.embedding_sum")
|
||||
cluster_usage = get(f"{prefix}.vq.layers.{i}._codebook.cluster_usage")
|
||||
books.append(embedding_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1))
|
||||
return torch.stack(books, dim=0) if n_layers > 1 else books[0]
|
||||
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
# --- quantizer: RVQ codebook decode ---
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_IN), get("decoder.quantizer.rvq_first.input_proj.weight").squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT), get("decoder.quantizer.rvq_first.output_proj.weight").squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), rvq_codebook("decoder.quantizer.rvq_first", 1))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_IN), get("decoder.quantizer.rvq_rest.input_proj.weight").squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_OUT), get("decoder.quantizer.rvq_rest.output_proj.weight").squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB), rvq_codebook("decoder.quantizer.rvq_rest", self._CODE_GEN_N_CODEBOOKS))
|
||||
|
||||
# --- pre_conv ---
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_PRE_CONV, suffix=".weight"), get("decoder.pre_conv.conv.weight"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_PRE_CONV, suffix=".bias"), get("decoder.pre_conv.conv.bias"))
|
||||
|
||||
# --- pre_transformer ---
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=".weight"), get("decoder.pre_transformer.input_proj.weight"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=".bias"), get("decoder.pre_transformer.input_proj.bias"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=".weight"), get("decoder.pre_transformer.output_proj.weight"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=".bias"), get("decoder.pre_transformer.output_proj.bias"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get("decoder.pre_transformer.norm.weight"))
|
||||
|
||||
tfm_layer_map = {
|
||||
"input_layernorm.weight": T.A_GEN_WAV_TFM_ATTN_NORM,
|
||||
"self_attn.q_proj.weight": T.A_GEN_WAV_TFM_ATTN_Q,
|
||||
"self_attn.k_proj.weight": T.A_GEN_WAV_TFM_ATTN_K,
|
||||
"self_attn.v_proj.weight": T.A_GEN_WAV_TFM_ATTN_V,
|
||||
"self_attn.o_proj.weight": T.A_GEN_WAV_TFM_ATTN_OUT,
|
||||
"self_attn_layer_scale.scale": T.A_GEN_WAV_TFM_ATTN_SCALE,
|
||||
"post_attention_layernorm.weight": T.A_GEN_WAV_TFM_FFN_NORM,
|
||||
"mlp.gate_proj.weight": T.A_GEN_WAV_TFM_FFN_GATE,
|
||||
"mlp.up_proj.weight": T.A_GEN_WAV_TFM_FFN_UP,
|
||||
"mlp.down_proj.weight": T.A_GEN_WAV_TFM_FFN_DOWN,
|
||||
"mlp_layer_scale.scale": T.A_GEN_WAV_TFM_FFN_SCALE,
|
||||
}
|
||||
assert wav_config is not None
|
||||
for bid in range(wav_config["num_hidden_layers"]):
|
||||
for key, tensor_id in tfm_layer_map.items():
|
||||
yield (self.format_tensor_name(tensor_id, bid), get(f"decoder.pre_transformer.layers.{bid}.{key}"))
|
||||
|
||||
# --- upsample: 2x (causal ConvTranspose1d + ConvNeXt block) ---
|
||||
up_map = {
|
||||
"0.conv.weight": (T.A_GEN_WAV_UP_CONV, ".weight"),
|
||||
"0.conv.bias": (T.A_GEN_WAV_UP_CONV, ".bias"),
|
||||
"1.dwconv.conv.weight": (T.A_GEN_WAV_UP_DWCONV, ".weight"),
|
||||
"1.dwconv.conv.bias": (T.A_GEN_WAV_UP_DWCONV, ".bias"),
|
||||
"1.norm.weight": (T.A_GEN_WAV_UP_NORM, ".weight"),
|
||||
"1.norm.bias": (T.A_GEN_WAV_UP_NORM, ".bias"),
|
||||
"1.pwconv1.weight": (T.A_GEN_WAV_UP_PW1, ".weight"),
|
||||
"1.pwconv1.bias": (T.A_GEN_WAV_UP_PW1, ".bias"),
|
||||
"1.pwconv2.weight": (T.A_GEN_WAV_UP_PW2, ".weight"),
|
||||
"1.pwconv2.bias": (T.A_GEN_WAV_UP_PW2, ".bias"),
|
||||
"1.gamma": (T.A_GEN_WAV_UP_GAMMA, ""),
|
||||
}
|
||||
for bid in range(len(wav_config["upsampling_ratios"])):
|
||||
for key, (tensor_id, suffix) in up_map.items():
|
||||
yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), get(f"decoder.upsample.{bid}.{key}"))
|
||||
|
||||
# --- DAC decoder ---
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), get("decoder.decoder.0.conv.weight"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get("decoder.decoder.0.conv.bias"))
|
||||
|
||||
n_dac_blocks = len(wav_config["upsample_rates"])
|
||||
for bid in range(n_dac_blocks):
|
||||
py = bid + 1 # decoder.decoder.0 is the entry conv, blocks start at 1
|
||||
|
||||
a, b = snake_fold(get(f"decoder.decoder.{py}.block.0.alpha"), get(f"decoder.decoder.{py}.block.0.beta"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), get(f"decoder.decoder.{py}.block.1.conv.weight"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(f"decoder.decoder.{py}.block.1.conv.bias"))
|
||||
|
||||
for xid in range(3):
|
||||
ridx = xid + 2 # block.2/3/4 are the 3 residual units
|
||||
|
||||
a1, b1 = snake_fold(get(f"decoder.decoder.{py}.block.{ridx}.act1.alpha"), get(f"decoder.decoder.{py}.block.{ridx}.act1.beta"))
|
||||
name1 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_ACT1].format(bid=bid, xid=xid)
|
||||
yield (name1 + ".alpha", a1)
|
||||
yield (name1 + ".beta", b1)
|
||||
|
||||
name_conv1 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_CONV1].format(bid=bid, xid=xid)
|
||||
yield (name_conv1 + ".weight", get(f"decoder.decoder.{py}.block.{ridx}.conv1.conv.weight"))
|
||||
yield (name_conv1 + ".bias", get(f"decoder.decoder.{py}.block.{ridx}.conv1.conv.bias"))
|
||||
|
||||
a2, b2 = snake_fold(get(f"decoder.decoder.{py}.block.{ridx}.act2.alpha"), get(f"decoder.decoder.{py}.block.{ridx}.act2.beta"))
|
||||
name2 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_ACT2].format(bid=bid, xid=xid)
|
||||
yield (name2 + ".alpha", a2)
|
||||
yield (name2 + ".beta", b2)
|
||||
|
||||
name_conv2 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_CONV2].format(bid=bid, xid=xid)
|
||||
yield (name_conv2 + ".weight", get(f"decoder.decoder.{py}.block.{ridx}.conv2.conv.weight"))
|
||||
yield (name_conv2 + ".bias", get(f"decoder.decoder.{py}.block.{ridx}.conv2.conv.bias"))
|
||||
|
||||
a5, b5 = snake_fold(get("decoder.decoder.5.alpha"), get("decoder.decoder.5.beta"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a5)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b5)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), get("decoder.decoder.6.conv.weight"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get("decoder.decoder.6.conv.bias"))
|
||||
|
|
@ -126,12 +126,8 @@ def parse_args() -> argparse.Namespace:
|
|||
help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-nextn", "--no-mtp", dest="no_mtp", action="store_true",
|
||||
help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dspark", action="store_true",
|
||||
help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.",
|
||||
"--no-mtp", action="store_true",
|
||||
help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mistral-format", action="store_true",
|
||||
|
|
@ -262,20 +258,13 @@ def main() -> None:
|
|||
from conversion.mistral import MistralModel
|
||||
model_class = MistralModel
|
||||
|
||||
if sum((args.mtp, args.no_mtp, args.dspark)) > 1:
|
||||
logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive")
|
||||
if args.mtp and args.no_mtp:
|
||||
logger.error("--mtp and --no-mtp are mutually exclusive")
|
||||
sys.exit(1)
|
||||
|
||||
if args.dspark:
|
||||
if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM":
|
||||
logger.error("--dspark is only supported for DeepseekV4ForCausalLM")
|
||||
sys.exit(1)
|
||||
from conversion.deepseek import DeepseekV4DSparkModel
|
||||
model_class = DeepseekV4DSparkModel
|
||||
|
||||
if args.mtp or args.no_mtp:
|
||||
if not model_class.supports_mtp_export:
|
||||
logger.error("--mtp / --no-nextn are not supported for %s", model_architecture)
|
||||
logger.error("--mtp / --no-mtp are not supported for %s", model_architecture)
|
||||
sys.exit(1)
|
||||
if args.no_mtp:
|
||||
model_class.no_mtp = True
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
# Release process
|
||||
|
||||
llama.cpp uses [semantic versioning](https://semver.org) (`MAJOR.MINOR.PATCH`).
|
||||
|
||||
## Version bump guidelines
|
||||
|
||||
| Change type | Version component |
|
||||
|---|---|
|
||||
| Breaking change to the public C API (`include/llama.h`) | `MAJOR` |
|
||||
| Backward-compatible features, model support, or API addition | `MINOR` |
|
||||
| Bug fix with no API change | `PATCH` |
|
||||
|
||||
The version is set in the three variables at the top of the root `CMakeLists.txt`:
|
||||
|
||||
```cmake
|
||||
set(LLAMA_VERSION_MAJOR 0)
|
||||
set(LLAMA_VERSION_MINOR 1)
|
||||
set(LLAMA_VERSION_PATCH 0)
|
||||
```
|
||||
|
||||
_A version bump should be included in the PR that introduces the change, or in a
|
||||
dedicated bump commit merged before the release is cut._
|
||||
|
||||
_TODO: add PR labels (`semver: patch`, `semver: minor`, `semver: major`) to help
|
||||
identify which PRs require a version bump before cutting a release._
|
||||
|
||||
## Making a release
|
||||
|
||||
Releases are created by running the [make-release](.github/workflows/make-release.yml)
|
||||
which is a manual workflow.
|
||||
|
||||
The workflow creates an annotated git tag (e.g. `v0.1.0`) and pushes it to the
|
||||
remote. No GitHub Release object is created, the tag is the release artifact.
|
||||
|
||||
## Building a release
|
||||
|
||||
By default, `LLAMA_BUILD_IS_DEV=ON` which appends a `-dev` suffix to `LLAMA_VERSION`,
|
||||
marking the build as a nightly/development build. Distributors building from a
|
||||
release tag must pass `-DLLAMA_BUILD_IS_DEV=OFF` to produce a clean version string
|
||||
(e.g. `0.1.0` instead of `0.1.0-dev`).
|
||||
|
||||
## How releases reach users
|
||||
Currently releases are not published to github releases, only nightly/development
|
||||
builds are available there. The way users can access releases are using the following
|
||||
channels:
|
||||
|
||||
- **llama-install.sh** — downloads pre-built binaries built from the release tag.
|
||||
- **Package managers** — consume the git tag directly.
|
||||
- **Build from source** — users clone the repo and check out the tag.
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
|
|||
-->
|
||||
<head>
|
||||
<script id="init-config">
|
||||
const LITEVER = 342;
|
||||
const LITEVER = 341;
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
var localflag = urlParams.get('local'); //this will be replaced automatically in embedded kcpp
|
||||
const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
|
||||
|
|
@ -4876,15 +4876,6 @@ Current version indicated by LITEVER below.
|
|||
"assistant_start":"[/INST]",
|
||||
"assistant_end":"</s>",
|
||||
},
|
||||
{
|
||||
"name":"Muse Glimmer",
|
||||
"system_start": "<|start|>system<|message|>",
|
||||
"system_end": "<|eot|>",
|
||||
"user_start": "<|start|>user<|message|>",
|
||||
"user_end": "<|eot|>",
|
||||
"assistant_start": "<|start|>assistant to=user<|message|>",
|
||||
"assistant_end": "<|eot|>"
|
||||
},
|
||||
{
|
||||
"name":"Phi-3 Mini",
|
||||
"system_start":"<|system|>\\n",
|
||||
|
|
@ -20991,7 +20982,7 @@ Current version indicated by LITEVER below.
|
|||
let st = get_instruct_starttag(true);
|
||||
let et = get_instruct_endtag(true);
|
||||
let me = get_my_multiplayer_chatname();
|
||||
seqs = [st]; //don't include the assistant tag, otherwise it breaks some think formats
|
||||
seqs = [st, et];
|
||||
if(localsettings.separate_end_tags)
|
||||
{
|
||||
if(get_instruct_endtag_end(true))
|
||||
|
|
@ -21013,7 +21004,7 @@ Current version indicated by LITEVER below.
|
|||
//the model just gets too confused and is likely to repeat the tag.
|
||||
}
|
||||
}
|
||||
//special case for GPT-OSS and Muse Glimmer, never use it as a stop sequence or stuff gets messed
|
||||
//special case for GPT-OSS, never use it as a stop sequence or stuff gets messed
|
||||
seqs = seqs.filter(itm => !itm.includes("<|start|>assistant"));
|
||||
if(!localsettings.includedefaultstops)
|
||||
{
|
||||
|
|
|
|||
3
examples/test-cmake/.gitignore
vendored
3
examples/test-cmake/.gitignore
vendored
|
|
@ -1,3 +0,0 @@
|
|||
llama-build-install
|
||||
install
|
||||
build
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
cmake_minimum_required(VERSION 3.14)
|
||||
project(llama-simple)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
|
||||
find_package(llama 0.1.0 REQUIRED)
|
||||
|
||||
add_executable(test-cmake test-cmake.cpp)
|
||||
target_link_libraries(test-cmake PRIVATE llama)
|
||||
target_compile_definitions(test-cmake PRIVATE
|
||||
LLAMA_BUILD_NUMBER=${LLAMA_BUILD_NUMBER}
|
||||
LLAMA_BUILD_COMMIT="${LLAMA_BUILD_COMMIT}"
|
||||
)
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
## cmake-test
|
||||
|
||||
This is just for manually testing/developing of a llama.cpp installation to
|
||||
enable troubleshooting issues and exploration. The idea is that this can be used
|
||||
after making changes to llama.cpp installation cmake configuration and then
|
||||
verify it locally.
|
||||
|
||||
### Usage
|
||||
The following will configure, build, and install llama.cpp
|
||||
|
||||
Configuring/build/install:
|
||||
```console
|
||||
./build-install.sh
|
||||
```
|
||||
The above command will create a directory named `install` in the current directory
|
||||
which will have the follwing files in its lib directory:
|
||||
```console
|
||||
(venv) $ ls install/lib/
|
||||
cmake libggml.so libllama-common.so.0 libllama.so.0.1.0 llama.cpp
|
||||
libggml-base.so libggml.so.0 libllama-common.so.0.1.0 libmtmd.so pkgconfig
|
||||
libggml-base.so.0 libggml.so.0.19.0 libllama.so libmtmd.so.0
|
||||
libggml-base.so.0.19.0 libllama-common.so libllama.so.0 libmtmd.so.0.1.0
|
||||
```
|
||||
|
||||
Build/run this project using the installation created above:
|
||||
```console
|
||||
(venv) $ ./build.sh
|
||||
-- Configuring done (0.0s)
|
||||
-- Generating done (0.0s)
|
||||
-- Build files have been written to: /path/to/llama.cpp/examples/test-cmake/build
|
||||
[100%] Built target test-cmake
|
||||
[test-cmake] Using llama.cpp version 0.1.0-dev-b10335
|
||||
[test-cmake] Initializing backend...
|
||||
load_backend: loaded CPU backend from /path/to/llama.cpp/examples/test-cmake/install/lib/llama.cpp/libggml-cpu-alderlake.so
|
||||
[test-cmake] Backend initialized.
|
||||
```
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
rm -rf llama-build-install install
|
||||
|
||||
cmake --fresh -S ../../. -B llama-build-install -DCMAKE_BUILD_TYPE=Release \
|
||||
-DBUILD_SHARED_LIBS=ON \
|
||||
-DGGML_BACKEND_DL=ON \
|
||||
-DGGML_CPU_ALL_VARIANTS=ON \
|
||||
-DLLAMA_TESTS_INSTALL=OFF \
|
||||
-DCMAKE_INSTALL_PREFIX="${PWD}/install" \
|
||||
-DGGML_BACKEND_DIR="${PWD}/install/lib/llama.cpp" \
|
||||
-DGGML_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
|
||||
-DLLAMA_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
|
||||
-DLLAMA_TOOLS_INSTALL=OFF
|
||||
|
||||
cmake --build llama-build-install --parallel 12
|
||||
cmake --install llama-build-install
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install"
|
||||
cmake --build build
|
||||
LD_LIBRARY_PATH="${PWD}/install/lib/llama.cpp:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
#include "llama.h"
|
||||
#include <cstdio>
|
||||
|
||||
int main(void) {
|
||||
printf("[test-cmake] version: %s, build: %d (%s)\n",
|
||||
llama_version(), LLAMA_BUILD_NUMBER, LLAMA_BUILD_COMMIT);
|
||||
printf("[test-cmake] Initializing backend...\n");
|
||||
llama_backend_init();
|
||||
printf("[test-cmake] Backend initialized.\n");
|
||||
llama_backend_free();
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -217,14 +217,6 @@ extern "C"
|
|||
{
|
||||
sdtype_abort_generation();
|
||||
}
|
||||
sd_info_outputs sd_get_ongoing_generation_info()
|
||||
{
|
||||
return sdtype_get_ongoing_generation_info();
|
||||
}
|
||||
void sd_request_ongoing_generation_preview()
|
||||
{
|
||||
sdtype_request_ongoing_generation_preview();
|
||||
}
|
||||
|
||||
bool whisper_load_model(const whisper_load_model_inputs inputs)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -154,8 +154,6 @@ extern "C" {
|
|||
bool buffer_from_host_ptr;
|
||||
// event synchronization
|
||||
bool events;
|
||||
// mmap is supported for loading
|
||||
bool mmap_support;
|
||||
};
|
||||
|
||||
// all the device properties
|
||||
|
|
|
|||
|
|
@ -2477,8 +2477,7 @@ extern "C" {
|
|||
struct ggml_tensor * A,
|
||||
struct ggml_tensor * B,
|
||||
struct ggml_tensor * C,
|
||||
struct ggml_tensor * ids,
|
||||
int64_t K);
|
||||
struct ggml_tensor * ids);
|
||||
|
||||
// partition into non-overlapping windows with padding if needed
|
||||
// example:
|
||||
|
|
@ -2807,12 +2806,6 @@ extern "C" {
|
|||
struct ggml_cgraph * cgraph,
|
||||
struct ggml_tensor * tensor);
|
||||
|
||||
// add the tensor and its parents to the graph without marking them for compute
|
||||
// the flag is set later, when the tensor is reached from a node that computes
|
||||
GGML_API void ggml_build_forward_order(
|
||||
struct ggml_cgraph * cgraph,
|
||||
struct ggml_tensor * tensor);
|
||||
|
||||
GGML_API void ggml_build_backward_expand(
|
||||
struct ggml_context * ctx, // context for gradient computation
|
||||
struct ggml_cgraph * cgraph,
|
||||
|
|
|
|||
|
|
@ -132,7 +132,6 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back
|
|||
/* .host_buffer = */ false, // Not implemented.
|
||||
/* .buffer_from_host_ptr = */ false, // Not implemented.
|
||||
/* .events = */ false, // Not implemented.
|
||||
/* .mmap_support = */ true,
|
||||
};
|
||||
for (ggml_backend_dev_t simple_dev : meta_dev_ctx->simple_devs) {
|
||||
ggml_backend_dev_props tmp_props;
|
||||
|
|
@ -141,7 +140,6 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back
|
|||
props->caps.host_buffer = props->caps.host_buffer && tmp_props.caps.host_buffer;
|
||||
props->caps.buffer_from_host_ptr = props->caps.buffer_from_host_ptr && tmp_props.caps.buffer_from_host_ptr;
|
||||
props->caps.events = props->caps.events && tmp_props.caps.events;
|
||||
props->caps.mmap_support = props->caps.mmap_support && tmp_props.caps.mmap_support;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -766,9 +766,8 @@ struct ggml_backend_sched_split {
|
|||
int backend_id;
|
||||
int i_start;
|
||||
int i_end;
|
||||
struct ggml_tensor ** inputs;
|
||||
struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS];
|
||||
int n_inputs;
|
||||
int inputs_capacity;
|
||||
// graph view of this split
|
||||
struct ggml_cgraph graph;
|
||||
};
|
||||
|
|
@ -807,9 +806,8 @@ struct ggml_backend_sched {
|
|||
int cur_copy;
|
||||
int next_copy;
|
||||
ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];
|
||||
struct ggml_tensor ** graph_inputs;
|
||||
struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS];
|
||||
int n_graph_inputs;
|
||||
int graph_inputs_capacity;
|
||||
|
||||
struct ggml_context * ctx;
|
||||
|
||||
|
|
@ -835,36 +833,6 @@ struct ggml_backend_sched {
|
|||
#define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]
|
||||
#define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id)
|
||||
|
||||
static void ggml_backend_sched_split_inputs_grow(struct ggml_backend_sched_split * split) {
|
||||
int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;
|
||||
if (split->inputs_capacity > 0) {
|
||||
new_cap = 2*split->inputs_capacity;
|
||||
GGML_LOG_WARN("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap);
|
||||
}
|
||||
auto * pnew = (struct ggml_tensor **) realloc((void *) split->inputs, new_cap * sizeof(struct ggml_tensor *));
|
||||
if (pnew == NULL) {
|
||||
GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *));
|
||||
GGML_ABORT("failed to grow split inputs container");
|
||||
}
|
||||
split->inputs = pnew;
|
||||
split->inputs_capacity = new_cap;
|
||||
}
|
||||
|
||||
static void ggml_backend_sched_graph_inputs_grow(ggml_backend_sched_t sched) {
|
||||
int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;
|
||||
if (sched->graph_inputs_capacity > 0) {
|
||||
new_cap = 2*sched->graph_inputs_capacity;
|
||||
GGML_LOG_WARN("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap);
|
||||
}
|
||||
auto * pnew = (struct ggml_tensor **) realloc((void *) sched->graph_inputs, new_cap * sizeof(struct ggml_tensor *));
|
||||
if (pnew == NULL) {
|
||||
GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *));
|
||||
GGML_ABORT("failed to grow graph inputs container");
|
||||
}
|
||||
sched->graph_inputs = pnew;
|
||||
sched->graph_inputs_capacity = new_cap;
|
||||
}
|
||||
|
||||
// returns the priority of the backend, lower id is higher priority
|
||||
static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {
|
||||
for (int i = 0; i < sched->n_backends; i++) {
|
||||
|
|
@ -1336,7 +1304,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
|
|||
}
|
||||
// check if the split has too many inputs
|
||||
// FIXME: count the number of inputs instead of only checking when full
|
||||
if (split->n_inputs >= split->inputs_capacity) {
|
||||
if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) {
|
||||
const size_t id = hash_id(src);
|
||||
int src_backend_id = sched->hv_tensor_backend_ids[id];
|
||||
bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);
|
||||
|
|
@ -1352,14 +1320,10 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
|
|||
split->i_end = i;
|
||||
i_split++;
|
||||
if (i_split >= sched->splits_capacity) {
|
||||
int old_cap = sched->splits_capacity;
|
||||
sched->splits_capacity *= 2;
|
||||
sched->splits = (ggml_backend_sched_split *)
|
||||
realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));
|
||||
GGML_ASSERT(sched->splits != NULL);
|
||||
for (int k = old_cap; k < sched->splits_capacity; k++) {
|
||||
memset(&sched->splits[k], 0, sizeof(struct ggml_backend_sched_split));
|
||||
}
|
||||
}
|
||||
split = &sched->splits[i_split];
|
||||
split->backend_id = node_backend_id;
|
||||
|
|
@ -1396,9 +1360,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
|
|||
SET_CAUSE(tensor_copy, "4.cpy");
|
||||
}
|
||||
int n_graph_inputs = sched->n_graph_inputs++;
|
||||
if (n_graph_inputs >= sched->graph_inputs_capacity) {
|
||||
ggml_backend_sched_graph_inputs_grow(sched);
|
||||
}
|
||||
GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);
|
||||
sched->graph_inputs[n_graph_inputs] = src;
|
||||
}
|
||||
}
|
||||
|
|
@ -1418,9 +1380,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
|
|||
SET_CAUSE(tensor_copy, "4.cpy");
|
||||
}
|
||||
int n_inputs = split->n_inputs++;
|
||||
if (n_inputs >= split->inputs_capacity) {
|
||||
ggml_backend_sched_split_inputs_grow(split);
|
||||
}
|
||||
GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);
|
||||
split->inputs[n_inputs] = src;
|
||||
}
|
||||
node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy);
|
||||
|
|
@ -1446,11 +1406,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
|
|||
sched->prev_leaf_backend_ids = tmp;
|
||||
}
|
||||
|
||||
int total_inputs = sched->n_graph_inputs;
|
||||
for (int i = 0; i < sched->n_splits; i++) {
|
||||
total_inputs += sched->splits[i].n_inputs;
|
||||
}
|
||||
int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies;
|
||||
int graph_size = std::max(graph->n_nodes, graph->n_leafs) + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sched->n_copies;
|
||||
|
||||
// remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]
|
||||
sched->debug_prev_graph_size = sched->debug_graph_size;
|
||||
|
|
@ -1837,9 +1793,6 @@ ggml_backend_sched_t ggml_backend_sched_new(
|
|||
sched->splits = (ggml_backend_sched_split *) calloc(initial_splits_capacity, sizeof(sched->splits[0]));
|
||||
sched->splits_capacity = initial_splits_capacity;
|
||||
|
||||
sched->graph_inputs_capacity = GGML_SCHED_MAX_SPLIT_INPUTS;
|
||||
sched->graph_inputs = (struct ggml_tensor **) calloc(sched->graph_inputs_capacity, sizeof(struct ggml_tensor *));
|
||||
|
||||
for (int b = 0; b < n_backends; b++) {
|
||||
sched->backends[b] = backends[b];
|
||||
sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]);
|
||||
|
|
@ -1872,11 +1825,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) {
|
|||
ggml_gallocr_free(sched->galloc);
|
||||
ggml_free(sched->ctx);
|
||||
ggml_hash_set_free(&sched->hash_set);
|
||||
for (int i = 0; i < sched->splits_capacity; i++) {
|
||||
free(sched->splits[i].inputs);
|
||||
}
|
||||
free(sched->splits);
|
||||
free(sched->graph_inputs);
|
||||
free(sched->hv_tensor_backend_ids);
|
||||
free(sched->hv_tensor_copies);
|
||||
free(sched->node_backend_ids);
|
||||
|
|
|
|||
|
|
@ -367,7 +367,6 @@ static void ggml_backend_blas_device_get_props(ggml_backend_dev_t dev, struct gg
|
|||
/* .host_buffer = */ false,
|
||||
/* .buffer_from_host_ptr = */ true,
|
||||
/* .events = */ false,
|
||||
/* .mmap_support = */ true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,81 @@
|
|||
#include "ggml-backend-impl.h"
|
||||
#include "ggml-feats.h"
|
||||
|
||||
#if defined(__aarch64__) || defined(_M_ARM64)
|
||||
#if defined(__aarch64__)
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <sys/auxv.h>
|
||||
#elif defined(__APPLE__)
|
||||
#include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_SVE2)
|
||||
#define HWCAP2_SVE2 (1 << 1)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_I8MM)
|
||||
#define HWCAP2_I8MM (1 << 13)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_SME)
|
||||
#define HWCAP2_SME (1 << 23)
|
||||
#endif
|
||||
|
||||
struct aarch64_features {
|
||||
// has_neon not needed, aarch64 has NEON guaranteed
|
||||
bool has_dotprod = false;
|
||||
bool has_fp16_va = false;
|
||||
bool has_sve = false;
|
||||
bool has_sve2 = false;
|
||||
bool has_i8mm = false;
|
||||
bool has_sme = false;
|
||||
bool has_sme2 = false;
|
||||
|
||||
aarch64_features() {
|
||||
#if defined(__linux__)
|
||||
uint32_t hwcap = getauxval(AT_HWCAP);
|
||||
uint32_t hwcap2 = getauxval(AT_HWCAP2);
|
||||
|
||||
has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
|
||||
has_fp16_va = !!(hwcap & HWCAP_FPHP);
|
||||
has_sve = !!(hwcap & HWCAP_SVE);
|
||||
has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
|
||||
has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
|
||||
has_sme = !!(hwcap2 & HWCAP2_SME);
|
||||
#elif defined(__APPLE__)
|
||||
int oldp = 0;
|
||||
size_t size = sizeof(oldp);
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, NULL, 0) == 0) {
|
||||
has_dotprod = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, NULL, 0) == 0) {
|
||||
has_i8mm = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, NULL, 0) == 0) {
|
||||
has_sme = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, NULL, 0) == 0) {
|
||||
has_sme2 = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
// Apple apparently does not implement SVE yet
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
static int ggml_backend_cpu_aarch64_score() {
|
||||
int score = 1;
|
||||
const ggml_feats_arch64_runtime_t af = ggml_feats_get_arch64_runtime();
|
||||
GGML_UNUSED(af);
|
||||
aarch64_features af;
|
||||
|
||||
#ifdef GGML_USE_DOTPROD
|
||||
if (!af.has_dotprod) { return 0; }
|
||||
score += 1<<1;
|
||||
#endif
|
||||
#ifdef GGML_USE_FP16_VECTOR_ARITHMETIC
|
||||
if (!af.has_fp16) { return 0; }
|
||||
if (!af.has_fp16_va) { return 0; }
|
||||
score += 1<<2;
|
||||
#endif
|
||||
#ifdef GGML_USE_SVE
|
||||
|
|
@ -38,4 +100,4 @@ static int ggml_backend_cpu_aarch64_score() {
|
|||
|
||||
GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score)
|
||||
|
||||
# endif // defined(__aarch64__) || defined(_M_ARM64)
|
||||
# endif // defined(__aarch64__)
|
||||
|
|
|
|||
|
|
@ -3443,7 +3443,7 @@ static bool ggml_thread_apply_priority(int32_t prio) {
|
|||
return true;
|
||||
}
|
||||
|
||||
#elif defined(__linux__)
|
||||
#elif defined(__gnu_linux__)
|
||||
// TODO: this may not work on BSD, to be verified
|
||||
|
||||
static bool ggml_thread_apply_affinity(const bool * mask) {
|
||||
|
|
@ -3630,11 +3630,6 @@ struct ggml_cplan ggml_graph_plan(
|
|||
n_threads = 1;
|
||||
#endif
|
||||
|
||||
#if defined(__wasi__)
|
||||
// WASI doesn't support parallelism yet
|
||||
n_threads = 1;
|
||||
#endif
|
||||
|
||||
size_t work_size = 0;
|
||||
|
||||
struct ggml_cplan cplan;
|
||||
|
|
|
|||
|
|
@ -397,7 +397,6 @@ static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggm
|
|||
/* .host_buffer = */ false,
|
||||
/* .buffer_from_host_ptr = */ true,
|
||||
/* .events = */ false,
|
||||
/* .mmap_support = */ true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -472,8 +471,6 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
|
|||
src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_CONV_2D:
|
||||
return ggml_is_contiguous(op->src[0]);
|
||||
case GGML_OP_SSM_SCAN:
|
||||
return ggml_get_op_params_i32(op, 0) == 1 || op->src[3]->ne[0] == 1;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8941,7 +8941,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
|
|||
for (int tk = 0; tk < kv_tile; tk++) {
|
||||
const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
|
||||
if (kv_type == GGML_TYPE_F16) {
|
||||
ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
|
||||
ggml_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
|
||||
} else {
|
||||
memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
|
||||
}
|
||||
|
|
@ -9644,13 +9644,11 @@ static void ggml_compute_forward_ssm_scan_f32(
|
|||
const int64_t ng = src4->ne[1];
|
||||
const int64_t nt = src1->ne[2]; // number of tokens per sequence
|
||||
const int64_t ns = src1->ne[3]; // number of sequences in the batch
|
||||
const int64_t K = ggml_get_op_params_i32(dst, 0);
|
||||
|
||||
// can't use ggml_nbytes because src1 is not necessarily contiguous
|
||||
const int64_t s_off = ggml_nelements(src1) * ggml_element_size(src1);
|
||||
|
||||
GGML_ASSERT(K >= 1);
|
||||
GGML_ASSERT(ggml_nelements(src1) + K*nc*nr*nh*ns == ggml_nelements(dst));
|
||||
GGML_ASSERT(ggml_nelements(src1) + nc*nr*nh*ns == ggml_nelements(dst));
|
||||
GGML_ASSERT(src0->nb[0] == sizeof(float));
|
||||
GGML_ASSERT(src1->nb[0] == sizeof(float));
|
||||
GGML_ASSERT(src2->nb[0] == sizeof(float));
|
||||
|
|
@ -9659,7 +9657,6 @@ static void ggml_compute_forward_ssm_scan_f32(
|
|||
GGML_ASSERT(src5->nb[0] == sizeof(float));
|
||||
GGML_ASSERT(src6->nb[0] == sizeof(int32_t));
|
||||
GGML_ASSERT(nh % ng == 0);
|
||||
GGML_ASSERT(src3->ne[0] == 1 || K == 1);
|
||||
|
||||
// heads per thread
|
||||
const int dh = (nh + nth - 1)/nth;
|
||||
|
|
@ -9834,13 +9831,6 @@ static void ggml_compute_forward_ssm_scan_f32(
|
|||
}
|
||||
}
|
||||
}
|
||||
const int64_t slot = nt - 1 - i2;
|
||||
if (K > 1 && slot > 0 && slot < K) {
|
||||
float * s_snapshot = (float *) ((char *) dst->data + s_off + (slot*ns + i3)*(src0->nb[3]));
|
||||
for (int h = ih0; h < ih1; ++h) {
|
||||
memcpy((char *) s_snapshot + h*src0->nb[2], (char *) s + h*src0->nb[2], src0->nb[2]);
|
||||
}
|
||||
}
|
||||
// use the output as the source when it's not the first token-wise iteration
|
||||
s0 = s;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -195,7 +195,6 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS> class tensor_
|
|||
case GGML_TYPE_Q4_K:
|
||||
case GGML_TYPE_Q6_K:
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_Q5_K:
|
||||
//case GGML_TYPE_MXFP4:
|
||||
|
|
@ -215,7 +214,6 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS> class tensor_
|
|||
case GGML_TYPE_Q4_K:
|
||||
case GGML_TYPE_Q6_K:
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_Q5_K:
|
||||
//case GGML_TYPE_MXFP4:
|
||||
|
|
|
|||
|
|
@ -634,8 +634,7 @@ template <typename T> struct block_reduce_policy<block_reduce_method::MAX, T> {
|
|||
};
|
||||
|
||||
template <block_reduce_method reduce_method_t, const unsigned int block_size_template = 0, typename T>
|
||||
static __device__ T block_reduce(T val, [[maybe_unused]] T * shared_vals) {
|
||||
// for multi-warp reductions, callers must not reuse shared_vals until all reads from this invocation have completed
|
||||
static __device__ T block_reduce(T val, T * shared_vals) {
|
||||
val = block_reduce_policy<reduce_method_t, T>::reduce(val);
|
||||
const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template;
|
||||
if (block_size > WARP_SIZE) {
|
||||
|
|
|
|||
|
|
@ -253,9 +253,9 @@ static void ggml_cpy_f32_q8_0_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {
|
||||
|
||||
GGML_ASSERT(ne % QK8_0 == 0);
|
||||
const int64_t num_blocks = (ne/QK8_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne / QK8_0;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
|
||||
cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>
|
||||
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
||||
|
|
@ -264,9 +264,9 @@ static void ggml_cpy_q8_0_f32_cuda(
|
|||
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,
|
||||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {
|
||||
|
||||
const int64_t num_blocks = (ne/QK8_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_q_f32<cpy_blck_q8_0_f32, QK8_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
|
||||
cpy_q_f32<cpy_blck_q8_0_f32, QK8_0><<<num_blocks, 1, 0, stream>>>
|
||||
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
||||
|
|
@ -276,9 +276,9 @@ static void ggml_cpy_f32_q4_0_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {
|
||||
|
||||
GGML_ASSERT(ne % QK4_0 == 0);
|
||||
const int64_t num_blocks = (ne/QK4_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne / QK4_0;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
|
||||
cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>
|
||||
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
||||
|
|
@ -289,9 +289,9 @@ static void ggml_cpy_q4_0_f32_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,
|
||||
const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,
|
||||
cudaStream_t stream) {
|
||||
const int64_t num_blocks = (ne/QK4_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_q_f32<cpy_blck_q_f32<dequantize_q4_0, QK4_0>, QK4_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(
|
||||
cpy_q_f32<cpy_blck_q_f32<dequantize_q4_0, QK4_0>, QK4_0><<<num_blocks, 1, 0, stream>>>(
|
||||
cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,
|
||||
ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
|
@ -302,9 +302,9 @@ static void ggml_cpy_f32_q4_1_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {
|
||||
|
||||
GGML_ASSERT(ne % QK4_1 == 0);
|
||||
const int64_t num_blocks = (ne/QK4_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne / QK4_1;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
|
||||
cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>
|
||||
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
||||
|
|
@ -315,9 +315,9 @@ static void ggml_cpy_q4_1_f32_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,
|
||||
const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,
|
||||
cudaStream_t stream) {
|
||||
const int64_t num_blocks = (ne/QK4_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_q_f32<cpy_blck_q_f32<dequantize_q4_1, QK4_1>, QK4_1><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(
|
||||
cpy_q_f32<cpy_blck_q_f32<dequantize_q4_1, QK4_1>, QK4_1><<<num_blocks, 1, 0, stream>>>(
|
||||
cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,
|
||||
ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
|
@ -328,9 +328,9 @@ static void ggml_cpy_f32_q5_0_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {
|
||||
|
||||
GGML_ASSERT(ne % QK5_0 == 0);
|
||||
const int64_t num_blocks = (ne/QK5_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne / QK5_0;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
|
||||
cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, 1, 0, stream>>>
|
||||
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
||||
|
|
@ -341,9 +341,9 @@ static void ggml_cpy_q5_0_f32_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,
|
||||
const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,
|
||||
cudaStream_t stream) {
|
||||
const int64_t num_blocks = (ne/QK5_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_q_f32<cpy_blck_q_f32<dequantize_q5_0, QK5_0>, QK5_0><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(
|
||||
cpy_q_f32<cpy_blck_q_f32<dequantize_q5_0, QK5_0>, QK5_0><<<num_blocks, 1, 0, stream>>>(
|
||||
cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,
|
||||
ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
|
@ -354,9 +354,9 @@ static void ggml_cpy_f32_q5_1_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {
|
||||
|
||||
GGML_ASSERT(ne % QK5_1 == 0);
|
||||
const int64_t num_blocks = (ne/QK5_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne / QK5_1;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
|
||||
cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, 1, 0, stream>>>
|
||||
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
||||
|
|
@ -367,9 +367,9 @@ static void ggml_cpy_q5_1_f32_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,
|
||||
const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,
|
||||
cudaStream_t stream) {
|
||||
const int64_t num_blocks = (ne/QK5_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_q_f32<cpy_blck_q_f32<dequantize_q5_1, QK5_1>, QK5_1><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>(
|
||||
cpy_q_f32<cpy_blck_q_f32<dequantize_q5_1, QK5_1>, QK5_1><<<num_blocks, 1, 0, stream>>>(
|
||||
cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,
|
||||
ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
|
@ -380,9 +380,9 @@ static void ggml_cpy_f32_iq4_nl_cuda(
|
|||
const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {
|
||||
|
||||
GGML_ASSERT(ne % QK4_NL == 0);
|
||||
const int64_t num_blocks = (ne/QK4_NL + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;
|
||||
const int64_t num_blocks = ne / QK4_NL;
|
||||
GGML_ASSERT(num_blocks <= INT_MAX);
|
||||
cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>
|
||||
cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>
|
||||
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1865,37 +1865,6 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
|
|||
ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst);
|
||||
}
|
||||
|
||||
// returns true when ggml_cuda_mul_mat_id takes the fallback path that requires stream synchronization
|
||||
// [TAG_MUL_MAT_ID_CUDA_GRAPHS]
|
||||
static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int cc) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
|
||||
if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (dst->ne[2] <= MMVQ_MAX_BATCH_SIZE) {
|
||||
if (ggml_is_quantized(src0->type)) {
|
||||
if (dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc)) {
|
||||
return false;
|
||||
}
|
||||
} else if (GGML_CUDA_CC_IS_AMD(cc)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[2], /*n_experts=*/src0->ne[2])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
|
|
@ -1938,7 +1907,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
|||
}
|
||||
|
||||
// note: this path should not be reached when recording CUDA graphs, because it requires stream synchronization
|
||||
GGML_ASSERT(ggml_cuda_mul_mat_id_needs_sync(dst, cc));
|
||||
// TODO: add asserts to verify this. should work with CUDA, HIP, etc.
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
GGML_ASSERT(nb12 % nb11 == 0);
|
||||
|
|
@ -2557,8 +2526,10 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
|
|||
// [TAG_MUL_MAT_ID_CUDA_GRAPHS]
|
||||
if (node->op == GGML_OP_MUL_MAT_ID) {
|
||||
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
|
||||
if (ggml_cuda_mul_mat_id_needs_sync(node, cc)) {
|
||||
// the mul_mat_id fallback path synchronizes the stream, so we cannot use CUDA graphs
|
||||
const int mmvq_mmid_max = get_mmvq_mmid_max_batch(node->src[0]->type, cc);
|
||||
if (!ggml_is_quantized(node->src[0]->type) || node->ne[2] > mmvq_mmid_max) {
|
||||
// under these conditions, the mul_mat_id operation will need to synchronize the stream, so we cannot use CUDA graphs
|
||||
// TODO: figure out a way to enable for larger batch sizes, without hurting performance
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/18958
|
||||
use_cuda_graph = false;
|
||||
#ifndef NDEBUG
|
||||
|
|
@ -2688,52 +2659,6 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope,
|
|||
return true;
|
||||
}
|
||||
|
||||
static bool ggml_cuda_should_fuse_rms_norm_mul_rope(const ggml_tensor * rms_norm,
|
||||
const ggml_tensor * mul,
|
||||
const ggml_tensor * rope) {
|
||||
if (rms_norm->op != GGML_OP_RMS_NORM || mul->op != GGML_OP_MUL || rope->op != GGML_OP_ROPE) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rms_norm->src[0]->type != GGML_TYPE_F32 || rms_norm->type != GGML_TYPE_F32 ||
|
||||
mul->src[0]->type != GGML_TYPE_F32 || mul->src[1]->type != GGML_TYPE_F32 ||
|
||||
mul->type != GGML_TYPE_F32 || rope->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rope->src[0] != mul) {
|
||||
return false;
|
||||
}
|
||||
|
||||
//if rms norm is the B operand, then we don't handle broadcast
|
||||
if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!ggml_are_same_shape(rms_norm, mul)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
//rms_norm kernel assumes contiguous rows
|
||||
if (!ggml_is_contiguous_rows(rms_norm->src[0]) ||
|
||||
!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// the fused kernel handles the norm/neox rope modes only
|
||||
const int mode = ((const int32_t *) rope->op_params)[2];
|
||||
if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int n_dims = ((const int32_t *) rope->op_params)[1];
|
||||
if (n_dims % 2 != 0 || rope->src[0]->ne[0] % 2 != 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// match gated_delta_net + the strided cpy that scatters its state snapshots into the cache
|
||||
// (slot i -> rollback group i, slot 0 newest), so the kernel can write them and skip the cpy.
|
||||
static int ggml_cuda_try_gdn_cache_fusion(
|
||||
|
|
@ -3063,36 +2988,6 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
|
|||
}
|
||||
}
|
||||
|
||||
std::initializer_list<enum ggml_op> rms_norm_mul_rope_ops = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE };
|
||||
std::initializer_list<enum ggml_op> rms_norm_mul_rope_set_rows_ops = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };
|
||||
|
||||
if (is_equal(rms_norm_mul_rope_set_rows_ops, ops) && ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 4 })) {
|
||||
const ggml_tensor * rms_norm = cgraph->nodes[node_idx];
|
||||
const ggml_tensor * mul = cgraph->nodes[node_idx + 1];
|
||||
const ggml_tensor * rope = cgraph->nodes[node_idx + 2];
|
||||
const ggml_tensor * view = cgraph->nodes[node_idx + 3];
|
||||
const ggml_tensor * set_rows = cgraph->nodes[node_idx + 4];
|
||||
|
||||
if (ggml_check_edges(cgraph, node_idx, {{1, 0, 0}, {2, 0, 1}, {3, 0, 2}, {4, 0, 3}}) &&
|
||||
ggml_cuda_should_fuse_rms_norm_mul_rope(rms_norm, mul, rope) &&
|
||||
ggml_cuda_should_fuse_rope_set_rows(rope, view, set_rows)) {
|
||||
int out_nodes[] = { node_idx + 4 };
|
||||
return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (is_equal(rms_norm_mul_rope_ops, ops) && ggml_can_fuse(cgraph, node_idx, ops)) {
|
||||
const ggml_tensor * rms_norm = cgraph->nodes[node_idx];
|
||||
const ggml_tensor * mul = cgraph->nodes[node_idx + 1];
|
||||
const ggml_tensor * rope = cgraph->nodes[node_idx + 2];
|
||||
|
||||
if (ggml_cuda_should_fuse_rms_norm_mul_rope(rms_norm, mul, rope)) {
|
||||
int out_nodes[] = { node_idx + 2 };
|
||||
return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::initializer_list<enum ggml_op> rope_set_rows_ops = { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS };
|
||||
|
||||
if (is_equal(rope_set_rows_ops, ops) && ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 2 })) {
|
||||
|
|
@ -3101,8 +2996,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
|
|||
const ggml_tensor * set_rows = cgraph->nodes[node_idx + 2];
|
||||
|
||||
if (ggml_cuda_should_fuse_rope_set_rows(rope, view, set_rows)) {
|
||||
int out_nodes[] = { node_idx + 2 };
|
||||
return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3954,16 +3848,6 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
|
|||
return fused_node_count - 1;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, {})) {
|
||||
ggml_cuda_op_rms_norm_mul_rope_fused(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 4]);
|
||||
return 4;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE }, {})) {
|
||||
ggml_cuda_op_rms_norm_mul_rope_fused(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2], nullptr);
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
|
||||
ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
|
||||
return 2;
|
||||
|
|
@ -4157,11 +4041,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud
|
|||
continue;
|
||||
}
|
||||
#ifndef NDEBUG
|
||||
// On integrated GPUs (APUs, e.g. RDNA3.5) the scheduler may place a
|
||||
// node's output on the host-visible buffer, which the compute path
|
||||
// handles. Allow that here, mirroring the src-tensor check below.
|
||||
assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) ||
|
||||
(integrated && ggml_backend_buft_is_cuda_host(node->buffer->buft)));
|
||||
assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));
|
||||
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
||||
if (node->src[j] != nullptr) {
|
||||
assert(node->src[j]->buffer);
|
||||
|
|
@ -4843,7 +4723,6 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
|
|||
/* .host_buffer = */ host_buffer,
|
||||
/* .buffer_from_host_ptr = */ false,
|
||||
/* .events = */ events,
|
||||
/* .mmap_support = */ props->type != GGML_BACKEND_DEVICE_TYPE_IGPU,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -5202,17 +5081,11 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
|||
(op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) &&
|
||||
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);
|
||||
case GGML_OP_SSM_SCAN: {
|
||||
const int32_t K = ggml_get_op_params_i32(op, 0);
|
||||
|
||||
if (op->src[3]->ne[0] == 1) {
|
||||
// Mamba2
|
||||
// (kernel only supports (d_state == 128 || d_state == 256) && d_head % 16 == 0)
|
||||
return (op->src[0]->ne[0] == 128 || op->src[0]->ne[0] == 256) && op->src[0]->ne[1] % 16 == 0;
|
||||
} else {
|
||||
if (K > 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Mamba
|
||||
// (kernel only supports d_state == 16, d_head == 1, n_head % 128 == 0, n_group == 1)
|
||||
return op->src[0]->ne[0] == 16 && op->src[0]->ne[1] == 1 && op->src[0]->ne[2] % 128 == 0 && op->src[4]->ne[1] == 1;
|
||||
|
|
@ -5234,7 +5107,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
|||
return max_bias == 0.0f;
|
||||
}
|
||||
case GGML_OP_ROLL:
|
||||
if(op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0])) {
|
||||
if(op->src[0]->type == GGML_TYPE_F32) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
|
@ -5345,7 +5218,6 @@ static bool ggml_backend_cuda_device_offload_op(ggml_backend_dev_t dev, const gg
|
|||
|
||||
static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_t dev) {
|
||||
#ifdef GGML_CUDA_NO_PEER_COPY
|
||||
GGML_UNUSED(dev);
|
||||
return nullptr;
|
||||
#else
|
||||
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *)dev->context;
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr
|
|||
tmp += xi * xi;
|
||||
}
|
||||
|
||||
tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum + 32);
|
||||
tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum);
|
||||
|
||||
const float variance = tmp / group_size;
|
||||
const float scale = rsqrtf(variance + eps);
|
||||
|
|
@ -297,7 +297,7 @@ static void group_norm_f32_cuda(
|
|||
group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
|
||||
} else {
|
||||
const dim3 block_dims(1024, 1, 1);
|
||||
group_norm_f32<1024><<<num_groups, block_dims, block_dims.x > WARP_SIZE ? 2 * 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps);
|
||||
group_norm_f32<1024><<<num_groups, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ struct __builtin_align__(32) float8 {
|
|||
float x; float y; float z; float w;
|
||||
float p; float q; float r; float s;
|
||||
};
|
||||
#endif
|
||||
|
||||
#if CUDART_VERSION >= 12080
|
||||
static __device__ __forceinline__ float nvfp4_native_scale_error(
|
||||
|
|
@ -48,7 +49,6 @@ static __device__ __forceinline__ float nvfp4_native_scale_error(
|
|||
return err;
|
||||
}
|
||||
#endif // CUDART_VERSION >= 12080
|
||||
#endif // defined(BLACKWELL_MMA_AVAILABLE)
|
||||
|
||||
__launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1)
|
||||
static __global__ void quantize_q8_1(
|
||||
|
|
|
|||
|
|
@ -670,238 +670,3 @@ void ggml_cuda_op_rope_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
|
|||
void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rope, ggml_tensor * set_rows) {
|
||||
ggml_cuda_op_rope_impl<true>(ctx, rope, set_rows);
|
||||
}
|
||||
|
||||
// fused RMS_NORM + MUL + ROPE (+ VIEW + SET_ROWS)
|
||||
// one block per row: block_reduce gives the norm scale, then each thread applies mul and rope to the elements it owns
|
||||
template <int block_size, bool has_ff, typename D>
|
||||
static __global__ void rms_norm_mul_rope_f32(
|
||||
const float * x, D * dst, const int ncols,
|
||||
const int64_t s01, const int64_t s02, const int64_t s03,
|
||||
const int64_t s1, const int64_t s2, const int64_t s3,
|
||||
const float eps,
|
||||
const float * mul,
|
||||
const int64_t mul_s01, const int64_t mul_s02, const int64_t mul_s03,
|
||||
const uint3 mul_ncols_packed, const uint3 mul_nrows_packed,
|
||||
const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed,
|
||||
const int n_dims, const int32_t * pos,
|
||||
const float freq_scale, const float ext_factor, const float attn_factor,
|
||||
const rope_corr_dims corr_dims, const float theta_scale,
|
||||
const float * freq_factors,
|
||||
const int64_t * row_indices, const int set_rows_stride,
|
||||
const bool is_neox) {
|
||||
ggml_cuda_pdl_lc();
|
||||
const int row = blockIdx.x;
|
||||
const int channel = blockIdx.y;
|
||||
const int sample = blockIdx.z;
|
||||
const int tid = threadIdx.x;
|
||||
|
||||
x += sample*s03 + channel*s02 + row*s01;
|
||||
|
||||
const uint32_t mul_row = fastmodulo(row, mul_nrows_packed);
|
||||
const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed);
|
||||
const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed);
|
||||
mul += mul_sample*mul_s03 + mul_channel*mul_s02 + mul_row*mul_s01;
|
||||
|
||||
float tmp = 0.0f;
|
||||
|
||||
ggml_cuda_pdl_sync();
|
||||
for (int col = tid; col < ncols; col += block_size) {
|
||||
const float xi = x[col];
|
||||
tmp += xi * xi;
|
||||
}
|
||||
|
||||
extern __shared__ float s_sum[];
|
||||
tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum);
|
||||
|
||||
const float scale = rsqrtf(tmp/ncols + eps);
|
||||
|
||||
int64_t idst = sample*s3 + channel*s2 + row*s1;
|
||||
if (set_rows_stride != 0) {
|
||||
idst = row*s1 + row_indices[channel]*set_rows_stride;
|
||||
}
|
||||
dst += idst;
|
||||
|
||||
for (int i0 = 2*tid; i0 < ncols; i0 += 2*block_size) {
|
||||
int ix0;
|
||||
int ix1;
|
||||
if (is_neox && i0 < n_dims) {
|
||||
ix0 = i0/2;
|
||||
ix1 = i0/2 + n_dims/2;
|
||||
} else {
|
||||
ix0 = i0 + 0;
|
||||
ix1 = i0 + 1;
|
||||
}
|
||||
|
||||
const float x0 = scale * x[ix0] * mul[fastmodulo(ix0, mul_ncols_packed)];
|
||||
const float x1 = scale * x[ix1] * mul[fastmodulo(ix1, mul_ncols_packed)];
|
||||
|
||||
if (i0 >= n_dims) {
|
||||
dst[ix0] = ggml_cuda_cast<D>(x0);
|
||||
dst[ix1] = ggml_cuda_cast<D>(x1);
|
||||
continue;
|
||||
}
|
||||
|
||||
const float theta_base = pos[channel]*powf(theta_scale, i0/2.0f);
|
||||
const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f;
|
||||
|
||||
float cos_theta;
|
||||
float sin_theta;
|
||||
rope_yarn<true>(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
dst[ix0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta);
|
||||
dst[ix1] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename D>
|
||||
static void rms_norm_mul_rope_cuda(
|
||||
const float * x, D * dst,
|
||||
const int ncols, const int nrows, const int nchannels, const int nsamples,
|
||||
const int64_t s01, const int64_t s02, const int64_t s03,
|
||||
const int64_t s1, const int64_t s2, const int64_t s3,
|
||||
const float eps,
|
||||
const float * mul,
|
||||
const int64_t mul_s01, const int64_t mul_s02, const int64_t mul_s03,
|
||||
const uint32_t mul_ncols, const uint32_t mul_nrows,
|
||||
const uint32_t mul_nchannels, const uint32_t mul_nsamples,
|
||||
const int n_dims, const int32_t * pos,
|
||||
const float freq_scale, const float freq_base, const float ext_factor, const float attn_factor,
|
||||
const rope_corr_dims corr_dims,
|
||||
const float * freq_factors,
|
||||
const int64_t * row_indices, const int set_rows_stride,
|
||||
const bool is_neox, cudaStream_t stream) {
|
||||
GGML_ASSERT(ncols % 2 == 0);
|
||||
|
||||
const dim3 blocks_num(nrows, nchannels, nsamples);
|
||||
|
||||
const float theta_scale = powf(freq_base, -2.0f/n_dims);
|
||||
|
||||
const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols);
|
||||
const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows);
|
||||
const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);
|
||||
const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples);
|
||||
|
||||
if (ncols < 1024) {
|
||||
const dim3 block_dims(256, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream};
|
||||
if (freq_factors == nullptr) {
|
||||
ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, false, D>, launch_params,
|
||||
x, dst, ncols, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
|
||||
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
|
||||
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
|
||||
freq_factors, row_indices, set_rows_stride, is_neox);
|
||||
} else {
|
||||
ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, true, D>, launch_params,
|
||||
x, dst, ncols, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
|
||||
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
|
||||
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
|
||||
freq_factors, row_indices, set_rows_stride, is_neox);
|
||||
}
|
||||
} else {
|
||||
const dim3 block_dims(1024, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream};
|
||||
if (freq_factors == nullptr) {
|
||||
ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<1024, false, D>, launch_params,
|
||||
x, dst, ncols, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
|
||||
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
|
||||
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
|
||||
freq_factors, row_indices, set_rows_stride, is_neox);
|
||||
} else {
|
||||
ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<1024, true, D>, launch_params,
|
||||
x, dst, ncols, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03,
|
||||
mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
|
||||
n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale,
|
||||
freq_factors, row_indices, set_rows_stride, is_neox);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx,
|
||||
ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows) {
|
||||
const ggml_tensor * x = rms_norm->src[0];
|
||||
const ggml_tensor * mul_src = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0];
|
||||
|
||||
float eps = 0.0f;
|
||||
memcpy(&eps, rms_norm->op_params, sizeof(float));
|
||||
GGML_ASSERT(eps >= 0.0f);
|
||||
|
||||
GGML_ASSERT(x->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(mul_src->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(rope->type == GGML_TYPE_F32);
|
||||
|
||||
void * dst_d = rope->data;
|
||||
ggml_type dst_type = rope->type;
|
||||
const int64_t * row_indices = nullptr;
|
||||
int set_rows_stride = 0;
|
||||
|
||||
if (set_rows != nullptr) {
|
||||
dst_d = set_rows->data;
|
||||
dst_type = set_rows->type;
|
||||
row_indices = (const int64_t *) set_rows->src[1]->data;
|
||||
set_rows_stride = set_rows->nb[1] / ggml_type_size(set_rows->type);
|
||||
}
|
||||
|
||||
const int n_dims = ((const int32_t *) rope->op_params)[1];
|
||||
const int mode = ((const int32_t *) rope->op_params)[2];
|
||||
const int n_ctx_orig = ((const int32_t *) rope->op_params)[4];
|
||||
|
||||
float freq_base;
|
||||
float freq_scale;
|
||||
float ext_factor;
|
||||
float attn_factor;
|
||||
float beta_fast;
|
||||
float beta_slow;
|
||||
|
||||
memcpy(&freq_base, (const int32_t *) rope->op_params + 5, sizeof(float));
|
||||
memcpy(&freq_scale, (const int32_t *) rope->op_params + 6, sizeof(float));
|
||||
memcpy(&ext_factor, (const int32_t *) rope->op_params + 7, sizeof(float));
|
||||
memcpy(&attn_factor, (const int32_t *) rope->op_params + 8, sizeof(float));
|
||||
memcpy(&beta_fast, (const int32_t *) rope->op_params + 9, sizeof(float));
|
||||
memcpy(&beta_slow, (const int32_t *) rope->op_params + 10, sizeof(float));
|
||||
|
||||
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
|
||||
|
||||
const int32_t * pos = (const int32_t *) rope->src[1]->data;
|
||||
|
||||
const float * freq_factors = rope->src[2] != nullptr ? (const float *) rope->src[2]->data : nullptr;
|
||||
|
||||
rope_corr_dims corr_dims;
|
||||
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims.v);
|
||||
|
||||
const size_t ts0 = ggml_type_size(x->type);
|
||||
GGML_ASSERT(x->nb[0] == ts0);
|
||||
const int64_t s01 = x->nb[1] / ts0;
|
||||
const int64_t s02 = x->nb[2] / ts0;
|
||||
const int64_t s03 = x->nb[3] / ts0;
|
||||
|
||||
const size_t ts_mul = ggml_type_size(mul_src->type);
|
||||
GGML_ASSERT(mul_src->nb[0] == ts_mul);
|
||||
const int64_t mul_s01 = mul_src->nb[1] / ts_mul;
|
||||
const int64_t mul_s02 = mul_src->nb[2] / ts_mul;
|
||||
const int64_t mul_s03 = mul_src->nb[3] / ts_mul;
|
||||
|
||||
const size_t ts_dst = ggml_type_size(rope->type);
|
||||
const int64_t s1 = rope->nb[1] / ts_dst;
|
||||
const int64_t s2 = rope->nb[2] / ts_dst;
|
||||
const int64_t s3 = rope->nb[3] / ts_dst;
|
||||
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
if (dst_type == GGML_TYPE_F32) {
|
||||
rms_norm_mul_rope_cuda((const float *) x->data, (float *) dst_d,
|
||||
x->ne[0], x->ne[1], x->ne[2], x->ne[3], s01, s02, s03, s1, s2, s3, eps,
|
||||
(const float *) mul_src->data, mul_s01, mul_s02, mul_s03,
|
||||
mul_src->ne[0], mul_src->ne[1], mul_src->ne[2], mul_src->ne[3],
|
||||
n_dims, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims,
|
||||
freq_factors, row_indices, set_rows_stride, is_neox, stream);
|
||||
} else if (dst_type == GGML_TYPE_F16) {
|
||||
rms_norm_mul_rope_cuda((const float *) x->data, (half *) dst_d,
|
||||
x->ne[0], x->ne[1], x->ne[2], x->ne[3], s01, s02, s03, s1, s2, s3, eps,
|
||||
(const float *) mul_src->data, mul_s01, mul_s02, mul_s03,
|
||||
mul_src->ne[0], mul_src->ne[1], mul_src->ne[2], mul_src->ne[3],
|
||||
n_dims, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims,
|
||||
freq_factors, row_indices, set_rows_stride, is_neox, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,5 +7,3 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
|||
void ggml_cuda_op_rope_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
|
||||
void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * set_rows);
|
||||
|
||||
void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows);
|
||||
|
|
|
|||
|
|
@ -116,11 +116,6 @@ static __global__ void soft_max_f32(
|
|||
vals[col] = val;
|
||||
}
|
||||
|
||||
if (block_size > WARP_SIZE) {
|
||||
// sync is needed as we reuse buf_iw across block_reduce invocations, see #26385
|
||||
// for block_size <= WARP_SIZE, block_reduce does not access buf_iw
|
||||
__syncthreads();
|
||||
}
|
||||
// find the sum of exps in the block
|
||||
tmp = block_reduce<block_reduce_method::SUM, block_size_template>(tmp, buf_iw);
|
||||
|
||||
|
|
@ -147,8 +142,6 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
|
|||
float * __restrict__ dst,
|
||||
float * __restrict__ tmp_maxs,
|
||||
float * __restrict__ tmp_sums,
|
||||
float * shared_vals_max,
|
||||
float * shared_vals_sum,
|
||||
const soft_max_params p) {
|
||||
namespace cg = cooperative_groups;
|
||||
|
||||
|
|
@ -161,6 +154,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
|
|||
float local_vals[n_elem_per_thread] = { -INFINITY, -INFINITY, -INFINITY, -INFINITY };
|
||||
float local_max = -INFINITY;
|
||||
const int step_size = gridDim.x * blockDim.x;
|
||||
__shared__ float shared_vals[32];
|
||||
|
||||
// Compute thread-local max
|
||||
for (int col = col_start; col < p.ncols;) {
|
||||
|
|
@ -177,7 +171,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
|
|||
}
|
||||
|
||||
// Compute CTA-level max
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals);
|
||||
|
||||
// Store CTA-level max to GMEM
|
||||
if (tid == 0) {
|
||||
|
|
@ -192,7 +186,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
|
|||
} else {
|
||||
local_max = -INFINITY;
|
||||
}
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals);
|
||||
|
||||
// Compute softmax dividends, accumulate divisor
|
||||
float tmp_expf = 0.0f;
|
||||
|
|
@ -215,7 +209,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
|
|||
}
|
||||
|
||||
// Reduce divisor within CTA
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals);
|
||||
|
||||
// Store CTA-level sum to GMEM
|
||||
if (tid == 0) {
|
||||
|
|
@ -229,7 +223,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
|
|||
} else {
|
||||
tmp_expf = 0.0f;
|
||||
}
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals);
|
||||
|
||||
// Divide dividend by global sum + store data
|
||||
for (int col = col_start; col < p.ncols;) {
|
||||
|
|
@ -316,11 +310,9 @@ __launch_bounds__(8*WARP_SIZE, 1) static __global__ void soft_max_f32_paralleliz
|
|||
// https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#grid-synchronization
|
||||
// https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#class-cluster-group
|
||||
{
|
||||
__shared__ float shared_vals[2][32];
|
||||
|
||||
for (int rowx = 0; rowx < p.ne01 * p.ne02 * p.ne03; rowx++) {
|
||||
soft_max_f32_parallelize_cols_single_row(x + int64_t(rowx) * p.ncols, dst + int64_t(rowx) * p.ncols, tmp_maxs,
|
||||
tmp_sums, shared_vals[0], shared_vals[1], p);
|
||||
tmp_sums, p);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ __global__ void __launch_bounds__(d_state, 1)
|
|||
const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3,
|
||||
const int src2_nb1, const int src2_nb2, const int src3_nb1,
|
||||
const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3,
|
||||
const int64_t s_off, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok, const int64_t K) {
|
||||
const int64_t s_off, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok) {
|
||||
const float * GGML_CUDA_RESTRICT src0 = src0_ptr;
|
||||
const float * GGML_CUDA_RESTRICT src1 = src1_ptr;
|
||||
const float * GGML_CUDA_RESTRICT src2 = src2_ptr;
|
||||
|
|
@ -217,16 +217,6 @@ __global__ void __launch_bounds__(d_state, 1)
|
|||
if (lane == 0) {
|
||||
y_warp[i * stride_y] = state_sum;
|
||||
}
|
||||
|
||||
// Slot 0 is the final state written below; slots 1..K-1 are rollback snapshots.
|
||||
const int64_t slot = n_tok - 1 - i;
|
||||
if (K > 1 && slot > 0 && slot < K) {
|
||||
float * s_snapshot_warp = (float *) ((char *) dst + s_off + (slot * gridDim.y + seq_idx) * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < c_factor; j++) {
|
||||
s_snapshot_warp[WARP_SIZE * j + lane] = state[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// write back the state
|
||||
|
|
@ -242,7 +232,7 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa
|
|||
const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2,
|
||||
const int src5_nb3, const int64_t s_off, const int64_t d_state, const int64_t head_dim,
|
||||
const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq,
|
||||
const int64_t K, cudaStream_t stream) {
|
||||
cudaStream_t stream) {
|
||||
// NOTE: if you change conditions here, be sure to update the corresponding supports_op condition!
|
||||
if (src3_nb1 == sizeof(float)) {
|
||||
// Mamba-2
|
||||
|
|
@ -255,7 +245,7 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa
|
|||
ggml_cuda_kernel_launch(ssm_scan_f32_group<128/WARP_SIZE, 128>, launch_params,
|
||||
src0, src1, src2, src3, src4, src5, src6, dst,
|
||||
src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,
|
||||
src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K);
|
||||
src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok);
|
||||
} else if (d_state == 256) { // Falcon-H1
|
||||
constexpr int threads = 256;
|
||||
constexpr int num_warps = threads/WARP_SIZE;
|
||||
|
|
@ -265,13 +255,12 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa
|
|||
ggml_cuda_kernel_launch(ssm_scan_f32_group<256/WARP_SIZE, 256>, launch_params,
|
||||
src0, src1, src2, src3, src4, src5, src6, dst,
|
||||
src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,
|
||||
src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K);
|
||||
src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok);
|
||||
} else {
|
||||
GGML_ABORT("doesn't support d_state!=(128 or 256).");
|
||||
}
|
||||
} else {
|
||||
// Mamba-1
|
||||
GGML_ASSERT(K == 1);
|
||||
constexpr int threads = 128;
|
||||
GGML_ASSERT(n_head % threads == 0);
|
||||
GGML_ASSERT(head_dim == 1);
|
||||
|
|
@ -780,12 +769,10 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|||
const int64_t ng = src4->ne[1]; // n_group
|
||||
const int64_t n_t = src1->ne[2]; // number of tokens per sequence
|
||||
const int64_t n_s = src1->ne[3]; // number of sequences in the batch
|
||||
const int32_t K_param = ggml_get_op_params_i32(dst, 0);
|
||||
const int64_t K = K_param > 0 ? K_param : 1;
|
||||
|
||||
const int64_t s_off = ggml_nelements(src1) * sizeof(float);
|
||||
|
||||
GGML_ASSERT(ggml_nelements(src1) + K*nc*nr*nh*n_s == ggml_nelements(dst));
|
||||
GGML_ASSERT(ggml_nelements(src1) + nc*nr*nh*n_s == ggml_nelements(dst));
|
||||
GGML_ASSERT(src0->nb[0] == sizeof(float));
|
||||
GGML_ASSERT(src1->nb[0] == sizeof(float));
|
||||
GGML_ASSERT(src2->nb[0] == sizeof(float));
|
||||
|
|
@ -793,7 +780,6 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|||
GGML_ASSERT(src4->nb[0] == sizeof(float));
|
||||
GGML_ASSERT(src5->nb[0] == sizeof(float));
|
||||
GGML_ASSERT(src6->nb[0] == sizeof(int32_t));
|
||||
GGML_ASSERT(src3->ne[0] == 1 || K == 1);
|
||||
|
||||
const float * src0_d = (const float *) src0->data;
|
||||
const float * src1_d = (const float *) src1->data;
|
||||
|
|
@ -828,7 +814,6 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|||
const bool is_mamba2 = (src3->nb[1] == sizeof(float));
|
||||
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
|
||||
const bool use_ssd = is_mamba2 && n_t > SSM_SSD_MIN_TOKENS
|
||||
&& K == 1
|
||||
&& n_t <= SSM_SSD_MAX_TOKENS
|
||||
&& GGML_CUDA_CC_IS_NVIDIA(cc)
|
||||
&& cc >= GGML_CUDA_CC_TURING
|
||||
|
|
@ -856,5 +841,5 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|||
ssm_scan_f32_cuda(src0_d, src1_d, src2_d, src3_d, src4_d, src5_d, src6_d, dst_d,
|
||||
src0->nb[2], src0->nb[3], src1->nb[2], src1->nb[3], src2->nb[1], src2->nb[2],
|
||||
src3->nb[1], src4->nb[2], src4->nb[3], src5->nb[2], src5->nb[3],
|
||||
s_off, nc, nr, nh, ng, n_t, n_s, K, stream);
|
||||
s_off, nc, nr, nh, ng, n_t, n_s, stream);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -141,57 +141,6 @@ static __global__ void rwkv_wkv7_f32(const int B, const int T, const int C, cons
|
|||
}
|
||||
}
|
||||
|
||||
template <int rows_per_block>
|
||||
static __global__ void __launch_bounds__(WARP_SIZE * rows_per_block, 2)
|
||||
rwkv_wkv7_f32_t1_warp_row(const int T, const int C, const int H, const float * r, const float * w, const float * k, const float * v, const float * a, const float * b, const float * s, float * dst) {
|
||||
constexpr int head_size = CUDA_WKV_BLOCK_SIZE;
|
||||
constexpr int half_head = head_size / 2;
|
||||
|
||||
const int lane = threadIdx.x;
|
||||
const int row = blockIdx.y * rows_per_block + threadIdx.y;
|
||||
const int bid = blockIdx.x;
|
||||
|
||||
const int batch_i = bid / H;
|
||||
const int head_i = bid % H;
|
||||
const int state_size = C * head_size;
|
||||
const int head_off = head_i * head_size;
|
||||
const int t = batch_i * C + head_off + row;
|
||||
|
||||
__shared__ float _r[head_size], _w[head_size], _k[head_size], _a[head_size], _b[head_size];
|
||||
|
||||
if (threadIdx.y == 0) {
|
||||
_r[lane] = r[batch_i * C + head_off + lane];
|
||||
_w[lane] = w[batch_i * C + head_off + lane];
|
||||
_k[lane] = k[batch_i * C + head_off + lane];
|
||||
_a[lane] = a[batch_i * C + head_off + lane];
|
||||
_b[lane] = b[batch_i * C + head_off + lane];
|
||||
|
||||
_r[lane + half_head] = r[batch_i * C + head_off + lane + half_head];
|
||||
_w[lane + half_head] = w[batch_i * C + head_off + lane + half_head];
|
||||
_k[lane + half_head] = k[batch_i * C + head_off + lane + half_head];
|
||||
_a[lane + half_head] = a[batch_i * C + head_off + lane + half_head];
|
||||
_b[lane + half_head] = b[batch_i * C + head_off + lane + half_head];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const int64_t state_base = batch_i * state_size + head_i * head_size * head_size + row * head_size;
|
||||
const float s0 = s[state_base + lane];
|
||||
const float s1 = s[state_base + lane + half_head];
|
||||
const float sa = warp_reduce_sum(_a[lane] * s0 + _a[lane + half_head] * s1);
|
||||
|
||||
const float vt = v[t];
|
||||
const float st0 = s0 * _w[lane] + _k[lane] * vt + sa * _b[lane];
|
||||
const float st1 = s1 * _w[lane + half_head] + _k[lane + half_head] * vt + sa * _b[lane + half_head];
|
||||
const float y = warp_reduce_sum(st0 * _r[lane] + st1 * _r[lane + half_head]);
|
||||
|
||||
dst[T * C + state_base + lane] = st0;
|
||||
dst[T * C + state_base + lane + half_head] = st1;
|
||||
|
||||
if (lane == 0) {
|
||||
dst[t] = y;
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const float * k_d = (const float *)dst->src[0]->data;
|
||||
const float * v_d = (const float *)dst->src[1]->data;
|
||||
|
|
@ -242,10 +191,7 @@ void ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
|
|||
GGML_ASSERT(C % H == 0);
|
||||
GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2);
|
||||
|
||||
if (T / B == 1 && C / H == CUDA_WKV_BLOCK_SIZE) {
|
||||
constexpr int rows_per_block = 4;
|
||||
rwkv_wkv7_f32_t1_warp_row<rows_per_block><<<dim3(B * H, CUDA_WKV_BLOCK_SIZE / rows_per_block), dim3(WARP_SIZE, rows_per_block), 0, stream>>>(T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d);
|
||||
} else if (C / H == CUDA_WKV_BLOCK_SIZE) {
|
||||
if (C / H == CUDA_WKV_BLOCK_SIZE) {
|
||||
rwkv_wkv7_f32<CUDA_WKV_BLOCK_SIZE><<<B * H, C / H, 0, stream>>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d);
|
||||
} else {
|
||||
rwkv_wkv7_f32<CUDA_WKV_BLOCK_SIZE * 2><<<B * H, C / H, 0, stream>>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d);
|
||||
|
|
|
|||
|
|
@ -1,166 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#if defined(__aarch64__) || defined(_M_ARM64)
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <sys/auxv.h>
|
||||
#include <sys/prctl.h>
|
||||
|
||||
#if !defined(HWCAP2_SVE2)
|
||||
#define HWCAP2_SVE2 (1ULL << 1)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_FPHP)
|
||||
#define HWCAP_FPHP (1 << 9)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_ASIMDHP)
|
||||
#define HWCAP_ASIMDHP (1 << 10)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_I8MM)
|
||||
#define HWCAP2_I8MM (1ULL << 13)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_ASIMDDP)
|
||||
#define HWCAP_ASIMDDP (1 << 20)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP_SVE)
|
||||
#define HWCAP_SVE (1 << 22)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_SME)
|
||||
#define HWCAP2_SME (1ULL << 23)
|
||||
#endif
|
||||
|
||||
#if !defined(HWCAP2_SME2)
|
||||
#define HWCAP2_SME2 (1ULL << 37)
|
||||
#endif
|
||||
|
||||
#if !defined(PR_SVE_GET_VL)
|
||||
#define PR_SVE_GET_VL 51
|
||||
#endif
|
||||
|
||||
#if !defined(PR_SVE_VL_LEN_MASK)
|
||||
#define PR_SVE_VL_LEN_MASK 0xffff
|
||||
#endif
|
||||
|
||||
#elif defined(__APPLE__)
|
||||
#include <sys/sysctl.h>
|
||||
#elif defined(_WIN32)
|
||||
#include <windows.h>
|
||||
|
||||
#if !defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE 67
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_SME_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_SME_INSTRUCTIONS_AVAILABLE 70
|
||||
#endif
|
||||
|
||||
#if !defined(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE)
|
||||
#define PF_ARM_SME2_INSTRUCTIONS_AVAILABLE 71
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
typedef struct ggml_feats_arch64_runtime {
|
||||
bool has_dotprod;
|
||||
bool has_fp16;
|
||||
bool has_sve;
|
||||
bool has_sve2;
|
||||
bool has_i8mm;
|
||||
bool has_sme;
|
||||
bool has_sme2;
|
||||
int sve_cnt;
|
||||
} ggml_feats_arch64_runtime_t;
|
||||
|
||||
static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) {
|
||||
ggml_feats_arch64_runtime_t runtime_feat = {};
|
||||
|
||||
#if defined(__linux__)
|
||||
const unsigned long hwcap = getauxval(AT_HWCAP);
|
||||
const unsigned long hwcap2 = getauxval(AT_HWCAP2);
|
||||
|
||||
runtime_feat.has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
|
||||
runtime_feat.has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);;
|
||||
runtime_feat.has_sve = !!(hwcap & HWCAP_SVE);
|
||||
runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
|
||||
runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
|
||||
runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME);
|
||||
runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2);
|
||||
|
||||
if (runtime_feat.has_sve) {
|
||||
const int vl = prctl(PR_SVE_GET_VL);
|
||||
if (vl >= 0) {
|
||||
runtime_feat.sve_cnt = vl & PR_SVE_VL_LEN_MASK;
|
||||
}
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
int oldp = 0;
|
||||
size_t size = sizeof(oldp);
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_dotprod = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_FP16", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_fp16 = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SVE", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_sve = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SVE2", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_sve2 = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_i8mm = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_sme = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, nullptr, 0) == 0) {
|
||||
runtime_feat.has_sme2 = static_cast<bool>(oldp);
|
||||
}
|
||||
|
||||
// Apple does not support userspace non-streaming SVE; keep SVE vector length unknown.
|
||||
runtime_feat.sve_cnt = 0;
|
||||
#elif defined (_WIN32)
|
||||
runtime_feat.has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_fp16 = IsProcessorFeaturePresent(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0;
|
||||
|
||||
// Windows exposes SVE feature presence, but not the runtime SVE vector length here.
|
||||
runtime_feat.sve_cnt = 0;
|
||||
#endif
|
||||
|
||||
return runtime_feat;
|
||||
}
|
||||
|
||||
#endif // defined(__aarch64__) || defined(_M_ARM64)
|
||||
|
|
@ -477,41 +477,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max(ggml_me
|
|||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer(
|
||||
ggml_metal_library_t lib,
|
||||
const ggml_tensor * op) {
|
||||
GGML_ASSERT(op->op == GGML_OP_LIGHTNING_INDEXER);
|
||||
|
||||
char name[256];
|
||||
|
||||
snprintf(name, 256, "kernel_lightning_indexer_%s", ggml_type_name(op->src[1]->type));
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, ggml_op op) {
|
||||
const char * name = nullptr;
|
||||
|
||||
switch (op) {
|
||||
case GGML_OP_DSV4_HC_COMB: name = "kernel_dsv4_hc_comb_f32"; break;
|
||||
case GGML_OP_DSV4_HC_PRE: name = "kernel_dsv4_hc_pre_f32"; break;
|
||||
case GGML_OP_DSV4_HC_POST: name = "kernel_dsv4_hc_post_f32"; break;
|
||||
default: GGML_ABORT("fatal error");
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);
|
||||
|
|
@ -953,11 +918,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
|
|||
nr0 = N_R0_IQ4_XS;
|
||||
smem = 32*sizeof(float);
|
||||
} break;
|
||||
case GGML_TYPE_TQ2_0:
|
||||
{
|
||||
nsg = N_SG_TQ2_0;
|
||||
nr0 = N_R0_TQ2_0;
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
GGML_LOG_ERROR("Asserting on type %d\n", (int) tsrc0);
|
||||
|
|
@ -1187,11 +1147,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m
|
|||
nr0 = N_R0_IQ4_XS;
|
||||
smem = 32*sizeof(float);
|
||||
} break;
|
||||
case GGML_TYPE_TQ2_0:
|
||||
{
|
||||
nsg = N_SG_TQ2_0;
|
||||
nr0 = N_R0_TQ2_0;
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
GGML_LOG_ERROR("Asserting on type %d\n", (int)op->src[2]->type);
|
||||
|
|
@ -2162,23 +2117,6 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_sgd(ggm
|
|||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_SILU_BACK);
|
||||
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
snprintf(base, 256, "kernel_silu_back_%s", ggml_type_name(op->src[0]->type));
|
||||
snprintf(name, 256, "%s", base);
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_memset(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
GGML_ASSERT(op->type == GGML_TYPE_I64);
|
||||
|
||||
|
|
|
|||
|
|
@ -117,7 +117,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag
|
|||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat (ggml_metal_library_t lib, enum ggml_type tsrc);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat (ggml_metal_library_t lib, enum ggml_type tsrc);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum_rows (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
|
@ -125,8 +124,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_bl
|
|||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_add (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, enum ggml_op op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
#import "ggml-impl.h"
|
||||
#import "ggml-backend-impl.h"
|
||||
#import "ggml-metal-impl.h"
|
||||
|
||||
#include <Foundation/Foundation.h>
|
||||
|
||||
|
|
@ -1144,14 +1143,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
|||
default:
|
||||
return false;
|
||||
}
|
||||
case GGML_OP_SILU_BACK:
|
||||
return (op->src[0]->type == GGML_TYPE_F32) &&
|
||||
(op->src[1]->type == GGML_TYPE_F32) &&
|
||||
(op->type == GGML_TYPE_F32) &&
|
||||
ggml_is_contiguous(op->src[0]) &&
|
||||
ggml_is_contiguous(op->src[1]) &&
|
||||
ggml_is_contiguous(op) &&
|
||||
ggml_are_same_shape(op->src[0], op->src[1]);
|
||||
case GGML_OP_GLU:
|
||||
switch (ggml_get_glu_op(op)) {
|
||||
case GGML_GLU_OP_REGLU:
|
||||
|
|
@ -1196,7 +1187,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
|||
case GGML_OP_MUL:
|
||||
case GGML_OP_DIV:
|
||||
case GGML_OP_ADD_ID:
|
||||
return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->src[0]->type == op->src[1]->type);
|
||||
case GGML_OP_ACC:
|
||||
return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_REPEAT:
|
||||
|
|
@ -1274,9 +1264,8 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
|||
case GGML_OP_ARGSORT:
|
||||
case GGML_OP_TOP_K:
|
||||
case GGML_OP_ARANGE:
|
||||
return true;
|
||||
case GGML_OP_ROLL:
|
||||
return ggml_is_contiguous(op->src[0]);
|
||||
return true;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
// for new head sizes, add checks here
|
||||
if (op->src[0]->ne[0] != 32 &&
|
||||
|
|
@ -1316,75 +1305,8 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
|||
return false;
|
||||
}
|
||||
return has_simdgroup_mm; // TODO: over-restricted for vec-kernels
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
if (op->src[0]->ne[0] != OP_LIGHTNING_INDEXER_DK ||
|
||||
op->src[0]->ne[1] != OP_LIGHTNING_INDEXER_NH) {
|
||||
return false;
|
||||
}
|
||||
if (!has_simdgroup_mm ||
|
||||
op->src[0]->type != GGML_TYPE_F32 ||
|
||||
op->src[2]->type != GGML_TYPE_F32 ||
|
||||
op->src[3]->type != GGML_TYPE_F16 ||
|
||||
op->type != GGML_TYPE_F32 ||
|
||||
!ggml_is_contiguous_rows(op->src[0]) ||
|
||||
!ggml_is_contiguous_rows(op->src[1]) ||
|
||||
!ggml_is_contiguous_rows(op->src[2]) ||
|
||||
!ggml_is_contiguous_rows(op->src[3])) {
|
||||
return false;
|
||||
}
|
||||
switch (op->src[1]->type) {
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_Q8_0:
|
||||
return true;
|
||||
case GGML_TYPE_BF16:
|
||||
return has_bfloat;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
case GGML_OP_DSV4_HC_COMB:
|
||||
return has_simdgroup_reduction &&
|
||||
op->src[0]->type == GGML_TYPE_F32 &&
|
||||
op->src[1]->type == GGML_TYPE_F32 &&
|
||||
op->src[2]->type == GGML_TYPE_F32 &&
|
||||
op->type == GGML_TYPE_F32 &&
|
||||
op->src[0]->ne[0] == 24 &&
|
||||
op->src[1]->ne[0] >= 3 &&
|
||||
op->src[2]->ne[0] == 24 &&
|
||||
ggml_is_contiguous_rows(op->src[0]) &&
|
||||
ggml_is_contiguous_rows(op->src[1]) &&
|
||||
ggml_is_contiguous_rows(op->src[2]);
|
||||
case GGML_OP_DSV4_HC_PRE:
|
||||
return has_simdgroup_reduction &&
|
||||
op->src[0]->type == GGML_TYPE_F32 &&
|
||||
op->src[1]->type == GGML_TYPE_F32 &&
|
||||
op->type == GGML_TYPE_F32 &&
|
||||
op->src[0]->ne[1] == 4 &&
|
||||
op->src[1]->ne[0] == 4 &&
|
||||
ggml_is_contiguous_rows(op->src[0]) &&
|
||||
ggml_is_contiguous_rows(op->src[1]);
|
||||
case GGML_OP_DSV4_HC_POST:
|
||||
return has_simdgroup_reduction &&
|
||||
op->src[0]->type == GGML_TYPE_F32 &&
|
||||
op->src[1]->type == GGML_TYPE_F32 &&
|
||||
op->src[2]->type == GGML_TYPE_F32 &&
|
||||
op->src[3]->type == GGML_TYPE_F32 &&
|
||||
op->type == GGML_TYPE_F32 &&
|
||||
op->src[1]->ne[1] == 4 &&
|
||||
op->src[2]->ne[0] == 4 &&
|
||||
op->src[3]->ne[0] == 4 &&
|
||||
op->src[3]->ne[1] == 4 &&
|
||||
ggml_is_contiguous_rows(op->src[0]) &&
|
||||
ggml_is_contiguous_rows(op->src[1]) &&
|
||||
ggml_is_contiguous_rows(op->src[2]) &&
|
||||
ggml_is_contiguous_rows(op->src[3]);
|
||||
case GGML_OP_SSM_SCAN:
|
||||
return has_simdgroup_reduction;
|
||||
case GGML_OP_SSM_CONV:
|
||||
case GGML_OP_SSM_SCAN:
|
||||
return has_simdgroup_reduction;
|
||||
case GGML_OP_RWKV_WKV6:
|
||||
case GGML_OP_RWKV_WKV7:
|
||||
|
|
@ -1414,7 +1336,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
|||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_IQ4_NL:
|
||||
case GGML_TYPE_TQ2_0:
|
||||
case GGML_TYPE_I32:
|
||||
return true;
|
||||
default:
|
||||
|
|
@ -1443,7 +1364,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
|||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_TQ2_0:
|
||||
switch (op->type) {
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_F16:
|
||||
|
|
@ -1479,7 +1399,6 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
|||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_IQ4_NL:
|
||||
case GGML_TYPE_TQ2_0:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -87,9 +87,6 @@
|
|||
#define N_R0_IQ4_XS 2
|
||||
#define N_SG_IQ4_XS 2
|
||||
|
||||
#define N_R0_TQ2_0 4
|
||||
#define N_SG_TQ2_0 2
|
||||
|
||||
// function constants offsets
|
||||
#define FC_FLASH_ATTN_EXT_PAD 100
|
||||
#define FC_FLASH_ATTN_EXT_BLK 200
|
||||
|
|
@ -115,13 +112,6 @@
|
|||
#define OP_FLASH_ATTN_EXT_VEC_NQPSG 1
|
||||
#define OP_FLASH_ATTN_EXT_VEC_NCPSG 32
|
||||
|
||||
#define OP_LIGHTNING_INDEXER_DK 128
|
||||
#define OP_LIGHTNING_INDEXER_NH 64
|
||||
#define OP_LIGHTNING_INDEXER_NHPTG 8
|
||||
#define OP_LIGHTNING_INDEXER_NKPSG 8
|
||||
#define OP_LIGHTNING_INDEXER_NSG 8
|
||||
#define OP_LIGHTNING_INDEXER_NBPTG 8
|
||||
|
||||
#define OP_UNARY_NUM_SCALE 10
|
||||
#define OP_UNARY_NUM_FILL 11
|
||||
#define OP_UNARY_NUM_CLAMP 12
|
||||
|
|
@ -880,7 +870,6 @@ typedef struct {
|
|||
int64_t n_group;
|
||||
int64_t n_seq_tokens;
|
||||
int64_t n_seqs;
|
||||
int64_t K;
|
||||
uint64_t s_off;
|
||||
uint64_t nb00;
|
||||
uint64_t nb01;
|
||||
|
|
@ -1182,66 +1171,6 @@ typedef struct {
|
|||
int64_t val;
|
||||
} ggml_metal_kargs_memset;
|
||||
|
||||
typedef struct {
|
||||
int32_t n_kv;
|
||||
int32_t n_batch;
|
||||
int32_t mask_ne3;
|
||||
uint64_t nb1;
|
||||
uint64_t nb3;
|
||||
uint64_t nbq1;
|
||||
uint64_t nbq2;
|
||||
uint64_t nbq3;
|
||||
uint64_t nbk2;
|
||||
uint64_t nbk3;
|
||||
uint64_t nbw1;
|
||||
uint64_t nbw3;
|
||||
uint64_t nbm1;
|
||||
uint64_t nbm3;
|
||||
} ggml_metal_kargs_lightning_indexer;
|
||||
|
||||
typedef struct {
|
||||
int32_t n_tokens;
|
||||
int32_t n_iter;
|
||||
uint64_t nb_m0;
|
||||
uint64_t nb_m1;
|
||||
uint64_t nb_s0;
|
||||
uint64_t nb_b0;
|
||||
uint64_t nb_d0;
|
||||
uint64_t nb_d1;
|
||||
uint64_t nb_d2;
|
||||
float eps;
|
||||
} ggml_metal_kargs_dsv4_hc_comb;
|
||||
|
||||
typedef struct {
|
||||
int32_t n_embd;
|
||||
int32_t n_tokens;
|
||||
uint64_t nb_x0;
|
||||
uint64_t nb_x1;
|
||||
uint64_t nb_x2;
|
||||
uint64_t nb_w0;
|
||||
uint64_t nb_w1;
|
||||
uint64_t nb_d0;
|
||||
uint64_t nb_d1;
|
||||
} ggml_metal_kargs_dsv4_hc_pre;
|
||||
|
||||
typedef struct {
|
||||
int32_t n_embd;
|
||||
int32_t n_tokens;
|
||||
uint64_t nb_x0;
|
||||
uint64_t nb_x1;
|
||||
uint64_t nb_r0;
|
||||
uint64_t nb_r1;
|
||||
uint64_t nb_r2;
|
||||
uint64_t nb_p0;
|
||||
uint64_t nb_p1;
|
||||
uint64_t nb_c0;
|
||||
uint64_t nb_c1;
|
||||
uint64_t nb_c2;
|
||||
uint64_t nb_d0;
|
||||
uint64_t nb_d1;
|
||||
uint64_t nb_d2;
|
||||
} ggml_metal_kargs_dsv4_hc_post;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne00;
|
||||
int32_t ne01;
|
||||
|
|
@ -1293,8 +1222,4 @@ typedef struct {
|
|||
int64_t np;
|
||||
} ggml_metal_kargs_opt_step_sgd;
|
||||
|
||||
typedef struct {
|
||||
int64_t ne;
|
||||
} ggml_metal_kargs_silu_back;
|
||||
|
||||
#endif // GGML_METAL_IMPL
|
||||
|
|
|
|||
|
|
@ -299,10 +299,6 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
|
|||
{
|
||||
n_fuse = ggml_metal_op_unary(ctx, idx);
|
||||
} break;
|
||||
case GGML_OP_SILU_BACK:
|
||||
{
|
||||
n_fuse = ggml_metal_op_silu_back(ctx, idx);
|
||||
} break;
|
||||
case GGML_OP_GLU:
|
||||
{
|
||||
n_fuse = ggml_metal_op_glu(ctx, idx);
|
||||
|
|
@ -320,16 +316,6 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
|
|||
{
|
||||
n_fuse = ggml_metal_op_cumsum(ctx, idx);
|
||||
} break;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
{
|
||||
n_fuse = ggml_metal_op_lightning_indexer(ctx, idx);
|
||||
} break;
|
||||
case GGML_OP_DSV4_HC_COMB:
|
||||
case GGML_OP_DSV4_HC_PRE:
|
||||
case GGML_OP_DSV4_HC_POST:
|
||||
{
|
||||
n_fuse = ggml_metal_op_dsv4_hc(ctx, idx);
|
||||
} break;
|
||||
case GGML_OP_SOFT_MAX:
|
||||
{
|
||||
n_fuse = ggml_metal_op_soft_max(ctx, idx);
|
||||
|
|
@ -1311,203 +1297,6 @@ int ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
ggml_metal_encoder_t enc = ctx->enc;
|
||||
|
||||
GGML_ASSERT(op->op == GGML_OP_LIGHTNING_INDEXER);
|
||||
|
||||
const ggml_tensor * q = op->src[0];
|
||||
const ggml_tensor * k = op->src[1];
|
||||
const ggml_tensor * w = op->src[2];
|
||||
const ggml_tensor * m = op->src[3];
|
||||
|
||||
GGML_ASSERT(q->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(k->type == GGML_TYPE_F32 ||
|
||||
k->type == GGML_TYPE_F16 ||
|
||||
k->type == GGML_TYPE_BF16 ||
|
||||
k->type == GGML_TYPE_Q4_0 ||
|
||||
k->type == GGML_TYPE_Q4_1 ||
|
||||
k->type == GGML_TYPE_Q5_0 ||
|
||||
k->type == GGML_TYPE_Q5_1 ||
|
||||
k->type == GGML_TYPE_Q8_0);
|
||||
GGML_ASSERT(w->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(m->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(op->type == GGML_TYPE_F32);
|
||||
|
||||
GGML_ASSERT(q->ne[0] == OP_LIGHTNING_INDEXER_DK);
|
||||
GGML_ASSERT(q->ne[1] == OP_LIGHTNING_INDEXER_NH);
|
||||
|
||||
ggml_metal_kargs_lightning_indexer args = {
|
||||
/*.n_kv =*/ (int32_t) k->ne[2],
|
||||
/*.n_batch =*/ (int32_t) q->ne[2],
|
||||
/*.mask_ne3 =*/ (int32_t) m->ne[3],
|
||||
/*.nb1 =*/ op->nb[1],
|
||||
/*.nb3 =*/ op->nb[3],
|
||||
/*.nbq1 =*/ q->nb[1],
|
||||
/*.nbq2 =*/ q->nb[2],
|
||||
/*.nbq3 =*/ q->nb[3],
|
||||
/*.nbk2 =*/ k->nb[2],
|
||||
/*.nbk3 =*/ k->nb[3],
|
||||
/*.nbw1 =*/ w->nb[1],
|
||||
/*.nbw3 =*/ w->nb[3],
|
||||
/*.nbm1 =*/ m->nb[1],
|
||||
/*.nbm3 =*/ m->nb[3],
|
||||
};
|
||||
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(q), 1);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(k), 2);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(w), 3);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(m), 4);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5);
|
||||
|
||||
const int nsg = OP_LIGHTNING_INDEXER_NSG;
|
||||
const int nkptg = OP_LIGHTNING_INDEXER_NKPSG*nsg;
|
||||
const int nbptg = OP_LIGHTNING_INDEXER_NBPTG;
|
||||
|
||||
auto pipeline = ggml_metal_library_get_pipeline_lightning_indexer(ctx->lib, op);
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc,
|
||||
(k->ne[2] + nkptg - 1)/nkptg,
|
||||
(q->ne[2] + nbptg - 1)/nbptg,
|
||||
q->ne[3], 32, nsg, 1);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
ggml_metal_encoder_t enc = ctx->enc;
|
||||
auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op->op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
|
||||
switch (op->op) {
|
||||
case GGML_OP_DSV4_HC_COMB:
|
||||
{
|
||||
const ggml_tensor * mixes = op->src[0];
|
||||
const ggml_tensor * scale = op->src[1];
|
||||
const ggml_tensor * base = op->src[2];
|
||||
|
||||
GGML_ASSERT(mixes->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(scale->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(base->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(op->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(mixes->ne[0] == 24);
|
||||
GGML_ASSERT(op->ne[0] == 4 && op->ne[1] == 4);
|
||||
|
||||
ggml_metal_kargs_dsv4_hc_comb args = {
|
||||
/*.n_tokens =*/ (int32_t) mixes->ne[1],
|
||||
/*.n_iter =*/ ggml_get_op_params_i32(op, 1),
|
||||
/*.nb_m0 =*/ mixes->nb[0],
|
||||
/*.nb_m1 =*/ mixes->nb[1],
|
||||
/*.nb_s0 =*/ scale->nb[0],
|
||||
/*.nb_b0 =*/ base->nb[0],
|
||||
/*.nb_d0 =*/ op->nb[0],
|
||||
/*.nb_d1 =*/ op->nb[1],
|
||||
/*.nb_d2 =*/ op->nb[2],
|
||||
/*.eps =*/ ggml_get_op_params_f32(op, 0),
|
||||
};
|
||||
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(mixes), 1);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(scale), 2);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(base), 3);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4);
|
||||
|
||||
// One SIMDgroup owns one 4x4 Sinkhorn matrix. Packing up to four
|
||||
// independent tokens per threadgroup keeps both decode and prompt
|
||||
// dispatches compact without any threadgroup-memory synchronization.
|
||||
const int nsg = std::min(4, args.n_tokens);
|
||||
ggml_metal_encoder_dispatch_threadgroups(
|
||||
enc, (args.n_tokens + nsg - 1)/nsg, 1, 1, 32, nsg, 1);
|
||||
} break;
|
||||
case GGML_OP_DSV4_HC_PRE:
|
||||
{
|
||||
const ggml_tensor * x = op->src[0];
|
||||
const ggml_tensor * weights = op->src[1];
|
||||
|
||||
GGML_ASSERT(x->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(weights->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(op->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(x->ne[1] == 4);
|
||||
|
||||
ggml_metal_kargs_dsv4_hc_pre args = {
|
||||
/*.n_embd =*/ (int32_t) x->ne[0],
|
||||
/*.n_tokens =*/ (int32_t) x->ne[2],
|
||||
/*.nb_x0 =*/ x->nb[0],
|
||||
/*.nb_x1 =*/ x->nb[1],
|
||||
/*.nb_x2 =*/ x->nb[2],
|
||||
/*.nb_w0 =*/ weights->nb[0],
|
||||
/*.nb_w1 =*/ weights->nb[1],
|
||||
/*.nb_d0 =*/ op->nb[0],
|
||||
/*.nb_d1 =*/ op->nb[1],
|
||||
};
|
||||
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(weights), 2);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3);
|
||||
|
||||
const int n_tiles = (args.n_embd + 31)/32;
|
||||
const int nsg = std::min(4, n_tiles);
|
||||
ggml_metal_encoder_dispatch_threadgroups(
|
||||
enc, (n_tiles + nsg - 1)/nsg, args.n_tokens, 1, 32, nsg, 1);
|
||||
} break;
|
||||
case GGML_OP_DSV4_HC_POST:
|
||||
{
|
||||
const ggml_tensor * x = op->src[0];
|
||||
const ggml_tensor * residual = op->src[1];
|
||||
const ggml_tensor * post = op->src[2];
|
||||
const ggml_tensor * comb = op->src[3];
|
||||
|
||||
GGML_ASSERT(x->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(residual->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(post->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(comb->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(op->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(residual->ne[1] == 4);
|
||||
|
||||
ggml_metal_kargs_dsv4_hc_post args = {
|
||||
/*.n_embd =*/ (int32_t) x->ne[0],
|
||||
/*.n_tokens =*/ (int32_t) x->ne[1],
|
||||
/*.nb_x0 =*/ x->nb[0],
|
||||
/*.nb_x1 =*/ x->nb[1],
|
||||
/*.nb_r0 =*/ residual->nb[0],
|
||||
/*.nb_r1 =*/ residual->nb[1],
|
||||
/*.nb_r2 =*/ residual->nb[2],
|
||||
/*.nb_p0 =*/ post->nb[0],
|
||||
/*.nb_p1 =*/ post->nb[1],
|
||||
/*.nb_c0 =*/ comb->nb[0],
|
||||
/*.nb_c1 =*/ comb->nb[1],
|
||||
/*.nb_c2 =*/ comb->nb[2],
|
||||
/*.nb_d0 =*/ op->nb[0],
|
||||
/*.nb_d1 =*/ op->nb[1],
|
||||
/*.nb_d2 =*/ op->nb[2],
|
||||
};
|
||||
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(residual), 2);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(post), 3);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4);
|
||||
ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5);
|
||||
|
||||
const int n_tiles = (args.n_embd + 31)/32;
|
||||
const int nsg = std::min(4, n_tiles);
|
||||
ggml_metal_encoder_dispatch_threadgroups(
|
||||
enc, (n_tiles + nsg - 1)/nsg, args.n_tokens, 1, 32, nsg, 1);
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
|
|
@ -1710,10 +1499,6 @@ int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) {
|
|||
const int64_t n_group = ne41;
|
||||
const int64_t n_seq_tokens = ne12;
|
||||
const int64_t n_seqs = ne13;
|
||||
const int64_t K = ggml_get_op_params_i32(op, 0);
|
||||
|
||||
GGML_ASSERT(K >= 1);
|
||||
GGML_ASSERT(ggml_nelements(op->src[1]) + K*d_state*d_inner*n_head*n_seqs == ggml_nelements(op));
|
||||
|
||||
ggml_metal_kargs_ssm_scan args = {
|
||||
/*.d_state =*/ d_state,
|
||||
|
|
@ -1722,7 +1507,6 @@ int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) {
|
|||
/*.n_group =*/ n_group,
|
||||
/*.n_seq_tokens =*/ n_seq_tokens,
|
||||
/*.n_seqs =*/ n_seqs,
|
||||
/*.K =*/ K,
|
||||
/*.s_off =*/ ggml_nelements(op->src[1]) * sizeof(float),
|
||||
/*.nb00 =*/ nb00,
|
||||
/*.nb01 =*/ nb01,
|
||||
|
|
@ -3413,6 +3197,9 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
|
|||
GGML_TENSOR_LOCALS( int32_t, ne, op, ne);
|
||||
GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);
|
||||
|
||||
GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);
|
||||
|
||||
GGML_ASSERT(ggml_is_contiguous_rows(op->src[0]));
|
||||
GGML_ASSERT(ggml_is_contiguous_rows(op->src[1]));
|
||||
|
||||
|
|
@ -3552,36 +3339,6 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
|
|||
return n_fuse;
|
||||
}
|
||||
|
||||
int ggml_metal_op_silu_back(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
ggml_metal_library_t lib = ctx->lib;
|
||||
ggml_metal_encoder_t enc = ctx->enc;
|
||||
|
||||
auto pipeline = ggml_metal_library_get_pipeline_silu_back(lib, op);
|
||||
|
||||
const int64_t ne = ggml_nelements(op);
|
||||
|
||||
ggml_metal_kargs_silu_back args = {
|
||||
/*.ne =*/ ne,
|
||||
};
|
||||
|
||||
int arg_idx{0};
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), arg_idx++);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), arg_idx++);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), arg_idx++);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), arg_idx++);
|
||||
|
||||
const int nth = std::min<int64_t>(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne);
|
||||
const int64_t n = (ne + nth - 1) / nth;
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, nth, 1, 1);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ggml_metal_op_l2_norm(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
|
|
@ -3821,7 +3578,7 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) {
|
|||
}
|
||||
|
||||
nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
|
||||
nth = std::min(nth, (args.ne00_t + 31)/32*32);
|
||||
nth = std::min(nth, args.ne00_t);
|
||||
|
||||
const size_t smem = pipeline.smem;
|
||||
|
||||
|
|
|
|||
|
|
@ -54,8 +54,6 @@ int ggml_metal_op_cumsum (ggml_metal_op_t ctx, int idx);
|
|||
int ggml_metal_op_get_rows (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_set_rows (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_diag (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_lightning_indexer (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_dsv4_hc (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_soft_max (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_ssm_conv (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_ssm_scan (ggml_metal_op_t ctx, int idx);
|
||||
|
|
@ -72,7 +70,6 @@ int ggml_metal_op_mul_mat_id (ggml_metal_op_t ctx, int idx);
|
|||
int ggml_metal_op_add_id (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_flash_attn_ext (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_bin (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_silu_back (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_l2_norm (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_group_norm (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_norm (ggml_metal_op_t ctx, int idx);
|
||||
|
|
|
|||
|
|
@ -681,7 +681,6 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
|
|||
/* .host_buffer = */ false,
|
||||
/* .buffer_from_host_ptr = */ true,
|
||||
/* .events = */ true,
|
||||
/* .mmap_support = */ true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -468,34 +468,6 @@ void quantize_iq4_nl(device const float * src, device block_iq4_nl & dst) {
|
|||
dst.d = sumq2 > 0 ? sumqx/sumq2 : d;
|
||||
}
|
||||
|
||||
void quantize_tq2_0(device const float * src, device block_tq2_0 & dst) {
|
||||
#pragma METAL fp math_mode(safe)
|
||||
float amax = 0.0f; // absolute max
|
||||
|
||||
for (int j = 0; j < QK_K; j++) {
|
||||
const float v = src[j];
|
||||
amax = MAX(amax, fabs(v));
|
||||
}
|
||||
|
||||
const float d = amax;
|
||||
const float id = d ? 1.0f/d : 0.0f;
|
||||
|
||||
dst.d = (half) d;
|
||||
|
||||
for (int j = 0; j < QK_K/4; j += 32) {
|
||||
for (int m = 0; m < 32; ++m) {
|
||||
uint8_t q = 0;
|
||||
for (int n = 0; n < 4; ++n) {
|
||||
// -1, 0, 1 -> 0, 1, 2
|
||||
int xi = (int)round(src[m + n*32] * id) + 1;
|
||||
q += (uint8_t)((xi & 3) << (2*n));
|
||||
}
|
||||
dst.qs[j + m] = q;
|
||||
}
|
||||
src += 4*32;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename type4x4>
|
||||
void dequantize_q4_1(device const block_q4_1 * xb, short il, thread type4x4 & reg) {
|
||||
device const uint16_t * qs = ((device const uint16_t *)xb + 2);
|
||||
|
|
@ -1049,25 +1021,6 @@ void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4
|
|||
}
|
||||
}
|
||||
|
||||
template <typename type4x4>
|
||||
void dequantize_tq2_0(device const block_tq2_0 * xb, short il, thread type4x4 & reg) {
|
||||
device const uint8_t * qs = xb->qs;
|
||||
const float d = xb->d;
|
||||
|
||||
float4x4 reg_f;
|
||||
|
||||
// 2 bits per element, 4 elements per byte, 128 elements per 32-byte group
|
||||
const short base = il * 16;
|
||||
for (int k = 0; k < 16; k++) {
|
||||
const int i = base + k;
|
||||
const int byte = ((i >> 7) & 1) * 32 + (i & 31);
|
||||
const int l = (i >> 5) & 3;
|
||||
reg_f[k/4][k%4] = d * (float)(((qs[byte] >> (2*l)) & 3) - 1);
|
||||
}
|
||||
|
||||
reg = (type4x4) reg_f;
|
||||
}
|
||||
|
||||
enum ggml_sort_order {
|
||||
GGML_SORT_ORDER_ASC,
|
||||
GGML_SORT_ORDER_DESC,
|
||||
|
|
@ -1302,20 +1255,6 @@ template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_un
|
|||
template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl<half, half, float>;
|
||||
template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4, half4, float4>;
|
||||
|
||||
kernel void kernel_silu_back_f32(
|
||||
constant ggml_metal_kargs_silu_back & args,
|
||||
device const float * dy,
|
||||
device const float * x,
|
||||
device float * dx,
|
||||
uint gid [[thread_position_in_grid]]) {
|
||||
if (gid >= args.ne) {
|
||||
return;
|
||||
}
|
||||
|
||||
const float s = 1.0f / (1.0f + exp(-x[gid]));
|
||||
dx[gid] = dy[gid] * s * (1.0f + x[gid] * (1.0f - s));
|
||||
}
|
||||
|
||||
// OP: 0 - add, 1 - sub, 2 - mul, 3 - div
|
||||
constant short FC_bin_op [[function_constant(FC_BIN + 0)]];
|
||||
constant short FC_bin_f [[function_constant(FC_BIN + 1)]];
|
||||
|
|
@ -1479,8 +1418,6 @@ typedef decltype(kernel_bin_fuse_impl<float, float, float>) kernel_bin_fuse_t;
|
|||
|
||||
template [[host_name("kernel_bin_fuse_f32_f32_f32")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<float, float, float>;
|
||||
template [[host_name("kernel_bin_fuse_f32_f32_f32_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<float4, float4, float4>;
|
||||
template [[host_name("kernel_bin_fuse_f16_f16_f16")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<half, half, half>;
|
||||
template [[host_name("kernel_bin_fuse_f16_f16_f16_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl<half4, half4, half4>;
|
||||
|
||||
kernel void kernel_add_id(
|
||||
constant ggml_metal_kargs_add_id & args,
|
||||
|
|
@ -2429,8 +2366,6 @@ kernel void kernel_ssm_scan_f32(
|
|||
const int32_t nh = args.n_head;
|
||||
const int32_t ng = args.n_group;
|
||||
const int32_t n_t = args.n_seq_tokens;
|
||||
const int32_t n_s = args.n_seqs;
|
||||
const int32_t K = args.K;
|
||||
|
||||
const int32_t s_off = args.s_off;
|
||||
|
||||
|
|
@ -2489,12 +2424,6 @@ kernel void kernel_ssm_scan_f32(
|
|||
// recurse
|
||||
s0 = s;
|
||||
|
||||
const int32_t slot = n_t - 1 - (i2 + t);
|
||||
if (slot > 0 && slot < K) {
|
||||
device float * s_snapshot = (device float *) ((device char *) s_buff + (int64_t) slot*n_s*args.nb03);
|
||||
s_snapshot[i] = s;
|
||||
}
|
||||
|
||||
B += args.ns42;
|
||||
C += args.ns52;
|
||||
}
|
||||
|
|
@ -8056,7 +7985,6 @@ template [[host_name("kernel_cpy_f32_q4_1")]] kernel cpy_f_q_t kernel_cpy_f32_
|
|||
template [[host_name("kernel_cpy_f32_q5_0")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK5_0, block_q5_0, quantize_q5_0>;
|
||||
template [[host_name("kernel_cpy_f32_q5_1")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK5_1, block_q5_1, quantize_q5_1>;
|
||||
template [[host_name("kernel_cpy_f32_iq4_nl")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK4_NL, block_iq4_nl, quantize_iq4_nl>;
|
||||
template [[host_name("kernel_cpy_f32_tq2_0")]] kernel cpy_f_q_t kernel_cpy_f32_q<QK_K, block_tq2_0, quantize_tq2_0>;
|
||||
|
||||
template<typename T4x4, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread T4x4 &)>
|
||||
kernel void kernel_cpy_q_f32(
|
||||
|
|
@ -8104,8 +8032,6 @@ template [[host_name("kernel_cpy_q5_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<
|
|||
template [[host_name("kernel_cpy_q5_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_q5_1, 2, dequantize_q5_1>;
|
||||
template [[host_name("kernel_cpy_q8_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_q8_0, 2, dequantize_q8_0>;
|
||||
|
||||
template [[host_name("kernel_cpy_tq2_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32<float4x4, block_tq2_0, QK_NL, dequantize_tq2_0>;
|
||||
|
||||
template [[host_name("kernel_cpy_q1_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q1_0, 8, dequantize_q1_0>;
|
||||
template [[host_name("kernel_cpy_q2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q2_0, 4, dequantize_q2_0>;
|
||||
template [[host_name("kernel_cpy_q4_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q4_0, 2, dequantize_q4_0>;
|
||||
|
|
@ -8114,8 +8040,6 @@ template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<
|
|||
template [[host_name("kernel_cpy_q5_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q5_1, 2, dequantize_q5_1>;
|
||||
template [[host_name("kernel_cpy_q8_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_q8_0, 2, dequantize_q8_0>;
|
||||
|
||||
template [[host_name("kernel_cpy_tq2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32<half4x4, block_tq2_0, QK_NL, dequantize_tq2_0>;
|
||||
|
||||
template<typename T>
|
||||
kernel void kernel_concat(
|
||||
constant ggml_metal_kargs_concat & args,
|
||||
|
|
@ -9882,121 +9806,6 @@ kernel void kernel_mul_mv_mxfp4_f32(
|
|||
kernel_mul_mv_mxfp4_f32_impl<N_R0_MXFP4, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
|
||||
}
|
||||
|
||||
template<int nr0, typename args_t>
|
||||
void kernel_mul_mv_tq2_0_f32_impl(
|
||||
args_t args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
threadgroup char * shmem,
|
||||
uint3 tgpig,
|
||||
ushort tiisg,
|
||||
ushort sgitg) {
|
||||
const short NSG = FC_mul_mv_nsg;
|
||||
|
||||
const int nb = args.ne00/QK_K;
|
||||
|
||||
const int r0 = tgpig.x;
|
||||
const int r1 = tgpig.y;
|
||||
const int im = tgpig.z;
|
||||
|
||||
const int first_row = (r0 * NSG + sgitg) * nr0;
|
||||
|
||||
const uint i12 = im%FC_mul_mv_ne12;
|
||||
const uint i13 = im/FC_mul_mv_ne12;
|
||||
|
||||
const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
|
||||
|
||||
device const float * y = (device const float *) (src1 + offset1);
|
||||
|
||||
device const block_tq2_0 * ax[nr0];
|
||||
for (int row = 0; row < nr0; ++row) {
|
||||
const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
|
||||
ax[row] = (device const block_tq2_0 *) ((device char *) src0 + offset0);
|
||||
}
|
||||
|
||||
float sumf[nr0] = {0.f};
|
||||
|
||||
// 8 threads per block, NBLOCK blocks per pass, 2 halves per block per pass
|
||||
constexpr short NBLOCK = 4;
|
||||
|
||||
constexpr short NB = N_SIMDWIDTH/NBLOCK; // threads per block
|
||||
|
||||
const short blk = tiisg / NB; // 0..NBLOCK-1, block handled by this thread
|
||||
const short htg = tiisg % NB; // 0..NB-1, thread within block (0..7)
|
||||
|
||||
// byte and y base offsets within the block (32 elements per thread, 4 per byte)
|
||||
device const float4 * yb4 = (device const float4 *)(y + 4*htg + blk*QK_K);
|
||||
|
||||
// hoisted per-byte coefficients (from y) and total y-sum, shared across rows
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/26980
|
||||
float4 coef[4];
|
||||
|
||||
for (int ib = blk; ib < nb; ib += NBLOCK) {
|
||||
FOR_UNROLL (short h0 = 0; h0 < 2; ++h0) {
|
||||
const float4 y0 = yb4[ 0 + 32*h0];
|
||||
const float4 y1 = yb4[ 8 + 32*h0];
|
||||
const float4 y2 = yb4[16 + 32*h0];
|
||||
const float4 y3 = yb4[24 + 32*h0];
|
||||
|
||||
float sumy = 0.f;
|
||||
FOR_UNROLL (short j = 0; j < 4; ++j) {
|
||||
coef[j] = float4(
|
||||
y0[j],
|
||||
y1[j] - 4.0f*y0[j],
|
||||
y2[j] - 4.0f*y1[j],
|
||||
y3[j] - 4.0f*y2[j]);
|
||||
|
||||
sumy += (y0[j] + y1[j]) + (y2[j] + y3[j]);
|
||||
}
|
||||
|
||||
FOR_UNROLL (short row = 0; row < nr0; ++row) {
|
||||
device const block_tq2_0 & xb = ax[row][ib];
|
||||
device const uchar * qs = xb.qs + 4*htg + 32*h0;
|
||||
|
||||
float sum = -sumy;
|
||||
FOR_UNROLL (short j = 0; j < 4; ++j) {
|
||||
// express the 2-bit field shifts (v>>2, v>>4, v>>6) as float floor ops
|
||||
const float v = (float)qs[j];
|
||||
|
||||
const float f0 = v;
|
||||
const float f1 = floor(v*0.25f); // v>>2
|
||||
const float f2 = floor(v*0.0625); // v>>4
|
||||
const float f3 = floor(v*0.015625); // v>>6
|
||||
|
||||
sum += coef[j][0]*f0 + coef[j][1]*f1 + coef[j][2]*f2 + coef[j][3]*f3;
|
||||
}
|
||||
|
||||
sumf[row] += xb.d * sum;
|
||||
}
|
||||
}
|
||||
|
||||
yb4 += QK_K * NBLOCK / 4;
|
||||
}
|
||||
|
||||
device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
|
||||
|
||||
for (int row = 0; row < nr0; ++row) {
|
||||
const float tot = simd_sum(sumf[row]);
|
||||
if (tiisg == 0 && first_row + row < args.ne01) {
|
||||
dst_f32[first_row + row] = tot;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[[host_name("kernel_mul_mv_tq2_0_f32")]]
|
||||
kernel void kernel_mul_mv_tq2_0_f32(
|
||||
constant ggml_metal_kargs_mul_mv & args,
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
|
||||
|
||||
kernel_mul_mv_tq2_0_f32_impl<N_R0_TQ2_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
|
||||
}
|
||||
|
||||
template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread float4x4 &)>
|
||||
kernel void kernel_get_rows_q(
|
||||
constant ggml_metal_kargs_get_rows & args,
|
||||
|
|
@ -10090,38 +9899,6 @@ template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get
|
|||
template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_q_t kernel_get_rows_q<block_iq1_m, QK_NL, dequantize_iq1_m>;
|
||||
template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_q_t kernel_get_rows_q<block_iq4_nl, 2, dequantize_iq4_nl>;
|
||||
template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_q_t kernel_get_rows_q<block_iq4_xs, QK_NL, dequantize_iq4_xs>;
|
||||
template [[host_name("kernel_get_rows_tq2_0")]] kernel get_rows_q_t kernel_get_rows_q<block_tq2_0, QK_NL, dequantize_tq2_0>;
|
||||
|
||||
template<typename TS, typename TI, short QK, typename block_q, void (*quantize_func)(device const float *, device block_q &)>
|
||||
kernel void kernel_set_rows_q(
|
||||
constant ggml_metal_kargs_set_rows & args,
|
||||
device const void * src0,
|
||||
device const void * src1,
|
||||
device float * dst,
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
uint tiitg[[thread_index_in_threadgroup]],
|
||||
uint3 tptg [[threads_per_threadgroup]]) {
|
||||
const int32_t i03 = tgpig.z;
|
||||
const int32_t i02 = tgpig.y;
|
||||
|
||||
const int32_t i12 = i03%args.ne12;
|
||||
const int32_t i11 = i02%args.ne11;
|
||||
|
||||
const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x;
|
||||
if (i01 >= args.ne01) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int32_t i10 = i01;
|
||||
const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0];
|
||||
|
||||
device block_q * dst_row = ( device block_q *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3);
|
||||
const device TS * src_row = (const device TS *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03);
|
||||
|
||||
for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) {
|
||||
quantize_func(src_row + QK*ind, dst_row[ind]);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename TS, typename TI, typename block_q, void (*quantize_func)(device const float *, device block_q &)>
|
||||
kernel void kernel_set_rows_q32(
|
||||
|
|
@ -10218,11 +9995,6 @@ template [[host_name("kernel_set_rows_f32_i32_q5_1")]] kernel set_rows_q32_t k
|
|||
template [[host_name("kernel_set_rows_f32_i64_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32<float, int64_t, block_iq4_nl, quantize_iq4_nl>;
|
||||
template [[host_name("kernel_set_rows_f32_i32_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32<float, int32_t, block_iq4_nl, quantize_iq4_nl>;
|
||||
|
||||
typedef decltype(kernel_set_rows_q<float, int64_t, QK_K, block_tq2_0, quantize_tq2_0>) set_rows_qK_t;
|
||||
|
||||
template [[host_name("kernel_set_rows_f32_i64_tq2_0")]] kernel set_rows_qK_t kernel_set_rows_q<float, int64_t, QK_K, block_tq2_0, quantize_tq2_0>;
|
||||
template [[host_name("kernel_set_rows_f32_i32_tq2_0")]] kernel set_rows_qK_t kernel_set_rows_q<float, int32_t, QK_K, block_tq2_0, quantize_tq2_0>;
|
||||
|
||||
kernel void kernel_diag_f32(
|
||||
constant ggml_metal_kargs_diag & args,
|
||||
device const char * src0,
|
||||
|
|
@ -10998,7 +10770,6 @@ template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mul_mm_t kernel_mul_m
|
|||
template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m, float, float4x4, float, float2x4>;
|
||||
template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl, float, float4x4, float, float2x4>;
|
||||
template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs, float, float4x4, float, float2x4>;
|
||||
template [[host_name("kernel_mul_mm_tq2_0_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_tq2_0, QK_NL, dequantize_tq2_0, float, float4x4, float, float2x4>;
|
||||
|
||||
template [[host_name("kernel_mul_mm_f32_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, float4x4, 1, dequantize_f32, float, float4x4, half, half2x4>;
|
||||
template [[host_name("kernel_mul_mm_f16_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, half4x4, 1, dequantize_f16, half, half4x4, half, half2x4>;
|
||||
|
|
@ -11024,7 +10795,6 @@ template [[host_name("kernel_mul_mm_iq1_s_f16")]] kernel mul_mm_t kernel_mul_m
|
|||
template [[host_name("kernel_mul_mm_iq1_m_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m, float, float4x4, half, half2x4>;
|
||||
template [[host_name("kernel_mul_mm_iq4_nl_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl, float, float4x4, half, half2x4>;
|
||||
template [[host_name("kernel_mul_mm_iq4_xs_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs, float, float4x4, half, half2x4>;
|
||||
template [[host_name("kernel_mul_mm_tq2_0_f16")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_tq2_0, QK_NL, dequantize_tq2_0, float, float4x4, half, half2x4>;
|
||||
|
||||
//
|
||||
// indirect matrix-matrix multiplication
|
||||
|
|
@ -11059,7 +10829,6 @@ template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mul_mm_id kernel_m
|
|||
template [[host_name("kernel_mul_mm_id_iq1_m_f32")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m, float, float4x4, float, float2x4>;
|
||||
template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl, float, float4x4, float, float2x4>;
|
||||
template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs, float, float4x4, float, float2x4>;
|
||||
template [[host_name("kernel_mul_mm_id_tq2_0_f32")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_tq2_0, QK_NL, dequantize_tq2_0, float, float4x4, float, float2x4>;
|
||||
|
||||
template [[host_name("kernel_mul_mm_id_f32_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, float4x4, 1, dequantize_f32, float, float4x4, half, half2x4>;
|
||||
template [[host_name("kernel_mul_mm_id_f16_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, half4x4, 1, dequantize_f16, half, half4x4, half, half2x4>;
|
||||
|
|
@ -11085,7 +10854,6 @@ template [[host_name("kernel_mul_mm_id_iq1_s_f16")]] kernel mul_mm_id kernel_m
|
|||
template [[host_name("kernel_mul_mm_id_iq1_m_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq1_m, QK_NL, dequantize_iq1_m, float, float4x4, half, half2x4>;
|
||||
template [[host_name("kernel_mul_mm_id_iq4_nl_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_nl, 2, dequantize_iq4_nl, float, float4x4, half, half2x4>;
|
||||
template [[host_name("kernel_mul_mm_id_iq4_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq4_xs, QK_NL, dequantize_iq4_xs, float, float4x4, half, half2x4>;
|
||||
template [[host_name("kernel_mul_mm_id_tq2_0_f16")]] kernel mul_mm_id kernel_mul_mm_id<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_tq2_0, QK_NL, dequantize_tq2_0, float, float4x4, half, half2x4>;
|
||||
|
||||
//
|
||||
// matrix-vector multiplication
|
||||
|
|
@ -11243,7 +11011,6 @@ template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t
|
|||
template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_impl <N_R0_IQ2_S>>>;
|
||||
template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL>>>;
|
||||
template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS>>>;
|
||||
template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_tq2_0_f32_impl <N_R0_TQ2_0>>>;
|
||||
|
||||
kernel void kernel_pool_2d_max_f32(
|
||||
constant ggml_metal_kargs_pool_2d & args,
|
||||
|
|
@ -11511,310 +11278,3 @@ kernel void kernel_count_equal(
|
|||
typedef decltype(kernel_count_equal<int32_t>) kernel_count_equal_t;
|
||||
|
||||
template [[host_name("kernel_count_equal_i32")]] kernel kernel_count_equal_t kernel_count_equal<int32_t>;
|
||||
|
||||
template<
|
||||
typename kd4x4_t,
|
||||
short nl_k,
|
||||
void (*deq_k)(device const kd4x4_t *, short, thread half4x4 &)>
|
||||
kernel void kernel_lightning_indexer(
|
||||
constant ggml_metal_kargs_lightning_indexer & args,
|
||||
device const char * q,
|
||||
device const char * k,
|
||||
device const char * w,
|
||||
device const char * m,
|
||||
device char * dst,
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
ushort tiitg[[thread_index_in_threadgroup]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
|
||||
constexpr short DK = OP_LIGHTNING_INDEXER_DK;
|
||||
constexpr short NH = OP_LIGHTNING_INDEXER_NH;
|
||||
constexpr short NHPTG = OP_LIGHTNING_INDEXER_NHPTG;
|
||||
constexpr short NKPSG = OP_LIGHTNING_INDEXER_NKPSG;
|
||||
constexpr short NSG = OP_LIGHTNING_INDEXER_NSG;
|
||||
constexpr short NBPTG = OP_LIGHTNING_INDEXER_NBPTG;
|
||||
|
||||
constexpr short DK4 = DK/4;
|
||||
constexpr short DK8 = DK/8;
|
||||
constexpr short DK16 = DK/16;
|
||||
|
||||
constexpr short NK = NKPSG*NSG; // keys per threadgroup
|
||||
constexpr short NTG = 32*NSG; // threads per threadgroup
|
||||
|
||||
const int i_stream = tgpig.z;
|
||||
const int i_kv_0 = tgpig.x*NK; // first key of this threadgroup
|
||||
const int i_kv = i_kv_0 + sgitg*NKPSG; // first key of this simdgroup
|
||||
|
||||
threadgroup half sk[NK * DK16 * 16];
|
||||
threadgroup half4x4 * sk4x4 = (threadgroup half4x4 *) sk;
|
||||
|
||||
for (short i = tiitg; i < NK*DK16; i += NTG) {
|
||||
const short ik = i/DK16;
|
||||
const short i16 = i%DK16;
|
||||
|
||||
half4x4 tmp;
|
||||
|
||||
if (i_kv_0 + ik < args.n_kv) {
|
||||
device const kd4x4_t * kr = (device const kd4x4_t *) (k + (i_kv_0 + ik)*args.nbk2 + i_stream*args.nbk3);
|
||||
|
||||
deq_k(kr + i16/nl_k, i16%nl_k, tmp);
|
||||
} else {
|
||||
FOR_UNROLL (short j = 0; j < 4; ++j) {
|
||||
tmp[j] = half4(0.0h);
|
||||
}
|
||||
}
|
||||
|
||||
sk4x4[i] = tmp;
|
||||
}
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
// K tile of this simdgroup, transposed to [DK, NKPSG]
|
||||
simdgroup_half8x8 mk[DK8];
|
||||
|
||||
FOR_UNROLL (short i = 0; i < DK8; ++i) {
|
||||
simdgroup_load(mk[i], sk + sgitg*NKPSG*DK + 8*i, DK, 0, true);
|
||||
}
|
||||
|
||||
threadgroup half4 sq4[NHPTG*DK4];
|
||||
threadgroup half * sq = (threadgroup half *) sq4;
|
||||
|
||||
threadgroup float sw [NHPTG];
|
||||
threadgroup float sqk[NSG*NHPTG*NKPSG];
|
||||
|
||||
const int i_batch_0 = tgpig.y*NBPTG;
|
||||
const int n_batch = min((int) NBPTG, args.n_batch - i_batch_0);
|
||||
|
||||
for (short ib = 0; ib < n_batch; ++ib) {
|
||||
const int i_batch = i_batch_0 + ib;
|
||||
|
||||
device const char * pq = q + i_batch*args.nbq2 + i_stream*args.nbq3;
|
||||
device const char * pw = w + i_batch*args.nbw1 + i_stream*args.nbw3;
|
||||
|
||||
float score = 0.0f;
|
||||
|
||||
FOR_UNROLL (short i_head = 0; i_head < NH; i_head += NHPTG) {
|
||||
// stage the Q tile [DK, NHPTG] and the (prescaled) head weights
|
||||
for (short i = tiitg; i < NHPTG*DK4; i += NTG) {
|
||||
const short ih = i/DK4;
|
||||
const short i4 = i%DK4;
|
||||
|
||||
device const float4 * q4 = (device const float4 *) (pq + (i_head + ih)*args.nbq1);
|
||||
|
||||
sq4[ih*DK4 + i4] = half4(q4[i4]);
|
||||
}
|
||||
|
||||
if (tiitg < NHPTG) {
|
||||
sw[tiitg] = ((device const float *) pw)[i_head + tiitg];
|
||||
}
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
simdgroup_float8x8 mqk = make_filled_simdgroup_matrix<float, 8>(0.0f);
|
||||
|
||||
FOR_UNROLL (short i = 0; i < DK8; ++i) {
|
||||
simdgroup_half8x8 mq;
|
||||
|
||||
simdgroup_load(mq, sq + 8*i, DK, 0, false);
|
||||
simdgroup_multiply_accumulate(mqk, mq, mk[i], mqk);
|
||||
}
|
||||
|
||||
threadgroup float * pqk = sqk + sgitg*NHPTG*NKPSG;
|
||||
|
||||
simdgroup_store(mqk, pqk, NKPSG, 0, false);
|
||||
simdgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
// one lane per key: ReLU, apply the head weight and accumulate over the head tile
|
||||
if (tiisg < NKPSG) {
|
||||
FOR_UNROLL (short ih = 0; ih < NHPTG; ++ih) {
|
||||
score += max(pqk[ih*NKPSG + tiisg], 0.0f)*sw[ih];
|
||||
}
|
||||
}
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
}
|
||||
|
||||
if (tiisg < NKPSG) {
|
||||
const int ik = i_kv + tiisg;
|
||||
if (ik < args.n_kv) {
|
||||
device const half * pm = (device const half *) (m + i_batch*args.nbm1 + (i_stream % args.mask_ne3)*args.nbm3);
|
||||
device float * pd = (device float *) (dst + i_batch*args.nb1 + i_stream*args.nb3);
|
||||
|
||||
pd[ik] = score + (float) pm[ik];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
typedef decltype(kernel_lightning_indexer<half4x4, 1, dequantize_f16>) kernel_lightning_indexer_t;
|
||||
|
||||
template [[host_name("kernel_lightning_indexer_f32")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<float4x4, 1, dequantize_f32>;
|
||||
template [[host_name("kernel_lightning_indexer_f16")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<half4x4, 1, dequantize_f16>;
|
||||
|
||||
#if defined(GGML_METAL_HAS_BF16)
|
||||
template [[host_name("kernel_lightning_indexer_bf16")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<bfloat4x4, 1, dequantize_bf16>;
|
||||
#endif
|
||||
|
||||
template [[host_name("kernel_lightning_indexer_q4_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q4_0, 2, dequantize_q4_0>;
|
||||
template [[host_name("kernel_lightning_indexer_q4_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q4_1, 2, dequantize_q4_1>;
|
||||
template [[host_name("kernel_lightning_indexer_q5_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q5_0, 2, dequantize_q5_0>;
|
||||
template [[host_name("kernel_lightning_indexer_q5_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q5_1, 2, dequantize_q5_1>;
|
||||
template [[host_name("kernel_lightning_indexer_q8_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer<block_q8_0, 2, dequantize_q8_0>;
|
||||
|
||||
kernel void kernel_dsv4_hc_comb_f32(
|
||||
constant ggml_metal_kargs_dsv4_hc_comb & args,
|
||||
device const char * mixes,
|
||||
device const char * scale,
|
||||
device const char * base,
|
||||
device char * dst,
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]],
|
||||
ushort3 ntg[[threads_per_threadgroup]]) {
|
||||
constexpr ushort hc = 4;
|
||||
constexpr ushort comb_offset = 2*hc;
|
||||
|
||||
const int it = tgpig.x*ntg.y + sgitg;
|
||||
if (it >= args.n_tokens) {
|
||||
return;
|
||||
}
|
||||
|
||||
float scale_lane = 0.0f;
|
||||
if (tiisg == 0) {
|
||||
scale_lane = *(device const float *) (scale + 2*args.nb_s0);
|
||||
}
|
||||
const float scale_comb = simd_shuffle(scale_lane, 0);
|
||||
|
||||
float v = 0.0f;
|
||||
if (tiisg < hc*hc) {
|
||||
v = *(device const float *) (mixes + (comb_offset + tiisg)*args.nb_m0 + it*args.nb_m1)*scale_comb
|
||||
+ *(device const float *) (base + (comb_offset + tiisg)*args.nb_b0);
|
||||
}
|
||||
|
||||
// Softmax across destinations (the four contiguous lanes for each source).
|
||||
float vmax = max(v, simd_shuffle_xor(v, 1));
|
||||
vmax = max(vmax, simd_shuffle_xor(vmax, 2));
|
||||
v = exp(v - vmax);
|
||||
|
||||
float sum = v + simd_shuffle_xor(v, 1);
|
||||
sum += simd_shuffle_xor(sum, 2);
|
||||
v = v/sum + args.eps;
|
||||
|
||||
// Normalize columns: equal destination indices are four lanes apart.
|
||||
sum = v + simd_shuffle_xor(v, 4);
|
||||
sum += simd_shuffle_xor(sum, 8);
|
||||
v /= sum + args.eps;
|
||||
|
||||
for (int i = 1; i < args.n_iter; ++i) {
|
||||
sum = v + simd_shuffle_xor(v, 1);
|
||||
sum += simd_shuffle_xor(sum, 2);
|
||||
v /= sum + args.eps;
|
||||
|
||||
sum = v + simd_shuffle_xor(v, 4);
|
||||
sum += simd_shuffle_xor(sum, 8);
|
||||
v /= sum + args.eps;
|
||||
}
|
||||
|
||||
if (tiisg < hc*hc) {
|
||||
const ushort idst = tiisg & 3;
|
||||
const ushort isrc = tiisg >> 2;
|
||||
*(device float *) (dst + idst*args.nb_d0 + isrc*args.nb_d1 + it*args.nb_d2) = v;
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_dsv4_hc_pre_f32(
|
||||
constant ggml_metal_kargs_dsv4_hc_pre & args,
|
||||
device const char * x,
|
||||
device const char * weights,
|
||||
device char * dst,
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]],
|
||||
ushort3 ntg[[threads_per_threadgroup]]) {
|
||||
constexpr ushort hc = 4;
|
||||
|
||||
const int it = tgpig.y;
|
||||
const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg;
|
||||
|
||||
float weight_lane = 0.0f;
|
||||
if (tiisg < hc) {
|
||||
weight_lane = *(device const float *) (weights + tiisg*args.nb_w0 + it*args.nb_w1);
|
||||
}
|
||||
|
||||
float w[hc];
|
||||
FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) {
|
||||
w[ih] = simd_shuffle(weight_lane, ih);
|
||||
}
|
||||
|
||||
if (i0 >= args.n_embd) {
|
||||
return;
|
||||
}
|
||||
|
||||
device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2;
|
||||
float result = 0.0f;
|
||||
FOR_UNROLL (ushort ih = 0; ih < hc; ++ih) {
|
||||
result = fma(*(device const float *) (xb + ih*args.nb_x1), w[ih], result);
|
||||
}
|
||||
|
||||
*(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = result;
|
||||
}
|
||||
|
||||
kernel void kernel_dsv4_hc_post_f32(
|
||||
constant ggml_metal_kargs_dsv4_hc_post & args,
|
||||
device const char * x,
|
||||
device const char * residual,
|
||||
device const char * post,
|
||||
device const char * comb,
|
||||
device char * dst,
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]],
|
||||
ushort3 ntg[[threads_per_threadgroup]]) {
|
||||
constexpr ushort hc = 4;
|
||||
|
||||
const int it = tgpig.y;
|
||||
const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg;
|
||||
|
||||
float coeff_lane = 0.0f;
|
||||
if (tiisg < hc) {
|
||||
coeff_lane = *(device const float *) (post + tiisg*args.nb_p0 + it*args.nb_p1);
|
||||
} else if (tiisg < hc + hc*hc) {
|
||||
const ushort idx = tiisg - hc;
|
||||
const ushort idst = idx & 3;
|
||||
const ushort isrc = idx >> 2;
|
||||
coeff_lane = *(device const float *) (comb + idst*args.nb_c0 + isrc*args.nb_c1 + it*args.nb_c2);
|
||||
}
|
||||
|
||||
float post_reg[hc];
|
||||
float comb_reg[hc][hc];
|
||||
FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
|
||||
post_reg[idst] = simd_shuffle(coeff_lane, idst);
|
||||
}
|
||||
FOR_UNROLL (ushort isrc = 0; isrc < hc; ++isrc) {
|
||||
FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
|
||||
comb_reg[isrc][idst] = simd_shuffle(coeff_lane, hc + idst + hc*isrc);
|
||||
}
|
||||
}
|
||||
|
||||
if (i0 >= args.n_embd) {
|
||||
return;
|
||||
}
|
||||
|
||||
const float xv = *(device const float *) (x + i0*args.nb_x0 + it*args.nb_x1);
|
||||
float result[hc];
|
||||
FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
|
||||
result[idst] = xv*post_reg[idst];
|
||||
}
|
||||
|
||||
device const char * rb = residual + i0*args.nb_r0 + it*args.nb_r2;
|
||||
FOR_UNROLL (ushort isrc = 0; isrc < hc; ++isrc) {
|
||||
const float rv = *(device const float *) (rb + isrc*args.nb_r1);
|
||||
FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
|
||||
result[idst] = fma(rv, comb_reg[isrc][idst], result[idst]);
|
||||
}
|
||||
}
|
||||
|
||||
FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) {
|
||||
*(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = result[idst];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1884,7 +1884,6 @@ static void ggml_backend_rpc_device_get_props(ggml_backend_dev_t dev, struct ggm
|
|||
/* .host_buffer = */ false,
|
||||
/* .buffer_from_host_ptr = */ false,
|
||||
/* .events = */ false,
|
||||
/* .mmap_support = */ true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,280 +0,0 @@
|
|||
#include "ggml-impl.h"
|
||||
#include "dsv4-hc.hpp"
|
||||
|
||||
#include <cmath>
|
||||
|
||||
static constexpr int DSV4_HC = 4;
|
||||
|
||||
static void dsv4_hc_pre_f32_sycl(
|
||||
const float * x, const float * weights, float * dst,
|
||||
int64_t n_embd, int64_t hc, int64_t n_tokens,
|
||||
int64_t sx0, int64_t sx1, int64_t sx2,
|
||||
int64_t sw0, int64_t sw1,
|
||||
int64_t sd0, int64_t sd1,
|
||||
queue_ptr stream) {
|
||||
const int64_t nr = n_embd * n_tokens;
|
||||
const int64_t block_size = 256;
|
||||
const int64_t num_blocks = (nr + block_size - 1) / block_size;
|
||||
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)),
|
||||
[=](sycl::nd_item<1> item) {
|
||||
const int64_t ir = item.get_global_id(0);
|
||||
if (ir >= nr) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t i0 = ir % n_embd;
|
||||
const int64_t it = ir / n_embd;
|
||||
|
||||
float sum = x[i0*sx0 + it*sx2] * weights[it*sw1];
|
||||
for (int64_t ih = 1; ih < hc; ++ih) {
|
||||
const float xv = x[i0*sx0 + ih*sx1 + it*sx2];
|
||||
const float wv = weights[ih*sw0 + it*sw1];
|
||||
sum += xv * wv;
|
||||
}
|
||||
|
||||
dst[i0*sd0 + it*sd1] = sum;
|
||||
});
|
||||
}
|
||||
|
||||
static void dsv4_hc_comb_norm_cols(float * comb, float eps) {
|
||||
for (int idst = 0; idst < DSV4_HC; ++idst) {
|
||||
float sum = eps;
|
||||
for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
|
||||
sum += comb[idst + DSV4_HC*isrc];
|
||||
}
|
||||
|
||||
const float inv_sum = 1.0f / sum;
|
||||
for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
|
||||
comb[idst + DSV4_HC*isrc] *= inv_sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void dsv4_hc_comb_norm_rows(float * comb, float eps) {
|
||||
for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
|
||||
float sum = eps;
|
||||
for (int idst = 0; idst < DSV4_HC; ++idst) {
|
||||
sum += comb[idst + DSV4_HC*isrc];
|
||||
}
|
||||
|
||||
const float inv_sum = 1.0f / sum;
|
||||
for (int idst = 0; idst < DSV4_HC; ++idst) {
|
||||
comb[idst + DSV4_HC*isrc] *= inv_sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void dsv4_hc_comb_f32_sycl(
|
||||
const float * mixes,
|
||||
const float * scale,
|
||||
const float * base,
|
||||
float * dst,
|
||||
int64_t n_tokens,
|
||||
int64_t sm0,
|
||||
int64_t sm1,
|
||||
int64_t ss0,
|
||||
int64_t sb0,
|
||||
int64_t sd0,
|
||||
int64_t sd1,
|
||||
int64_t sd2,
|
||||
float eps,
|
||||
int32_t n_iter,
|
||||
queue_ptr stream) {
|
||||
constexpr int comb_offset = 2*DSV4_HC;
|
||||
|
||||
const int64_t block_size = 256;
|
||||
const int64_t num_blocks = (n_tokens + block_size - 1) / block_size;
|
||||
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)),
|
||||
[=](sycl::nd_item<1> item_ct1) {
|
||||
const int64_t it = item_ct1.get_global_id(0);
|
||||
|
||||
if (it >= n_tokens) {
|
||||
return;
|
||||
}
|
||||
|
||||
const float scale_comb = scale[2*ss0];
|
||||
float comb[DSV4_HC*DSV4_HC];
|
||||
|
||||
for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
|
||||
float max = -INFINITY;
|
||||
for (int idst = 0; idst < DSV4_HC; ++idst) {
|
||||
const int idx = idst + DSV4_HC*isrc;
|
||||
const float v = mixes[(comb_offset + idx)*sm0 + it*sm1] * scale_comb + base[(comb_offset + idx)*sb0];
|
||||
comb[idx] = v;
|
||||
max = fmaxf(max, v);
|
||||
}
|
||||
|
||||
float sum = 0.0f;
|
||||
for (int idst = 0; idst < DSV4_HC; ++idst) {
|
||||
const int idx = idst + DSV4_HC*isrc;
|
||||
const float v = expf(comb[idx] - max);
|
||||
comb[idx] = v;
|
||||
sum += v;
|
||||
}
|
||||
|
||||
const float inv_sum = 1.0f / sum;
|
||||
for (int idst = 0; idst < DSV4_HC; ++idst) {
|
||||
const int idx = idst + DSV4_HC*isrc;
|
||||
comb[idx] = comb[idx] * inv_sum + eps;
|
||||
}
|
||||
}
|
||||
|
||||
dsv4_hc_comb_norm_cols(comb, eps);
|
||||
for (int32_t i = 1; i < n_iter; ++i) {
|
||||
dsv4_hc_comb_norm_rows(comb, eps);
|
||||
dsv4_hc_comb_norm_cols(comb, eps);
|
||||
}
|
||||
|
||||
for (int isrc = 0; isrc < DSV4_HC; ++isrc) {
|
||||
for (int idst = 0; idst < DSV4_HC; ++idst) {
|
||||
const int idx = idst + DSV4_HC*isrc;
|
||||
dst[idst*sd0 + isrc*sd1 + it*sd2] = comb[idx];
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
static void dsv4_hc_post_f32_sycl(
|
||||
const float * x, const float * residual, const float * post, const float * comb, float * dst,
|
||||
int64_t n_embd, int64_t hc, int64_t n_tokens,
|
||||
int64_t sx0, int64_t sx1,
|
||||
int64_t sr0, int64_t sr1, int64_t sr2,
|
||||
int64_t sp0, int64_t sp1,
|
||||
int64_t sc0, int64_t sc1, int64_t sc2,
|
||||
int64_t sd0, int64_t sd1, int64_t sd2,
|
||||
queue_ptr stream) {
|
||||
const int64_t nr = n_embd * hc * n_tokens;
|
||||
const int64_t block_size = 256;
|
||||
const int64_t num_blocks = (nr + block_size - 1) / block_size;
|
||||
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)),
|
||||
[=](sycl::nd_item<1> item) {
|
||||
const int64_t ir = item.get_global_id(0);
|
||||
if (ir >= nr) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t i0 = ir % n_embd;
|
||||
const int64_t idst = (ir / n_embd) % hc;
|
||||
const int64_t it = ir / (n_embd * hc);
|
||||
|
||||
float sum = x[i0*sx0 + it*sx1] * post[idst*sp0 + it*sp1];
|
||||
for (int64_t isrc = 0; isrc < hc; ++isrc) {
|
||||
sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2];
|
||||
}
|
||||
|
||||
dst[i0*sd0 + idst*sd1 + it*sd2] = sum;
|
||||
});
|
||||
}
|
||||
|
||||
void ggml_sycl_op_dsv4_hc_pre(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
|
||||
const ggml_tensor * x = dst->src[0];
|
||||
const ggml_tensor * weights = dst->src[1];
|
||||
|
||||
GGML_ASSERT(x->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(weights->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
|
||||
GGML_TENSOR_LOCALS(size_t, nbx, x, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbw, weights, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbd, dst, nb);
|
||||
|
||||
const int64_t n_embd = x->ne[0];
|
||||
const int64_t hc = x->ne[1];
|
||||
const int64_t n_tokens = x->ne[2];
|
||||
|
||||
queue_ptr stream = ctx.stream();
|
||||
|
||||
dsv4_hc_pre_f32_sycl(
|
||||
(const float *) x->data, (const float *) weights->data, (float *) dst->data,
|
||||
n_embd, hc, n_tokens,
|
||||
nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float),
|
||||
nbw0 / sizeof(float), nbw1 / sizeof(float),
|
||||
nbd0 / sizeof(float), nbd1 / sizeof(float),
|
||||
stream);
|
||||
}
|
||||
|
||||
void ggml_sycl_op_dsv4_hc_comb(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3);
|
||||
|
||||
const ggml_tensor * mixes = dst->src[0];
|
||||
const ggml_tensor * scale = dst->src[1];
|
||||
const ggml_tensor * base = dst->src[2];
|
||||
|
||||
GGML_ASSERT(mixes->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(scale->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(base->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
|
||||
constexpr int64_t hc_mix_dim = (2 + DSV4_HC)*DSV4_HC;
|
||||
|
||||
GGML_ASSERT(mixes->ne[0] == hc_mix_dim);
|
||||
GGML_ASSERT(dst->ne[0] == DSV4_HC);
|
||||
GGML_ASSERT(dst->ne[1] == DSV4_HC);
|
||||
GGML_ASSERT(dst->ne[2] == mixes->ne[1]);
|
||||
GGML_ASSERT(scale->ne[0] >= 3);
|
||||
GGML_ASSERT(base->ne[0] == hc_mix_dim);
|
||||
|
||||
GGML_TENSOR_LOCALS(size_t, nbm, mixes, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbs, scale, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbb, base, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbd, dst, nb);
|
||||
|
||||
const int64_t n_tokens = mixes->ne[1];
|
||||
const float eps = ggml_get_op_params_f32(dst, 0);
|
||||
const int32_t n_iter = ggml_get_op_params_i32(dst, 1);
|
||||
|
||||
queue_ptr stream = ctx.stream();
|
||||
|
||||
dsv4_hc_comb_f32_sycl(
|
||||
(const float *) mixes->data, (const float *) scale->data, (const float *) base->data, (float *) dst->data,
|
||||
n_tokens,
|
||||
nbm0 / sizeof(float), nbm1 / sizeof(float),
|
||||
nbs0 / sizeof(float),
|
||||
nbb0 / sizeof(float),
|
||||
nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float),
|
||||
eps, n_iter, stream);
|
||||
}
|
||||
|
||||
void ggml_sycl_op_dsv4_hc_post(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/4);
|
||||
const ggml_tensor * x = dst->src[0];
|
||||
const ggml_tensor * residual = dst->src[1];
|
||||
const ggml_tensor * post = dst->src[2];
|
||||
const ggml_tensor * comb = dst->src[3];
|
||||
|
||||
GGML_ASSERT(x->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(residual->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(post->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(comb->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
|
||||
GGML_TENSOR_LOCALS(size_t, nbx, x, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbr, residual, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbp, post, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbc, comb, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbd, dst, nb);
|
||||
|
||||
const int64_t n_embd = x->ne[0];
|
||||
const int64_t n_tokens = x->ne[1];
|
||||
const int64_t hc = residual->ne[1];
|
||||
|
||||
queue_ptr stream = ctx.stream();
|
||||
|
||||
dsv4_hc_post_f32_sycl(
|
||||
(const float *) x->data, (const float *) residual->data,
|
||||
(const float *) post->data, (const float *) comb->data, (float *) dst->data,
|
||||
n_embd, hc, n_tokens,
|
||||
nbx0 / sizeof(float), nbx1 / sizeof(float),
|
||||
nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float),
|
||||
nbp0 / sizeof(float), nbp1 / sizeof(float),
|
||||
nbc0 / sizeof(float), nbc1 / sizeof(float), nbc2 / sizeof(float),
|
||||
nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float),
|
||||
stream);
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
#ifndef GGML_SYCL_DSV4_HC_HPP
|
||||
#define GGML_SYCL_DSV4_HC_HPP
|
||||
|
||||
#include "common.hpp"
|
||||
|
||||
void ggml_sycl_op_dsv4_hc_pre(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
||||
void ggml_sycl_op_dsv4_hc_comb(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
||||
void ggml_sycl_op_dsv4_hc_post(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
||||
|
||||
#endif // GGML_SYCL_DSV4_HC_HPP
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
#include "lightning-indexer.hpp"
|
||||
#include "dequantize.hpp"
|
||||
|
||||
static void lightning_indexer_f32_sycl(
|
||||
const char * q, const char * k, const char * w, const char * m, float * dst,
|
||||
int64_t n_embd, int64_t n_head, int64_t n_batch, int64_t n_stream, int64_t n_kv,
|
||||
int64_t nem3,
|
||||
int64_t nbq1, int64_t nbq2, int64_t nbq3,
|
||||
int64_t nbk2, int64_t nbk3,
|
||||
int64_t nbw1, int64_t nbw3,
|
||||
int64_t nbm1, int64_t nbm3,
|
||||
int64_t nb1, int64_t nb3,
|
||||
ggml_type k_type,
|
||||
queue_ptr stream) {
|
||||
|
||||
constexpr int64_t LANES = WARP_SIZE;
|
||||
constexpr int64_t ELEMS_PER_LANE = 8;
|
||||
constexpr int64_t ROWS_PER_BLOCK = 4;
|
||||
constexpr int64_t BLOCK_SIZE = ROWS_PER_BLOCK * LANES;
|
||||
|
||||
const int64_t n_rows = n_batch * n_stream * n_kv;
|
||||
const int64_t n_blocks = (n_rows + ROWS_PER_BLOCK - 1) / ROWS_PER_BLOCK;
|
||||
|
||||
stream->parallel_for(
|
||||
sycl::nd_range<1>(
|
||||
sycl::range<1>(n_blocks * BLOCK_SIZE),
|
||||
sycl::range<1>(BLOCK_SIZE)),
|
||||
[=](sycl::nd_item<1> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
const int64_t ir = item.get_global_id(0);
|
||||
const int64_t lane = ir % LANES;
|
||||
const int64_t row = ir / LANES;
|
||||
if (row >= n_rows) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t i_bs = row / n_kv;
|
||||
const int64_t i_kv = row % n_kv;
|
||||
const int64_t i_batch = i_bs / n_stream;
|
||||
const int64_t i_stream = i_bs % n_stream;
|
||||
|
||||
// load K row slice into registers (row is contiguous, nbk0 == type size)
|
||||
const char * k_base = k + i_kv*nbk2 + i_stream*nbk3;
|
||||
float k_local[ELEMS_PER_LANE];
|
||||
if (k_type == GGML_TYPE_F16) {
|
||||
const sycl::half * k_row = (const sycl::half *) k_base;
|
||||
#pragma unroll
|
||||
for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
|
||||
k_local[j] = static_cast<float>(k_row[lane*ELEMS_PER_LANE + j]);
|
||||
}
|
||||
} else if (k_type == GGML_TYPE_F32) {
|
||||
const float * k_row = (const float *) k_base;
|
||||
#pragma unroll
|
||||
for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
|
||||
k_local[j] = k_row[lane*ELEMS_PER_LANE + j];
|
||||
}
|
||||
} else {
|
||||
const int64_t lane_base = lane * ELEMS_PER_LANE;
|
||||
switch (k_type) {
|
||||
case GGML_TYPE_BF16: {
|
||||
const sycl::ext::oneapi::bfloat16 * k_row = (const sycl::ext::oneapi::bfloat16 *) k_base;
|
||||
#pragma unroll
|
||||
for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
|
||||
k_local[j] = static_cast<float>(k_row[lane_base + j]);
|
||||
}
|
||||
} break;
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1: {
|
||||
#pragma unroll
|
||||
for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
|
||||
const int64_t idx = lane_base + j;
|
||||
const int64_t ib = idx / QK4_0;
|
||||
const int iqs = idx % (QK4_0/2);
|
||||
dfloat2 kv;
|
||||
if (k_type == GGML_TYPE_Q4_0) {
|
||||
dequantize_q4_0(k_base, ib, iqs, kv);
|
||||
} else if (k_type == GGML_TYPE_Q4_1) {
|
||||
dequantize_q4_1(k_base, ib, iqs, kv);
|
||||
} else if (k_type == GGML_TYPE_Q5_0) {
|
||||
dequantize_q5_0(k_base, ib, iqs, kv);
|
||||
} else {
|
||||
dequantize_q5_1(k_base, ib, iqs, kv);
|
||||
}
|
||||
k_local[j] = (idx % QK4_0) < (QK4_0/2) ? static_cast<float>(kv.x()) : static_cast<float>(kv.y());
|
||||
}
|
||||
} break;
|
||||
case GGML_TYPE_Q8_0: {
|
||||
#pragma unroll
|
||||
for (int64_t pair = 0; pair < ELEMS_PER_LANE / 2; ++pair) {
|
||||
const int64_t elem0 = lane_base + 2 * pair;
|
||||
dfloat2 kv;
|
||||
dequantize_q8_0(k_base, elem0 / QK8_0, elem0 % QK8_0, kv);
|
||||
k_local[2 * pair + 0] = static_cast<float>(kv.x());
|
||||
k_local[2 * pair + 1] = static_cast<float>(kv.y());
|
||||
}
|
||||
} break;
|
||||
case GGML_TYPE_IQ4_NL: {
|
||||
#pragma unroll
|
||||
for (int64_t pair = 0; pair < ELEMS_PER_LANE / 2; ++pair) {
|
||||
const int64_t elem0 = lane_base + 2 * pair;
|
||||
dfloat2 kv;
|
||||
dequantize_iq4_nl(k_base, elem0 / QK4_NL, elem0 % QK4_NL, kv);
|
||||
k_local[2 * pair + 0] = static_cast<float>(kv.x());
|
||||
k_local[2 * pair + 1] = static_cast<float>(kv.y());
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
#pragma unroll
|
||||
for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
|
||||
k_local[j] = 0.0f;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const char * q_base = q + i_batch*nbq2 + i_stream*nbq3;
|
||||
const float * w_base = (const float *) (w + i_batch*nbw1 + i_stream*nbw3);
|
||||
|
||||
float score = 0.0f;
|
||||
for (int64_t h = 0; h < n_head; ++h) {
|
||||
const float * q_row = (const float *) (q_base + h*nbq1);
|
||||
float dot = 0.0f;
|
||||
#pragma unroll
|
||||
for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) {
|
||||
const int64_t i = lane*ELEMS_PER_LANE + j;
|
||||
if (i < n_embd) {
|
||||
dot += q_row[i] * k_local[j];
|
||||
}
|
||||
}
|
||||
dot = sycl::reduce_over_group(item.get_sub_group(), dot, sycl::plus<float>());
|
||||
if (lane == 0) {
|
||||
score += sycl::max(dot, 0.0f) * w_base[h];
|
||||
}
|
||||
}
|
||||
|
||||
if (lane == 0) {
|
||||
const sycl::half * m_base = (const sycl::half *) (m + i_batch*nbm1 + (i_stream % nem3)*nbm3);
|
||||
// flat-index store: storing through a strided base pointer
|
||||
// hangs/misroutes writes on this stack when n_batch*n_stream > 1
|
||||
const int64_t dst_idx = i_kv + i_batch*(nb1/sizeof(float)) + i_stream*(nb3/sizeof(float));
|
||||
dst[dst_idx] = score + static_cast<float>(m_base[i_kv]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void ggml_sycl_op_lightning_indexer(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/4);
|
||||
const ggml_tensor * q = dst->src[0];
|
||||
const ggml_tensor * k = dst->src[1];
|
||||
const ggml_tensor * w = dst->src[2]; // weights
|
||||
const ggml_tensor * m = dst->src[3]; // mask
|
||||
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( q->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( w->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT( m->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_BF16 ||
|
||||
k->type == GGML_TYPE_Q8_0 || k->type == GGML_TYPE_Q5_1 || k->type == GGML_TYPE_Q5_0 ||
|
||||
k->type == GGML_TYPE_Q4_1 || k->type == GGML_TYPE_Q4_0 || k->type == GGML_TYPE_IQ4_NL);
|
||||
|
||||
GGML_TENSOR_LOCALS(int64_t, neq, q, ne);
|
||||
GGML_TENSOR_LOCALS(size_t, nbq, q, nb);
|
||||
GGML_TENSOR_LOCALS(int64_t, nek, k, ne);
|
||||
GGML_TENSOR_LOCALS(size_t, nbk, k, nb);
|
||||
GGML_TENSOR_LOCALS(size_t, nbw, w, nb);
|
||||
GGML_TENSOR_LOCALS(int64_t, nem, m, ne);
|
||||
GGML_TENSOR_LOCALS(size_t, nbm, m, nb);
|
||||
GGML_TENSOR_LOCALS(int64_t, ne, dst, ne);
|
||||
GGML_TENSOR_LOCALS(size_t, nb, dst, nb);
|
||||
|
||||
// input rows must be contiguous
|
||||
GGML_ASSERT(nbq0 == ggml_type_size(q->type));
|
||||
GGML_ASSERT(nbk0 == ggml_type_size(k->type));
|
||||
GGML_ASSERT(nbm0 == ggml_type_size(m->type));
|
||||
GGML_ASSERT(nb0 == ggml_type_size(dst->type));
|
||||
|
||||
const int64_t n_embd = neq0;
|
||||
const int64_t n_head = neq1;
|
||||
const int64_t n_batch = neq2;
|
||||
const int64_t n_stream = neq3;
|
||||
const int64_t n_kv = nek2;
|
||||
|
||||
GGML_ASSERT(n_embd == WARP_SIZE * 8);
|
||||
|
||||
lightning_indexer_f32_sycl(
|
||||
(const char *) q->data, (const char *) k->data,
|
||||
(const char *) w->data, (const char *) m->data, (float *) dst->data,
|
||||
n_embd, n_head, n_batch, n_stream, n_kv, nem3,
|
||||
nbq1, nbq2, nbq3,
|
||||
nbk2, nbk3,
|
||||
nbw1, nbw3,
|
||||
nbm1, nbm3,
|
||||
nb1, nb3,
|
||||
k->type,
|
||||
ctx.stream());
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
#ifndef GGML_SYCL_LIGHTNING_INDEXER_HPP
|
||||
#define GGML_SYCL_LIGHTNING_INDEXER_HPP
|
||||
|
||||
#include "common.hpp"
|
||||
|
||||
void ggml_sycl_op_lightning_indexer(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
||||
|
||||
#endif // GGML_SYCL_LIGHTNING_INDEXER_HPP
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -608,20 +608,6 @@ vec2 get_dm(uint ib, uint a_offset) {
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(DATA_A_TQ2_0)
|
||||
vec2 dequantize(uint ib, uint iqs, uint a_offset) {
|
||||
// elem e -> byte qs[(e/128)*32 + e%32], bits 2*((e%128)/32); w = q - 1 (d applied via get_dm)
|
||||
const uint qsi = (iqs / 128) * 32 + (iqs % 32); // iqs even -> qsi, qsi+1 in same group/level
|
||||
const uint shift = 2 * ((iqs % 128) / 32);
|
||||
|
||||
const uvec2 qs = uvec2(data_a[a_offset + ib].qs[qsi], data_a[a_offset + ib].qs[qsi + 1]);
|
||||
return vec2((qs >> shift) & 3) - 1.0;
|
||||
}
|
||||
vec2 get_dm(uint ib, uint a_offset) {
|
||||
return vec2(float(data_a[a_offset + ib].d), 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(DATA_A_Q3_K)
|
||||
vec2 dequantize(uint ib, uint iqs, uint a_offset) {
|
||||
iqs /= 2;
|
||||
|
|
|
|||
|
|
@ -247,44 +247,6 @@ f16vec4 dequantFuncQ8_0_v(const in decodeBufQ8_0 bl, const in uint blockCoords[2
|
|||
return f16vec4(vec4(qi) * vec4(float(d)));
|
||||
}
|
||||
|
||||
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0 {
|
||||
block_tq2_0 block;
|
||||
};
|
||||
|
||||
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufTQ2_0_packed16 {
|
||||
block_tq2_0_packed16 block;
|
||||
};
|
||||
|
||||
float16_t dequantFuncTQ2_0(const in decodeBufTQ2_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
|
||||
{
|
||||
decodeBufTQ2_0_packed16 bl16 = decodeBufTQ2_0_packed16(bl);
|
||||
const uint idx = coordInBlock[1];
|
||||
|
||||
const uint qsshift = (idx & 0x60) >> 4; // 0,2,4,6
|
||||
|
||||
uint qs = uint32_t(bl16.block.qs[((idx & 0x80) >> 3) + ((idx & 0x1E) >> 1)]);
|
||||
qs = (qs >> qsshift) & 0x0303;
|
||||
qs = unpack8(qs)[idx & 1];
|
||||
|
||||
return bl.block.d * (float16_t(int(qs)) - float16_t(1.0));
|
||||
}
|
||||
|
||||
f16vec4 dequantFuncTQ2_0_v(const in decodeBufTQ2_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
|
||||
{
|
||||
const uint idx = coordInBlock[1];
|
||||
|
||||
const uint qsshift = (idx & 0x60) >> 4; // 0,2,4,6
|
||||
const uint qsi = ((idx & 0x80) >> 2) + (idx & 0x1C); // byte index of 4-aligned group
|
||||
|
||||
const uint qsw = (uint(bl.block.qs[qsi]))
|
||||
| (uint(bl.block.qs[qsi + 1]) << 8)
|
||||
| (uint(bl.block.qs[qsi + 2]) << 16)
|
||||
| (uint(bl.block.qs[qsi + 3]) << 24);
|
||||
const u8vec4 q = unpack8((qsw >> qsshift) & 0x03030303);
|
||||
|
||||
return bl.block.d * (f16vec4(q) - f16vec4(1.0));
|
||||
}
|
||||
|
||||
layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ2_K {
|
||||
block_q2_K block;
|
||||
};
|
||||
|
|
@ -1406,9 +1368,6 @@ f16vec4 dequantFuncNVFP4_v(const in decodeBufNVFP4 bl, const in uint blockCoords
|
|||
#elif defined(DATA_A_Q8_0)
|
||||
#define dequantFuncA dequantFuncQ8_0
|
||||
#define dequantFuncA_v dequantFuncQ8_0_v
|
||||
#elif defined(DATA_A_TQ2_0)
|
||||
#define dequantFuncA dequantFuncTQ2_0
|
||||
#define dequantFuncA_v dequantFuncTQ2_0_v
|
||||
#elif defined(DATA_A_Q2_K)
|
||||
#define dequantFuncA dequantFuncQ2_K
|
||||
#define dequantFuncA_v dequantFuncQ2_K_v
|
||||
|
|
|
|||
|
|
@ -1,31 +0,0 @@
|
|||
#version 450
|
||||
|
||||
#include "dequant_head.glsl"
|
||||
|
||||
layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
|
||||
layout (binding = 1) writeonly buffer D {D_TYPE data_b[];};
|
||||
|
||||
void main() {
|
||||
[[unroll]] for (uint wgy = 0; wgy < 256; wgy++) {
|
||||
const uint i = gl_WorkGroupID.x * 256 + wgy;
|
||||
if (i >= p.nel / QUANT_K) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uint tid = gl_LocalInvocationID.x;
|
||||
const uint ip = tid / 32; // group 0,1 (128 elems each)
|
||||
const uint il = tid - 32 * ip; // byte in group 0..31
|
||||
|
||||
const uint y_idx = i * QUANT_K + 128 * ip + il;
|
||||
|
||||
const uint8_t qs = data_a[i].qs[32 * ip + il];
|
||||
|
||||
const FLOAT_TYPE d = FLOAT_TYPE(data_a[i].d);
|
||||
data_b[y_idx + 0] = D_TYPE(d * FLOAT_TYPE(int((qs >> 0) & 3) - 1));
|
||||
data_b[y_idx + 32] = D_TYPE(d * FLOAT_TYPE(int((qs >> 2) & 3) - 1));
|
||||
data_b[y_idx + 64] = D_TYPE(d * FLOAT_TYPE(int((qs >> 4) & 3) - 1));
|
||||
data_b[y_idx + 96] = D_TYPE(d * FLOAT_TYPE(int((qs >> 6) & 3) - 1));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
#version 450
|
||||
|
||||
#extension GL_EXT_control_flow_attributes : require
|
||||
|
||||
#define BLOCK_SIZE 64
|
||||
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout(push_constant) uniform Parameters {
|
||||
uint B;
|
||||
uint T;
|
||||
uint C;
|
||||
uint H;
|
||||
float scale;
|
||||
};
|
||||
|
||||
layout(binding = 0) readonly buffer KBuf { A_TYPE k[]; };
|
||||
layout(binding = 1) readonly buffer VBuf { A_TYPE v[]; };
|
||||
layout(binding = 2) readonly buffer QBuf { A_TYPE q[]; };
|
||||
layout(binding = 3) readonly buffer GBuf { A_TYPE g[]; };
|
||||
layout(binding = 4) readonly buffer StateBuf { A_TYPE state_in[]; };
|
||||
layout(binding = 5) buffer DstBuf { A_TYPE dst[]; };
|
||||
|
||||
shared A_TYPE _k[BLOCK_SIZE], _q[BLOCK_SIZE], _g[BLOCK_SIZE];
|
||||
|
||||
void main() {
|
||||
const uint head_size = BLOCK_SIZE;
|
||||
const uint batch_id = gl_WorkGroupID.x / H;
|
||||
const uint head_id = gl_WorkGroupID.x % H;
|
||||
const uint tid = gl_LocalInvocationID.x;
|
||||
|
||||
const uint state_size = C * head_size;
|
||||
const uint n_seq_tokens = T / B;
|
||||
|
||||
if (batch_id >= B || head_id >= H) {
|
||||
return;
|
||||
}
|
||||
|
||||
// state[i] holds column tid of this head's state matrix: S[i][tid]
|
||||
A_TYPE state[BLOCK_SIZE];
|
||||
[[unroll]] for (uint i = 0; i < head_size; i++) {
|
||||
state[i] = state_in[batch_id * state_size + head_id * head_size * head_size
|
||||
+ i * head_size + tid];
|
||||
}
|
||||
|
||||
const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid;
|
||||
const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid;
|
||||
|
||||
for (uint t = start_t; t < end_t; t += C) {
|
||||
barrier();
|
||||
_k[tid] = k[t];
|
||||
_q[tid] = q[t];
|
||||
_g[tid] = g[t];
|
||||
barrier();
|
||||
|
||||
const A_TYPE v_val = v[t];
|
||||
A_TYPE y = 0.0;
|
||||
|
||||
[[unroll]] for (uint i = 0; i < head_size; i += 4) {
|
||||
vec4 k_vec = vec4(_k[i], _k[i+1], _k[i+2], _k[i+3]);
|
||||
vec4 q_vec = vec4(_q[i], _q[i+1], _q[i+2], _q[i+3]);
|
||||
vec4 g_vec = vec4(_g[i], _g[i+1], _g[i+2], _g[i+3]);
|
||||
vec4 s_vec = vec4(state[i], state[i+1], state[i+2], state[i+3]);
|
||||
|
||||
vec4 kv = k_vec * v_val;
|
||||
|
||||
s_vec = s_vec * g_vec + kv;
|
||||
y += dot(q_vec, s_vec);
|
||||
|
||||
state[i] = s_vec.x;
|
||||
state[i+1] = s_vec.y;
|
||||
state[i+2] = s_vec.z;
|
||||
state[i+3] = s_vec.w;
|
||||
}
|
||||
|
||||
dst[t] = y * scale;
|
||||
}
|
||||
|
||||
[[unroll]] for (uint i = 0; i < head_size; i++) {
|
||||
dst[T * C + batch_id * state_size + head_id * head_size * head_size
|
||||
+ i * head_size + tid] = state[i];
|
||||
}
|
||||
}
|
||||
|
|
@ -1,102 +0,0 @@
|
|||
#version 450
|
||||
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
||||
|
||||
#include "mul_mat_vec_base.glsl"
|
||||
|
||||
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
||||
|
||||
// ternary TQ2_0: w = (q - 1) * d. Same qs group/level layout as q2_K, but a
|
||||
// single f16 scale per 256-block and no mins:
|
||||
// sum_e b_e * (q_e - 1) * d = d * (sum_e b_e * q_e - sum_e b_e)
|
||||
void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
||||
const uint y_idx = i * QUANT_K + y_offset;
|
||||
|
||||
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
||||
const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
|
||||
if (i >= num_blocks_per_row) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const uint32_t qs_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 8]) << 16);
|
||||
const vec4 qs_u32_0 = vec4(unpack8(qs_u32 & 0x03030303));
|
||||
const vec4 qs_u32_2 = vec4(unpack8((qs_u32 >> 2) & 0x03030303));
|
||||
const vec4 qs_u32_4 = vec4(unpack8((qs_u32 >> 4) & 0x03030303));
|
||||
const vec4 qs_u32_6 = vec4(unpack8((qs_u32 >> 6) & 0x03030303));
|
||||
|
||||
const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib0 + i].d);
|
||||
|
||||
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
||||
vec2 b0 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]);
|
||||
vec2 b16 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]);
|
||||
vec2 b32 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]);
|
||||
vec2 b48 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]);
|
||||
vec2 b64 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 32]);
|
||||
vec2 b80 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 40]);
|
||||
vec2 b96 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 48]);
|
||||
vec2 b112 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 56]);
|
||||
|
||||
FLOAT_TYPE sumq = FLOAT_TYPE(0.0);
|
||||
FLOAT_TYPE sumb = FLOAT_TYPE(0.0);
|
||||
[[unroll]] for (int l = 0; l < 2; ++l) {
|
||||
sumq = fma(FLOAT_TYPE(b0[l]), FLOAT_TYPE(qs_u32_0[l ]),
|
||||
fma(FLOAT_TYPE(b16[l]), FLOAT_TYPE(qs_u32_0[l+2]),
|
||||
fma(FLOAT_TYPE(b32[l]), FLOAT_TYPE(qs_u32_2[l ]),
|
||||
fma(FLOAT_TYPE(b48[l]), FLOAT_TYPE(qs_u32_2[l+2]),
|
||||
fma(FLOAT_TYPE(b64[l]), FLOAT_TYPE(qs_u32_4[l ]),
|
||||
fma(FLOAT_TYPE(b80[l]), FLOAT_TYPE(qs_u32_4[l+2]),
|
||||
fma(FLOAT_TYPE(b96[l]), FLOAT_TYPE(qs_u32_6[l ]),
|
||||
fma(FLOAT_TYPE(b112[l]), FLOAT_TYPE(qs_u32_6[l+2]), sumq))))))));
|
||||
sumb += FLOAT_TYPE(b0[l]) + FLOAT_TYPE(b16[l]) + FLOAT_TYPE(b32[l]) + FLOAT_TYPE(b48[l])
|
||||
+ FLOAT_TYPE(b64[l]) + FLOAT_TYPE(b80[l]) + FLOAT_TYPE(b96[l]) + FLOAT_TYPE(b112[l]);
|
||||
}
|
||||
temp[j][n] = fma(d, sumq - sumb, temp[j][n]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
||||
uint a_offset, b_offset, d_offset;
|
||||
get_offsets(a_offset, b_offset, d_offset);
|
||||
|
||||
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
||||
|
||||
// 16 threads are used to process each block
|
||||
const uint it_size = gl_WorkGroupSize.x/16;
|
||||
const uint tid = gl_LocalInvocationID.x;
|
||||
const uint itid = tid%16; // 0...15
|
||||
const uint ix = tid/16;
|
||||
|
||||
const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
|
||||
const uint v_in = itid - 8*v_im; // 0...7
|
||||
|
||||
const uint l0 = 2*v_in; // 0...15
|
||||
const uint q_offset = 32*v_im + l0;
|
||||
const uint y_offset = 128*v_im + l0;
|
||||
|
||||
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
||||
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
||||
temp[j][i] = FLOAT_TYPE(0);
|
||||
}
|
||||
}
|
||||
|
||||
for (uint i0 = 0; i0 < num_blocks_per_row; i0 += it_size)
|
||||
calc_superblock(a_offset, b_offset, v_im, q_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows);
|
||||
|
||||
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
||||
}
|
||||
|
||||
void main() {
|
||||
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
||||
|
||||
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
||||
if (first_row + NUM_ROWS <= p.stride_d) {
|
||||
compute_outputs(first_row, NUM_ROWS);
|
||||
} else {
|
||||
if (first_row >= p.stride_d) {
|
||||
return;
|
||||
}
|
||||
compute_outputs(first_row, p.stride_d - first_row);
|
||||
}
|
||||
}
|
||||
|
|
@ -119,13 +119,10 @@ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working wit
|
|||
#endif
|
||||
|
||||
#ifdef COOPMAT
|
||||
layout(constant_id = 12) const uint SHMEM_STRIDE_PAD = 4;
|
||||
layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false;
|
||||
#define SHMEM_STRIDE (BK / 2 + 4)
|
||||
#else
|
||||
const uint SHMEM_STRIDE_PAD = 1;
|
||||
const bool APPLY_SLM_A_RESHAPE = false;
|
||||
#define SHMEM_STRIDE (BK / 2 + 1)
|
||||
#endif
|
||||
#define SHMEM_STRIDE (BK / 2 + SHMEM_STRIDE_PAD)
|
||||
|
||||
shared FLOAT_TYPEV2 buf_a[BM * SHMEM_STRIDE];
|
||||
shared FLOAT_TYPEV2 buf_b[BN * SHMEM_STRIDE];
|
||||
|
|
@ -305,7 +302,7 @@ void main() {
|
|||
[[unroll]] for (uint i = 0; i < BK; i += TK) {
|
||||
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
|
||||
// Load from shared into cache
|
||||
coopMatLoad(cache_a, buf_a, a_shmem_index(warp_r * WM + cm_row * TM, i / 2), a_shmem_stride(), gl_CooperativeMatrixLayoutRowMajor);
|
||||
coopMatLoad(cache_a, buf_a, (warp_r * WM + cm_row * TM) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutRowMajor);
|
||||
|
||||
[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
|
||||
coopMatLoad(cache_b, buf_b, (warp_c * WN + cm_col * TN) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue