vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-19 13:25:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

old gpu fattn fix

Browse source
This commit is contained in:
Concedo 2025-08-25 11:06:48 +08:00
parent b0fe1ac01a
commit d2c9fdddcb
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
ggml/src/ggml-cuda/fattn.cu
View file

@ -415,6 +415,11 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
return BEST_FATTN_KERNEL_WMMA_F16;
}
//kcpp: always force WMMA for older gpus, fix issues like "FlashAttention without tensor cores only supports head sizes 64 and 128."
if (ggml_cuda_highest_compiled_arch(cc) <= GGML_CUDA_CC_TURING || cc == GGML_CUDA_CC_TURING) {
return BEST_FATTN_KERNEL_WMMA_F16;
}
// If there is no suitable kernel for tensor cores or small batch sizes, use the generic kernel for large batch sizes:
if (prec == GGML_PREC_DEFAULT && fast_fp16_available(cc)) {
return BEST_FATTN_KERNEL_TILE_F16;