vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-07-09 17:08:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'upstream' into concedo_experimental

Browse source 
# Conflicts:
#	ggml/src/ggml-cuda/fattn.cu
This commit is contained in:
Concedo 2026-07-07 21:00:37 +08:00
commit dad4ff5737
1 changed files with 22 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

38
ggml/src/ggml-cuda/fattn.cu
View file

@ -338,6 +338,26 @@ enum best_fattn_kernel {
BEST_FATTN_KERNEL_MMA_F16 = 400, BEST_FATTN_KERNEL_MMA_F16 = 400,
}; };
static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) {
switch (type) {
case GGML_TYPE_F32:
case GGML_TYPE_F16:
return true;
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
#ifndef GGML_CUDA_FA_ALL_QUANTS
return false;
#endif // GGML_CUDA_FA_ALL_QUANTS
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q5_1: // kcpp: support q5_1 kv
case GGML_TYPE_Q8_0:
case GGML_TYPE_BF16:
return true;
default:
return false;
}
}
static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const ggml_tensor * dst) { static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const ggml_tensor * dst) {
#ifndef FLASH_ATTN_AVAILABLE #ifndef FLASH_ATTN_AVAILABLE
GGML_UNUSED(device); GGML_UNUSED(dst); GGML_UNUSED(device); GGML_UNUSED(dst);
@ -428,22 +448,8 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
} }
#endif // GGML_CUDA_FA_ALL_QUANTS #endif // GGML_CUDA_FA_ALL_QUANTS
switch (K->type) { if (!ggml_cuda_fattn_kv_type_supported(K->type) || !ggml_cuda_fattn_kv_type_supported(V->type)) {
case GGML_TYPE_F32: return BEST_FATTN_KERNEL_NONE;
case GGML_TYPE_F16:
break;
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
#ifndef GGML_CUDA_FA_ALL_QUANTS
return BEST_FATTN_KERNEL_NONE;
#endif // GGML_CUDA_FA_ALL_QUANTS
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q5_1: //kcpp: support q5_1 kv
case GGML_TYPE_Q8_0:
case GGML_TYPE_BF16:
break;
default:
return BEST_FATTN_KERNEL_NONE;
} }
if (mask && mask->ne[2] != 1) { if (mask && mask->ne[2] != 1) {