vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

no repacking for avx2 for kcpp because it breaks 4_0_4_4 quants

Browse source
This commit is contained in:
Concedo 2025-03-22 00:33:27 +08:00
parent 7030ebf401
commit ae670dbe0e
2 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

10
ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp
View file

@ -5642,11 +5642,11 @@ static const ggml::cpu::tensor_traits * ggml_aarch64_get_optimal_repack_type(con
}
}
} else if (cur->type == GGML_TYPE_Q4_K) {
if (ggml_cpu_has_avx2()) {
if (cur->ne[1] % 8 == 0) {
return &ggml::cpu::aarch64::q4_K_8x8_q8_K;
}
}
// if (ggml_cpu_has_avx2()) { //we shall just use the regular avx2 handling, no repacking otherwise massive slowdown with gpu
// if (cur->ne[1] % 8 == 0) {
// return &ggml::cpu::aarch64::q4_K_8x8_q8_K;
// }
// }
} else if (cur->type == GGML_TYPE_IQ4_NL) {
if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {
if (cur->ne[1] % 4 == 0) {

2
src/llama-model.cpp
View file

@ -286,7 +286,7 @@ static buft_list_t make_cpu_buft_list(const std::vector<ggml_backend_dev_t> & de
// add extra buffer types, only if no GPU device is present
// ref: https://github.com/ggml-org/llama.cpp/issues/12481#issuecomment-2743136094
if (!has_gpu_device) {
if (true) { //kcpp needs this to be true, otherwise 4_0_4_4 quants will break. avx2 repacking dont affect us cause we disabled it
auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);
auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)