vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-02 04:33:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

opencl: skip the Adreno KQ/KQV image kernels for multi-stream batches (#26189)

Browse source 
The Adreno KQ/KQV image1d kernels (ggml_cl_mul_mat_kq_kqv_adreno) ignore
dim 3 entirely: the sub-buffer covers only nb02*ne02 bytes and the kernel
receives no ne03/ne13/nb03/nb13 arguments. With the unified KV cache,
multi-sequence batches (e.g. llama-perplexity with its default -b 2048,
n_seq=4, or a multi-slot llama-server) present KQ/KQV as 4D tensors with
ne3 = n_stream, so every stream past the first reads the first stream's
K/V and produces garbage. Flash attention masks the bug where it is
enabled; devices where FA is declined (e.g. Adreno 740) hit it with
default settings.

Route ne03/ne13 > 1 to the general path, which handles dim 3, and honor
view_offs when creating the sub-buffers (currently always 0 for tensors
reaching this function, but the function would silently misread any
future view).

Llama-3.2-1B-Instruct Q4_0, wiki.test.raw, 8 chunks, -ngl 99:
- Adreno 740, default:            PPL 1817.64 -> 15.61
- Adreno 740, -fa 0:              PPL 1941.64 -> 15.61
- Adreno 840, -fa 0:              PPL 1943.90 -> 15.50
- single-stream (-b 512) results unchanged (15.6090)
- test-backend-ops -o MUL_MAT on 740: identical before/after (909 OK,
  12 pre-existing q6_K failures)
This commit is contained in:
Hongqiang Wang 2026-07-28 11:04:42 -07:00 • committed by GitHub
parent 7e1e28cae3
commit 8190848bb3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
ggml/src/ggml-opencl/ggml-opencl.cpp
View file

@ -15675,7 +15675,7 @@ static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_ten
// <--------------------------------------------> //
extra0 = src0->view_src ? (ggml_tensor_extra_cl *)src0->view_src->extra : (ggml_tensor_extra_cl *)src0->extra;
region.origin = (extra0->offset);
region.origin = (extra0->offset + src0->view_offs);
if (nb01 > nb02) {
// KQ
region.size = nb01 * ne01;
@ -15691,7 +15691,7 @@ static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_ten
// create sub-buffer for B
// <--------------------------------------------> //
region.origin = (extra1->offset);
region.origin = (extra1->offset + src1->view_offs);
region.size = nb10 * ne10 * ne11 * ne12;
B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
CL_CHECK(status);
@ -15712,7 +15712,7 @@ static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_ten
// create sub-buffer for output C
// <--------------------------------------------> //
region.origin = (extrad->offset);
region.origin = (extrad->offset + dst->view_offs);
region.size = ne0 * ne1 * dst->ne[2] * dst->nb[0]; // size of C in bytes
D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
CL_CHECK(status);
@ -18591,6 +18591,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){
if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && (ne12 % ne02) == 0 &&
// the KQ/KQV image kernels do not handle dim 3 (multi-stream batches)
ne03 == 1 && ne13 == 1 &&
// dst is wrapped with image1d_buffer, the size limit applies, also src0
(ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4 <= backend_ctx->image_max_buffer_size)) {
// For KQ