vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-07-10 01:18:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix stale tensor-split params for draft models (#24814)

Browse source 
* meta: fix tensor split metadata for GQA attention

* Tidied the code a bit to match existing style

* Revert "Tidied the code a bit to match existing style"

This reverts commit b90c6c6300091fe09e2350a3d4edcfcf15db8d2e.

* Reverted the ggml-backend-meta asset hack.
This commit is contained in:
Al G 2026-07-05 19:39:36 +01:00 • committed by GitHub
parent 3e5036fbfb
commit 2da6686176
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 8 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/llama-model.cpp
View file

@ -1012,9 +1012,17 @@ struct llama_model::impl {
std::vector<layer_dev> dev_layer;
bool has_tensor_overrides;
std::vector<float> tensor_split_owned;
};
llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {
if (params.tensor_split != nullptr) {
// llama_model_params stores tensor_split as a borrowed pointer, but the model
// may need it later for tensor-parallel KV-cache split metadata.
pimpl->tensor_split_owned.assign(params.tensor_split, params.tensor_split + llama_max_devices());
this->params.tensor_split = pimpl->tensor_split_owned.data();
}
pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern;
}