vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

rpc: fix register position (#11424)

Browse source 
Signed-off-by: thxCode <thxcode0824@gmail.com>
This commit is contained in:
Frank Mai 2025-01-26 23:20:34 +08:00 • committed by GitHub
parent 2cc9b8c32c
commit 1d8ee06000
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 13 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/llama.cpp
View file

@ -9405,6 +9405,7 @@ static struct llama_model * llama_model_load_from_file_impl(
model->devices.push_back(*dev);
}
} else {
std::vector<ggml_backend_dev_t> rpc_servers;
// use all available devices
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
@ -9415,10 +9416,19 @@ static struct llama_model * llama_model_load_from_file_impl(
break;
case GGML_BACKEND_DEVICE_TYPE_GPU:
model->devices.push_back(dev);
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
if (ggml_backend_reg_name(reg) == std::string("RPC")) {
rpc_servers.push_back(dev);
} else {
model->devices.push_back(dev);
}
break;
}
}
// add RPC servers at the front of the list
if (!rpc_servers.empty()) {
model->devices.insert(model->devices.begin(), rpc_servers.begin(), rpc_servers.end());
}
}
// if using single GPU mode, remove all except the main GPU