vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-27 09:13:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

common: share thread pools when n_threads differ (#27138)

Browse source
This commit is contained in:
Niklas Wenzel 2026-08-18 16:23:43 +02:00 • committed by GitHub
parent 0596704284
commit 04b569142d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 15 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

16
common/common.cpp
View file

@ -1750,6 +1750,18 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo
return tpp;
}
namespace {
bool can_share_threadpool(const ggml_threadpool_params & tpp1, const ggml_threadpool_params & tpp2) {
// n_threads does not matter -> we'll use what's larger
ggml_threadpool_params tpp_comparison = tpp1;
tpp_comparison.n_threads = tpp2.n_threads;
return ggml_threadpool_params_match(&tpp_comparison, &tpp2);
}
} // namespace
common_threadpools::~common_threadpools() {
if (!free_fn) {
return;
@ -1778,7 +1790,9 @@ void common_threadpools::init(llama_context * ctx, const common_params & params)
struct ggml_threadpool_params tpp =
ggml_threadpool_params_from_cpu_params(params.cpuparams);
if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
if (can_share_threadpool(tpp, tpp_batch)) {
tpp.n_threads = std::max(tpp.n_threads, tpp_batch.n_threads);
} else {
threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
if (!threadpool_batch) {
COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads);