From d166fb9f6e94483bf0cf222eaeb95fd1a04df9d8 Mon Sep 17 00:00:00 2001 From: Lander-Hatsune Date: Mon, 10 Mar 2025 22:03:04 +0800 Subject: [PATCH] cpuinfer: filter repeated backend instantiation --- ktransformers/operators/cpuinfer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ktransformers/operators/cpuinfer.py b/ktransformers/operators/cpuinfer.py index 74b8fa8..bf66efb 100644 --- a/ktransformers/operators/cpuinfer.py +++ b/ktransformers/operators/cpuinfer.py @@ -727,8 +727,13 @@ class CPUInferKVCache: class CPUInfer: cpuinfer = None + cur_backend_thread_num = 0 + def __init__(self, thread_num): - CPUInfer.cpuinfer = cpuinfer_ext.CPUInfer(thread_num) + if thread_num > CPUInfer.cur_backend_thread_num: + CPUInfer.cur_backend_thread_num = thread_num + del CPUInfer.cpuinfer + CPUInfer.cpuinfer = cpuinfer_ext.CPUInfer(thread_num) def submit(self, task): CPUInfer.cpuinfer.submit(task)