vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-06 20:49:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

cpuinfer: filter repeated backend instantiation

Browse source
This commit is contained in:
Lander-Hatsune 2025-03-10 22:03:04 +08:00
parent 09c043d8a6
commit d166fb9f6e
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
ktransformers/operators/cpuinfer.py
View file

@ -727,8 +727,13 @@ class CPUInferKVCache:
class CPUInfer:
cpuinfer = None
cur_backend_thread_num = 0
def __init__(self, thread_num):
CPUInfer.cpuinfer = cpuinfer_ext.CPUInfer(thread_num)
if thread_num > CPUInfer.cur_backend_thread_num:
CPUInfer.cur_backend_thread_num = thread_num
del CPUInfer.cpuinfer
CPUInfer.cpuinfer = cpuinfer_ext.CPUInfer(thread_num)
def submit(self, task):
CPUInfer.cpuinfer.submit(task)