vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-09 22:05:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix-singleton

Browse source
This commit is contained in:
Atream 2025-03-14 04:16:53 +00:00
parent 7f57769c23
commit 6f43bbe55f
4 changed files with 11 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

4
ktransformers/operators/experts.py
View file

@ -120,7 +120,7 @@ class KExpertsCPU(KExpertsBase):
output_gpu_map:dict = {} # Manage output tensor buffer on different gpu
#stream_map:dict = {} # Manage cuda stream on different gpu
#gguf_loader:GGUFLoader = None
CPU_INFER = CPUInfer(Config().cpu_infer)
CPU_INFER = None
def __init__(
self,
key: str,
@ -133,6 +133,8 @@ class KExpertsCPU(KExpertsBase):
**kwargs
):
super().__init__(key, gguf_loader, config, orig_module, device, **kwargs)
if KExpertsCPU.CPU_INFER is None:
KExpertsCPU.CPU_INFER = CPUInfer(Config().cpu_infer)
#if KExpertsCPU.gguf_loader is None:
# KExpertsCPU.gguf_loader = GGUFLoader("/mnt/data/model/DeepseekV3-q4km-gguf")
self.gguf_loader = gguf_loader