vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-10 15:29:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix-singleton

Browse source
This commit is contained in:
Atream 2025-03-14 04:16:53 +00:00
parent 7f57769c23
commit 6f43bbe55f
4 changed files with 11 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

5
ktransformers/operators/dynamic_attention.py
View file

@ -26,6 +26,7 @@ import json
class DynamicScaledDotProductAttention:
remaining_length: int
cpu_infer = None
def __init__(
self,
@ -180,7 +181,9 @@ class DynamicScaledDotProductAttention:
self.preselect_block_num = 0 # block_num before preselect
self.evict_tokens = 0
self.cpu_infer = CPUInfer(threads_num)
if DynamicScaledDotProductAttention.cpu_infer is None:
DynamicScaledDotProductAttention.cpu_infer = CPUInfer(threads_num)
self.cpu_infer = DynamicScaledDotProductAttention.cpu_infer
self.local_thread = CPUInferKVCache(
self.layer_num,
self.kv_head_num,