vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-07 21:19:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix deduplicate_and_sort cudagraphs

Browse source
This commit is contained in:
qiyuxinlin 2025-05-15 04:09:34 +00:00
parent d35d61f6a1
commit b40f13abeb
2 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ktransformers/operators/experts.py
View file

@ -45,7 +45,7 @@ def generate_cuda_graphs(chunk_size: int) -> list:
base_list = [1, 2, 3, Config().max_batch_size, 64, 256, 512, chunk_size]
if chunk_size <= 1024:
return base_list
return deduplicate_and_sort(base_list)
multiples = [i for i in range(1024, chunk_size + 1, 1024)]

2
ktransformers/server/balance_serve/inference/model_runner.py
View file

@ -45,7 +45,7 @@ def generate_cuda_graphs(chunk_size: int) -> list:
base_list = [1, 2, 3, Config().max_batch_size, 64, 256, 512, chunk_size]
if chunk_size <= 1024:
return base_list
return deduplicate_and_sort(base_list)
multiples = [i for i in range(1024, chunk_size + 1, 1024)]