vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-07 04:59:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1305 from kvcache-ai/update-readme

Browse source 
fix deduplicate_and_sort cudagraphs
This commit is contained in:
wang jiahao 2025-05-15 12:10:20 +08:00 • committed by GitHub
commit 8caecf37d8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ktransformers/operators/experts.py
View file

@ -45,7 +45,7 @@ def generate_cuda_graphs(chunk_size: int) -> list:
base_list = [1, 2, 3, Config().max_batch_size, 64, 256, 512, chunk_size] base_list = [1, 2, 3, Config().max_batch_size, 64, 256, 512, chunk_size]
if chunk_size <= 1024: if chunk_size <= 1024:
return base_list return deduplicate_and_sort(base_list)
multiples = [i for i in range(1024, chunk_size + 1, 1024)] multiples = [i for i in range(1024, chunk_size + 1, 1024)]

2
ktransformers/server/balance_serve/inference/model_runner.py
View file

@ -45,7 +45,7 @@ def generate_cuda_graphs(chunk_size: int) -> list:
base_list = [1, 2, 3, Config().max_batch_size, 64, 256, 512, chunk_size] base_list = [1, 2, 3, Config().max_batch_size, 64, 256, 512, chunk_size]
if chunk_size <= 1024: if chunk_size <= 1024:
return base_list return deduplicate_and_sort(base_list)
multiples = [i for i in range(1024, chunk_size + 1, 1024)] multiples = [i for i in range(1024, chunk_size + 1, 1024)]