vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-09 22:05:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into feat-chunk-prefill-flashinfer

Browse source
This commit is contained in:
Atream 2025-03-01 11:35:09 +00:00
commit fa03ea48dd
3 changed files with 10 additions and 146 deletions
Download patch file Download diff file Expand all files Collapse all files

6
ktransformers/server/backend/interfaces/transformers.py
View file

@ -328,6 +328,12 @@ class TransformersInterface(BackendInterfaceBase):
@torch.no_grad
def generate(self):
self.args.max_new_tokens = min(self.args.max_new_tokens, self.args.cache_lens - self.seq_length)
if(self.args.max_new_tokens <= 0):
logger.warning("max_new_tokens is less than 0")
yield self.streamer.end()
return
logger.info(f"max_new_tokens: {self.args.max_new_tokens}")
self.profiler.set_counter("decode", 0)
for i in range(1, self.args.max_new_tokens):