vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-09 13:55:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix according to upstream changes

Browse source
This commit is contained in:
lazymio 2025-02-27 18:11:35 +08:00
parent 26f7b4af11
commit b121ca4df8
No known key found for this signature in database
GPG key ID: DFF27E34A47CB873
1 changed files with 1 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
ktransformers/server/backend/interfaces/ktransformers.py
View file

@ -201,10 +201,9 @@ class KTransformersInterface(TransformersInterface):
else:
logits = self.model(inputs_embeds=inputs_embeds, return_dict=False)[0]
self.prepare_logits_wrapper(input_ids, device, temperature, top_p)
if flashinfer_enabled:
MLAWrapperSingleton.reset_buffer()
self.prepare_logits_wrapper(input_ids, device)
self.prepare_logits_wrapper(input_ids, device, temperature, top_p)
next_token = self.logits_to_token(logits[0, -1, :])
yield self.append_new_tokens(next_token)