vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-10 15:29:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix temperature=0, flashinfer sample error

Browse source
This commit is contained in:
dongjw 2025-04-07 12:30:47 +08:00
parent aac0c91d02
commit ec03bcbd7f
1 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ktransformers/server/backend/interfaces/balance_serve.py
View file

@ -367,6 +367,8 @@ class BalanceServeInterface(BackendInterfaceBase):
#@TODO add server #@TODO add server
stop_criteria = [self.tokenizer.encode(self.tokenizer.eos_token, add_special_tokens=False),self.tokenizer.encode("<|im_end|>")] stop_criteria = [self.tokenizer.encode(self.tokenizer.eos_token, add_special_tokens=False),self.tokenizer.encode("<|im_end|>")]
query_add.stop_criteria = stop_criteria query_add.stop_criteria = stop_criteria
if temperature == 0:
temperature = 0.0001
query_add.sample_options.temperature = temperature query_add.sample_options.temperature = temperature
if top_p == 0: if top_p == 0:
top_p = 0.0001 top_p = 0.0001