vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-09 13:55:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix top_p = 0 bug

Browse source
This commit is contained in:
dongjw 2025-04-01 20:38:33 +08:00
parent aeabd783b0
commit 5c7ed7b579
2 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ktransformers/server/backend/interfaces/balance_serve.py
View file

@ -368,6 +368,8 @@ class BalanceServeInterface(BackendInterfaceBase):
stop_criteria = [self.tokenizer.encode(self.tokenizer.eos_token, add_special_tokens=False),self.tokenizer.encode("<|im_end|>")] stop_criteria = [self.tokenizer.encode(self.tokenizer.eos_token, add_special_tokens=False),self.tokenizer.encode("<|im_end|>")]
query_add.stop_criteria = stop_criteria query_add.stop_criteria = stop_criteria
query_add.sample_options.temperature = temperature query_add.sample_options.temperature = temperature
if top_p == 0:
top_p = 0.0001
query_add.sample_options.top_p = top_p query_add.sample_options.top_p = top_p
query_add.estimated_length = min(self.args.cache_lens, query_length+self.args.max_new_tokens) query_add.estimated_length = min(self.args.cache_lens, query_length+self.args.max_new_tokens)
query_id = self.sched_client.add_query(query_add) query_id = self.sched_client.add_query(query_add)

2
ktransformers/server/backend/interfaces/transformers.py
View file

@ -208,6 +208,8 @@ class TransformersInterface(BackendInterfaceBase):
temperature = self.model.generation_config.temperature temperature = self.model.generation_config.temperature
if top_p is None: if top_p is None:
top_p = self.model.generation_config.top_p top_p = self.model.generation_config.top_p
if top_p == 0:
top_p = 0.0001
generation_config, model_kwargs = self.model._prepare_generation_config( generation_config, model_kwargs = self.model._prepare_generation_config(
None, max_length=self.args.max_new_tokens, None, max_length=self.args.max_new_tokens,
do_sample=True, do_sample=True,