vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-11 07:44:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

⚡ update force_think config

Browse source
This commit is contained in:
liam 2025-02-12 12:10:16 +08:00
parent e536e1420d
commit 6f3a39be08
2 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ktransformers/server/args.py
View file

@ -90,7 +90,7 @@ class ArgumentParser:
# user config # user config
parser.add_argument("--user_secret_key", type=str, default=self.cfg.user_secret_key) parser.add_argument("--user_secret_key", type=str, default=self.cfg.user_secret_key)
parser.add_argument("--user_algorithm", type=str, default=self.cfg.user_algorithm) parser.add_argument("--user_algorithm", type=str, default=self.cfg.user_algorithm)
parser.add_argument("--force_think", type=bool, default=self.cfg.force_think) parser.add_argument("--force_think", type=bool, default=self.cfg.user_force_think)
# web config # web config
parser.add_argument("--web_cross_domain", type=bool, default=self.cfg.web_cross_domain) parser.add_argument("--web_cross_domain", type=bool, default=self.cfg.web_cross_domain)

4
ktransformers/server/backend/interfaces/transformers.py
View file

@ -324,7 +324,7 @@ class TransformersInterface(BackendInterfaceBase):
#input_ids = torch.tensor([[6366]], device=input_ids.device) #input_ids = torch.tensor([[6366]], device=input_ids.device)
else: else:
raise ValueError("local_messages should be List or str") raise ValueError("local_messages should be List or str")
if Config().force_think: if Config().user_force_think:
token_thinks = torch.tensor([self.tokenizer.encode("<think>\\n",add_special_tokens=False)]) token_thinks = torch.tensor([self.tokenizer.encode("<think>\\n",add_special_tokens=False)])
input_ids = torch.cat( input_ids = torch.cat(
[input_ids, token_thinks], dim=1 [input_ids, token_thinks], dim=1
@ -333,7 +333,7 @@ class TransformersInterface(BackendInterfaceBase):
self.profiler.pause_timer("tokenize") self.profiler.pause_timer("tokenize")
self.profiler.create_and_start_timer("prefill") self.profiler.create_and_start_timer("prefill")
if Config().force_think: if Config().user_force_think:
print("<think>\n") print("<think>\n")
yield "<think>\n" yield "<think>\n"
for t in self.prefill(input_ids, self.check_is_new(thread_id)): for t in self.prefill(input_ids, self.check_is_new(thread_id)):