vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-06 04:30:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

modify moeinfer param

Browse source
This commit is contained in:
Azure 2025-02-06 14:07:38 +00:00
parent ee24a27001
commit 027b11266c
2 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ktransformers/local_chat.py
View file

@ -81,8 +81,10 @@ def local_chat():
content = "hi" content = "hi"
else: else:
content = open(config.prompt_file, "r").read() content = open(config.prompt_file, "r").read()
print("User: ", content)
elif os.path.isfile(content): elif os.path.isfile(content):
content = open(content, "r").read() content = open(content, "r").read()
print("User: ", content)
messages = his_content + [{"role": "user", "content": content}] messages = his_content + [{"role": "user", "content": content}]
async def async_inference(messages): async def async_inference(messages):

2
ktransformers/operators/experts.py
View file

@ -163,7 +163,7 @@ class KExpertsCPU(KExpertsBase):
self.config.hidden_size, self.config.hidden_size,
self.config.moe_intermediate_size, self.config.moe_intermediate_size,
64, 64,
10, 1024,
1024, 1024,
gate_ptr, gate_ptr,
up_ptr, up_ptr,