vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-10 15:29:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

GLM4 and SmallThinker

Browse source
This commit is contained in:
qiyuxinlin 2025-07-25 16:56:36 +00:00
parent c7307aa0ae
commit 9e1560bb82
7 changed files with 58 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

14
ktransformers/server/args.py
View file

@ -138,17 +138,11 @@ class ArgumentParser:
self.cfg.server_ip = args.host
self.cfg.server_port = args.port
self.cfg.user_force_think = args.force_think
if args.model_name == "Qwen3MoeForCausalLM":
model_config = Qwen3MoeConfig.from_pretrained(args.model_dir, trust_remote_code=True)
elif args.model_name == "Glm4MoeForCausalLM":
model_config = Glm4MoeConfig.from_pretrained(args.model_dir, trust_remote_code=True)
elif args.model_name == "SmallThinkerForCausalLM":
model_config = SmallthinkerConfig.from_pretrained(args.model_dir, trust_remote_code=True)
model_config._attn_implementation = "eager"
else:
try:
model_config = AutoConfig.from_pretrained(args.model_dir, trust_remote_code=True)
except:
try:
model_config = AutoConfig.from_pretrained(args.model_dir, trust_remote_code=True)
model_config = Glm4MoeConfig.from_pretrained(args.model_dir, trust_remote_code=True)
except:
raise ValueError(f"Model {args.model_name} not supported. Please check your model directory or model name.")