vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-11 15:54:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

revert using FP16

Browse source
This commit is contained in:
rnwang04 2025-07-01 14:24:27 +08:00
parent 495ae37478
commit 5b5deda420
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
ktransformers/local_chat.py
View file

@ -79,8 +79,9 @@ def local_chat(
if mode == 'long_context': if mode == 'long_context':
assert config.architectures[0] == "LlamaForCausalLM", "only LlamaForCausalLM support long_context mode" assert config.architectures[0] == "LlamaForCausalLM", "only LlamaForCausalLM support long_context mode"
torch.set_default_dtype(torch.float16) torch.set_default_dtype(torch.float16)
elif xpu_fp16_model(config): # elif xpu_fp16_model(config):
torch.set_default_dtype(torch.float16) # # using FP16 may cause accuracy issues, triggering core dumped during runtime
# torch.set_default_dtype(torch.float16)
else: else:
torch.set_default_dtype(config.torch_dtype) torch.set_default_dtype(config.torch_dtype)