vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-08 13:39:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1409 from rnwang04/fix_fp16

Browse source 
revert using FP16 in XPU
This commit is contained in:
aubreyli 2025-07-01 15:00:41 +08:00 • committed by GitHub
commit f96aab3c85
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 3 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ktransformers/local_chat.py
View file

@ -79,8 +79,9 @@ def local_chat(
if mode == 'long_context': if mode == 'long_context':
assert config.architectures[0] == "LlamaForCausalLM", "only LlamaForCausalLM support long_context mode" assert config.architectures[0] == "LlamaForCausalLM", "only LlamaForCausalLM support long_context mode"
torch.set_default_dtype(torch.float16) torch.set_default_dtype(torch.float16)
elif xpu_fp16_model(config): # elif xpu_fp16_model(config):
torch.set_default_dtype(torch.float16) # # using FP16 may cause accuracy issues, triggering core dumped during runtime
# torch.set_default_dtype(torch.float16)
else: else:
torch.set_default_dtype(config.torch_dtype) torch.set_default_dtype(config.torch_dtype)