vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-09 13:55:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable support for Intel XPU devices, add support for DeepSeek V2/V3 first

Browse source
This commit is contained in:
rnwang04 2025-05-14 14:28:22 +00:00
parent 333351c7c8
commit 142fb7ce6c
22 changed files with 673 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

8
ktransformers/operators/models.py
View file

@ -647,6 +647,13 @@ class KDeepseekV2Model(BaseInjectedModule):
if position_ids is None:
position_ids = cache_position.unsqueeze(0)
if inputs_embeds.device.type == "xpu" and position_ids is not None:
cos, sin = self.layers[0].self_attn.rotary_emb(inputs_embeds,
position_ids)
position_embeddings = (cos, sin)
else:
position_embeddings = None
if per_layer_prefill_flag:
causal_mask = None
else:
@ -737,6 +744,7 @@ class KDeepseekV2Model(BaseInjectedModule):
output_attentions=output_attentions,
use_cache=use_cache,
cache_position=cache_position,
position_embeddings=position_embeddings,
)
t5 = time.time()
if per_layer_prefill_flag: