mirror of
https://github.com/kvcache-ai/ktransformers.git
synced 2025-09-10 15:29:39 +00:00
optimize gguf dequant, save mem, support Q2_K
use marlin for lm_head, lm_head only calc last token for prefill extend context window to 19K for DeepSeek-V3/R1 within 24GB VRAM
This commit is contained in:
parent
7e1fe256c8
commit
5ec33d046d
27 changed files with 435 additions and 259 deletions
|
@ -1699,7 +1699,7 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel):
|
|||
)
|
||||
|
||||
hidden_states = outputs[0]
|
||||
logits = self.lm_head(hidden_states.to(self.lm_head.weight.device))
|
||||
logits = self.lm_head(hidden_states[:,-1:,:])
|
||||
logits = logits.float()
|
||||
|
||||
loss = None
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue