mirror of
https://github.com/kvcache-ai/ktransformers.git
synced 2026-04-28 11:49:51 +00:00
toy support for experts on GPU, no CUDA Graph
This commit is contained in:
parent
1548c99234
commit
c189d55bd1
6 changed files with 199 additions and 65 deletions
|
|
@ -713,6 +713,8 @@
|
|||
generate_device: "cuda:7"
|
||||
prefill_device: "cuda:7"
|
||||
|
||||
# don't inject lm_head if already inject marlin experts
|
||||
|
||||
# For final modules (model.norm and lm_head), ensure they are on GPU 7 (as in your original config)
|
||||
- match:
|
||||
name: "(^model\\.layers\\.(4[5-9]|5[0-9]|60)\\.)|(^model\\.norm)|(^lm_head)"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue