vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2026-05-02 13:41:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

smallthink run

Browse source
This commit is contained in:
qiyuxinlin 2025-07-24 15:08:29 +00:00
parent 590fcb41cd
commit 71c1d4eed7
7 changed files with 123 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

14
ktransformers/configs/config.yaml
View file

@ -21,12 +21,12 @@ user:
model:
# type: transformers
# type: balance_serve
type: ktransformers
type: balance_serve
# type: ktransformers
name: DeepSeek-Coder-V2-Instruct
path: deepseek-ai/DeepSeek-V2-Lite-Chat
gguf_path: ./DeepSeek-V2-Lite-Chat-GGUF
name: SmallthinkerForCausalLM
path: /mnt/data/models/Smallthinker-21B
gguf_path: /mnt/data/models/Smallthinker-21B
device: cuda:0
cache_lens: 16384
@ -67,7 +67,7 @@ attn:
page_size: 256
chunk_size: 256
kvc2:
gpu_only: false
gpu_only: true
utilization_percentage: 1.0
cpu_memory_size_GB: 500
disk_path: /mnt/data/kvc
disk_path: /home/wjh/kvc