vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2026-04-30 21:00:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

✨: refactor local_chat and fix message slice bug in server

Browse source
This commit is contained in:
liam 2024-10-21 22:49:05 +08:00
parent 43fc7f44a6
commit dd1d8667f3
13 changed files with 549 additions and 405 deletions
Download patch file Download diff file Expand all files Collapse all files

14
ktransformers/configs/config.yaml
View file

@ -7,7 +7,7 @@ log:
server:
ip: 0.0.0.0
port: 12456
port: 10002
db:
type: "sqllite"
@ -24,10 +24,13 @@ model:
type: ktransformers
name: DeepSeek-Coder-V2-Instruct
path: /mnt/data/model/DeepSeek-Coder-V2-Instruct/
gguf_path: /mnt/data/model/DeepSeek-Coder-V2-GGUF-WJH/
# path: /mnt/data/model/DeepSeek-Coder-V2-Instruct/
path: deepseek-ai/DeepSeek-V2-Lite-Chat
# gguf_path: /mnt/data/model/DeepSeek-Coder-V2-GGUF-WJH/
gguf_path: ./DeepSeek-V2-Lite-Chat-GGUF
device: cuda:0
cache_lens: 8192
web:
mount: False
@ -50,4 +53,7 @@ long_context:
head_select_mode: SHARED
preselect_block_count: 32
layer_step: 1
token_step: 100
token_step:
local_chat:
prompt_file: "./ktransformers/p.txt"