vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-10 06:14:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #719 from kvcache-ai/fix-use-generation-json

Browse source 
use generation config from json file in official repo
This commit is contained in:
Atream 2025-02-27 19:49:41 +08:00 • committed by GitHub
commit 85e2cc7bf4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 57 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

5
ktransformers/util/utils.py
View file

@ -183,8 +183,9 @@ def prefill_and_generate(model, tokenizer, inputs, max_new_tokens=10000, use_cud
inputs_embeds = inputs_embeds, cache_position=cache_position, past_key_values=past_key_values, return_dict=False, use_cache=True
)[0][:,-1,:].unsqueeze(0).clone().to(torch_device)
generation_config, model_kwargs = model._prepare_generation_config(
None, max_length=max_new_tokens,
do_sample=True, top_k=5, top_p=0.85, temperature=0.1 # change this to modify generate config
None, do_sample=True
# change this to modify generate config
#top_k=5, top_p=0.85, temperature=0.1
)
try: # transformers==4.43
logits_warper = (