vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2025-09-09 13:55:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert repetition_penalty as it is not in API spec

Browse source
This commit is contained in:
lazymio 2025-02-24 21:30:03 +08:00
parent 05ad288453
commit 76487c4dcb
No known key found for this signature in database
GPG key ID: DFF27E34A47CB873
5 changed files with 12 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

4
ktransformers/server/api/openai/endpoints/chat.py
View file

@ -28,13 +28,13 @@ async def chat_completion(request:Request,create:ChatCompletionCreate):
if create.stream:
async def inner():
chunk = ChatCompletionChunk(id=id,object='chat.completion.chunk',created=int(time()))
async for token in interface.inference(input_message,id,create.temperature,create.top_p,create.repetition_penalty):
async for token in interface.inference(input_message,id,create.temperature,create.top_p):
chunk.set_token(token)
yield chunk
return chat_stream_response(request,inner())
else:
comp = ChatCompletionObject(id=id,object='chat.completion',created=int(time()))
comp.usage = Usage(completion_tokens=1, prompt_tokens=1, total_tokens=2)
async for token in interface.inference(input_message,id,create.temperature,create.top_p,create.repetition_penalty):
async for token in interface.inference(input_message,id,create.temperature,create.top_p):
comp.append_token(token)
return comp

4
ktransformers/server/api/openai/legacy/completions.py
View file

@ -20,7 +20,7 @@ async def create_completion(request:Request,create:CompletionCreate):
if create.stream:
async def inner():
async for token in interface.inference(create.prompt,id,create.temperature,create.top_p,create.repetition_penalty):
async for token in interface.inference(create.prompt,id,create.temperature,create.top_p):
d = {'choices':[{'delta':{'content':token}}]}
yield f"data:{json.dumps(d)}\n\n"
d = {'choices':[{'delta':{'content':''},'finish_reason':''}]}
@ -28,6 +28,6 @@ async def create_completion(request:Request,create:CompletionCreate):
return stream_response(request,inner())
else:
comp = CompletionObject(id=id,object='text_completion',created=int(time()))
async for token in interface.inference(create.prompt,id,create.temperature,create.top_p,create.repetition_penalty):
async for token in interface.inference(create.prompt,id,create.temperature,create.top_p):
comp.append_token(token)
return comp