vrr/kvcache-ai-ktransformers
2
0
Fork 0
mirror of https://github.com/kvcache-ai/ktransformers.git synced 2026-05-05 15:40:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #842 from BITcyman/fix-openai_chat_completion

Browse source 
[fix] thread context bug
This commit is contained in:
Atream 2025-03-07 22:56:19 +08:00 • committed by GitHub
commit 09c043d8a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 11 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

17
ktransformers/server/backend/base.py
View file

@ -15,6 +15,7 @@ from ktransformers.server.schemas.assistants.assistants import AssistantObject
from ktransformers.server.schemas.assistants.messages import MessageCreate, MessageObject, Role from ktransformers.server.schemas.assistants.messages import MessageCreate, MessageObject, Role
from ktransformers.server.schemas.assistants.runs import RunObject from ktransformers.server.schemas.assistants.runs import RunObject
from ktransformers.server.schemas.assistants.threads import ThreadObject from ktransformers.server.schemas.assistants.threads import ThreadObject
from ktransformers.server.schemas.endpoints.chat import RawUsage
from ktransformers.server.schemas.base import ObjectID, Order from ktransformers.server.schemas.base import ObjectID, Order
from ktransformers.server.utils.multi_timer import Profiler from ktransformers.server.utils.multi_timer import Profiler
@ -142,12 +143,16 @@ class ThreadContext:
yield reply_message.stream_response_with_event(MessageObject.Status.in_progress) yield reply_message.stream_response_with_event(MessageObject.Status.in_progress)
yield self.run.stream_response_with_event(RunObject.Status.in_progress) yield self.run.stream_response_with_event(RunObject.Status.in_progress)
async for token, finish_reason in self.interface.inference(local_messages,self.thread.id): async for res in self.interface.inference(local_messages,self.thread.id):
if self.run.status == RunObject.Status.cancelling: if isinstance(res, RawUsage):
logger.warn(f'Run {self.run.id} cancelling') raw_usage = res
break else:
yield reply_message.append_message_delta(token) token, finish_reason = res
response_str_count+=1 if self.run.status == RunObject.Status.cancelling:
logger.warn(f'Run {self.run.id} cancelling')
break
yield reply_message.append_message_delta(token)
response_str_count+=1
if self.run.status == RunObject.Status.cancelling: if self.run.status == RunObject.Status.cancelling:
yield self.run.stream_response_with_event(RunObject.Status.cancelled) yield self.run.stream_response_with_event(RunObject.Status.cancelled)