From 0cce215de4e097884f65450b738ab974e7de1b57 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Wed, 5 Aug 2026 23:54:41 -0500 Subject: [PATCH] fix(ai): harden compatible chat parsing (#40798) --- packages/ai/src/protocols/openai-chat.ts | 130 +++++++++++++----- .../provider/openai-compatible-chat.test.ts | 104 +++++++++++++- 2 files changed, 200 insertions(+), 34 deletions(-) diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index f758cdc260b..26a2d4b19d8 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -67,6 +67,12 @@ const OpenAIChatAssistantToolCall = Schema.Struct({ }) type OpenAIChatAssistantToolCall = Schema.Schema.Type +// Intentionally omit Gemini's provider-specific `extra_content.google.thought_signature` +// extension until direct Google OpenAI-compatible routing is supported here: +// https://github.com/vercel/ai/issues/11590 +// https://github.com/vercel/ai/pull/11745 +// https://ai.google.dev/gemini-api/docs/thought-signatures#openai + const OpenAIChatUserContent = Schema.Union([ Schema.Struct({ type: Schema.Literal("text"), @@ -145,22 +151,33 @@ export type OpenAIChatBody = Schema.Schema.Type // The event schema is one decoded SSE `data:` payload. `Framing.sse` splits the // byte stream into strings, then `Protocol.jsonEvent` decodes each string into // this provider-native event shape. -const OpenAIChatUsage = Schema.Struct({ - prompt_tokens: Schema.optional(Schema.Number), - completion_tokens: Schema.optional(Schema.Number), - total_tokens: Schema.optional(Schema.Number), - prompt_tokens_details: optionalNull( - Schema.Struct({ - cached_tokens: Schema.optional(Schema.Number), - cache_write_tokens: Schema.optional(Schema.Number), - }), - ), - completion_tokens_details: optionalNull( - Schema.Struct({ - reasoning_tokens: Schema.optional(Schema.Number), - }), - ), -}) +const OpenAIChatUsage = Schema.StructWithRest( + Schema.Struct({ + prompt_tokens: optionalNull(Schema.Number), + completion_tokens: optionalNull(Schema.Number), + total_tokens: optionalNull(Schema.Number), + prompt_tokens_details: optionalNull( + Schema.StructWithRest( + Schema.Struct({ + cached_tokens: optionalNull(Schema.Number), + cache_write_tokens: optionalNull(Schema.Number), + }), + [Schema.Record(Schema.String, Schema.Unknown)], + ), + ), + completion_tokens_details: optionalNull( + Schema.StructWithRest( + Schema.Struct({ + reasoning_tokens: optionalNull(Schema.Number), + accepted_prediction_tokens: optionalNull(Schema.Number), + rejected_prediction_tokens: optionalNull(Schema.Number), + }), + [Schema.Record(Schema.String, Schema.Unknown)], + ), + ), + }), + [Schema.Record(Schema.String, Schema.Unknown)], +) const OpenAIChatToolCallDeltaFunction = Schema.Struct({ name: optionalNull(Schema.String), @@ -168,7 +185,7 @@ const OpenAIChatToolCallDeltaFunction = Schema.Struct({ }) const OpenAIChatToolCallDelta = Schema.Struct({ - index: Schema.Number, + index: optionalNull(Schema.Number), id: optionalNull(Schema.String), function: optionalNull(OpenAIChatToolCallDeltaFunction), }) @@ -222,6 +239,8 @@ export interface ParserState { readonly reasoningDetails: Array readonly reasoningDetailsObserved: boolean readonly reasoningEmitted: boolean + readonly latestToolIndex?: number + readonly nextToolIndex: number } // ============================================================================= @@ -559,22 +578,34 @@ const mapFinishReason = (reason: string | null | undefined): FinishReason => { // satisfied on both sides. const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => { if (!usage) return undefined - const cached = usage.prompt_tokens_details?.cached_tokens - const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens - const reasoning = usage.completion_tokens_details?.reasoning_tokens - const nonCached = ProviderShared.subtractTokens(usage.prompt_tokens, ProviderShared.sumTokens(cached, cacheWrite)) + const input = usage.prompt_tokens ?? undefined + const output = usage.completion_tokens ?? undefined + const cached = usage.prompt_tokens_details?.cached_tokens ?? undefined + const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens ?? undefined + const reasoning = usage.completion_tokens_details?.reasoning_tokens ?? undefined + const nonCached = ProviderShared.subtractTokens(input, ProviderShared.sumTokens(cached, cacheWrite)) return new Usage({ - inputTokens: usage.prompt_tokens, - outputTokens: usage.completion_tokens, + inputTokens: input, + outputTokens: output, nonCachedInputTokens: nonCached, cacheReadInputTokens: cached, cacheWriteInputTokens: cacheWrite, reasoningTokens: reasoning, - totalTokens: ProviderShared.totalTokens(usage.prompt_tokens, usage.completion_tokens, usage.total_tokens), + totalTokens: ProviderShared.totalTokens(input, output, usage.total_tokens ?? undefined), providerMetadata: { openai: usage }, }) } +const toolIndexByID = ( + tools: ParserState["tools"], + pendingTools: ParserState["pendingTools"], + id: string | undefined, +) => { + if (!id) return undefined + const entry = Object.entries({ ...pendingTools, ...tools }).find(([, tool]) => tool?.id === id) + return entry ? Number(entry[0]) : undefined +} + const reasoningDelta = ( delta: Schema.Schema.Type | null | undefined, configuredField?: string, @@ -657,17 +688,34 @@ const step = (state: ParserState, event: OpenAIChatEvent) => const events: LLMEvent[] = [] const usage = mapUsage(event.usage) ?? state.usage const choice = event.choices?.[0] - const finishReason = choice?.finish_reason - ? { normalized: mapFinishReason(choice.finish_reason), raw: choice.native_finish_reason ?? choice.finish_reason } - : state.finishReason + const rawFinishReason = choice?.finish_reason + const finishReason = + rawFinishReason !== undefined && rawFinishReason !== null + ? { normalized: mapFinishReason(rawFinishReason), raw: choice?.native_finish_reason ?? rawFinishReason } + : state.finishReason const delta = choice?.delta const toolDeltas = delta?.tool_calls ?? [] let tools = state.tools let pendingTools = state.pendingTools + let latestToolIndex = state.latestToolIndex + let nextToolIndex = state.nextToolIndex let lifecycle = state.lifecycle const reasoning = reasoningDelta(delta, state.reasoningField) + const hasLateContent = + Boolean(delta?.content) || + reasoning !== undefined || + (Array.isArray(delta?.reasoning_details) && delta.reasoning_details.length > 0) || + toolDeltas.some( + (tool) => Boolean(tool.id) || Boolean(tool.function?.name) || Boolean(tool.function?.arguments), + ) + if (state.finishReason !== undefined) { + if (hasLateContent) + return yield* ProviderShared.eventError(ADAPTER, "OpenAI Chat received content after the finish reason") + return [{ ...state, usage }, events] as const + } + const reasoningField = state.reasoningField ?? (!state.lifecycle.text.has("text-0") ? reasoning?.field : undefined) const detailDelta = Array.isArray(delta?.reasoning_details) ? delta.reasoning_details : undefined if (detailDelta !== undefined) appendReasoningDetails(state.reasoningDetails, detailDelta) @@ -694,24 +742,37 @@ const step = (state: ParserState, event: OpenAIChatEvent) => lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.content) } - for (const tool of toolDeltas) { - const current = tools[tool.index] - const pending = pendingTools[tool.index] + // Compatible providers may omit indexes. Prefer durable identity, then use + // batch position for parallel deltas or the latest call for sparse chunks. + for (const [position, tool] of toolDeltas.entries()) { + const matched = toolIndexByID(tools, pendingTools, tool.id || undefined) + const fallback = toolDeltas.length > 1 ? position : (latestToolIndex ?? position) + const fallbackTool = tools[fallback] ?? pendingTools[fallback] + const index = + tool.index ?? matched ?? + (tool.id && fallbackTool?.id && fallbackTool.id !== tool.id ? nextToolIndex : fallback) + const current = tools[index] + const pending = pendingTools[index] const id = current?.id ?? pending?.id ?? (tool.id || undefined) const name = current?.name ?? pending?.name ?? (tool.function?.name || undefined) const text = `${pending?.input ?? ""}${tool.function?.arguments ?? ""}` + latestToolIndex = index + nextToolIndex = Math.max(nextToolIndex, index + 1) if (!current && (!id || !name)) { - pendingTools = { ...pendingTools, [tool.index]: { id: id || undefined, name: name || undefined, input: text } } + pendingTools = { + ...pendingTools, + [index]: { id: id || undefined, name: name || undefined, input: text }, + } continue } if (pending) { pendingTools = { ...pendingTools } - delete pendingTools[tool.index] + delete pendingTools[index] } const result = ToolStream.appendOrStart( ADAPTER, tools, - tool.index, + index, { id: id || undefined, name: name || undefined, text }, "OpenAI Chat tool call delta is missing id or name", ) @@ -743,6 +804,8 @@ const step = (state: ParserState, event: OpenAIChatEvent) => reasoningDetails: state.reasoningDetails, reasoningDetailsObserved, reasoningEmitted, + latestToolIndex, + nextToolIndex, }, events, ] as const @@ -799,6 +862,7 @@ export const protocol = Protocol.make({ reasoningDetails: [], reasoningDetailsObserved: false, reasoningEmitted: false, + nextToolIndex: 0, }), step, onHalt: finishEvents, diff --git a/packages/ai/test/provider/openai-compatible-chat.test.ts b/packages/ai/test/provider/openai-compatible-chat.test.ts index 25ac410bed3..526e9edce2a 100644 --- a/packages/ai/test/provider/openai-compatible-chat.test.ts +++ b/packages/ai/test/provider/openai-compatible-chat.test.ts @@ -7,7 +7,7 @@ import { compileRequest } from "../../src/route/client" import * as OpenAICompatible from "../../src/providers/openai-compatible" import * as OpenAICompatibleChat from "../../src/protocols/openai-compatible-chat" import { it } from "../lib/effect" -import { dynamicResponse } from "../lib/http" +import { dynamicResponse, fixedResponse } from "../lib/http" import { sseEvents } from "../lib/sse" const Json = Schema.fromJsonString(Schema.Unknown) @@ -253,4 +253,106 @@ describe("OpenAI-compatible Chat route", () => { }) }), ) + + it.effect("accepts nullable usage and preserves provider fields", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + deltaChunk({ content: "Hello" }), + deltaChunk({}, "stop"), + usageChunk({ + prompt_tokens: null, + completion_tokens: null, + total_tokens: null, + prompt_tokens_details: { cached_tokens: null, vendor_cache_tokens: 3 }, + completion_tokens_details: { + reasoning_tokens: null, + accepted_prediction_tokens: null, + rejected_prediction_tokens: null, + }, + cost: "0.001", + }), + ), + ), + ), + ) + + expect(response.usage).toMatchObject({ + inputTokens: undefined, + outputTokens: undefined, + totalTokens: undefined, + providerMetadata: { + openai: { + prompt_tokens: null, + completion_tokens: null, + total_tokens: null, + prompt_tokens_details: { cached_tokens: null, vendor_cache_tokens: 3 }, + cost: "0.001", + }, + }, + }) + }), + ) + + it.effect("assembles indexless parallel tool calls across sparse chunks", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate( + LLMRequest.update(request, { + tools: [ToolDefinition.make({ name: "weather", description: "Get weather", inputSchema: { type: "object" } })], + }), + ).pipe( + Effect.provide( + fixedResponse( + sseEvents( + deltaChunk({ + tool_calls: [ + { id: "call_paris", function: { name: "weather", arguments: '{"city":"' } }, + { index: null, id: "call_london", function: { name: "weather", arguments: '{"city":"' } }, + ], + }), + deltaChunk({ tool_calls: [{ function: { arguments: 'London"}' } }] }), + deltaChunk({ tool_calls: [{ id: "call_paris", function: { arguments: 'Paris"}' } }] }), + deltaChunk({}, "tool_calls"), + ), + ), + ), + ) + + expect(response.toolCalls).toMatchObject([ + { id: "call_paris", name: "weather", input: { city: "Paris" } }, + { id: "call_london", name: "weather", input: { city: "London" } }, + ]) + }), + ) + + it.effect("treats an empty finish reason as terminal", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(request).pipe( + Effect.provide(fixedResponse(sseEvents(deltaChunk({ content: "Hello" }), deltaChunk({}, "")))), + ) + + expect(response.finishReason).toEqual({ normalized: "unknown", raw: "" }) + }), + ) + + it.effect("rejects content after a terminal chunk", () => + Effect.gen(function* () { + const error = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + deltaChunk({ content: "Hello" }), + deltaChunk({}, "stop"), + deltaChunk({ tool_calls: [{ index: 0, id: "call_1", function: { name: "lookup", arguments: "{}" } }] }), + ), + ), + ), + Effect.flip, + ) + + expect(error.message).toContain("OpenAI Chat received content after the finish reason") + }), + ) })