From 8e7190f795a3280acb05995f4cbaa3b360edba5e Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:32:19 -0500 Subject: [PATCH] fix(ai): respect completed response item text (#45854) --- packages/ai/src/protocols/open-responses.ts | 122 ++++++---- .../provider/open-responses-finals.test.ts | 209 ++++++++++++++++++ .../provider/open-responses-lifecycle.test.ts | 12 +- .../openai-compatible-responses.test.ts | 24 +- .../ai/test/provider/openai-responses.test.ts | 34 +-- 5 files changed, 315 insertions(+), 86 deletions(-) create mode 100644 packages/ai/test/provider/open-responses-finals.test.ts diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index 069e6e18f97..79fec2d2b8b 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -1,4 +1,4 @@ -import { Effect, Schema } from "effect" +import { Effect, Option, Schema } from "effect" import type { Content } from "@opencode-ai/schema/tool" import { HttpTransport } from "../route/transport/index.js" import { Protocol } from "../route/protocol.js" @@ -845,6 +845,21 @@ const onOutputTextDone = (state: ParserState, event: Event, id: string): StepRes return [{ ...state, lifecycle: Lifecycle.textEnd(state.lifecycle, events, id) }, events] } +const decodeMessagePart = Schema.decodeUnknownOption( + Schema.Union([OpenResponsesOutputText, Schema.Struct({ type: Schema.tag("refusal"), refusal: Schema.String })]), +) + +const decodeSummaryPart = Schema.decodeUnknownOption(OpenResponsesReasoningSummaryText) + +const decodeReasoningPart = Schema.decodeUnknownOption( + Schema.Struct({ type: Schema.tag("reasoning_text"), text: Schema.String }), +) + +const joinReasoningText = (parts: ReadonlyArray) => { + if (!parts.some((part) => part !== undefined && part.length > 0)) return undefined + return parts.filter((part) => part !== undefined).join("\n\n") +} + export const outputItemID = (state: ParserState, event: Event) => event.output_index === undefined ? event.item_id : (state.outputItems[event.output_index] ?? event.item_id) @@ -1065,24 +1080,33 @@ const onFunctionCallArgumentsDelta = Effect.fn("OpenResponses.onFunctionCallArgu return [{ ...state, lifecycle, tools: result.tools }, events] satisfies StepResult }) -const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* (state: ParserState, event: Event) { - const item = event.item +const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* ( + state: ParserState, + item: Event["item"], +) { if (!item) return [state, NO_EVENTS] satisfies StepResult if (item.type === "message" && item.id !== undefined) { + const message = state.message?.id === item.id ? state.message : undefined const itemPhase = messagePhase(item.phase) - const phase = itemPhase === undefined && state.message?.id === item.id ? state.message.phase : itemPhase + const phase = itemPhase === undefined ? message?.phase : itemPhase + const parts: ReadonlyArray = Array.isArray(item.content) ? item.content : [] + const content: string[] = [] + for (const part of parts) { + const decoded = Option.getOrUndefined(decodeMessagePart(part)) + if (!decoded) continue + content.push(decoded.type === "output_text" ? decoded.text : decoded.refusal) + } + const text = content.length > 0 ? content.join("") : undefined + const metadata = providerMetadata(state, { itemId: item.id, ...(phase === undefined ? {} : { phase }) }) const events: LLMEvent[] = [] + const lifecycle = + message && text ? Lifecycle.textStart(state.lifecycle, events, item.id, metadata) : state.lifecycle return [ { ...state, - lifecycle: Lifecycle.textEnd( - state.lifecycle, - events, - item.id, - providerMetadata(state, { itemId: item.id, ...(phase === undefined ? {} : { phase }) }), - ), - message: state.message?.id === item.id ? undefined : state.message, + lifecycle: Lifecycle.textEnd(lifecycle, events, item.id, metadata, text), + message: message ? undefined : state.message, }, events, ] satisfies StepResult @@ -1137,17 +1161,33 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* ( } if (isReasoningItem(item)) { - const events: LLMEvent[] = [] + if (state.reasoningItems[item.id]?.open === false) return [state, NO_EVENTS] satisfies StepResult const metadata = reasoningMetadata(state, item) + const summaryParts: ReadonlyArray = Array.isArray(item.summary) ? item.summary : [] + const summary: Array = [] + for (const part of summaryParts) { + const decoded = Option.getOrUndefined(decodeSummaryPart(part)) + // Keep missing entries so the array still matches the provider's summary indexes. + summary.push(decoded?.text) + } + const reasoningParts: ReadonlyArray = Array.isArray(item.content) ? item.content : [] + const content: string[] = [] + for (const part of reasoningParts) { + const decoded = Option.getOrUndefined(decodeReasoningPart(part)) + if (decoded) content.push(decoded.text) + } + const itemText = joinReasoningText(summary) ?? joinReasoningText(content) + const events: LLMEvent[] = [] const reasoningItem = state.reasoningItems[item.id] if (reasoningItem) { - if (!reasoningItem.open) return [state, NO_EVENTS] satisfies StepResult - const lifecycle = Object.entries(reasoningItem.summaryParts) - .filter((entry) => entry[1] === "active" || entry[1] === "can-conclude") - .reduce( - (lifecycle, entry) => Lifecycle.reasoningEnd(lifecycle, events, `${item.id}:${entry[0]}`, metadata), - state.lifecycle, - ) + const fragments = Object.entries(reasoningItem.summaryParts) + let lifecycle = state.lifecycle + for (const [index, status] of fragments) { + if (status === "concluded") continue + // Do not repeat earlier summaries that were already emitted as separate fragments. + const finalText = fragments.length === 1 ? itemText : summary[Number(index)] + lifecycle = Lifecycle.reasoningEnd(lifecycle, events, `${item.id}:${index}`, metadata, finalText || undefined) + } return [ { ...state, @@ -1167,7 +1207,13 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* ( if (!state.lifecycle.reasoning.has(item.id)) { const lifecycle = Lifecycle.stepStart(state.lifecycle, events) events.push(LLMEvent.reasoningStart({ id: item.id, providerMetadata: metadata })) - events.push(LLMEvent.reasoningEnd({ id: item.id, providerMetadata: metadata })) + events.push( + LLMEvent.reasoningEnd({ + id: item.id, + providerMetadata: metadata, + text: itemText, + }), + ) return [ { ...state, @@ -1195,32 +1241,24 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* ( }) const onResponseFinish = Effect.fn("OpenResponses.onResponseFinish")(function* (state: ParserState, event: Event) { - const reconciled = - event.type === "response.completed" - ? yield* Effect.reduce( - event.response?.output ?? [], - () => [state, NO_EVENTS] satisfies StepResult, - ([current, events], item) => { - const id = item.id ?? (item.type === "function_call" ? item.call_id : undefined) - if ( - id === undefined || - ((item.type !== "function_call" || !current.tools[id]) && - (item.type !== "reasoning" || !current.reasoningItems[id]?.open)) - ) - return Effect.succeed([current, events] satisfies StepResult) - return onOutputItemDone(current, { type: "response.output_item.done", item }).pipe( - Effect.map(([next, emitted]) => [next, [...events, ...emitted]] satisfies StepResult), - ) - }, - ) - : ([state, NO_EVENTS] satisfies StepResult) - const current = reconciled[0] + let current = state + const events: LLMEvent[] = [] + if (event.type === "response.completed") { + for (const item of event.response?.output ?? []) { + const id = item.id ?? (item.type === "function_call" ? item.call_id : undefined) + if (id === undefined) continue + if (item.type !== "function_call" || !current.tools[id]) continue + const [next, emitted] = yield* onOutputItemDone(current, item) + current = next + events.push(...emitted) + } + } // Some compatible providers omit output_item.done even after completing the response. const pending = event.type === "response.completed" ? yield* ToolStream.finishAll(current.id, current.tools) : { tools: current.tools, events: NO_EVENTS } - const events: LLMEvent[] = [...reconciled[1], ...pending.events] + events.push(...pending.events) const hasFunctionCall = pending.events.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)) || current.hasFunctionCall @@ -1346,7 +1384,7 @@ export const step = (state: ParserState, input: Event) => { if (event.type === "response.output_item.done") { if (event.item?.type === "message" && event.item.id === undefined) return ProviderShared.eventError(state.id, `${event.type} message is missing id`) - return onOutputItemDone(state, event) + return onOutputItemDone(state, event.item) } if (event.type === "response.completed" || event.type === "response.incomplete") return onResponseFinish(state, event) if (event.type === "response.failed") return providerFailure(event, `${state.name} response failed`) diff --git a/packages/ai/test/provider/open-responses-finals.test.ts b/packages/ai/test/provider/open-responses-finals.test.ts new file mode 100644 index 00000000000..d46a335ed7a --- /dev/null +++ b/packages/ai/test/provider/open-responses-finals.test.ts @@ -0,0 +1,209 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { LLM, LLMEvent } from "../../src/index.js" +import { OpenResponses } from "../../src/protocols/open-responses.js" +import { configure } from "../../src/providers/openai-compatible-responses.js" +import { LLMClient } from "../../src/route.js" +import { it } from "../lib/effect.js" +import { fixedResponse } from "../lib/http.js" +import { sseEvents } from "../lib/sse.js" + +const request = LLM.request({ + model: configure({ apiKey: "test-key", baseURL: "https://responses.example.test/v1" }).model("example-model"), + prompt: "Respond.", +}) +const completed = { type: "response.completed", response: { id: "resp_1" } } +const generate = (...events: OpenResponses.Event[]) => + LLMClient.generate(request).pipe(Effect.provide(fixedResponse(sseEvents(...events)))) + +describe("Open Responses completed item text", () => { + ;["Draft expanded", "D", "Replacement", ""].forEach((text) => { + it.effect(`replaces streamed text with completed item text ${JSON.stringify(text)}`, () => + Effect.gen(function* () { + const response = yield* generate( + { type: "response.output_item.added", item: { type: "message", id: "msg_1", phase: "commentary" } }, + { type: "response.output_text.delta", item_id: "msg_1", delta: "Draft" }, + { type: "response.output_text.done", item_id: "msg_1", text: "Part final" }, + { + type: "response.output_item.done", + item: { type: "message", id: "msg_1", phase: "final_answer", content: [{ type: "output_text", text }] }, + }, + completed, + ) + expect(response.text).toBe(text) + expect(response.events.filter(LLMEvent.is.textDelta).map((event) => event.text)).toEqual(["Draft"]) + expect(response.events.filter(LLMEvent.is.textEnd)).toEqual([ + { + type: "text-end", + id: "msg_1", + text, + providerMetadata: { "openai-compatible": { itemId: "msg_1", phase: "final_answer" } }, + }, + ]) + }), + ) + }) + + it.effect("joins completed text and refusal parts without streamed text", () => + Effect.gen(function* () { + const response = yield* generate( + { type: "response.output_item.added", item: { type: "message", id: "msg_1" } }, + { + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1", + content: [ + { type: "output_text", text: "Answer. " }, + { type: "refusal", refusal: "Cannot help." }, + ], + }, + }, + completed, + ) + expect(response.text).toBe("Answer. Cannot help.") + expect(response.events.filter(LLMEvent.is.textStart)).toHaveLength(1) + expect(response.events.filter(LLMEvent.is.textEnd)).toHaveLength(1) + }), + ) + + it.effect("does not create an empty text fragment for an empty completed message", () => + Effect.gen(function* () { + const response = yield* generate( + { type: "response.output_item.added", item: { type: "message", id: "msg_1" } }, + { type: "response.output_text.done", item_id: "msg_1", text: "" }, + { + type: "response.output_item.done", + item: { type: "message", id: "msg_1", content: [{ type: "output_text", text: "" }] }, + }, + completed, + ) + expect(response.message.content).toEqual([]) + expect(response.events.filter(LLMEvent.is.textStart)).toEqual([]) + }), + ) +}) + +describe("Open Responses completed item reasoning", () => { + ;[ + { + name: "summary", + summary: [ + { type: "summary_text", text: "Final" }, + { type: "summary_text", text: "summary" }, + ], + content: [{ type: "reasoning_text", text: "Raw" }], + text: "Final\n\nsummary", + }, + { + name: "raw text", + summary: [ + { type: "summary_text", text: "" }, + { type: "summary_text", text: "" }, + ], + content: [{ type: "reasoning_text", text: "Raw" }], + text: "Raw", + }, + { + name: "streamed fallback", + summary: [ + { type: "summary_text", text: "" }, + { type: "summary_text", text: "" }, + ], + content: [ + { type: "reasoning_text", text: "" }, + { type: "reasoning_text", text: "" }, + ], + text: "Draft", + }, + ].forEach((fixture) => { + it.effect(`uses ${fixture.name} at item completion`, () => + Effect.gen(function* () { + const response = yield* generate( + { type: "response.output_item.added", item: { type: "reasoning", id: "rs_1" } }, + { type: "response.reasoning_summary_text.delta", item_id: "rs_1", delta: "Draft" }, + { type: "response.reasoning_summary_text.done", item_id: "rs_1", text: "Part final" }, + { + type: "response.output_item.done", + item: { + type: "reasoning", + id: "rs_1", + summary: fixture.summary, + content: fixture.content, + encrypted_content: "encrypted", + }, + }, + completed, + ) + expect(response.reasoning).toBe(fixture.text) + expect(response.events.filter(LLMEvent.is.reasoningEnd)).toHaveLength(1) + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: "encrypted" }, + }) + }), + ) + }) + + it.effect("replaces only the still-open summary without repeating earlier text", () => + Effect.gen(function* () { + const response = yield* generate( + { type: "response.output_item.added", item: { type: "reasoning", id: "rs_1" } }, + { type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 0, delta: "First " }, + { type: "response.reasoning_summary_part.added", item_id: "rs_1", summary_index: 1 }, + { type: "response.reasoning_summary_text.delta", item_id: "rs_1", summary_index: 1, delta: "draft" }, + { + type: "response.output_item.done", + item: { + type: "reasoning", + id: "rs_1", + summary: [ + { type: "summary_text", text: "First " }, + { type: "summary_text", text: "final" }, + ], + }, + }, + completed, + ) + expect(response.reasoning).toBe("First final") + expect(response.events.filter(LLMEvent.is.reasoningEnd).map((event) => event.text)).toEqual([undefined, "final"]) + }), + ) +}) +;["response.completed", "response.incomplete"].forEach((type) => { + it.effect(`keeps streamed text when part finals are followed by ${type} without item completion`, () => + Effect.gen(function* () { + const response = yield* generate( + { type: "response.output_item.added", item: { type: "message", id: "msg_1" } }, + { type: "response.output_text.delta", item_id: "msg_1", content_index: 0, delta: "Hel" }, + { type: "response.output_text.delta", item_id: "msg_1", content_index: 1, delta: "world" }, + { type: "response.output_text.done", item_id: "msg_1", content_index: 0, text: "Hello " }, + { + type: "response.content_part.done", + item_id: "msg_1", + content_index: 0, + part: { type: "output_text", text: "Hello " }, + }, + { type: "response.output_item.added", item: { type: "reasoning", id: "rs_1" } }, + { type: "response.reasoning_summary_text.delta", item_id: "rs_1", delta: "Draft" }, + { type: "response.reasoning_summary_text.done", item_id: "rs_1", text: "Part final" }, + { + type: "response.reasoning_summary_part.done", + item_id: "rs_1", + summary_index: 0, + part: { type: "summary_text", text: "Part final" }, + }, + { + type, + response: { + id: "resp_1", + incomplete_details: type === "response.incomplete" ? { reason: "max_output_tokens" } : undefined, + }, + }, + ) + expect(response.text).toBe("Helworld") + expect(response.reasoning).toBe("Draft") + expect(response.events.filter(LLMEvent.is.textEnd).map((event) => event.text)).toEqual([undefined]) + expect(response.events.filter(LLMEvent.is.reasoningEnd).map((event) => event.text)).toEqual([undefined]) + }), + ) +}) diff --git a/packages/ai/test/provider/open-responses-lifecycle.test.ts b/packages/ai/test/provider/open-responses-lifecycle.test.ts index 065288c0bf3..dfd3c8e4a4a 100644 --- a/packages/ai/test/provider/open-responses-lifecycle.test.ts +++ b/packages/ai/test/provider/open-responses-lifecycle.test.ts @@ -129,7 +129,7 @@ describe("Open Responses basic-item lifecycles", () => { }), ) - it.effect("preserves done-only encrypted reasoning without replaying its summary or late events", () => + it.effect("preserves done-only reasoning text and encryption without replaying late events", () => Effect.gen(function* () { const item = { type: "reasoning", @@ -157,6 +157,7 @@ describe("Open Responses basic-item lifecycles", () => { { type: "reasoning-end", id: "rs_1", + text: "Not streamed", providerMetadata: { "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, }, ]) @@ -301,7 +302,7 @@ describe("Open Responses basic-item lifecycles", () => { ) }) - it.effect("recovers pending items in completed output order with terminal encrypted metadata", () => + it.effect("recovers pending calls without reconciling terminal reasoning", () => Effect.gen(function* () { const events = yield* collect( { @@ -325,11 +326,6 @@ describe("Open Responses basic-item lifecycles", () => { }, ) expect(events.slice(5, -2)).toEqual([ - { - type: "reasoning-end", - id: "rs_1:0", - providerMetadata: { "openai-compatible": { itemId: "rs_1", reasoningEncryptedContent: "terminal-state" } }, - }, { type: "tool-input-end", id: "call_1", @@ -341,8 +337,10 @@ describe("Open Responses basic-item lifecycles", () => { id: "call_1", name: "lookup", input: { query: "final" }, + providerExecuted: undefined, providerMetadata: { "openai-compatible": { itemId: "fc_1" } }, }, + { type: "reasoning-end", id: "rs_1:0" }, ]) }), ) diff --git a/packages/ai/test/provider/openai-compatible-responses.test.ts b/packages/ai/test/provider/openai-compatible-responses.test.ts index 7b687968d63..ced45842d26 100644 --- a/packages/ai/test/provider/openai-compatible-responses.test.ts +++ b/packages/ai/test/provider/openai-compatible-responses.test.ts @@ -406,7 +406,7 @@ describe("Open Responses-compatible route", () => { }) routings.forEach((routing) => { - it.effect(`preserves reasoning summary boundaries and terminal metadata with ${routing.name}`, () => + it.effect(`preserves reasoning summary boundaries without terminal reconciliation with ${routing.name}`, () => Effect.gen(function* () { const address = { item_id: routing.item_id, output_index: routing.output_index } const response = yield* LLMClient.generate(request).pipe( @@ -444,21 +444,18 @@ describe("Open Responses-compatible route", () => { type: "reasoning", text: "Second.", providerMetadata: { - "openai-compatible": { itemId: routing.id, reasoningEncryptedContent: "final-state" }, + "openai-compatible": { itemId: routing.id, reasoningEncryptedContent: null }, }, }, ]) expect(response.events.filter(LLMEvent.is.reasoningEnd)).toEqual([ - expect.objectContaining({ + { + type: "reasoning-end", id: `${routing.id}:0`, + text: undefined, providerMetadata: { "openai-compatible": { itemId: routing.id } }, - }), - expect.objectContaining({ - id: `${routing.id}:1`, - providerMetadata: { - "openai-compatible": { itemId: routing.id, reasoningEncryptedContent: "final-state" }, - }, - }), + }, + { type: "reasoning-end", id: `${routing.id}:1` }, ]) }), ) @@ -671,7 +668,7 @@ describe("Open Responses-compatible route", () => { }), ) - it.effect("preserves terminal reasoning metadata when item completion is missing", () => + it.effect("ignores terminal reasoning output when item completion is missing", () => Effect.gen(function* () { const model = configure({ apiKey: "test-key", @@ -697,8 +694,9 @@ describe("Open Responses-compatible route", () => { ), ) - expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({ - providerMetadata: { "openai-compatible": { itemId: "rs_raw", reasoningEncryptedContent: "raw-state" } }, + expect(response.events.find((event) => event.type === "reasoning-end")).toEqual({ + type: "reasoning-end", + id: "rs_raw:0", }) }), ) diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index ad681eb6753..c2d3347c00d 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -2554,7 +2554,7 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("preserves terminal reasoning metadata when output item completion is missing", () => + it.effect("ignores terminal reasoning output when item completion is missing", () => Effect.gen(function* () { const response = yield* LLMClient.generate( LLMRequest.update(request, { providerOptions: { store: false } }), @@ -2595,29 +2595,13 @@ describe("OpenAI Responses route", () => { expect(response.reasoning).toBe("Checked the diff.") expect(response.events.filter((event) => event.type === "reasoning-end")).toEqual([ - { - type: "reasoning-end", - id: "rs_1:0", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "terminal-state" } }, - }, + { type: "reasoning-end", id: "rs_1:0" }, ]) expect(response.message.content).toContainEqual({ type: "reasoning", text: "Checked the diff.", - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "terminal-state" } }, + providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: null } }, }) - - const prepared = yield* compileRequest( - LLM.request({ model, messages: [response.message], providerOptions: { store: false } }), - ) - expect(prepared.body.input).toEqual([ - { - type: "reasoning", - id: "rs_1", - summary: [{ type: "summary_text", text: "Checked the diff." }], - encrypted_content: "terminal-state", - }, - ]) }), ) @@ -2644,7 +2628,7 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("reconciles pending reasoning and function calls in completed output order", () => + it.effect("recovers pending function calls without reconciling terminal reasoning", () => Effect.gen(function* () { const response = yield* LLMClient.generate( LLMRequest.update(request, { providerOptions: { store: false } }), @@ -2682,14 +2666,15 @@ describe("OpenAI Responses route", () => { ), ) - expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({ - providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "terminal-state" } }, + expect(response.events.find((event) => event.type === "reasoning-end")).toEqual({ + type: "reasoning-end", + id: "rs_1:0", }) expect(response.events.filter(LLMEvent.is.toolCall)).toEqual([ expect.objectContaining({ id: "call_1", input: { query: "weather" } }), ]) - expect(response.events.findIndex((event) => event.type === "reasoning-end")).toBeLessThan( - response.events.findIndex(LLMEvent.is.toolCall), + expect(response.events.findIndex(LLMEvent.is.toolCall)).toBeLessThan( + response.events.findIndex((event) => event.type === "reasoning-end"), ) expect(response.finishReason.normalized).toBe("tool-calls") }), @@ -3019,6 +3004,7 @@ describe("OpenAI Responses route", () => { { type: "reasoning-end", id: "rs_1:0", + text: "Checked the diff.", providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, }, ])