diff --git a/packages/ai/src/protocols/index.ts b/packages/ai/src/protocols/index.ts index 7f8127d23e4..00ee4538753 100644 --- a/packages/ai/src/protocols/index.ts +++ b/packages/ai/src/protocols/index.ts @@ -8,3 +8,4 @@ export * as OpenAICompatibleResponses from "./openai-compatible-responses.js" export * as OpenAIResponses from "./openai-responses.js" export * as OpenResponses from "./open-responses.js" export * as OpenResponsesChannel from "./open-responses-channel.js" +export * as XAIResponses from "./xai-responses.js" diff --git a/packages/ai/src/protocols/open-responses-channel.ts b/packages/ai/src/protocols/open-responses-channel.ts index 142fdffd7a7..163437c5a67 100644 --- a/packages/ai/src/protocols/open-responses-channel.ts +++ b/packages/ai/src/protocols/open-responses-channel.ts @@ -10,6 +10,7 @@ import { } from "../route/transport/index.js" import * as ProviderShared from "./shared.js" import { OpenResponses } from "./open-responses.js" +import { OpenResponsesContinuation } from "./open-responses-continuation.js" const WebSocketResponseCreate = Schema.StructWithRest(Schema.Struct({ type: Schema.tag("response.create") }), [ Schema.Record(Schema.String, Schema.Unknown), @@ -25,11 +26,6 @@ export interface Options { readonly enabled?: (url: string) => boolean readonly url?: (url: string) => string readonly headers?: (headers: Headers.Headers) => Headers.Headers - readonly driver?: (input: { - readonly request: Readonly> - readonly message: string - readonly base: WebSocketChannelDriver - }) => WebSocketChannelDriver } export interface Prepared { @@ -158,7 +154,13 @@ export const transport = (options: Options): Transport { } } -export const OpenAIResponsesChannel = { driver } as const +export const OpenResponsesContinuation = { driver } as const diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index 9ed589fdbde..18c0f7817c6 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -42,7 +42,12 @@ const OpenResponsesInputImage = Schema.Struct({ const OpenResponsesInputFile = Schema.Struct({ type: Schema.tag("input_file"), filename: Schema.String, - file_data: Schema.String, + file_data: Schema.optional(Schema.String), + file_url: Schema.optional(Schema.String), +}) +const OpenResponsesInputVideo = Schema.Struct({ + type: Schema.tag("input_video"), + video_url: Schema.String, }) const MediaInput = Schema.Union([OpenResponsesInputImage, OpenResponsesInputFile]) export type MediaInput = Schema.Schema.Type @@ -53,7 +58,7 @@ const OpenResponsesOutputText = Schema.Struct({ text: Schema.String, }) -export const MessagePhase = Schema.Literals(["commentary", "final_answer"]) +export const MessagePhase = Schema.NullOr(Schema.Literals(["commentary", "final_answer"])) type MessagePhase = Schema.Schema.Type const OpenResponsesReasoningSummaryText = Schema.Struct({ @@ -80,6 +85,7 @@ const OpenResponsesFunctionCallOutputContent = Schema.Union([ OpenResponsesInputText, OpenResponsesInputImage, OpenResponsesInputFile, + OpenResponsesInputVideo, ]) const OpenResponsesFunctionCallOutput = Schema.Union([ @@ -322,7 +328,6 @@ export interface Extension { readonly media: ProviderShared.NormalizedMedia readonly request: LLMRequest }) => MediaInput | undefined - readonly messagePhase?: (value: unknown) => MessagePhase | null | undefined } const BASE: Extension = { id: ADAPTER, name: NAME } @@ -421,18 +426,25 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* ( part: MediaPart, request: LLMRequest, extension: Extension, + target: "message" | "tool-result", ) { const media = ProviderShared.normalizeMedia(part) const extended = extension.lowerMedia?.({ part, media, request }) if (extended) return extended + const url = + typeof part.data === "string" && (part.data.startsWith("https://") || part.data.startsWith("http://")) + ? part.data + : undefined if (!media.mime.startsWith("image/")) { + if (target === "tool-result" && media.mime.startsWith("video/")) + return { type: "input_video" as const, video_url: url ?? media.dataUrl } return { type: "input_file" as const, filename: part.filename ?? (media.mime === "application/pdf" ? "document.pdf" : "file"), - file_data: media.base64, + ...(url ? { file_url: url } : { file_data: media.base64 }), } } - return { type: "input_image" as const, image_url: media.dataUrl } + return { type: "input_image" as const, image_url: url ?? media.dataUrl } }) const lowerUserContent = Effect.fnUntraced(function* ( @@ -441,10 +453,17 @@ const lowerUserContent = Effect.fnUntraced(function* ( extension: Extension, ) { if (part.type === "text") return { type: "input_text" as const, text: part.text } - if (part.type === "media") return yield* lowerMedia(part, request, extension) + if (part.type === "media") return yield* lowerMessageMedia(part, request, extension) return yield* ProviderShared.unsupportedContent(extension.name, "user", ["text", "media"]) }) +const lowerMessageMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMRequest, extension: Extension) { + const lowered = yield* lowerMedia(part, request, extension, "message") + if (lowered.type === "input_video") + return yield* ProviderShared.invalidRequest(`${extension.name} user messages do not support input_video`) + return lowered +}) + // Tool results may carry structured text, images, and files. Keep media as provider-native // content instead of JSON-stringifying base64 into a prompt string. const lowerToolResultContentItem = Effect.fnUntraced(function* ( @@ -457,6 +476,20 @@ const lowerToolResultContentItem = Effect.fnUntraced(function* ( { type: "media", mediaType: item.mime, data: item.uri, filename: item.name }, request, extension, + "tool-result", + ) +}) + +const lowerHostedToolResultContentItem = Effect.fnUntraced(function* ( + item: Content, + request: LLMRequest, + extension: Extension, +) { + if (item.type === "text") return { type: "input_text" as const, text: item.text } + return yield* lowerMessageMedia( + { type: "media", mediaType: item.mime, data: item.uri, filename: item.name }, + request, + extension, ) }) @@ -509,7 +542,7 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques >((groups, part) => { const metadata = part.providerMetadata?.[providerMetadataKey] const id = itemID(part.providerMetadata, providerMetadataKey) - const phase = ProviderShared.isRecord(metadata) ? messagePhase(metadata.phase, extension) : undefined + const phase = ProviderShared.isRecord(metadata) ? messagePhase(metadata.phase) : undefined const group = groups.at(-1) if (group && group.id === id && group.phase === phase) group.parts.push(part) else groups.push({ id, phase, parts: [part] }) @@ -566,7 +599,9 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques const content: ReadonlyArray = part.result.value input.push({ role: "user", - content: yield* Effect.forEach(content, (item) => lowerToolResultContentItem(item, request, extension)), + content: yield* Effect.forEach(content, (item) => + lowerHostedToolResultContentItem(item, request, extension), + ), }) } if (itemID) hostedToolReferences.add(itemID) @@ -1158,15 +1193,15 @@ export const initial = (request: LLMRequest, extension: Extension = BASE): Parse tools: ToolStream.empty(), lifecycle: Lifecycle.initial(), messageItems: new Set(), - messagePhase: (value) => messagePhase(value, extension), + messagePhase, messagePhases: {}, reasoningItems: {}, store: OpenResponsesOptions.resolve(request).store, }) -const messagePhase = (value: unknown, extension: Extension): MessagePhase | null | undefined => { - if (value === "commentary" || value === "final_answer") return value - return extension.messagePhase?.(value) +const messagePhase = (value: unknown): MessagePhase | undefined => { + if (value === null || value === "commentary" || value === "final_answer") return value + return undefined } export const protocol = Protocol.make({ diff --git a/packages/ai/src/protocols/openai-responses.ts b/packages/ai/src/protocols/openai-responses.ts index 63275f6e506..10ca6364fd7 100644 --- a/packages/ai/src/protocols/openai-responses.ts +++ b/packages/ai/src/protocols/openai-responses.ts @@ -5,14 +5,13 @@ import { Auth } from "../route/auth.js" import { Endpoint } from "../route/endpoint.js" import { Protocol } from "../route/protocol.js" import { HttpTransport } from "../route/transport/index.js" -import { LLMEvent, LLMRequest, type JsonSchema, type ToolDefinition } from "../schema/index.js" +import { LLMRequest, type JsonSchema, type ToolDefinition } from "../schema/index.js" import { OpenResponses } from "./open-responses.js" import { optionalArray, ProviderShared } from "./shared.js" -import { Lifecycle } from "./utils/lifecycle.js" import { OpenAIImage } from "./utils/openai-image.js" +import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js" import { ToolSchemaProjection } from "./utils/tool-schema.js" -import { OpenResponsesChannel, type Options } from "./open-responses-channel.js" -import { OpenAIResponsesChannel } from "./openai-responses-channel.js" +import { OpenResponsesChannel } from "./open-responses-channel.js" const ADAPTER = "openai-responses" const NAME = "OpenAI Responses" @@ -40,20 +39,8 @@ const OpenAIResponsesToolChoice = Schema.Union([ Schema.Struct({ type: Schema.tag("image_generation") }), ]) -const OpenAIResponsesInputItem = Schema.Union([ - Schema.Struct({ - type: Schema.tag("message"), - id: Schema.optionalKey(Schema.String), - role: Schema.tag("assistant"), - content: Schema.Array(Schema.Struct({ type: Schema.tag("output_text"), text: Schema.String })), - phase: Schema.optionalKey(Schema.NullOr(OpenResponses.MessagePhase)), - }), - OpenResponses.InputItem, -]) - const OpenAIResponsesCoreFields = { ...OpenResponses.coreFields, - input: Schema.Array(OpenAIResponsesInputItem), tools: optionalArray(OpenAIResponsesTools), tool_choice: Schema.optional(OpenAIResponsesToolChoice), } @@ -67,7 +54,6 @@ export type OpenAIResponsesBody = Schema.Schema.Type const extension = { id: ADAPTER, name: NAME, - messagePhase: (value: unknown) => (value === null ? null : undefined), } satisfies OpenResponses.Extension const nativeImageToolInput = (tool: ToolDefinition) => { @@ -119,46 +105,7 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request: } satisfies OpenAIResponsesBody }) -type HostedToolData = OpenResponses.StreamItem & { - readonly id: string - readonly status?: string - readonly action?: unknown - readonly queries?: unknown - readonly results?: unknown - readonly code?: string - readonly container_id?: string - readonly outputs?: unknown - readonly server_label?: string - readonly output?: unknown - readonly result?: string - readonly output_format?: "png" | "jpeg" | "webp" - readonly error?: unknown -} - -const HOSTED_TOOLS = { - web_search_call: { name: "web_search", input: (item) => item.action ?? {} }, - web_search_preview_call: { name: "web_search_preview", input: (item) => item.action ?? {} }, - file_search_call: { name: "file_search", input: (item) => ({ queries: item.queries ?? [] }) }, - code_interpreter_call: { - name: "code_interpreter", - input: (item) => ({ code: item.code, container_id: item.container_id }), - }, - computer_use_call: { name: "computer_use", input: (item) => item.action ?? {} }, - image_generation_call: { name: "image_generation", input: () => ({}) }, - mcp_call: { - name: "mcp", - input: (item) => ({ server_label: item.server_label, name: item.name, arguments: item.arguments }), - }, - local_shell_call: { name: "local_shell", input: (item) => item.action ?? {} }, -} as const satisfies Record unknown }> - -type HostedToolType = keyof typeof HOSTED_TOOLS -type HostedToolItem = HostedToolData & { readonly type: HostedToolType } - -const isHostedToolItem = (item: OpenResponses.StreamItem): item is HostedToolItem => - item.type in HOSTED_TOOLS && typeof item.id === "string" && item.id.length > 0 - -const hostedToolResult = Effect.fn("OpenAIResponses.hostedToolResult")(function* (item: HostedToolItem) { +const hostedToolResult = Effect.fn("OpenAIResponses.hostedToolResult")(function* (item: ResponsesHostedTools.Item) { const isError = item.error !== undefined && item.error !== null if (item.type === "image_generation_call" && item.result) { yield* Effect.fromResult(Encoding.decodeBase64(item.result)).pipe( @@ -179,32 +126,22 @@ const hostedToolResult = Effect.fn("OpenAIResponses.hostedToolResult")(function* return isError ? { type: "error" as const, value: item.error } : { type: "json" as const, value: item } }) -const onHostedToolDone = Effect.fn("OpenAIResponses.onHostedToolDone")(function* ( - state: OpenResponses.ParserState, - item: HostedToolItem, -) { - const tool = HOSTED_TOOLS[item.type] - const providerMetadata = OpenResponses.providerMetadata(state, { itemId: item.id }) - const events: LLMEvent[] = [] - const lifecycle = Lifecycle.stepStart(state.lifecycle, events) - events.push( - LLMEvent.toolCall({ - id: item.id, - name: tool.name, - input: tool.input(item), - providerExecuted: true, - providerMetadata, - }), - LLMEvent.toolResult({ - id: item.id, - name: tool.name, - result: yield* hostedToolResult(item), - providerExecuted: true, - providerMetadata, - }), - ) - return [{ ...state, lifecycle }, events] satisfies OpenResponses.StepResult -}) +const HOSTED_TOOLS = { + web_search_call: { name: "web_search", input: (item) => item.action ?? {} }, + web_search_preview_call: { name: "web_search_preview", input: (item) => item.action ?? {} }, + file_search_call: { name: "file_search", input: (item) => ({ queries: item.queries ?? [] }) }, + code_interpreter_call: { + name: "code_interpreter", + input: (item) => ({ code: item.code, container_id: item.container_id }), + }, + computer_use_call: { name: "computer_use", input: (item) => item.action ?? {} }, + image_generation_call: { name: "image_generation", input: () => ({}), result: hostedToolResult }, + mcp_call: { + name: "mcp", + input: (item) => ({ server_label: item.server_label, name: item.name, arguments: item.arguments }), + }, + local_shell_call: { name: "local_shell", input: (item) => item.action ?? {} }, +} as const satisfies ResponsesHostedTools.Definitions const step = (state: OpenResponses.ParserState, event: OpenResponses.Event) => { if (event.type === "response.reasoning_text.delta" || event.type === "response.reasoning_summary.delta") @@ -215,8 +152,8 @@ const step = (state: OpenResponses.ParserState, event: OpenResponses.Event) => { return event.item_id ? Effect.succeed(OpenResponses.onReasoningDone(state, event)) : ProviderShared.eventError(ADAPTER, `${event.type} is missing item_id`) - if (event.type === "response.output_item.done" && event.item && isHostedToolItem(event.item)) - return onHostedToolDone(state, event.item) + if (event.type === "response.output_item.done" && event.item && ResponsesHostedTools.isItem(event.item, HOSTED_TOOLS)) + return ResponsesHostedTools.onDone(state, event.item, HOSTED_TOOLS) return OpenResponses.step(state, event) } @@ -238,11 +175,7 @@ const endpoint = Endpoint.path(PATH, { baseURL: DEFAULT_BAS const auth = Auth.none export const httpTransport = HttpTransport.sseJson.with() -export const channelTransport = (options: Omit) => - OpenResponsesChannel.transport({ - ...options, - driver: (input) => OpenAIResponsesChannel.driver({ id: options.id, name: options.name, ...input }), - }) +export const channelTransport = OpenResponsesChannel.transport export const transport = channelTransport({ id: ADAPTER, name: NAME, diff --git a/packages/ai/src/protocols/utils/responses-hosted-tools.ts b/packages/ai/src/protocols/utils/responses-hosted-tools.ts new file mode 100644 index 00000000000..0635a94bb90 --- /dev/null +++ b/packages/ai/src/protocols/utils/responses-hosted-tools.ts @@ -0,0 +1,70 @@ +import { Effect } from "effect" +import { LLMEvent, type AIError, type ToolResultPart } from "../../schema/index.js" +import { OpenResponses } from "../open-responses.js" +import { Lifecycle } from "./lifecycle.js" + +export type Item = OpenResponses.StreamItem & { + readonly id: string + readonly status?: string + readonly action?: unknown + readonly queries?: unknown + readonly results?: unknown + readonly code?: string + readonly container_id?: string + readonly outputs?: unknown + readonly server_label?: string + readonly output?: unknown + readonly result?: string + readonly output_format?: "png" | "jpeg" | "webp" + readonly error?: unknown +} + +export interface Definition { + readonly name: string + readonly input: (item: Item) => unknown + readonly result?: (item: Item) => Effect.Effect +} + +export type Definitions = Readonly> + +export const isItem = (item: OpenResponses.StreamItem, tools: Tools): item is Item => + item.type in tools && typeof item.id === "string" && item.id.length > 0 + +export const onDone: ( + state: OpenResponses.ParserState, + item: Item, + tools: Definitions, +) => Effect.Effect = Effect.fn("ResponsesHostedTools.onDone")(function* ( + state, + item, + tools, +) { + const tool = tools[item.type] + if (!tool) return [state, []] satisfies OpenResponses.StepResult + const providerMetadata = OpenResponses.providerMetadata(state, { itemId: item.id }) + const events: LLMEvent[] = [] + const lifecycle = Lifecycle.stepStart(state.lifecycle, events) + events.push( + LLMEvent.toolCall({ + id: item.id, + name: tool.name, + input: tool.input(item), + providerExecuted: true, + providerMetadata, + }), + LLMEvent.toolResult({ + id: item.id, + name: tool.name, + result: tool.result + ? yield* tool.result(item) + : item.error !== undefined && item.error !== null + ? { type: "error", value: item.error } + : { type: "json", value: item }, + providerExecuted: true, + providerMetadata, + }), + ) + return [{ ...state, lifecycle }, events] satisfies OpenResponses.StepResult +}) + +export * as ResponsesHostedTools from "./responses-hosted-tools.js" diff --git a/packages/ai/src/protocols/xai-responses.ts b/packages/ai/src/protocols/xai-responses.ts new file mode 100644 index 00000000000..01ab748946a --- /dev/null +++ b/packages/ai/src/protocols/xai-responses.ts @@ -0,0 +1,55 @@ +import { Effect } from "effect" +import { Protocol } from "../route/protocol.js" +import { OpenResponses } from "./open-responses.js" +import { ProviderShared } from "./shared.js" +import { ResponsesHostedTools } from "./utils/responses-hosted-tools.js" + +const ADAPTER = "xai-responses" +const NAME = "xAI Responses" + +const extension = { + id: ADAPTER, + name: NAME, +} satisfies OpenResponses.Extension + +const HOSTED_TOOLS = { + web_search_call: { name: "web_search", input: (item) => item.action ?? {} }, + x_search_call: { name: "x_search", input: (item) => item.action ?? {} }, + file_search_call: { name: "file_search", input: (item) => ({ queries: item.queries ?? [] }) }, + code_interpreter_call: { + name: "code_interpreter", + input: (item) => ({ code: item.code, container_id: item.container_id }), + }, + image_generation_call: { name: "image_generation", input: () => ({}) }, + mcp_call: { + name: "mcp", + input: (item) => ({ server_label: item.server_label, name: item.name, arguments: item.arguments }), + }, +} as const satisfies ResponsesHostedTools.Definitions + +const step = (state: OpenResponses.ParserState, event: OpenResponses.Event) => { + if (event.type === "response.reasoning_text.delta" || event.type === "response.reasoning_summary.delta") + return event.item_id + ? Effect.succeed(OpenResponses.onReasoningDelta(state, event, event.item_id)) + : ProviderShared.eventError(ADAPTER, `${event.type} is missing item_id`) + if (event.type === "response.reasoning_text.done" || event.type === "response.reasoning_summary.done") + return event.item_id + ? Effect.succeed(OpenResponses.onReasoningDone(state, event)) + : ProviderShared.eventError(ADAPTER, `${event.type} is missing item_id`) + if (event.type === "response.output_item.done" && event.item && ResponsesHostedTools.isItem(event.item, HOSTED_TOOLS)) + return ResponsesHostedTools.onDone(state, event.item, HOSTED_TOOLS) + return OpenResponses.step(state, event) +} + +export const protocol = Protocol.make({ + id: ADAPTER, + body: OpenResponses.protocol.body, + stream: { + event: OpenResponses.protocol.stream.event, + initial: (request) => OpenResponses.initial(request, extension), + step, + terminal: OpenResponses.terminal, + }, +}) + +export * as XAIResponses from "./xai-responses.js" diff --git a/packages/ai/src/providers/xai.ts b/packages/ai/src/providers/xai.ts index 65b32ab521d..9bca92783a9 100644 --- a/packages/ai/src/providers/xai.ts +++ b/packages/ai/src/providers/xai.ts @@ -5,7 +5,8 @@ import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js" import * as OpenAICompatibleProfiles from "./openai-compatible-profile.js" import * as OpenAICompatibleChat from "../protocols/openai-compatible-chat.js" import * as OpenAIChat from "../protocols/openai-chat.js" -import * as OpenAIResponses from "../protocols/openai-responses.js" +import { OpenResponsesChannel } from "../protocols/open-responses-channel.js" +import { XAIResponses } from "../protocols/xai-responses.js" import { XAIImages } from "../protocols/xai-images.js" import type { OpenAIOptionsInput } from "./openai-options.js" import type { ProviderPackage } from "../provider-package.js" @@ -34,9 +35,9 @@ const responsesRoute = Route.make({ id: "openai-responses", provider: id, providerMetadataKey: "xai", - protocol: OpenAIResponses.protocol, + protocol: XAIResponses.protocol, endpoint: Endpoint.path("/responses", { baseURL: OpenAICompatibleProfiles.profiles.xai.baseURL }), - transport: OpenAIResponses.channelTransport({ + transport: OpenResponsesChannel.transport({ id: "openai-responses", name: "xAI Responses", rotateAfterMs: RESPONSES_WEBSOCKET_ROTATE_AFTER_MS, diff --git a/packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json b/packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json index 0ecf4cee845..bd39c70cbd7 100644 --- a/packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json +++ b/packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json @@ -5,12 +5,13 @@ "prefix:pdf", "pdf", "provider:xai", - "protocol:openai-responses", + "protocol:xai-responses", "tool", "tool-result" ], "name": "pdf/xai-tool-result", - "recordedAt": "2026-07-22T18:15:43.608Z" + "recordedAt": "2026-07-22T18:15:43.608Z", + "protocol": "xai-responses" }, "interactions": [ { diff --git a/packages/ai/test/fixtures/recordings/pdf/xai-user-input.json b/packages/ai/test/fixtures/recordings/pdf/xai-user-input.json index 99c8d179c23..f5fa00a5d46 100644 --- a/packages/ai/test/fixtures/recordings/pdf/xai-user-input.json +++ b/packages/ai/test/fixtures/recordings/pdf/xai-user-input.json @@ -5,11 +5,12 @@ "prefix:pdf", "pdf", "provider:xai", - "protocol:openai-responses", + "protocol:xai-responses", "user-input" ], "name": "pdf/xai-user-input", - "recordedAt": "2026-07-22T18:15:42.429Z" + "recordedAt": "2026-07-22T18:15:42.429Z", + "protocol": "xai-responses" }, "interactions": [ { diff --git a/packages/ai/test/provider/openai-compatible-responses.test.ts b/packages/ai/test/provider/openai-compatible-responses.test.ts index 82c2c20b0a4..e1968d6272b 100644 --- a/packages/ai/test/provider/openai-compatible-responses.test.ts +++ b/packages/ai/test/provider/openai-compatible-responses.test.ts @@ -93,7 +93,7 @@ describe("Open Responses-compatible route", () => { }), ) - it.effect("omits OpenAI-only nullable phases from the Open Responses baseline", () => + it.effect("preserves nullable phases in the forgiving Open Responses baseline", () => Effect.gen(function* () { const model = configure({ apiKey: "test-key", @@ -113,7 +113,9 @@ describe("Open Responses-compatible route", () => { ) expect(prepared.body).toMatchObject({ - input: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "Unclassified." }] }], + input: [ + { type: "message", role: "assistant", content: [{ type: "output_text", text: "Unclassified." }], phase: null }, + ], }) }), ) diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index a39bf7f8444..aed7afe4cc3 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -28,7 +28,7 @@ import * as Azure from "../../src/providers/azure.js" import * as OpenAI from "../../src/providers/openai.js" import * as XAI from "../../src/providers/xai.js" import * as OpenAIResponses from "../../src/protocols/openai-responses.js" -import { OpenAIResponsesChannel } from "../../src/protocols/openai-responses-channel.js" +import { OpenResponsesContinuation } from "../../src/protocols/open-responses-continuation.js" import * as ProviderShared from "../../src/protocols/shared.js" import { continuationRequest, nativeOpenAIResponsesContinuation } from "../continuation-scenarios.js" import { it } from "../lib/effect.js" @@ -68,7 +68,7 @@ const baseChannelDriver = (message: string): WebSocketChannelDriver => ({ const continuationDriver = (request: Readonly>) => { const message = ProviderShared.encodeJson(request) - return OpenAIResponsesChannel.driver({ + return OpenResponsesContinuation.driver({ id: "openai-responses", name: "OpenAI Responses", request, @@ -1363,6 +1363,60 @@ describe("OpenAI Responses route", () => { }), ) + it.effect("lowers remote tool-result media URLs without base64 wrapping", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.assistant([ToolCallPart.make({ id: "call_1", name: "fetch", input: {} })]), + Message.tool({ + id: "call_1", + name: "fetch", + resultType: "content", + result: [ + { type: "file", uri: "https://example.com/image.png", mime: "image/png" }, + { type: "file", uri: "https://example.com/report.pdf", mime: "application/pdf", name: "report.pdf" }, + ], + }), + ], + }), + ) + + expect(expectToolOutput(prepared.body).output).toEqual([ + { type: "input_image", image_url: "https://example.com/image.png" }, + { type: "input_file", filename: "report.pdf", file_url: "https://example.com/report.pdf" }, + ]) + }), + ) + + it.effect("lowers tool-result videos as input_video", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.assistant([ToolCallPart.make({ id: "call_1", name: "record", input: {} })]), + Message.tool({ + id: "call_1", + name: "record", + resultType: "content", + result: [ + { type: "file", uri: "data:video/mp4;base64,AAECAw==", mime: "video/mp4" }, + { type: "file", uri: "https://example.com/demo.mp4", mime: "video/mp4" }, + ], + }), + ], + }), + ) + + expect(expectToolOutput(prepared.body).output).toEqual([ + { type: "input_video", video_url: "data:video/mp4;base64,AAECAw==" }, + { type: "input_video", video_url: "https://example.com/demo.mp4" }, + ]) + }), + ) + it.effect("prepares the composed native continuation request", () => Effect.gen(function* () { const prepared = yield* compileRequest( @@ -2775,6 +2829,37 @@ describe("OpenAI Responses route", () => { }), ) + it.effect("lowers remote user media URLs without base64 wrapping", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.user([ + { type: "media", mediaType: "image/png", data: "https://example.com/image.png" }, + { + type: "media", + mediaType: "application/pdf", + data: "https://example.com/report.pdf", + filename: "report.pdf", + }, + ]), + ], + }), + ) + + expect(prepared.body.input).toEqual([ + { + role: "user", + content: [ + { type: "input_image", image_url: "https://example.com/image.png" }, + { type: "input_file", filename: "report.pdf", file_url: "https://example.com/report.pdf" }, + ], + }, + ]) + }), + ) + it.effect("fails with a typed rate limit for provider error frames", () => Effect.gen(function* () { const error = yield* LLMClient.generate(request).pipe( diff --git a/packages/ai/test/provider/pdf.recorded.test.ts b/packages/ai/test/provider/pdf.recorded.test.ts index 635f6d99c5b..5a668ec5b30 100644 --- a/packages/ai/test/provider/pdf.recorded.test.ts +++ b/packages/ai/test/provider/pdf.recorded.test.ts @@ -64,7 +64,7 @@ const targets: ReadonlyArray<{ id: "xai", name: "xAI Grok 4.5", provider: "xai", - protocol: "openai-responses", + protocol: "xai-responses", requires: "XAI_API_KEY", filename: "verification.pdf", maxTokens: 40, diff --git a/packages/ai/test/provider/xai-responses.test.ts b/packages/ai/test/provider/xai-responses.test.ts new file mode 100644 index 00000000000..4f9d14048be --- /dev/null +++ b/packages/ai/test/provider/xai-responses.test.ts @@ -0,0 +1,77 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { LLM, LLMEvent } from "../../src/index.js" +import { XAI } from "../../src/providers.js" +import { OpenResponses } from "../../src/protocols/open-responses.js" +import { OpenAIResponses } from "../../src/protocols/openai-responses.js" +import { XAIResponses } from "../../src/protocols/xai-responses.js" +import { LLMClient } from "../../src/route.js" +import { compileRequest } from "../../src/route/client.js" +import { it } from "../lib/effect.js" +import { fixedResponse } from "../lib/http.js" +import { sseEvents } from "../lib/sse.js" + +const model = XAI.configure({ apiKey: "test", baseURL: "https://api.x.ai/v1" }).responses("grok-4.6") + +describe("xAI Responses route", () => { + it.effect("extends the Open Responses baseline directly", () => + Effect.gen(function* () { + expect(XAIResponses.protocol.body).toBe(OpenResponses.protocol.body) + expect(XAIResponses.protocol.body).not.toBe(OpenAIResponses.protocol.body) + + const prepared = yield* compileRequest(LLM.request({ model, prompt: "Hello" })) + expect(prepared.protocol).toBe("xai-responses") + }), + ) + + it.effect("parses xAI reasoning text events", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(LLM.request({ model, prompt: "Think" })).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { type: "response.reasoning_text.delta", item_id: "reasoning_1", delta: "Considering." }, + { type: "response.reasoning_text.done", item_id: "reasoning_1" }, + { + type: "response.output_item.done", + item: { type: "reasoning", id: "reasoning_1", encrypted_content: "opaque" }, + }, + { type: "response.completed", response: { id: "response_1" } }, + ), + ), + ), + ) + + expect(response.message.content.find((part) => part.type === "reasoning")).toMatchObject({ + type: "reasoning", + text: "Considering.", + providerMetadata: { xai: { itemId: "reasoning_1", reasoningEncryptedContent: "opaque" } }, + }) + }), + ) + + it.effect("parses xAI hosted tool items", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(LLM.request({ model, prompt: "Search X" })).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { + type: "response.output_item.done", + item: { type: "x_search_call", id: "x_search_1", status: "completed", action: { query: "news" } }, + }, + { type: "response.completed", response: { id: "response_1" } }, + ), + ), + ), + ) + + expect(response.events.find(LLMEvent.is.toolCall)).toMatchObject({ + id: "x_search_1", + name: "x_search", + input: { query: "news" }, + providerExecuted: true, + }) + }), + ) +})