fix(ai): preserve responses image detail

This commit is contained in:
Shoubhit Dash 2026-08-31 20:52:22 +05:30
parent ffb9728068
commit 0d045152fc
3 changed files with 69 additions and 1 deletions

View file

@ -39,6 +39,7 @@ const OpenResponsesInputText = Schema.Struct({
const OpenResponsesInputImage = Schema.Struct({
type: Schema.tag("input_image"),
image_url: Schema.String,
detail: Schema.optional(Schema.String),
})
const OpenResponsesInputFile = Schema.Struct({
type: Schema.tag("input_file"),
@ -501,7 +502,13 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* (
...(url ? { file_url: url } : { file_data: media.dataUrl }),
}
}
return { type: "input_image" as const, image_url: url ?? media.dataUrl }
return {
type: "input_image" as const,
image_url: url ?? media.dataUrl,
detail: yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenResponsesInputImage.fields.detail))(
part.providerMetadata?.[request.model.route.providerMetadataKey ?? "openresponses"]?.detail,
),
}
})
const lowerUserContent = Effect.fnUntraced(function* (

View file

@ -52,6 +52,7 @@ export const MediaPart = Schema.Struct({
filename: Schema.optional(Schema.String),
cache: Schema.optional(CacheHint),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Content.Media" })
export type MediaPart = Schema.Schema.Type<typeof MediaPart>

View file

@ -0,0 +1,60 @@
import { expect } from "bun:test"
import { Effect, Schema } from "effect"
import { LLM, Message } from "../../src/index.js"
import { OpenAI, Azure, XAI } from "../../src/providers.js"
import { compileRequest } from "../../src/route/client.js"
import { it } from "../lib/effect.js"
for (const model of [
OpenAI.configure({ apiKey: "test" }).responses("fixture"),
Azure.configure({ apiKey: "test", resourceName: "test" }).responses("fixture"),
XAI.configure({ apiKey: "test" }).responses("fixture"),
]) {
it.effect(`${model.provider} preserves image detail through message serialization and lowering`, () =>
Effect.gen(function* () {
const details = [undefined, "low", "high", "auto"]
const message = Message.user(
details.map((detail) => ({
type: "media",
mediaType: "image/png",
data: "https://example.com/image.png",
providerMetadata:
detail === undefined ? undefined : { [model.route.providerMetadataKey ?? model.provider]: { detail } },
})),
)
const codec = Schema.fromJsonString(Message)
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [Schema.decodeSync(codec)(Schema.encodeSync(codec)(message))],
}),
)
expect(prepared.body.input[0].content).toEqual(
details.map((detail) => ({
type: "input_image",
image_url: "https://example.com/image.png",
detail,
})),
)
}),
)
}
it.effect("rejects malformed image detail instead of silently discarding it", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
LLM.request({
model: OpenAI.configure({ apiKey: "test" }).responses("fixture"),
messages: [
Message.user({
type: "media",
mediaType: "image/png",
data: "https://example.com/image.png",
providerMetadata: { openai: { detail: 42 } },
}),
],
}),
).pipe(Effect.flip)
expect(error.reason._tag).toBe("InvalidRequest")
}),
)