mirror of
https://github.com/anomalyco/opencode.git
synced 2026-08-30 14:02:38 +00:00
fix(core): normalize SDK file data (#45679)
This commit is contained in:
Kit Langton
GitHub
parent
5634ef1bb6
commit
134cdda333
2 changed files with 132 additions and 7 deletions
|
|
@ -482,8 +482,7 @@ function toolMessage(input: LLMRequest["messages"][number]) {
|
|||
const value = part.result.value.filter((item) => {
|
||||
if (item.type !== "file") return true
|
||||
if (!item.mime.startsWith("image/") && item.mime !== "application/pdf") return true
|
||||
const data = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(item.uri)?.[1] ?? item.uri
|
||||
media.push({ type: "file", mediaType: item.mime, data, filename: item.name })
|
||||
media.push({ type: "file", mediaType: item.mime, data: fileData(item.uri), filename: item.name })
|
||||
return false
|
||||
})
|
||||
return toolResultPart({
|
||||
|
|
@ -507,7 +506,7 @@ function text(part: ContentPart) {
|
|||
function userPart(part: ContentPart): UserContent {
|
||||
if (part.type === "text") return [{ type: "text", text: part.text }]
|
||||
if (part.type === "media")
|
||||
return [{ type: "file", mediaType: part.mediaType, data: part.data, filename: part.filename }]
|
||||
return [{ type: "file", mediaType: part.mediaType, data: fileData(part.data), filename: part.filename }]
|
||||
return []
|
||||
}
|
||||
|
||||
|
|
@ -516,7 +515,7 @@ function assistantPart(part: ContentPart): AssistantContent {
|
|||
case "text":
|
||||
return [{ type: "text", text: part.text, providerOptions: metadataProviderOptions(part.providerMetadata) }]
|
||||
case "media":
|
||||
return [{ type: "file", mediaType: part.mediaType, data: part.data, filename: part.filename }]
|
||||
return [{ type: "file", mediaType: part.mediaType, data: fileData(part.data), filename: part.filename }]
|
||||
case "reasoning":
|
||||
return [{ type: "reasoning", text: part.text, providerOptions: metadataProviderOptions(part.providerMetadata) }]
|
||||
case "tool-call":
|
||||
|
|
@ -535,6 +534,15 @@ function assistantPart(part: ContentPart): AssistantContent {
|
|||
}
|
||||
}
|
||||
|
||||
function fileData(data: Extract<ContentPart, { type: "media" }>["data"]) {
|
||||
if (typeof data !== "string") return data
|
||||
const base64 = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(data)?.[1]
|
||||
if (base64 !== undefined) return base64
|
||||
if (!URL.canParse(data)) return data
|
||||
const url = new URL(data)
|
||||
return url.protocol === "http:" || url.protocol === "https:" ? url : data
|
||||
}
|
||||
|
||||
function toolResultPart(part: ContentPart): ToolResultContent[] {
|
||||
if (part.type !== "tool-result") return []
|
||||
return [
|
||||
|
|
|
|||
|
|
@ -375,7 +375,100 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () =>
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("moves a tool image through the real Mistral provider as a user message", () =>
|
||||
it.effect("normalizes file data across AI SDK prompt parts", () =>
|
||||
Effect.gen(function* () {
|
||||
const aisdk = yield* AISDK.Service
|
||||
yield* aisdk.hook.sdk((event) => {
|
||||
event.sdk = { languageModel: () => ({ provider: event.model.providerID }) }
|
||||
})
|
||||
|
||||
const resolved = yield* aisdk.model(model("opaque-provider"))
|
||||
const bytes = new Uint8Array([0, 1, 2, 3])
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: resolved,
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "media", mediaType: "image/png", data: bytes, filename: "bytes.png" },
|
||||
{ type: "media", mediaType: "image/png", data: "AAAA", filename: "base64.png" },
|
||||
{
|
||||
type: "media",
|
||||
mediaType: "image/png",
|
||||
data: "data:image/png;charset=utf-8;base64,AQID",
|
||||
filename: "inline.png",
|
||||
},
|
||||
{ type: "media", mediaType: "image/png", data: "https://example.com/image.png" },
|
||||
{ type: "media", mediaType: "image/png", data: "s3://bucket/image.png" },
|
||||
]),
|
||||
Message.assistant({
|
||||
type: "media",
|
||||
mediaType: "application/pdf",
|
||||
data: "http://example.com/document.pdf",
|
||||
filename: "document.pdf",
|
||||
}),
|
||||
Message.tool({
|
||||
id: "call_1",
|
||||
name: "screenshot",
|
||||
result: {
|
||||
type: "content",
|
||||
value: [{ type: "file", uri: "data:image/png;base64,BAUG", mime: "image/png", name: "tool.png" }],
|
||||
},
|
||||
}),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.prompt).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "file", mediaType: "image/png", data: bytes, filename: "bytes.png" },
|
||||
{ type: "file", mediaType: "image/png", data: "AAAA", filename: "base64.png" },
|
||||
{ type: "file", mediaType: "image/png", data: "AQID", filename: "inline.png" },
|
||||
{
|
||||
type: "file",
|
||||
mediaType: "image/png",
|
||||
data: new URL("https://example.com/image.png"),
|
||||
filename: undefined,
|
||||
},
|
||||
{ type: "file", mediaType: "image/png", data: "s3://bucket/image.png", filename: undefined },
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "file",
|
||||
mediaType: "application/pdf",
|
||||
data: new URL("http://example.com/document.pdf"),
|
||||
filename: "document.pdf",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "tool",
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
toolCallId: "call_1",
|
||||
toolName: "screenshot",
|
||||
output: { type: "text", value: "Media attached in the following user message." },
|
||||
providerOptions: undefined,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Attached media from tool result:" },
|
||||
{ type: "file", mediaType: "image/png", data: "BAUG", filename: "tool.png" },
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("normalizes user and tool media through the real Mistral provider", () =>
|
||||
Effect.gen(function* () {
|
||||
const aisdk = yield* AISDK.Service
|
||||
let body: { messages?: unknown[] } | undefined
|
||||
|
|
@ -415,7 +508,14 @@ it.effect("moves a tool image through the real Mistral provider as a user messag
|
|||
LLM.request({
|
||||
model: resolved,
|
||||
messages: [
|
||||
Message.user("Inspect the screenshot."),
|
||||
Message.user([
|
||||
{ type: "text", text: "Inspect the attachments." },
|
||||
{ type: "media", mediaType: "image/png", data: new Uint8Array([0, 1, 2, 3]) },
|
||||
{ type: "media", mediaType: "image/png", data: "AQID" },
|
||||
{ type: "media", mediaType: "image/png", data: "data:image/png;base64,BAUG" },
|
||||
{ type: "media", mediaType: "image/png", data: "http://example.com/image.png" },
|
||||
{ type: "media", mediaType: "application/pdf", data: "https://example.com/document.pdf" },
|
||||
]),
|
||||
Message.assistant({ type: "tool-call", id: "call_1", name: "screenshot", input: {} }),
|
||||
Message.tool({
|
||||
type: "tool-result",
|
||||
|
|
@ -426,6 +526,12 @@ it.effect("moves a tool image through the real Mistral provider as a user messag
|
|||
value: [
|
||||
{ type: "text", text: "Screenshot captured" },
|
||||
{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "screen.png" },
|
||||
{
|
||||
type: "file",
|
||||
uri: "https://example.com/tool-document.pdf",
|
||||
mime: "application/pdf",
|
||||
name: "tool-document.pdf",
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
|
|
@ -434,7 +540,17 @@ it.effect("moves a tool image through the real Mistral provider as a user messag
|
|||
).pipe(Effect.provide(client))
|
||||
|
||||
expect(body?.messages).toEqual([
|
||||
{ role: "user", content: [{ type: "text", text: "Inspect the screenshot." }] },
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Inspect the attachments." },
|
||||
{ type: "image_url", image_url: "data:image/png;base64,AAECAw==" },
|
||||
{ type: "image_url", image_url: "data:image/png;base64,AQID" },
|
||||
{ type: "image_url", image_url: "data:image/png;base64,BAUG" },
|
||||
{ type: "image_url", image_url: "http://example.com/image.png" },
|
||||
{ type: "document_url", document_url: "https://example.com/document.pdf" },
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: "",
|
||||
|
|
@ -457,6 +573,7 @@ it.effect("moves a tool image through the real Mistral provider as a user messag
|
|||
content: [
|
||||
{ type: "text", text: "Attached media from tool result:" },
|
||||
{ type: "image_url", image_url: "data:image/png;base64,AAAA" },
|
||||
{ type: "document_url", document_url: "https://example.com/tool-document.pdf" },
|
||||
],
|
||||
},
|
||||
])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue