vrr/opencode
2
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-07-10 03:18:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat(llm): add video and audio media support to Gemini protocol (#31889)

Browse source 
Co-authored-by: Aiden Cline <63023139+rekram1-node@users.noreply.github.com>
This commit is contained in:
Tommy D. Rossi 2026-06-22 17:03:19 +02:00 • committed by GitHub
parent 41d1279b6f
commit d5980b47e9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 8 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
packages/llm/src/protocols/gemini.ts
View file

@ -21,7 +21,7 @@ import { GeminiToolSchema } from "./utils/gemini-tool-schema"
import { Lifecycle } from "./utils/lifecycle"
const ADAPTER = "gemini"
const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
const MEDIA_MIMES = new Set<string>(ProviderShared.MEDIA_MIMES)
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// =============================================================================
@ -182,7 +182,7 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
if (part.type === "text") return { text: part.text }
const media = yield* ProviderShared.validateMedia("Gemini", part, IMAGE_MIMES)
const media = yield* ProviderShared.validateMedia("Gemini", part, MEDIA_MIMES)
return { inlineData: { mimeType: media.mime, data: media.base64 } }
})
@ -275,7 +275,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
})
for (const item of content) {
if (item.type === "text") continue
const media = yield* ProviderShared.validateToolFile("Gemini", item, IMAGE_MIMES)
const media = yield* ProviderShared.validateToolFile("Gemini", item, MEDIA_MIMES)
parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } })
}
}

7
packages/llm/src/protocols/shared.ts
View file

@ -188,8 +188,11 @@ export const parseToolInput = (route: string, name: string, raw: string) =>
parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const
export const MAX_MEDIA_ENCODED_BYTES = 8 * 1024 * 1024
export const MAX_MEDIA_DECODED_BYTES = 6 * 1024 * 1024
export const VIDEO_MIMES = ["video/mp4", "video/webm", "video/quicktime"] as const
export const AUDIO_MIMES = ["audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac"] as const
export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES] as const
export const MAX_MEDIA_ENCODED_BYTES = 28 * 1024 * 1024
export const MAX_MEDIA_DECODED_BYTES = 20 * 1024 * 1024
const base64Pattern = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/