mirror of
https://github.com/anomalyco/opencode.git
synced 2026-08-30 00:21:59 +00:00
fix(ai): isolate native provider metadata ownership (#45155)
This commit is contained in:
Aiden Cline
GitHub
parent
33909f48d7
commit
94bd9f6c8b
24 changed files with 603 additions and 136 deletions
|
|
@ -406,6 +406,7 @@ const AnthropicEvent = Schema.Struct({
|
|||
type AnthropicEvent = Schema.Schema.Type<typeof AnthropicEvent>
|
||||
|
||||
interface ParserState {
|
||||
readonly providerMetadataKey: string
|
||||
readonly tools: ToolStream.State<number>
|
||||
readonly reasoningSignatures: Readonly<Record<number, string>>
|
||||
readonly usage?: Usage
|
||||
|
|
@ -440,18 +441,18 @@ const cacheControl = (breakpoints: Cache.Breakpoints, cache: CacheHint | undefin
|
|||
return Cache.ttlBucket(cache.ttlSeconds) === "1h" ? EPHEMERAL_1H : EPHEMERAL_5M
|
||||
}
|
||||
|
||||
const anthropicMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ anthropic: metadata })
|
||||
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
|
||||
|
||||
const signatureFromMetadata = (metadata: ProviderMetadata | undefined): string | undefined => {
|
||||
const anthropic = metadata?.anthropic
|
||||
if (!ProviderShared.isRecord(anthropic)) return undefined
|
||||
return typeof anthropic.signature === "string" ? anthropic.signature : undefined
|
||||
const signatureFromMetadata = (metadata: ProviderMetadata | undefined, key: string): string | undefined => {
|
||||
const provider = metadata?.[key]
|
||||
if (!ProviderShared.isRecord(provider)) return undefined
|
||||
return typeof provider.signature === "string" ? provider.signature : undefined
|
||||
}
|
||||
|
||||
const redactedDataFromMetadata = (metadata: ProviderMetadata | undefined): string | undefined => {
|
||||
const anthropic = metadata?.anthropic
|
||||
if (!ProviderShared.isRecord(anthropic)) return undefined
|
||||
return typeof anthropic.redactedData === "string" ? anthropic.redactedData : undefined
|
||||
const redactedDataFromMetadata = (metadata: ProviderMetadata | undefined, key: string): string | undefined => {
|
||||
const provider = metadata?.[key]
|
||||
if (!ProviderShared.isRecord(provider)) return undefined
|
||||
return typeof provider.redactedData === "string" ? provider.redactedData : undefined
|
||||
}
|
||||
|
||||
const lowerTool = (breakpoints: Cache.Breakpoints, tool: ToolDefinition, inputSchema: JsonSchema): AnthropicTool => ({
|
||||
|
|
@ -511,13 +512,16 @@ const serverToolResultType = (name: string): AnthropicServerToolResultType | und
|
|||
return undefined
|
||||
}
|
||||
|
||||
const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult")(function* (part: ToolResultPart) {
|
||||
const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult")(function* (
|
||||
part: ToolResultPart,
|
||||
providerMetadataKey: string,
|
||||
) {
|
||||
const wireType = serverToolResultType(part.name)
|
||||
if (!wireType)
|
||||
return yield* invalid(`Anthropic Messages does not know how to round-trip server tool result for ${part.name}`)
|
||||
// Prefer the provider-owned replay payload; fall back to the result value for
|
||||
// histories constructed directly from provider events.
|
||||
const payload = part.providerMetadata?.anthropic?.["result"] ?? part.result.value
|
||||
const payload = part.providerMetadata?.[providerMetadataKey]?.["result"] ?? part.result.value
|
||||
return {
|
||||
type: wireType,
|
||||
tool_use_id: scrubToolCallID(part.id),
|
||||
|
|
@ -804,6 +808,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
|
|||
breakpoints: Cache.Breakpoints,
|
||||
) {
|
||||
const messages: AnthropicMessage[] = []
|
||||
const providerMetadataKey = request.model.route.providerMetadataKey ?? String(request.model.provider)
|
||||
|
||||
for (const [index, message] of request.messages.entries()) {
|
||||
if (message.role === "system") {
|
||||
|
|
@ -849,8 +854,8 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
|
|||
if (part.type === "reasoning") {
|
||||
// A signature marks visible thinking; only signature-less parts carrying
|
||||
// redactedData round-trip as opaque redacted_thinking blocks.
|
||||
const signature = part.encrypted ?? signatureFromMetadata(part.providerMetadata)
|
||||
const redactedData = redactedDataFromMetadata(part.providerMetadata)
|
||||
const signature = part.encrypted ?? signatureFromMetadata(part.providerMetadata, providerMetadataKey)
|
||||
const redactedData = redactedDataFromMetadata(part.providerMetadata, providerMetadataKey)
|
||||
if (signature === undefined && redactedData !== undefined) {
|
||||
content.push({ type: "redacted_thinking", data: redactedData })
|
||||
continue
|
||||
|
|
@ -879,7 +884,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
|
|||
continue
|
||||
}
|
||||
if (part.type === "tool-result" && part.providerExecuted) {
|
||||
content.push(yield* lowerServerToolResult(part))
|
||||
content.push(yield* lowerServerToolResult(part, providerMetadataKey))
|
||||
continue
|
||||
}
|
||||
return yield* invalid(
|
||||
|
|
@ -1069,7 +1074,7 @@ const mapFinishReason = (reason: string | null | undefined): FinishReason => {
|
|||
// inclusive `inputTokens` the rest of the contract expects. Extended
|
||||
// thinking tokens are included in `output_tokens`; newer responses also
|
||||
// expose that subset through `output_tokens_details.thinking_tokens`.
|
||||
const mapUsage = (usage: AnthropicUsage | undefined): Usage | undefined => {
|
||||
const mapUsage = (usage: AnthropicUsage | undefined, providerMetadataKey: string): Usage | undefined => {
|
||||
if (!usage) return undefined
|
||||
const nonCached = usage.input_tokens ?? undefined
|
||||
const cacheRead = usage.cache_read_input_tokens ?? undefined
|
||||
|
|
@ -1083,7 +1088,7 @@ const mapUsage = (usage: AnthropicUsage | undefined): Usage | undefined => {
|
|||
cacheWriteInputTokens: cacheWrite,
|
||||
reasoningTokens: usage.output_tokens_details?.thinking_tokens,
|
||||
totalTokens: ProviderShared.totalTokens(inputTokens, usage.output_tokens, undefined),
|
||||
providerMetadata: { anthropic: usage },
|
||||
providerMetadata: { [providerMetadataKey]: usage },
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1092,7 +1097,7 @@ const mapUsage = (usage: AnthropicUsage | undefined): Usage | undefined => {
|
|||
// field prefers `right` when defined, falls back to `left`. `inputTokens` is
|
||||
// recomputed from the merged breakdown so the inclusive total stays
|
||||
// consistent with `nonCached + cacheRead + cacheWrite`.
|
||||
const mergeUsage = (left: Usage | undefined, right: Usage | undefined) => {
|
||||
const mergeUsage = (left: Usage | undefined, right: Usage | undefined, providerMetadataKey: string) => {
|
||||
if (!left) return right
|
||||
if (!right) return left
|
||||
const nonCachedInputTokens = right.nonCachedInputTokens ?? left.nonCachedInputTokens
|
||||
|
|
@ -1110,7 +1115,9 @@ const mergeUsage = (left: Usage | undefined, right: Usage | undefined) => {
|
|||
reasoningTokens,
|
||||
totalTokens: ProviderShared.totalTokens(inputTokens, outputTokens, undefined),
|
||||
providerMetadata: {
|
||||
anthropic: mergeJsonRecords(left.providerMetadata?.["anthropic"], right.providerMetadata?.["anthropic"]) ?? {},
|
||||
[providerMetadataKey]:
|
||||
mergeJsonRecords(left.providerMetadata?.[providerMetadataKey], right.providerMetadata?.[providerMetadataKey]) ??
|
||||
{},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -1128,7 +1135,7 @@ const SERVER_TOOL_RESULT_NAMES: Record<AnthropicServerToolResultType, string> =
|
|||
|
||||
const isServerToolResultType = (type: string): type is AnthropicServerToolResultType => type in SERVER_TOOL_RESULT_NAMES
|
||||
|
||||
const serverToolResultEvent = (block: AnthropicStreamBlock): LLMEvent | undefined => {
|
||||
const serverToolResultEvent = (block: AnthropicStreamBlock, providerMetadataKey: string): LLMEvent | undefined => {
|
||||
if (!block.type || !isServerToolResultType(block.type)) return undefined
|
||||
const errorPayload =
|
||||
typeof block.content === "object" && block.content !== null && "type" in block.content
|
||||
|
|
@ -1142,7 +1149,7 @@ const serverToolResultEvent = (block: AnthropicStreamBlock): LLMEvent | undefine
|
|||
providerExecuted: true,
|
||||
// The complete payload is irreducible provider replay state: subsequent
|
||||
// stateless requests must round-trip the typed result block verbatim.
|
||||
providerMetadata: anthropicMetadata({ blockType: block.type, result: block.content }),
|
||||
providerMetadata: providerMetadata(providerMetadataKey, { blockType: block.type, result: block.content }),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1151,8 +1158,8 @@ type StepResult = readonly [ParserState, ReadonlyArray<LLMEvent>]
|
|||
const NO_EVENTS: StepResult["1"] = []
|
||||
|
||||
const onMessageStart = (state: ParserState, event: AnthropicEvent): StepResult => {
|
||||
const usage = mapUsage(event.message?.usage)
|
||||
return [usage ? { ...state, usage: mergeUsage(state.usage, usage) } : state, NO_EVENTS]
|
||||
const usage = mapUsage(event.message?.usage, state.providerMetadataKey)
|
||||
return [usage ? { ...state, usage: mergeUsage(state.usage, usage, state.providerMetadataKey) } : state, NO_EVENTS]
|
||||
}
|
||||
|
||||
const onContentBlockStart = (
|
||||
|
|
@ -1204,14 +1211,16 @@ const onContentBlockStart = (
|
|||
if (block.type === "thinking" && block.thinking !== undefined) {
|
||||
const events: LLMEvent[] = []
|
||||
const id = `reasoning-${event.index ?? 0}`
|
||||
const providerMetadata =
|
||||
block.signature === undefined ? undefined : anthropicMetadata({ signature: block.signature })
|
||||
const lifecycle = Lifecycle.reasoningStart(state.lifecycle, events, id, providerMetadata)
|
||||
const metadata =
|
||||
block.signature === undefined
|
||||
? undefined
|
||||
: providerMetadata(state.providerMetadataKey, { signature: block.signature })
|
||||
const lifecycle = Lifecycle.reasoningStart(state.lifecycle, events, id, metadata)
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
lifecycle: block.thinking
|
||||
? Lifecycle.reasoningDelta(lifecycle, events, id, block.thinking, providerMetadata)
|
||||
? Lifecycle.reasoningDelta(lifecycle, events, id, block.thinking, metadata)
|
||||
: lifecycle,
|
||||
reasoningSignatures:
|
||||
event.index === undefined || block.signature === undefined
|
||||
|
|
@ -1234,14 +1243,14 @@ const onContentBlockStart = (
|
|||
state.lifecycle,
|
||||
events,
|
||||
`reasoning-${event.index ?? 0}`,
|
||||
anthropicMetadata({ redactedData: block.data }),
|
||||
providerMetadata(state.providerMetadataKey, { redactedData: block.data }),
|
||||
),
|
||||
},
|
||||
events,
|
||||
]
|
||||
}
|
||||
|
||||
const result = serverToolResultEvent(block)
|
||||
const result = serverToolResultEvent(block, state.providerMetadataKey)
|
||||
if (!result) return [state, NO_EVENTS]
|
||||
const events: LLMEvent[] = []
|
||||
return [{ ...state, lifecycle: Lifecycle.stepStart(state.lifecycle, events) }, [...events, result]]
|
||||
|
|
@ -1321,7 +1330,7 @@ const onContentBlockStop = Effect.fn("AnthropicMessages.onContentBlockStop")(fun
|
|||
Lifecycle.textEnd(state.lifecycle, events, `text-${event.index}`),
|
||||
events,
|
||||
`reasoning-${event.index}`,
|
||||
signature === undefined ? undefined : anthropicMetadata({ signature }),
|
||||
signature === undefined ? undefined : providerMetadata(state.providerMetadataKey, { signature }),
|
||||
)
|
||||
events.push(...resultEvents)
|
||||
const reasoningSignatures = { ...state.reasoningSignatures }
|
||||
|
|
@ -1333,7 +1342,7 @@ const onMessageDelta = (
|
|||
state: ParserState,
|
||||
event: AnthropicEvent & { readonly delta?: AnthropicStreamDelta },
|
||||
): StepResult => {
|
||||
const usage = mergeUsage(state.usage, mapUsage(event.usage))
|
||||
const usage = mergeUsage(state.usage, mapUsage(event.usage, state.providerMetadataKey), state.providerMetadataKey)
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
|
|
@ -1346,7 +1355,7 @@ const onMessageDelta = (
|
|||
providerMetadata:
|
||||
event.delta?.stop_sequence === null || event.delta?.stop_sequence === undefined
|
||||
? undefined
|
||||
: anthropicMetadata({ stopSequence: event.delta.stop_sequence }),
|
||||
: providerMetadata(state.providerMetadataKey, { stopSequence: event.delta.stop_sequence }),
|
||||
},
|
||||
},
|
||||
NO_EVENTS,
|
||||
|
|
@ -1472,7 +1481,8 @@ export const protocol = Protocol.make({
|
|||
},
|
||||
stream: {
|
||||
event: Protocol.jsonEvent(AnthropicEvent),
|
||||
initial: () => ({
|
||||
initial: (request) => ({
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
tools: ToolStream.empty<number>(),
|
||||
reasoningSignatures: {},
|
||||
lifecycle: Lifecycle.initial(),
|
||||
|
|
|
|||
|
|
@ -258,19 +258,21 @@ const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
|||
tool: (name) => ({ tool: { name } }) as const,
|
||||
})
|
||||
|
||||
const bedrockMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ bedrock: metadata })
|
||||
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
|
||||
|
||||
const reasoningSignature = (part: ReasoningPart) => {
|
||||
const bedrock = part.providerMetadata?.bedrock
|
||||
const reasoningSignature = (part: ReasoningPart, providerMetadataKey: string) => {
|
||||
const metadata = part.providerMetadata?.[providerMetadataKey]
|
||||
return (
|
||||
part.encrypted ??
|
||||
(ProviderShared.isRecord(bedrock) && typeof bedrock.signature === "string" ? bedrock.signature : undefined)
|
||||
(ProviderShared.isRecord(metadata) && typeof metadata.signature === "string" ? metadata.signature : undefined)
|
||||
)
|
||||
}
|
||||
|
||||
const reasoningRedactedData = (part: ReasoningPart) => {
|
||||
const bedrock = part.providerMetadata?.bedrock
|
||||
return ProviderShared.isRecord(bedrock) && typeof bedrock.redactedData === "string" ? bedrock.redactedData : undefined
|
||||
const reasoningRedactedData = (part: ReasoningPart, providerMetadataKey: string) => {
|
||||
const metadata = part.providerMetadata?.[providerMetadataKey]
|
||||
return ProviderShared.isRecord(metadata) && typeof metadata.redactedData === "string"
|
||||
? metadata.redactedData
|
||||
: undefined
|
||||
}
|
||||
|
||||
const lowerToolCall = (part: ToolCallPart): BedrockToolUseBlock => ({
|
||||
|
|
@ -318,6 +320,7 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
|
|||
breakpoints: BedrockCache.Breakpoints,
|
||||
) {
|
||||
const messages: BedrockMessage[] = []
|
||||
const providerMetadataKey = request.model.route.providerMetadataKey ?? String(request.model.provider)
|
||||
|
||||
for (const message of request.messages) {
|
||||
if (message.role === "system") {
|
||||
|
|
@ -365,8 +368,8 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
|
|||
continue
|
||||
}
|
||||
if (part.type === "reasoning") {
|
||||
const signature = reasoningSignature(part)
|
||||
const redactedData = reasoningRedactedData(part)
|
||||
const signature = reasoningSignature(part, providerMetadataKey)
|
||||
const redactedData = reasoningRedactedData(part, providerMetadataKey)
|
||||
if (signature === undefined && redactedData !== undefined) {
|
||||
content.push({ reasoningContent: { redactedContent: redactedData } })
|
||||
continue
|
||||
|
|
@ -466,7 +469,7 @@ const mapFinishReason = (reason: string): FinishReason => {
|
|||
|
||||
// AWS reports inputTokens separately from cache reads and writes.
|
||||
// Bedrock does not break reasoning out of outputTokens for current models.
|
||||
const mapUsage = (usage: BedrockUsageSchema | undefined): Usage | undefined => {
|
||||
const mapUsage = (usage: BedrockUsageSchema | undefined, providerMetadataKey: string): Usage | undefined => {
|
||||
if (!usage) return undefined
|
||||
const inputTokens = ProviderShared.sumTokens(
|
||||
usage.inputTokens,
|
||||
|
|
@ -480,11 +483,12 @@ const mapUsage = (usage: BedrockUsageSchema | undefined): Usage | undefined => {
|
|||
cacheReadInputTokens: usage.cacheReadInputTokens,
|
||||
cacheWriteInputTokens: usage.cacheWriteInputTokens,
|
||||
totalTokens: ProviderShared.totalTokens(inputTokens, usage.outputTokens, usage.totalTokens),
|
||||
providerMetadata: { bedrock: usage },
|
||||
providerMetadata: { [providerMetadataKey]: usage },
|
||||
})
|
||||
}
|
||||
|
||||
interface ParserState {
|
||||
readonly providerMetadataKey: string
|
||||
readonly tools: ToolStream.State<number>
|
||||
// Bedrock splits the finish into `messageStop` (carries `stopReason`) and
|
||||
// `metadata` (carries usage). Hold the terminal event in state so `onHalt`
|
||||
|
|
@ -541,20 +545,14 @@ const step = (state: ParserState, event: BedrockEvent) =>
|
|||
const reasoning = event.contentBlockDelta.delta.reasoningContent
|
||||
const events: LLMEvent[] = []
|
||||
const redactedData = reasoning.redactedContent ?? reasoning.data
|
||||
const providerMetadata = reasoning.signature
|
||||
? bedrockMetadata({ signature: reasoning.signature })
|
||||
const metadata = reasoning.signature
|
||||
? providerMetadata(state.providerMetadataKey, { signature: reasoning.signature })
|
||||
: redactedData !== undefined
|
||||
? bedrockMetadata({ redactedData })
|
||||
? providerMetadata(state.providerMetadataKey, { redactedData })
|
||||
: undefined
|
||||
const lifecycle =
|
||||
reasoning.text !== undefined || providerMetadata !== undefined
|
||||
? Lifecycle.reasoningDelta(
|
||||
state.lifecycle,
|
||||
events,
|
||||
`reasoning-${index}`,
|
||||
reasoning.text ?? "",
|
||||
providerMetadata,
|
||||
)
|
||||
reasoning.text !== undefined || metadata !== undefined
|
||||
? Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${index}`, reasoning.text ?? "", metadata)
|
||||
: state.lifecycle
|
||||
return [
|
||||
{
|
||||
|
|
@ -596,7 +594,7 @@ const step = (state: ParserState, event: BedrockEvent) =>
|
|||
events,
|
||||
`reasoning-${index}`,
|
||||
state.reasoningSignatures[index]
|
||||
? bedrockMetadata({ signature: state.reasoningSignatures[index] })
|
||||
? providerMetadata(state.providerMetadataKey, { signature: state.reasoningSignatures[index] })
|
||||
: undefined,
|
||||
)
|
||||
events.push(...resultEvents)
|
||||
|
|
@ -633,7 +631,7 @@ const step = (state: ParserState, event: BedrockEvent) =>
|
|||
}
|
||||
|
||||
if (event.metadata) {
|
||||
const usage = mapUsage(event.metadata.usage) ?? state.pendingFinish?.usage
|
||||
const usage = mapUsage(event.metadata.usage, state.providerMetadataKey) ?? state.pendingFinish?.usage
|
||||
return [
|
||||
{
|
||||
...state,
|
||||
|
|
@ -698,7 +696,8 @@ export const protocol = Protocol.make({
|
|||
},
|
||||
stream: {
|
||||
event: BedrockEvent,
|
||||
initial: () => ({
|
||||
initial: (request) => ({
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
tools: ToolStream.empty<number>(),
|
||||
pendingFinish: undefined,
|
||||
hasToolCalls: false,
|
||||
|
|
|
|||
|
|
@ -229,6 +229,7 @@ type GeminiEvent = Schema.Schema.Type<typeof GeminiEvent>
|
|||
|
||||
interface ParserState {
|
||||
readonly route: string
|
||||
readonly providerMetadataKey: string
|
||||
readonly finishReason?: string
|
||||
readonly hasToolCalls: boolean
|
||||
readonly promptFeedback?: GeminiPromptFeedback
|
||||
|
|
@ -285,22 +286,23 @@ const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPar
|
|||
return { inlineData: { mimeType: media.mime, data: media.base64 } }
|
||||
})
|
||||
|
||||
const googleMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ google: metadata })
|
||||
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
|
||||
|
||||
const thoughtSignature = (providerMetadata: ProviderMetadata | undefined) => {
|
||||
const google = providerMetadata?.google
|
||||
return ProviderShared.isRecord(google) && typeof google.thoughtSignature === "string"
|
||||
? google.thoughtSignature
|
||||
const thoughtSignature = (metadata: ProviderMetadata | undefined, key: string) => {
|
||||
const value = metadata?.[key]
|
||||
return ProviderShared.isRecord(value) && typeof value.thoughtSignature === "string"
|
||||
? value.thoughtSignature
|
||||
: undefined
|
||||
}
|
||||
|
||||
const lowerToolCall = (part: ToolCallPart, omitIds: boolean) => ({
|
||||
const lowerToolCall = (part: ToolCallPart, omitIds: boolean, metadataKey: string) => ({
|
||||
functionCall: { ...(omitIds ? {} : { id: part.id }), name: part.name, args: part.input },
|
||||
thoughtSignature: thoughtSignature(part.providerMetadata),
|
||||
thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey),
|
||||
})
|
||||
|
||||
const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMRequest) {
|
||||
const contents: GeminiContent[] = []
|
||||
const metadataKey = request.model.route.providerMetadataKey ?? String(request.model.provider)
|
||||
const omitCallIds = omitsFunctionCallIds(request.model.id)
|
||||
const legacyToolMedia = routesLegacyToolMedia(request.model.id)
|
||||
let pendingMedia: GeminiInlineDataPart[] | undefined
|
||||
|
|
@ -342,15 +344,19 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
|
|||
if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"]))
|
||||
return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"])
|
||||
if (part.type === "text") {
|
||||
parts.push({ text: part.text, thoughtSignature: thoughtSignature(part.providerMetadata) })
|
||||
parts.push({ text: part.text, thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey) })
|
||||
continue
|
||||
}
|
||||
if (part.type === "reasoning") {
|
||||
parts.push({ text: part.text, thought: true, thoughtSignature: thoughtSignature(part.providerMetadata) })
|
||||
parts.push({
|
||||
text: part.text,
|
||||
thought: true,
|
||||
thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (part.type === "tool-call") {
|
||||
const lowered = lowerToolCall(part, omitCallIds)
|
||||
const lowered = lowerToolCall(part, omitCallIds, metadataKey)
|
||||
const signature = lowered.thoughtSignature
|
||||
parts.push({
|
||||
...lowered,
|
||||
|
|
@ -498,7 +504,7 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
|
|||
// `cachedContentTokenCount` subset. `candidatesTokenCount` is *exclusive*
|
||||
// of `thoughtsTokenCount` — visible-only, not a total — so we sum the two
|
||||
// to produce the inclusive `outputTokens` the rest of the contract expects.
|
||||
const mapUsage = (usage: GeminiUsage | undefined) => {
|
||||
const mapUsage = (usage: GeminiUsage | undefined, metadataKey: string) => {
|
||||
if (!usage) return undefined
|
||||
// Explicit provider nulls decode as `null`; normalize to `undefined` so the
|
||||
// token arithmetic below treats them like absent counts.
|
||||
|
|
@ -519,7 +525,7 @@ const mapUsage = (usage: GeminiUsage | undefined) => {
|
|||
cacheReadInputTokens: cached,
|
||||
reasoningTokens: thoughts,
|
||||
totalTokens: ProviderShared.totalTokens(promptTokens, outputTokens, usage.totalTokenCount ?? undefined),
|
||||
providerMetadata: { google: usage },
|
||||
providerMetadata: providerMetadata(metadataKey, usage),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -567,14 +573,14 @@ const finish = (state: ParserState): ReadonlyArray<LLMEvent> => {
|
|||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
googleMetadata({ thoughtSignature: state.reasoningSignature }),
|
||||
providerMetadata(state.providerMetadataKey, { thoughtSignature: state.reasoningSignature }),
|
||||
)
|
||||
if (state.textSignature !== undefined)
|
||||
lifecycle = Lifecycle.textEnd(
|
||||
lifecycle,
|
||||
events,
|
||||
"text-0",
|
||||
googleMetadata({ thoughtSignature: state.textSignature }),
|
||||
providerMetadata(state.providerMetadataKey, { thoughtSignature: state.textSignature }),
|
||||
)
|
||||
Lifecycle.finish(lifecycle, events, {
|
||||
reason: {
|
||||
|
|
@ -584,7 +590,9 @@ const finish = (state: ParserState): ReadonlyArray<LLMEvent> => {
|
|||
},
|
||||
usage: state.usage,
|
||||
providerMetadata:
|
||||
state.promptFeedback === undefined ? undefined : googleMetadata({ promptFeedback: state.promptFeedback }),
|
||||
state.promptFeedback === undefined
|
||||
? undefined
|
||||
: providerMetadata(state.providerMetadataKey, { promptFeedback: state.promptFeedback }),
|
||||
})
|
||||
return events
|
||||
}
|
||||
|
|
@ -593,7 +601,9 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
|||
const nextState = {
|
||||
...state,
|
||||
promptFeedback: event.promptFeedback ?? state.promptFeedback,
|
||||
usage: event.usageMetadata ? (mapUsage(event.usageMetadata) ?? state.usage) : state.usage,
|
||||
usage: event.usageMetadata
|
||||
? (mapUsage(event.usageMetadata, state.providerMetadataKey) ?? state.usage)
|
||||
: state.usage,
|
||||
}
|
||||
const candidate = event.candidates?.[0]
|
||||
if (!candidate?.content)
|
||||
|
|
@ -637,7 +647,7 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
|||
events,
|
||||
"reasoning-0",
|
||||
part.text,
|
||||
signature ? googleMetadata({ thoughtSignature: signature }) : undefined,
|
||||
signature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: signature }) : undefined,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
|
@ -645,14 +655,16 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
|||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined,
|
||||
reasoningSignature
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature })
|
||||
: undefined,
|
||||
)
|
||||
lifecycle = Lifecycle.textDelta(
|
||||
lifecycle,
|
||||
events,
|
||||
"text-0",
|
||||
part.text,
|
||||
textSignature ? googleMetadata({ thoughtSignature: textSignature }) : undefined,
|
||||
textSignature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: textSignature }) : undefined,
|
||||
)
|
||||
textSignature = undefined
|
||||
continue
|
||||
|
|
@ -672,7 +684,9 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
|||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined,
|
||||
reasoningSignature
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature })
|
||||
: undefined,
|
||||
)
|
||||
lifecycle = Lifecycle.stepStart(lifecycle, events)
|
||||
events.push(
|
||||
|
|
@ -681,7 +695,7 @@ const step = (state: ParserState, event: GeminiEvent) => {
|
|||
name: part.functionCall.name,
|
||||
input,
|
||||
providerMetadata: part.thoughtSignature
|
||||
? googleMetadata({ thoughtSignature: part.thoughtSignature })
|
||||
? providerMetadata(state.providerMetadataKey, { thoughtSignature: part.thoughtSignature })
|
||||
: undefined,
|
||||
}),
|
||||
)
|
||||
|
|
@ -720,6 +734,7 @@ export const protocol = Protocol.make({
|
|||
event: Protocol.jsonEvent(GeminiEvent),
|
||||
initial: (request) => ({
|
||||
route: `${request.model.provider}/${request.model.route.id}`,
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
hasToolCalls: false,
|
||||
lifecycle: Lifecycle.initial(),
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -253,6 +253,7 @@ interface PendingToolDelta {
|
|||
}
|
||||
|
||||
export interface ParserState {
|
||||
readonly providerMetadataKey: string
|
||||
readonly tools: ToolStream.State<number>
|
||||
readonly pendingTools: Partial<Record<number, PendingToolDelta>>
|
||||
readonly toolCallEvents: ReadonlyArray<LLMEvent>
|
||||
|
|
@ -324,17 +325,18 @@ const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart
|
|||
const openAICompatibleReasoningContent = (native: unknown) =>
|
||||
isRecord(native) && typeof native.reasoning_content === "string" ? native.reasoning_content : undefined
|
||||
|
||||
const reasoningField = (part: ReasoningPart) => {
|
||||
const field = part.providerMetadata?.openai?.reasoningField
|
||||
const reasoningField = (part: ReasoningPart, providerMetadataKey: string) => {
|
||||
const field = part.providerMetadata?.[providerMetadataKey]?.reasoningField
|
||||
return typeof field === "string" ? field : undefined
|
||||
}
|
||||
|
||||
const reasoningDetails = (parts: ReadonlyArray<ReasoningPart>, native: unknown) => {
|
||||
const reasoningDetails = (parts: ReadonlyArray<ReasoningPart>, native: unknown, providerMetadataKey: string) => {
|
||||
const observed = parts.flatMap((part) => {
|
||||
const details = part.providerMetadata?.openai?.reasoningDetails
|
||||
const details = part.providerMetadata?.[providerMetadataKey]?.reasoningDetails
|
||||
return Array.isArray(details) ? details : []
|
||||
})
|
||||
if (parts.some((part) => Array.isArray(part.providerMetadata?.openai?.reasoningDetails))) return observed
|
||||
if (parts.some((part) => Array.isArray(part.providerMetadata?.[providerMetadataKey]?.reasoningDetails)))
|
||||
return observed
|
||||
if (isRecord(native) && Array.isArray(native.reasoning_details)) return native.reasoning_details
|
||||
}
|
||||
|
||||
|
|
@ -366,7 +368,7 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
|
|||
message: OpenAIChatRequestMessage,
|
||||
configuredField: string | undefined,
|
||||
requireReasoning: boolean,
|
||||
options: LoweringOptions,
|
||||
options: LoweringOptions & { readonly providerMetadataKey: string },
|
||||
) {
|
||||
const content: TextPart[] = []
|
||||
const reasoning: ReasoningPart[] = []
|
||||
|
|
@ -388,10 +390,14 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
|
|||
}
|
||||
}
|
||||
const text = reasoning.map((part) => part.text).join("")
|
||||
const details = reasoningDetails(reasoning, message.native?.openaiCompatible)
|
||||
const observedField = reasoning.map(reasoningField).find((value) => value !== undefined)
|
||||
const details = reasoningDetails(reasoning, message.native?.openaiCompatible, options.providerMetadataKey)
|
||||
const observedField = reasoning
|
||||
.map((part) => reasoningField(part, options.providerMetadataKey))
|
||||
.find((value) => value !== undefined)
|
||||
const nativeReasoning = openAICompatibleReasoningContent(message.native?.openaiCompatible)
|
||||
const fullyStructured = reasoning.every((part) => Array.isArray(part.providerMetadata?.openai?.reasoningDetails))
|
||||
const fullyStructured = reasoning.every((part) =>
|
||||
Array.isArray(part.providerMetadata?.[options.providerMetadataKey]?.reasoningDetails),
|
||||
)
|
||||
const field = (() => {
|
||||
if (configuredField !== undefined && (requireReasoning || reasoning.length > 0 || nativeReasoning !== undefined))
|
||||
return configuredField
|
||||
|
|
@ -459,7 +465,7 @@ const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (
|
|||
message: OpenAIChatRequestMessage,
|
||||
reasoningField: string | undefined,
|
||||
requireReasoning: boolean,
|
||||
options: LoweringOptions,
|
||||
options: LoweringOptions & { readonly providerMetadataKey: string },
|
||||
) {
|
||||
if (message.role === "user") return [yield* lowerUserMessage(message, options)]
|
||||
if (message.role === "assistant")
|
||||
|
|
@ -495,6 +501,7 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
|
|||
const mistral = ["mistral", "devstral", "codestral", "pixtral", "mixtral"].some((family) => modelID.includes(family))
|
||||
const lowering = {
|
||||
...options,
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
toolCallID: (id: string) => {
|
||||
if (mistral)
|
||||
return id
|
||||
|
|
@ -820,7 +827,7 @@ const mapFinishReason = Effect.fn("OpenAIChat.mapFinishReason")(function* (event
|
|||
// Providers differ on cache-hit location: OpenAI uses
|
||||
// `prompt_tokens_details.cached_tokens`, DeepSeek uses
|
||||
// `prompt_cache_hit_tokens`, and Zai uses top-level `cached_tokens`.
|
||||
const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => {
|
||||
const mapUsage = (usage: OpenAIChatEvent["usage"], providerMetadataKey: string): Usage | undefined => {
|
||||
if (!usage) return undefined
|
||||
const input = usage.prompt_tokens ?? undefined
|
||||
const output = usage.completion_tokens ?? undefined
|
||||
|
|
@ -839,7 +846,7 @@ const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => {
|
|||
cacheWriteInputTokens: cacheWrite,
|
||||
reasoningTokens: reasoning,
|
||||
totalTokens: ProviderShared.totalTokens(input, output, usage.total_tokens ?? undefined),
|
||||
providerMetadata: { openai: usage },
|
||||
providerMetadata: { [providerMetadataKey]: usage },
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -913,8 +920,12 @@ const conflictingReasoningTextDetails = (previous: Record<string, unknown>, curr
|
|||
const conflictingDetailValue = (previous: unknown, current: unknown) =>
|
||||
previous !== undefined && previous !== null && current !== undefined && current !== null && previous !== current
|
||||
|
||||
const reasoningMetadata = (field: ParserState["reasoningField"], details?: ReadonlyArray<unknown>) => ({
|
||||
openai: {
|
||||
const reasoningMetadata = (
|
||||
providerMetadataKey: string,
|
||||
field: ParserState["reasoningField"],
|
||||
details?: ReadonlyArray<unknown>,
|
||||
) => ({
|
||||
[providerMetadataKey]: {
|
||||
...(field ? { reasoningField: field } : {}),
|
||||
...(details ? { reasoningDetails: details } : {}),
|
||||
},
|
||||
|
|
@ -941,7 +952,10 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
|||
// Moonshot (and a few other OpenAI-compatible providers) attach usage to
|
||||
// `choice.usage` instead of the top-level `usage` field.
|
||||
const choiceUsage = (choice as unknown as { usage?: OpenAIChatEvent["usage"] })?.usage
|
||||
const usage = mapUsage(event.usage) ?? (choiceUsage ? mapUsage(choiceUsage) : undefined) ?? state.usage
|
||||
const usage =
|
||||
mapUsage(event.usage, state.providerMetadataKey) ??
|
||||
(choiceUsage ? mapUsage(choiceUsage, state.providerMetadataKey) : undefined) ??
|
||||
state.usage
|
||||
const rawFinishReason = choice?.finish_reason
|
||||
const finishReason = rawFinishReason
|
||||
? {
|
||||
|
|
@ -979,7 +993,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
|||
const detailDelta = Array.isArray(delta?.reasoning_details) ? delta.reasoning_details : undefined
|
||||
if (detailDelta !== undefined) appendReasoningDetails(state.reasoningDetails, detailDelta)
|
||||
const reasoningDetailsObserved = state.reasoningDetailsObserved || detailDelta !== undefined
|
||||
const deltaMetadata = reasoningMetadata(reasoningField)
|
||||
const deltaMetadata = reasoningMetadata(state.providerMetadataKey, reasoningField)
|
||||
const text = detailDelta?.length ? (detailText(detailDelta) ?? reasoning?.text) : reasoning?.text
|
||||
if (text !== undefined) lifecycle = Lifecycle.reasoningDelta(lifecycle, events, "reasoning-0", text, deltaMetadata)
|
||||
else if (
|
||||
|
|
@ -995,7 +1009,11 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
|||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningMetadata(reasoningField, reasoningDetailsObserved ? state.reasoningDetails : undefined),
|
||||
reasoningMetadata(
|
||||
state.providerMetadataKey,
|
||||
reasoningField,
|
||||
reasoningDetailsObserved ? state.reasoningDetails : undefined,
|
||||
),
|
||||
)
|
||||
lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.content)
|
||||
}
|
||||
|
|
@ -1005,7 +1023,11 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
|||
lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningMetadata(reasoningField, reasoningDetailsObserved ? state.reasoningDetails : undefined),
|
||||
reasoningMetadata(
|
||||
state.providerMetadataKey,
|
||||
reasoningField,
|
||||
reasoningDetailsObserved ? state.reasoningDetails : undefined,
|
||||
),
|
||||
)
|
||||
lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.refusal)
|
||||
}
|
||||
|
|
@ -1066,6 +1088,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
|||
|
||||
return [
|
||||
{
|
||||
providerMetadataKey: state.providerMetadataKey,
|
||||
tools: finished?.tools ?? tools,
|
||||
pendingTools,
|
||||
toolCallEvents: finished?.events ?? state.toolCallEvents,
|
||||
|
|
@ -1109,12 +1132,18 @@ const finishEvents = Effect.fn("OpenAIChat.finishEvents")(function* (state: Pars
|
|||
}
|
||||
: { normalized: hasToolCalls ? ("tool-calls" as const) : ("stop" as const) }
|
||||
const metadata = reasoningMetadata(
|
||||
state.providerMetadataKey,
|
||||
state.reasoningField,
|
||||
state.reasoningDetailsObserved ? state.reasoningDetails : undefined,
|
||||
)
|
||||
const started =
|
||||
state.reasoningDetailsObserved && !state.reasoningEmitted
|
||||
? Lifecycle.reasoningStart(state.lifecycle, events, "reasoning-0", reasoningMetadata(state.reasoningField))
|
||||
? Lifecycle.reasoningStart(
|
||||
state.lifecycle,
|
||||
events,
|
||||
"reasoning-0",
|
||||
reasoningMetadata(state.providerMetadataKey, state.reasoningField),
|
||||
)
|
||||
: state.lifecycle
|
||||
const ended = Lifecycle.reasoningEnd(started, events, "reasoning-0", metadata)
|
||||
const lifecycle = toolCallEvents.length ? Lifecycle.stepStart(ended, events) : ended
|
||||
|
|
@ -1141,6 +1170,7 @@ export const protocol = Protocol.make({
|
|||
stream: {
|
||||
event: Protocol.jsonEvent(OpenAIChatEvent),
|
||||
initial: (request) => ({
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
|
||||
tools: ToolStream.empty<number>(),
|
||||
pendingTools: {},
|
||||
toolCallEvents: [],
|
||||
|
|
|
|||
|
|
@ -23,13 +23,14 @@ export interface Settings extends ProviderPackage.Settings {
|
|||
readonly baseURL?: string
|
||||
readonly credentials?: Credentials
|
||||
readonly region?: string
|
||||
readonly topP?: number
|
||||
readonly providerOptions?: OpenAIProviderOptionsInput
|
||||
}
|
||||
|
||||
const responsesRoute = Route.make({
|
||||
id: "bedrock-mantle-responses",
|
||||
provider: id,
|
||||
providerMetadataKey: OpenAIResponses.route.providerMetadataKey,
|
||||
providerMetadataKey: "mantle",
|
||||
protocol: OpenAIResponses.protocol,
|
||||
endpoint: OpenAIResponses.route.endpoint,
|
||||
auth: OpenAIResponses.route.auth,
|
||||
|
|
@ -40,6 +41,7 @@ const responsesRoute = Route.make({
|
|||
const chatRoute = OpenAIChat.route.with({
|
||||
id: "bedrock-mantle-chat",
|
||||
provider: id,
|
||||
providerMetadataKey: "mantle",
|
||||
})
|
||||
|
||||
export const routes = [responsesRoute, chatRoute]
|
||||
|
|
@ -94,6 +96,7 @@ const config = (settings: Settings): Config => {
|
|||
apiKey: settings.auth === "sigv4" ? undefined : settings.apiKey,
|
||||
baseURL: settings.baseURL,
|
||||
credentials: settings.credentials,
|
||||
generation: settings.topP === undefined ? undefined : { topP: settings.topP },
|
||||
headers: settings.headers === undefined ? undefined : { ...settings.headers },
|
||||
http: settings.body === undefined ? undefined : { body: { ...settings.body } },
|
||||
providerOptions: settings.providerOptions,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ const configuredRoute = (input: Config) => {
|
|||
return BedrockConverse.route.with({
|
||||
...rest,
|
||||
provider: id,
|
||||
providerMetadataKey: "bedrock",
|
||||
endpoint: { baseURL: baseURL ?? bedrockBaseURL(resolvedRegion) },
|
||||
auth: apiKey === undefined ? BedrockConverse.sigV4Auth(credentials) : Auth.bearer(apiKey),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ export interface Settings extends ProviderPackage.Settings {
|
|||
const route = OpenAICompatibleChat.route.with({
|
||||
id: "google-vertex-chat",
|
||||
provider: id,
|
||||
providerMetadataKey: "vertex",
|
||||
})
|
||||
|
||||
export const routes = [route]
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ export interface Settings extends ProviderPackage.Settings {
|
|||
const route = OpenAICompatibleResponses.route.with({
|
||||
id: "google-vertex-responses",
|
||||
provider: id,
|
||||
providerMetadataKey: "vertex",
|
||||
providerOptions: { store: false },
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ const protocol = {
|
|||
const route = Route.make({
|
||||
id: "google-vertex-gemini",
|
||||
provider: id,
|
||||
providerMetadataKey: "google",
|
||||
providerMetadataKey: "vertex",
|
||||
protocol,
|
||||
endpoint: Endpoint.path(({ request }) => {
|
||||
const model = String(request.model.id)
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ const bodyOptions = (input: unknown) => {
|
|||
export const route = Route.make({
|
||||
id: ADAPTER,
|
||||
provider: profile.provider,
|
||||
providerMetadataKey: "openrouter",
|
||||
protocol,
|
||||
endpoint: Endpoint.path("/chat/completions", { baseURL: profile.baseURL }),
|
||||
framing: Framing.sse,
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ export interface RouteDefaultsInput {
|
|||
export interface RoutePatch<Body, Prepared> extends RouteDefaultsInput {
|
||||
readonly id?: string
|
||||
readonly provider?: string | ProviderID
|
||||
readonly providerMetadataKey?: string
|
||||
readonly auth?: Auth.Definition
|
||||
readonly transport?: Transport<Body, Prepared, unknown>
|
||||
readonly endpoint?: EndpointPatch<Body>
|
||||
|
|
@ -289,11 +290,16 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
|
|||
defaults: routeInput.defaults ?? {},
|
||||
body: protocol.body,
|
||||
with: (patch: RoutePatch<Body, Prepared>) => {
|
||||
const { id, provider, auth, transport, endpoint, ...defaults } = patch
|
||||
const { id, provider, providerMetadataKey, auth, transport, endpoint, ...defaults } = patch
|
||||
return build({
|
||||
...routeInput,
|
||||
id: id ?? routeInput.id,
|
||||
provider: provider ?? routeInput.provider,
|
||||
providerMetadataKey:
|
||||
providerMetadataKey ??
|
||||
(provider !== undefined && String(provider) !== String(routeInput.provider)
|
||||
? String(provider)
|
||||
: routeInput.providerMetadataKey),
|
||||
auth: auth ?? routeInput.auth,
|
||||
endpoint: endpoint ? Endpoint.merge(routeInput.endpoint, endpoint) : routeInput.endpoint,
|
||||
transport: (transport as Transport<Body, Prepared, Frame> | undefined) ?? routeInput.transport,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
|
|||
import { Effect } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { CacheHint, LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage } from "../../src/index.js"
|
||||
import { Auth, LLMClient } from "../../src/route.js"
|
||||
import { Auth, Endpoint, LLMClient, Route } from "../../src/route.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
import * as AnthropicMessages from "../../src/protocols/anthropic-messages.js"
|
||||
import { GoogleVertexMessages } from "../../src/providers.js"
|
||||
|
|
@ -810,6 +810,99 @@ describe("Anthropic Messages route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("round-trips compatible provider metadata in its own namespace", () =>
|
||||
Effect.gen(function* () {
|
||||
const compatible = Route.make({
|
||||
id: "custom-anthropic-messages",
|
||||
provider: "custom-anthropic",
|
||||
protocol: AnthropicMessages.protocol,
|
||||
endpoint: Endpoint.path("/messages", { baseURL: "https://compatible.test/v1" }),
|
||||
auth: Auth.header("x-api-key", "test"),
|
||||
framing: AnthropicMessages.framing,
|
||||
}).model({ id: "custom-model" })
|
||||
const result = [
|
||||
{
|
||||
type: "web_search_result",
|
||||
url: "https://example.com",
|
||||
citations: [{ type: "web_search_result_location", cited_text: "Example" }],
|
||||
},
|
||||
]
|
||||
const response = yield* LLMClient.generate(LLM.request({ model: compatible, prompt: "Search." })).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{ type: "message_start", message: { usage: { input_tokens: 5, custom_start: true } } },
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "thinking", thinking: "Thinking." } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "signature_delta", signature: "custom_sig" } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 1,
|
||||
content_block: { type: "redacted_thinking", data: "custom_redacted" },
|
||||
},
|
||||
{ type: "content_block_stop", index: 1 },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 2,
|
||||
content_block: {
|
||||
type: "server_tool_use",
|
||||
id: "custom_tool",
|
||||
name: "web_search",
|
||||
input: { query: "example" },
|
||||
},
|
||||
},
|
||||
{ type: "content_block_stop", index: 2 },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 3,
|
||||
content_block: { type: "web_search_tool_result", tool_use_id: "custom_tool", content: result },
|
||||
},
|
||||
{ type: "content_block_stop", index: 3 },
|
||||
{
|
||||
type: "message_delta",
|
||||
delta: { stop_reason: "end_turn", stop_sequence: "custom_stop" },
|
||||
usage: { output_tokens: 2, custom_terminal: true },
|
||||
},
|
||||
{ type: "message_stop" },
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.message.content).toMatchObject([
|
||||
{ type: "reasoning", text: "Thinking.", providerMetadata: { "custom-anthropic": { signature: "custom_sig" } } },
|
||||
{ type: "reasoning", text: "", providerMetadata: { "custom-anthropic": { redactedData: "custom_redacted" } } },
|
||||
{ type: "tool-call", id: "custom_tool", providerExecuted: true },
|
||||
{
|
||||
type: "tool-result",
|
||||
providerExecuted: true,
|
||||
providerMetadata: { "custom-anthropic": { blockType: "web_search_tool_result", result } },
|
||||
},
|
||||
])
|
||||
expect(response.usage?.providerMetadata).toEqual({
|
||||
"custom-anthropic": { input_tokens: 5, custom_start: true, output_tokens: 2, custom_terminal: true },
|
||||
})
|
||||
expect(response.events.at(-1)).toMatchObject({
|
||||
providerMetadata: { "custom-anthropic": { stopSequence: "custom_stop" } },
|
||||
})
|
||||
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({ model: compatible, messages: [response.message], cache: "none" }),
|
||||
)
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "thinking", thinking: "Thinking.", signature: "custom_sig" },
|
||||
{ type: "redacted_thinking", data: "custom_redacted" },
|
||||
{ type: "server_tool_use", id: "custom_tool", name: "web_search", input: { query: "example" } },
|
||||
{ type: "web_search_tool_result", tool_use_id: "custom_tool", content: result },
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses text, reasoning, and usage stream fixtures", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
|
|
|
|||
|
|
@ -569,6 +569,57 @@ describe("Bedrock Converse route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("round-trips reassigned provider reasoning and usage metadata in its own namespace", () =>
|
||||
Effect.gen(function* () {
|
||||
const compatible = model.route.with({ provider: "custom-bedrock" }).model({ id: model.id })
|
||||
const redactedData = "cmVkYWN0ZWQtdGhpbmtpbmc="
|
||||
const response = yield* LLMClient.generate(LLMRequest.update(baseRequest, { model: compatible })).pipe(
|
||||
Effect.provide(
|
||||
fixedBytes(
|
||||
eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { text: "Let me think." } } }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { signature: "custom_sig" } } }],
|
||||
["contentBlockStop", { contentBlockIndex: 0 }],
|
||||
[
|
||||
"contentBlockDelta",
|
||||
{ contentBlockIndex: 1, delta: { reasoningContent: { redactedContent: redactedData } } },
|
||||
],
|
||||
["contentBlockStop", { contentBlockIndex: 1 }],
|
||||
["messageStop", { stopReason: "end_turn" }],
|
||||
["metadata", { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } }],
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.message.content).toEqual([
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Let me think.",
|
||||
providerMetadata: { "custom-bedrock": { signature: "custom_sig" } },
|
||||
},
|
||||
{ type: "reasoning", text: "", providerMetadata: { "custom-bedrock": { redactedData } } },
|
||||
])
|
||||
expect(response.usage?.providerMetadata).toEqual({
|
||||
"custom-bedrock": { inputTokens: 5, outputTokens: 2, totalTokens: 7 },
|
||||
})
|
||||
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({ model: compatible, messages: [response.message], cache: "none" }),
|
||||
)
|
||||
expect(prepared.body.messages).toEqual([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ reasoningContent: { reasoningText: { text: "Let me think.", signature: "custom_sig" } } },
|
||||
{ reasoningContent: { redactedContent: redactedData } },
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves reasoning signatures when contentBlockStop is missing", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(
|
||||
|
|
|
|||
|
|
@ -36,6 +36,23 @@ describe("Amazon Bedrock Mantle provider", () => {
|
|||
protocol: "openai-responses",
|
||||
body: { model: "openai.gpt-oss-120b", store: false },
|
||||
})
|
||||
expect(provider.model("openai.gpt-oss-120b").route.providerMetadataKey).toBe("mantle")
|
||||
expect(provider.responses("openai.gpt-oss-120b").route.providerMetadataKey).toBe("mantle")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves configured top-p generation defaults for Chat and Responses", () =>
|
||||
Effect.gen(function* () {
|
||||
const settings = { apiKey: "test-key", topP: 0.8 }
|
||||
const chat = yield* compileRequest(
|
||||
LLM.request({ model: AmazonBedrockMantle.chatModel("openai.gpt-oss-safeguard-20b", settings), prompt: "Hi" }),
|
||||
)
|
||||
const responses = yield* compileRequest(
|
||||
LLM.request({ model: AmazonBedrockMantle.responsesModel("openai.gpt-oss-120b", settings), prompt: "Hi" }),
|
||||
)
|
||||
|
||||
expect(chat.body.top_p).toBe(0.8)
|
||||
expect(responses.body.top_p).toBe(0.8)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -107,6 +124,9 @@ describe("Amazon Bedrock Mantle provider", () => {
|
|||
LLM.request({ model, messages: [response.message, Message.user("Continue.")] }),
|
||||
)
|
||||
|
||||
expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({
|
||||
mantle: { itemId: "msg_95d4d0af4350432a", reasoningEncryptedContent: "mantle-state" },
|
||||
})
|
||||
expect(prepared.body.input).toEqual([
|
||||
{
|
||||
type: "reasoning",
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ describe("Cloudflare", () => {
|
|||
expect(response.reasoning).toBe("Thinking")
|
||||
expect(response.events.filter(LLMEvent.is.reasoningDelta)).toHaveLength(2)
|
||||
expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({
|
||||
openai: { reasoningField: "reasoning", reasoningDetails: merged },
|
||||
"cloudflare-ai-gateway": { reasoningField: "reasoning", reasoningDetails: merged },
|
||||
})
|
||||
|
||||
const replay = yield* compileRequest(LLM.request({ model, messages: [response.message] }))
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { GoogleVertex, GoogleVertexChat, GoogleVertexMessages, GoogleVertexRespo
|
|||
import { LLMClient } from "../../src/route.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
import { it } from "../lib/effect.js"
|
||||
import { dynamicResponse } from "../lib/http.js"
|
||||
import { dynamicResponse, fixedResponse } from "../lib/http.js"
|
||||
import { deltaChunk, finishChunk } from "../lib/openai-chunks.js"
|
||||
import { sseEvents } from "../lib/sse.js"
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ describe("Google Vertex providers", () => {
|
|||
id: "call_1",
|
||||
name: "lookup",
|
||||
input: { query: "weather" },
|
||||
providerMetadata: { google: { functionCallId: "provider_call_1" } },
|
||||
providerMetadata: { vertex: { functionCallId: "provider_call_1" } },
|
||||
}),
|
||||
]),
|
||||
Message.tool({
|
||||
|
|
@ -97,7 +97,7 @@ describe("Google Vertex providers", () => {
|
|||
name: "lookup",
|
||||
result: "sunny",
|
||||
resultType: "text",
|
||||
providerMetadata: { google: { functionCallId: "provider_call_1" } },
|
||||
providerMetadata: { vertex: { functionCallId: "provider_call_1" } },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
|
|
@ -122,6 +122,91 @@ describe("Google Vertex providers", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("round-trips Vertex Gemini metadata through signed content, tool calls, and usage", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = GoogleVertex.configure({
|
||||
accessToken: "vertex-token",
|
||||
project: "vertex-project",
|
||||
}).model("gemini-3.5-flash")
|
||||
const response = yield* LLMClient.generate(LLM.request({ model, prompt: "Check the weather." })).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: "model",
|
||||
parts: [
|
||||
{ text: "Thinking.", thought: true, thoughtSignature: "reasoning_sig" },
|
||||
{ text: "Checking.", thoughtSignature: "text_sig" },
|
||||
{
|
||||
functionCall: { id: "provider_call_1", name: "lookup", args: { query: "weather" } },
|
||||
thoughtSignature: "tool_sig",
|
||||
},
|
||||
],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
promptFeedback: { blockReasonMessage: "Reviewed" },
|
||||
usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 2, thoughtsTokenCount: 1 },
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
const reasoning = response.events.find((event) => event.type === "reasoning-end")
|
||||
const text = response.events.find((event) => event.type === "text-delta")
|
||||
const toolCall = response.toolCalls[0]
|
||||
|
||||
expect(reasoning?.providerMetadata).toEqual({ vertex: { thoughtSignature: "reasoning_sig" } })
|
||||
expect(text?.providerMetadata).toEqual({ vertex: { thoughtSignature: "text_sig" } })
|
||||
expect(toolCall).toMatchObject({
|
||||
id: "provider_call_1",
|
||||
providerMetadata: { vertex: { thoughtSignature: "tool_sig" } },
|
||||
})
|
||||
expect(response.usage?.providerMetadata).toEqual({
|
||||
vertex: { promptTokenCount: 5, candidatesTokenCount: 2, thoughtsTokenCount: 1 },
|
||||
})
|
||||
expect(response.events.at(-1)?.providerMetadata).toEqual({
|
||||
vertex: { promptFeedback: { blockReasonMessage: "Reviewed" } },
|
||||
})
|
||||
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
Message.assistant([
|
||||
{ type: "reasoning", text: "Thinking.", providerMetadata: reasoning?.providerMetadata },
|
||||
{ type: "text", text: "Checking.", providerMetadata: text?.providerMetadata },
|
||||
ToolCallPart.make({
|
||||
id: toolCall.id,
|
||||
name: toolCall.name,
|
||||
input: toolCall.input,
|
||||
providerMetadata: toolCall.providerMetadata,
|
||||
}),
|
||||
]),
|
||||
Message.tool({ id: toolCall.id, name: toolCall.name, result: "sunny", resultType: "text" }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.contents).toEqual([
|
||||
{
|
||||
role: "model",
|
||||
parts: [
|
||||
{ text: "Thinking.", thought: true, thoughtSignature: "reasoning_sig" },
|
||||
{ text: "Checking.", thoughtSignature: "text_sig" },
|
||||
{ functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" },
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ functionResponse: { name: "lookup", response: { name: "lookup", content: "sunny" } } }],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("projects Anthropic Messages onto the Vertex raw-predict API", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = GoogleVertexMessages.configure({
|
||||
|
|
|
|||
|
|
@ -2,13 +2,80 @@ import { describe, expect } from "bun:test"
|
|||
import { ConfigProvider, Effect } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, Message, ToolDefinition } from "../../src/index.js"
|
||||
import { Cerebras, DeepInfra, Groq, TogetherAI } from "../../src/providers/index.js"
|
||||
import {
|
||||
AmazonBedrock,
|
||||
AmazonBedrockMantle,
|
||||
Anthropic,
|
||||
AnthropicCompatible,
|
||||
Azure,
|
||||
Cerebras,
|
||||
CloudflareAIGateway,
|
||||
CloudflareWorkersAI,
|
||||
DeepInfra,
|
||||
Google,
|
||||
GoogleVertex,
|
||||
GoogleVertexChat,
|
||||
GoogleVertexMessages,
|
||||
GoogleVertexResponses,
|
||||
Groq,
|
||||
OpenAI,
|
||||
OpenAICompatible,
|
||||
OpenAICompatibleResponses,
|
||||
OpenRouter,
|
||||
TogetherAI,
|
||||
XAI,
|
||||
} from "../../src/providers/index.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
import { it } from "../lib/effect.js"
|
||||
import { dynamicResponse } from "../lib/http.js"
|
||||
import { sseEvents } from "../lib/sse.js"
|
||||
|
||||
describe("native OpenAI-compatible providers", () => {
|
||||
it.effect("assigns provider-owned metadata namespaces across native routes", () =>
|
||||
Effect.gen(function* () {
|
||||
const vertex = { project: "project", accessToken: "token" }
|
||||
const providers = [
|
||||
[OpenAI.configure({ apiKey: "test" }).chat("model"), "openai"],
|
||||
[OpenAI.configure({ apiKey: "test" }).responses("model"), "openai"],
|
||||
[Azure.configure({ resourceName: "resource", apiKey: "test" }).chat("model"), "azure"],
|
||||
[Azure.configure({ resourceName: "resource", apiKey: "test" }).responses("model"), "azure"],
|
||||
[AmazonBedrock.configure({ apiKey: "test" }).model("model"), "bedrock"],
|
||||
[AmazonBedrockMantle.configure({ apiKey: "test" }).chat("model"), "mantle"],
|
||||
[AmazonBedrockMantle.configure({ apiKey: "test" }).responses("model"), "mantle"],
|
||||
[Google.configure({ apiKey: "test" }).model("model"), "google"],
|
||||
[GoogleVertex.configure(vertex).model("model"), "vertex"],
|
||||
[GoogleVertexChat.configure(vertex).model("model"), "vertex"],
|
||||
[GoogleVertexResponses.configure(vertex).model("model"), "vertex"],
|
||||
[GoogleVertexMessages.configure(vertex).model("model"), "anthropic"],
|
||||
[Anthropic.configure({ apiKey: "test" }).model("model"), "anthropic"],
|
||||
[
|
||||
AnthropicCompatible.configure({ baseURL: "https://example.test/v1", provider: "minimax" }).model("model"),
|
||||
"minimax",
|
||||
],
|
||||
[
|
||||
OpenAICompatible.configure({ baseURL: "https://example.test/v1", provider: "custom" }).model("model"),
|
||||
"custom",
|
||||
],
|
||||
[
|
||||
OpenAICompatibleResponses.configure({ baseURL: "https://example.test/v1", provider: "custom" }).model(
|
||||
"model",
|
||||
),
|
||||
"custom",
|
||||
],
|
||||
[Cerebras.configure({ apiKey: "test" }).model("model"), "cerebras"],
|
||||
[DeepInfra.configure({ apiKey: "test" }).model("model"), "deepinfra"],
|
||||
[TogetherAI.configure({ apiKey: "test" }).model("model"), "togetherai"],
|
||||
[CloudflareAIGateway.configure({ accountId: "account" }).model("model"), "cloudflare-ai-gateway"],
|
||||
[CloudflareWorkersAI.configure({ accountId: "account" }).model("model"), "cloudflare-workers-ai"],
|
||||
[OpenRouter.configure({ apiKey: "test" }).model("model"), "openrouter"],
|
||||
[XAI.configure({ apiKey: "test" }).chat("model"), "xai"],
|
||||
[XAI.configure({ apiKey: "test" }).responses("model"), "xai"],
|
||||
] as const
|
||||
|
||||
for (const [model, key] of providers) expect(model.route.providerMetadataKey).toBe(key)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves native Together AI and Cerebras provider and route identities", () =>
|
||||
Effect.gen(function* () {
|
||||
const together = TogetherAI.configure({ apiKey: "fixture" }).model("meta-llama/Llama-3.3-70B")
|
||||
|
|
|
|||
|
|
@ -68,11 +68,13 @@ for (const item of cases) {
|
|||
expect(response.text.replaceAll(",", "").trim()).toBe("37887")
|
||||
expect(response.reasoning.length).toBeGreaterThan(0)
|
||||
expect(response.events.some(LLMEvent.is.reasoningDelta)).toBe(true)
|
||||
const metadata = response.message.content.find((part) => part.type === "reasoning")?.providerMetadata
|
||||
expect(metadata?.openai?.reasoningField).toBe(item.structured ? "reasoning" : "reasoning_content")
|
||||
expect(Array.isArray(metadata?.openai?.reasoningDetails)).toBe(item.structured)
|
||||
const metadata = response.message.content.find((part) => part.type === "reasoning")?.providerMetadata?.[
|
||||
item.model.route.providerMetadataKey ?? String(item.model.provider)
|
||||
]
|
||||
expect(metadata?.reasoningField).toBe(item.structured ? "reasoning" : "reasoning_content")
|
||||
expect(Array.isArray(metadata?.reasoningDetails)).toBe(item.structured)
|
||||
if (!item.structured) return
|
||||
const details = metadata?.openai?.reasoningDetails
|
||||
const details = metadata?.reasoningDetails
|
||||
if (!Array.isArray(details)) return
|
||||
expect(
|
||||
details.some(
|
||||
|
|
@ -126,7 +128,11 @@ for (const item of cases) {
|
|||
).toMatch(/^Paris is sunny\.?$/)
|
||||
const details = events
|
||||
.filter(LLMEvent.is.reasoningEnd)
|
||||
.map((event) => event.providerMetadata?.openai?.reasoningDetails)
|
||||
.map(
|
||||
(event) =>
|
||||
event.providerMetadata?.[item.model.route.providerMetadataKey ?? String(item.model.provider)]
|
||||
?.reasoningDetails,
|
||||
)
|
||||
.find(Array.isArray)
|
||||
expect(Array.isArray(details)).toBe(item.structured)
|
||||
if (!item.structured || !Array.isArray(details)) return
|
||||
|
|
|
|||
|
|
@ -903,6 +903,70 @@ describe("OpenAI Chat route", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("uses the configured provider metadata namespace for reasoning and usage", () =>
|
||||
Effect.gen(function* () {
|
||||
const selected = LanguageModel.update(model, {
|
||||
route: { ...model.route, providerMetadataKey: "vendor" },
|
||||
})
|
||||
const details = [{ type: "reasoning.text", text: "thinking", signature: "signed" }]
|
||||
const response = yield* LLMClient.generate(LLMRequest.update(request, { model: selected })).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{ choices: [{ delta: { reasoning: "thinking", reasoning_details: details } }] },
|
||||
deltaChunk({ content: "Hello" }),
|
||||
deltaChunk({}, "stop"),
|
||||
usageChunk({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({
|
||||
vendor: { reasoningField: "reasoning", reasoningDetails: details },
|
||||
})
|
||||
expect(response.usage?.providerMetadata).toEqual({
|
||||
vendor: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },
|
||||
})
|
||||
|
||||
const replay = yield* compileRequest(LLM.request({ model: selected, messages: [response.message] }))
|
||||
expect(replay.body.messages).toEqual([
|
||||
{ role: "assistant", content: "Hello", reasoning: "thinking", reasoning_details: details },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("falls back to the selected provider for the metadata namespace", () =>
|
||||
Effect.gen(function* () {
|
||||
const compatible = model.route.with({ provider: "deepseek" }).model({ id: "deepseek-chat" })
|
||||
const selected = LanguageModel.update(compatible, {
|
||||
route: { ...compatible.route, providerMetadataKey: undefined },
|
||||
})
|
||||
const response = yield* LLMClient.generate(LLMRequest.update(request, { model: selected })).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
deltaChunk({ reasoning_content: "thinking" }),
|
||||
deltaChunk({ content: "Hello" }),
|
||||
deltaChunk({}, "stop"),
|
||||
usageChunk({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({
|
||||
deepseek: { reasoningField: "reasoning_content" },
|
||||
})
|
||||
expect(response.usage?.providerMetadata).toEqual({
|
||||
deepseek: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },
|
||||
})
|
||||
|
||||
const replay = yield* compileRequest(LLM.request({ model: selected, messages: [response.message] }))
|
||||
expect(replay.body.messages).toEqual([{ role: "assistant", content: "Hello", reasoning_content: "thinking" }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses and replays a configured custom reasoning field", () =>
|
||||
Effect.gen(function* () {
|
||||
const custom = LanguageModel.update(model, { compatibility: { reasoningField: "vendor_reasoning" } })
|
||||
|
|
|
|||
|
|
@ -437,7 +437,7 @@ describe("OpenAI-compatible Chat route", () => {
|
|||
outputTokens: undefined,
|
||||
totalTokens: undefined,
|
||||
providerMetadata: {
|
||||
openai: {
|
||||
deepseek: {
|
||||
prompt_tokens: null,
|
||||
completion_tokens: null,
|
||||
total_tokens: null,
|
||||
|
|
|
|||
|
|
@ -195,19 +195,19 @@ describe("Open Responses-compatible route", () => {
|
|||
model,
|
||||
messages: [
|
||||
Message.assistant([
|
||||
{ type: "text", text: "Kept.", providerMetadata: { openresponses: { itemId: "history_1" } } },
|
||||
{ type: "text", text: "Kept.", providerMetadata: { "openai-compatible": { itemId: "history_1" } } },
|
||||
{
|
||||
type: "text",
|
||||
text: "Long.",
|
||||
providerMetadata: { openresponses: { itemId: `history_${"a".repeat(64)}` } },
|
||||
providerMetadata: { "openai-compatible": { itemId: `history_${"a".repeat(64)}` } },
|
||||
},
|
||||
{
|
||||
type: "text",
|
||||
text: "Opaque.",
|
||||
providerMetadata: { openresponses: { itemId: "provider_value/with+symbols" } },
|
||||
providerMetadata: { "openai-compatible": { itemId: "provider_value/with+symbols" } },
|
||||
},
|
||||
{ type: "text", text: "No suffix.", providerMetadata: { openresponses: { itemId: "msg_" } } },
|
||||
{ type: "text", text: "No prefix.", providerMetadata: { openresponses: { itemId: "_item" } } },
|
||||
{ type: "text", text: "No suffix.", providerMetadata: { "openai-compatible": { itemId: "msg_" } } },
|
||||
{ type: "text", text: "No prefix.", providerMetadata: { "openai-compatible": { itemId: "_item" } } },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
|
|
@ -267,7 +267,7 @@ describe("Open Responses-compatible route", () => {
|
|||
name: item.type,
|
||||
result: { type: "json", value: item },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openresponses: { itemId: item.id } },
|
||||
providerMetadata: { example: { itemId: item.id } },
|
||||
}),
|
||||
),
|
||||
}),
|
||||
|
|
@ -302,7 +302,7 @@ describe("Open Responses-compatible route", () => {
|
|||
)
|
||||
|
||||
expect(response.message.content).toEqual([
|
||||
{ type: "text", text: "Indexed", providerMetadata: { openresponses: { itemId: "msg_1" } } },
|
||||
{ type: "text", text: "Indexed", providerMetadata: { "openai-compatible": { itemId: "msg_1" } } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
|
@ -368,7 +368,7 @@ describe("Open Responses-compatible route", () => {
|
|||
),
|
||||
)
|
||||
|
||||
const metadata = { openresponses: { itemId: routing.id } }
|
||||
const metadata = { "openai-compatible": { itemId: routing.id } }
|
||||
if (fixture.item.type === "function_call") {
|
||||
expect(response.toolCalls).toEqual([
|
||||
expect.objectContaining({
|
||||
|
|
@ -386,7 +386,7 @@ describe("Open Responses-compatible route", () => {
|
|||
type: "reasoning",
|
||||
text: "Preserved",
|
||||
providerMetadata: {
|
||||
openresponses: { itemId: routing.id, reasoningEncryptedContent: "encrypted-state" },
|
||||
"openai-compatible": { itemId: routing.id, reasoningEncryptedContent: "encrypted-state" },
|
||||
},
|
||||
},
|
||||
])
|
||||
|
|
@ -438,22 +438,26 @@ describe("Open Responses-compatible route", () => {
|
|||
{
|
||||
type: "reasoning",
|
||||
text: "First.",
|
||||
providerMetadata: { openresponses: { itemId: routing.id } },
|
||||
providerMetadata: { "openai-compatible": { itemId: routing.id } },
|
||||
},
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Second.",
|
||||
providerMetadata: { openresponses: { itemId: routing.id, reasoningEncryptedContent: "final-state" } },
|
||||
providerMetadata: {
|
||||
"openai-compatible": { itemId: routing.id, reasoningEncryptedContent: "final-state" },
|
||||
},
|
||||
},
|
||||
])
|
||||
expect(response.events.filter(LLMEvent.is.reasoningEnd)).toEqual([
|
||||
expect.objectContaining({
|
||||
id: `${routing.id}:0`,
|
||||
providerMetadata: { openresponses: { itemId: routing.id } },
|
||||
providerMetadata: { "openai-compatible": { itemId: routing.id } },
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: `${routing.id}:1`,
|
||||
providerMetadata: { openresponses: { itemId: routing.id, reasoningEncryptedContent: "final-state" } },
|
||||
providerMetadata: {
|
||||
"openai-compatible": { itemId: routing.id, reasoningEncryptedContent: "final-state" },
|
||||
},
|
||||
}),
|
||||
])
|
||||
}),
|
||||
|
|
@ -483,7 +487,7 @@ describe("Open Responses-compatible route", () => {
|
|||
id: "call_1",
|
||||
name: "lookup",
|
||||
input: { query: "complete" },
|
||||
providerMetadata: { openresponses: { itemId: "" } },
|
||||
providerMetadata: { "openai-compatible": { itemId: "" } },
|
||||
}),
|
||||
])
|
||||
}),
|
||||
|
|
@ -510,7 +514,7 @@ describe("Open Responses-compatible route", () => {
|
|||
)
|
||||
|
||||
expect(response.message.content).toEqual([
|
||||
{ type: "text", text: "Before after", providerMetadata: { openresponses: { itemId: "msg_1" } } },
|
||||
{ type: "text", text: "Before after", providerMetadata: { "openai-compatible": { itemId: "msg_1" } } },
|
||||
])
|
||||
expect(response.events.map((event) => event.type)).toEqual([
|
||||
"step-start",
|
||||
|
|
@ -662,7 +666,7 @@ describe("Open Responses-compatible route", () => {
|
|||
|
||||
expect(response.events.find(LLMEvent.is.toolCall)).toMatchObject({
|
||||
input: { query: "complete" },
|
||||
providerMetadata: { openresponses: { itemId: "item_1" } },
|
||||
providerMetadata: { example: { itemId: "item_1" } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
|
@ -694,7 +698,7 @@ describe("Open Responses-compatible route", () => {
|
|||
)
|
||||
|
||||
expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({
|
||||
providerMetadata: { openresponses: { itemId: "rs_raw", reasoningEncryptedContent: "raw-state" } },
|
||||
providerMetadata: { "openai-compatible": { itemId: "rs_raw", reasoningEncryptedContent: "raw-state" } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
|
@ -743,7 +747,7 @@ describe("Open Responses-compatible route", () => {
|
|||
Message.assistant({
|
||||
type: "text",
|
||||
text: "Unclassified.",
|
||||
providerMetadata: { openresponses: { phase: null } },
|
||||
providerMetadata: { "openai-compatible": { phase: null } },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
|
|
@ -802,7 +806,7 @@ describe("Open Responses-compatible route", () => {
|
|||
{
|
||||
type: "text",
|
||||
text: "I can't help with that.",
|
||||
providerMetadata: { openresponses: { itemId: "msg_refusal" } },
|
||||
providerMetadata: { example: { itemId: "msg_refusal" } },
|
||||
},
|
||||
])
|
||||
|
||||
|
|
@ -891,7 +895,7 @@ describe("Open Responses-compatible route", () => {
|
|||
|
||||
expect(response.toolCalls).toEqual([])
|
||||
expect(response.events.find(LLMEvent.is.finish)).toMatchObject({
|
||||
providerMetadata: { openresponses: { responseId: "resp_1" } },
|
||||
providerMetadata: { example: { responseId: "resp_1" } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ describe("OpenRouter", () => {
|
|||
{
|
||||
type: "reasoning",
|
||||
text: "Thinking",
|
||||
providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } },
|
||||
providerMetadata: { openrouter: { reasoningField: "reasoning", reasoningDetails: details } },
|
||||
},
|
||||
]),
|
||||
],
|
||||
|
|
@ -328,7 +328,7 @@ describe("OpenRouter", () => {
|
|||
Message.assistant({
|
||||
type: "reasoning",
|
||||
text: "Thinking",
|
||||
providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } },
|
||||
providerMetadata: { openrouter: { reasoningField: "reasoning", reasoningDetails: details } },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
|
|
@ -354,7 +354,7 @@ describe("OpenRouter", () => {
|
|||
Message.assistant({
|
||||
type: "reasoning",
|
||||
text: "AB",
|
||||
providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } },
|
||||
providerMetadata: { openrouter: { reasoningField: "reasoning", reasoningDetails: details } },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -40,4 +40,13 @@ describe("Route.with", () => {
|
|||
"x-patch": "patch",
|
||||
})
|
||||
})
|
||||
|
||||
test("assigns metadata ownership to a replacement provider and preserves explicit overrides", () => {
|
||||
const route = OpenAIChat.route.with({ provider: "azure" })
|
||||
const overridden = route.with({ providerMetadataKey: "custom-azure" }).with({ headers: { "x-test": "value" } })
|
||||
|
||||
expect(route.providerMetadataKey).toBe("azure")
|
||||
expect(overridden.providerMetadataKey).toBe("custom-azure")
|
||||
expect(overridden.defaults).not.toHaveProperty("providerMetadataKey")
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1126,7 +1126,7 @@ describe("ModelResolver", () => {
|
|||
const mantle = yield* ModelResolver.fromCatalogModel(
|
||||
model(Provider.aisdk("@ai-sdk/amazon-bedrock/mantle"), {
|
||||
modelID: "openai.gpt-oss-120b",
|
||||
settings: { region: "us-east-1" },
|
||||
settings: { region: "us-east-1", topP: 0.6 },
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -1158,6 +1158,7 @@ describe("ModelResolver", () => {
|
|||
expect(bedrock.route.defaults.generation).toEqual({ topP: 0.8 })
|
||||
expect(bedrock.route.defaults.http?.body).toEqual({ serviceTier: { type: "priority" } })
|
||||
expect(mantle.route.id).toBe("bedrock-mantle-responses")
|
||||
expect(mantle.route.defaults.generation).toEqual({ topP: 0.6 })
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue