From 309c4fe6f0eb56b3fe63257bb95f2b1b0ee40f81 Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Wed, 29 Jul 2026 18:05:56 +0530 Subject: [PATCH] feat(ai): infer model provider options (#39493) --- packages/ai/src/llm.ts | 27 ++++++---- packages/ai/src/provider-package.ts | 9 ++-- packages/ai/src/route/client.ts | 28 +++++++--- packages/ai/src/schema/options.ts | 32 ++++++----- packages/ai/test/llm-option-types.types.ts | 62 ++++++++++++++++++++++ 5 files changed, 125 insertions(+), 33 deletions(-) create mode 100644 packages/ai/test/llm-option-types.types.ts diff --git a/packages/ai/src/llm.ts b/packages/ai/src/llm.ts index 8b6f904a5b4..77ec766e9af 100644 --- a/packages/ai/src/llm.ts +++ b/packages/ai/src/llm.ts @@ -9,25 +9,28 @@ import { LLMRequest, LLMResponse, Message, + Model, SystemPart, ToolChoice, ToolDefinition, type ContentPart, + type ModelProviderOptions, } from "./schema" import { make as makeTool, toDefinitions, type ToolSchema } from "./tool" /** Input accepted by `LLM.request`, normalized into the canonical `LLMRequest` class. */ -export type RequestInput = Omit< +export type RequestInput = Omit< ConstructorParameters[0], - "system" | "messages" | "tools" | "toolChoice" | "generation" | "http" | "providerOptions" + "model" | "system" | "messages" | "tools" | "toolChoice" | "generation" | "http" | "providerOptions" > & { + readonly model: SelectedModel readonly system?: string | SystemPart | ReadonlyArray readonly prompt?: string | ContentPart | ReadonlyArray readonly messages?: ReadonlyArray readonly tools?: ReadonlyArray readonly toolChoice?: ToolChoice.Input readonly generation?: GenerationOptions.Input - readonly providerOptions?: ConstructorParameters[0]["providerOptions"] + readonly providerOptions?: NoInfer> readonly http?: HttpOptions.Input } @@ -35,7 +38,7 @@ export const generate = LLMClient.generate export const stream = LLMClient.stream -export const request = (input: RequestInput) => { +export const request = (input: RequestInput) => { const { system: requestSystem, prompt, @@ -63,7 +66,7 @@ const GENERATE_OBJECT_TOOL_NAME = "generate_object" const GENERATE_OBJECT_TOOL_DESCRIPTION = "Return the structured result by calling this tool." -type GenerateObjectBase = Omit +type GenerateObjectBase = Omit, "tools" | "toolChoice"> export class GenerateObjectResponse { constructor( @@ -80,11 +83,13 @@ export class GenerateObjectResponse { } } -export interface GenerateObjectOptions> extends GenerateObjectBase { +export interface GenerateObjectOptions, SelectedModel extends Model = Model> + extends GenerateObjectBase { readonly schema: S } -export interface GenerateObjectDynamicOptions extends GenerateObjectBase { +export interface GenerateObjectDynamicOptions + extends GenerateObjectBase { /** Raw JSON Schema object describing the expected output shape. */ readonly jsonSchema: JsonSchema.JsonSchema } @@ -137,11 +142,11 @@ const runGenerateObject = Effect.fn("LLM.generateObject")(function* ( * 2. `jsonSchema: JsonSchema.JsonSchema` — `.object` is `unknown`. Use when * the schema is only available at runtime (MCP, plugin manifests). Caller validates. */ -export function generateObject>( - options: GenerateObjectOptions, +export function generateObject>( + options: GenerateObjectOptions, ): Effect.Effect>, LLMError> -export function generateObject( - options: GenerateObjectDynamicOptions, +export function generateObject( + options: GenerateObjectDynamicOptions, ): Effect.Effect, LLMError> export function generateObject(options: GenerateObjectOptions> | GenerateObjectDynamicOptions) { if ("schema" in options) { diff --git a/packages/ai/src/provider-package.ts b/packages/ai/src/provider-package.ts index fd878a9d06c..4f7dc0dcad6 100644 --- a/packages/ai/src/provider-package.ts +++ b/packages/ai/src/provider-package.ts @@ -1,4 +1,4 @@ -import type { Model } from "./schema" +import type { Model, ProviderOptions } from "./schema" export interface Settings extends Readonly> { readonly headers?: Readonly> @@ -9,8 +9,11 @@ export interface Settings extends Readonly> { } } -export interface Definition { - readonly model: (modelID: string, settings: ProviderSettings) => Model +export interface Definition< + ProviderSettings extends Settings = Settings, + Options extends ProviderOptions = ProviderOptions, +> { + readonly model: (modelID: string, settings: ProviderSettings) => Model } export * as ProviderPackage from "./provider-package" diff --git a/packages/ai/src/route/client.ts b/packages/ai/src/route/client.ts index 048f82f5542..98d0cab83cc 100644 --- a/packages/ai/src/route/client.ts +++ b/packages/ai/src/route/client.ts @@ -45,7 +45,9 @@ export interface Route { readonly defaults: RouteDefaults readonly body: RouteBody readonly with: (patch: RoutePatch) => Route - readonly model: (input: RouteMappedModelInput) => Model + readonly model: ( + input: RouteMappedModelInput, + ) => Model readonly prepareTransport: (body: Body, request: LLMRequest) => Effect.Effect readonly streamPrepared: ( prepared: Prepared, @@ -62,9 +64,15 @@ export type AnyRoute = Route export type HttpOptionsInput = HttpOptions.Input -export type RouteModelInput = Omit +export type RouteModelInput = Omit< + Model.Input, + "provider" | "route" +> -export type RouteRoutedModelInput = Omit +export type RouteRoutedModelInput = Omit< + Model.Input, + "route" +> export interface RouteDefaults { readonly headers?: Record @@ -90,14 +98,19 @@ export interface RoutePatch extends RouteDefaultsInput { readonly endpoint?: EndpointPatch } -type RouteMappedModelInput = RouteModelInput | RouteRoutedModelInput +type RouteMappedModelInput = + | RouteModelInput + | RouteRoutedModelInput -const makeRouteModel = (route: AnyRoute, mapped: RouteMappedModelInput) => { +const makeRouteModel = ( + route: AnyRoute, + mapped: RouteMappedModelInput, +) => { const provider = route.provider ?? ("provider" in mapped ? mapped.provider : undefined) if (!provider) throw new Error(`Route.model(${route.id}) requires a provider`) if (!endpointBaseURL(route.endpoint)) throw new Error(`Route.model(${route.id}) requires an endpoint baseURL — configure it on the route first`) - return Model.make({ + return Model.make({ ...mapped, provider, route, @@ -284,7 +297,8 @@ function makeFromTransport( defaults: mergeRouteDefaults(route.defaults, defaults), }) }, - model: (input) => makeRouteModel(route, input), + model: (input: RouteMappedModelInput) => + makeRouteModel(route, input), prepareTransport: (body, request) => routeInput.transport.prepare({ body, diff --git a/packages/ai/src/schema/options.ts b/packages/ai/src/schema/options.ts index c80bce0fda4..668954d1cf5 100644 --- a/packages/ai/src/schema/options.ts +++ b/packages/ai/src/schema/options.ts @@ -139,15 +139,17 @@ export class ModelDefaults extends Schema.Class("LLM.ModelDefault generation: Schema.optional(GenerationOptions), providerOptions: Schema.optional(ProviderOptions), http: Schema.optional(HttpOptions), -}) {} +}) { + declare protected readonly _ModelDefaults: void +} export namespace ModelDefaults { - export type Input = + export type Input = | ModelDefaults | { readonly limits?: ModelLimits.Input readonly generation?: GenerationOptions.Input - readonly providerOptions?: ProviderOptions + readonly providerOptions?: Options readonly http?: HttpOptions.Input } @@ -178,7 +180,8 @@ export namespace ModelCompatibility { export const make = (input: Input) => (input instanceof ModelCompatibility ? input : new ModelCompatibility(input)) } -export class Model { +export class Model { + declare protected readonly _ProviderOptions: Options readonly id: ModelID readonly provider: ProviderID readonly route: AnyRoute @@ -193,8 +196,8 @@ export class Model { this.compatibility = input.compatibility } - static make(input: Model.Input) { - return new Model({ + static make(input: Model.Input) { + return new Model({ id: ModelID.make(input.id), provider: ProviderID.make(input.provider), route: input.route, @@ -203,7 +206,7 @@ export class Model { }) } - static input(model: Model): Model.ConstructorInput { + static input(model: Model): Model.ConstructorInput { return { id: model.id, provider: model.provider, @@ -213,9 +216,9 @@ export class Model { } } - static update(model: Model, patch: Partial) { + static update(model: Model, patch: Partial>) { if (Object.keys(patch).length === 0) return model - return Model.make({ + return Model.make({ ...Model.input(model), ...patch, }) @@ -231,15 +234,20 @@ export namespace Model { readonly compatibility?: ModelCompatibility } - export type Input = Omit & { + export type Input = Omit< + ConstructorInput, + "id" | "provider" | "defaults" | "compatibility" + > & { readonly id: string | ModelID readonly provider: string | ProviderID - readonly defaults?: ModelDefaults.Input + readonly defaults?: ModelDefaults.Input readonly compatibility?: ModelCompatibility.Input } } -export type ModelInput = Model.Input +export type ModelInput = Model.Input + +export type ModelProviderOptions = SelectedModel extends Model ? Options : never export const ModelSchema = Schema.declare((value): value is Model => value instanceof Model, { expected: "LLM.Model" }) diff --git a/packages/ai/test/llm-option-types.types.ts b/packages/ai/test/llm-option-types.types.ts new file mode 100644 index 00000000000..d13ebdcc437 --- /dev/null +++ b/packages/ai/test/llm-option-types.types.ts @@ -0,0 +1,62 @@ +import { Schema } from "effect" +import { LLM, Model, type ModelProviderOptions, type ProviderOptions } from "../src" +import { OpenAIChat } from "../src/protocols" + +interface ExampleOptions { + readonly [key: string]: unknown + readonly mode?: "fast" | "thorough" +} + +type ExampleProviderOptions = ProviderOptions & { + readonly example?: ExampleOptions +} + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://example.com/v1" } }) + .model({ id: "example" }) + +LLM.request({ model, prompt: "Hello", providerOptions: { example: { mode: "fast" } } }) +LLM.request({ model, prompt: "Hello", providerOptions: { future: { option: true } } }) + +LLM.request({ + model, + prompt: "Hello", + // @ts-expect-error Known provider options preserve their value types. + providerOptions: { example: { mode: "slow" } }, +}) + +LLM.generateObject({ + model, + prompt: "Hello", + schema: Schema.Struct({ answer: Schema.String }), + providerOptions: { example: { mode: "thorough" } }, +}) + +LLM.generateObject({ + model, + prompt: "Hello", + jsonSchema: { type: "object" }, + // @ts-expect-error Dynamic object generation uses the selected model's provider options. + providerOptions: { example: { mode: false } }, +}) + +declare const generic: Model +LLM.request({ model: generic, prompt: "Hello", providerOptions: { arbitrary: { option: true } } }) + +const options: ModelProviderOptions = { example: { mode: "fast" } } +void options + +model.route.model({ + id: "example-with-defaults", + defaults: { + // @ts-expect-error Low-level model defaults preserve known provider option types. + providerOptions: { example: { mode: 1 } }, + }, +}) + +Model.update(model, { + defaults: { + // @ts-expect-error Updating a model cannot contradict its provider option type. + providerOptions: { example: { mode: "slow" } }, + }, +})