feat(ai): infer model provider options (#39493)

packages/ai/src/llm.ts

@@ -9,25 +9,28 @@ import {
   LLMRequest,
   LLMResponse,
   Message,
+  Model,
   SystemPart,
   ToolChoice,
   ToolDefinition,
   type ContentPart,
+  type ModelProviderOptions,
 } from "./schema"
 import { make as makeTool, toDefinitions, type ToolSchema } from "./tool"
 
 /** Input accepted by `LLM.request`, normalized into the canonical `LLMRequest` class. */
-export type RequestInput = Omit<
+export type RequestInput<SelectedModel extends Model = Model> = Omit<
   ConstructorParameters<typeof LLMRequest>[0],
-  "system" | "messages" | "tools" | "toolChoice" | "generation" | "http" | "providerOptions"
+  "model" | "system" | "messages" | "tools" | "toolChoice" | "generation" | "http" | "providerOptions"
 > & {
+  readonly model: SelectedModel
   readonly system?: string | SystemPart | ReadonlyArray<SystemPart>
   readonly prompt?: string | ContentPart | ReadonlyArray<ContentPart>
   readonly messages?: ReadonlyArray<Message | Message.Input>
   readonly tools?: ReadonlyArray<ToolDefinition.Input>
   readonly toolChoice?: ToolChoice.Input
   readonly generation?: GenerationOptions.Input
-  readonly providerOptions?: ConstructorParameters<typeof LLMRequest>[0]["providerOptions"]
+  readonly providerOptions?: NoInfer<ModelProviderOptions<SelectedModel>>
   readonly http?: HttpOptions.Input
 }
 
@@ -35,7 +38,7 @@ export const generate = LLMClient.generate
 
 export const stream = LLMClient.stream
 
-export const request = (input: RequestInput) => {
+export const request = <const SelectedModel extends Model>(input: RequestInput<SelectedModel>) => {
   const {
     system: requestSystem,
     prompt,
@@ -63,7 +66,7 @@ const GENERATE_OBJECT_TOOL_NAME = "generate_object"
 
 const GENERATE_OBJECT_TOOL_DESCRIPTION = "Return the structured result by calling this tool."
 
-type GenerateObjectBase = Omit<RequestInput, "tools" | "toolChoice">
+type GenerateObjectBase<SelectedModel extends Model = Model> = Omit<RequestInput<SelectedModel>, "tools" | "toolChoice">
 
 export class GenerateObjectResponse<T> {
   constructor(
@@ -80,11 +83,13 @@ export class GenerateObjectResponse<T> {
   }
 }
 
-export interface GenerateObjectOptions<S extends ToolSchema<any>> extends GenerateObjectBase {
+export interface GenerateObjectOptions<S extends ToolSchema<any>, SelectedModel extends Model = Model>
+  extends GenerateObjectBase<SelectedModel> {
   readonly schema: S
 }
 
-export interface GenerateObjectDynamicOptions extends GenerateObjectBase {
+export interface GenerateObjectDynamicOptions<SelectedModel extends Model = Model>
+  extends GenerateObjectBase<SelectedModel> {
   /** Raw JSON Schema object describing the expected output shape. */
   readonly jsonSchema: JsonSchema.JsonSchema
 }
@@ -137,11 +142,11 @@ const runGenerateObject = Effect.fn("LLM.generateObject")(function* (
  * 2. `jsonSchema: JsonSchema.JsonSchema` — `.object` is `unknown`. Use when
  *    the schema is only available at runtime (MCP, plugin manifests). Caller validates.
  */
-export function generateObject<S extends ToolSchema<any>>(
-  options: GenerateObjectOptions<S>,
+export function generateObject<const SelectedModel extends Model, S extends ToolSchema<any>>(
+  options: GenerateObjectOptions<S, SelectedModel>,
 ): Effect.Effect<GenerateObjectResponse<Schema.Schema.Type<S>>, LLMError>
-export function generateObject(
-  options: GenerateObjectDynamicOptions,
+export function generateObject<const SelectedModel extends Model>(
+  options: GenerateObjectDynamicOptions<SelectedModel>,
 ): Effect.Effect<GenerateObjectResponse<unknown>, LLMError>
 export function generateObject(options: GenerateObjectOptions<ToolSchema<any>> | GenerateObjectDynamicOptions) {
   if ("schema" in options) {

packages/ai/src/provider-package.ts

@@ -1,4 +1,4 @@
-import type { Model } from "./schema"
+import type { Model, ProviderOptions } from "./schema"
 
 export interface Settings extends Readonly<Record<string, unknown>> {
   readonly headers?: Readonly<Record<string, string>>
@@ -9,8 +9,11 @@ export interface Settings extends Readonly<Record<string, unknown>> {
   }
 }
 
-export interface Definition<ProviderSettings extends Settings = Settings> {
-  readonly model: (modelID: string, settings: ProviderSettings) => Model
+export interface Definition<
+  ProviderSettings extends Settings = Settings,
+  Options extends ProviderOptions = ProviderOptions,
+> {
+  readonly model: (modelID: string, settings: ProviderSettings) => Model<Options>
 }
 
 export * as ProviderPackage from "./provider-package"

packages/ai/src/route/client.ts

@@ -45,7 +45,9 @@ export interface Route<Body, Prepared = unknown> {
   readonly defaults: RouteDefaults
   readonly body: RouteBody<Body>
   readonly with: (patch: RoutePatch<Body, Prepared>) => Route<Body, Prepared>
-  readonly model: (input: RouteMappedModelInput) => Model
+  readonly model: <Options extends ProviderOptions = ProviderOptions>(
+    input: RouteMappedModelInput<Options>,
+  ) => Model<Options>
   readonly prepareTransport: (body: Body, request: LLMRequest) => Effect.Effect<Prepared, LLMError>
   readonly streamPrepared: (
     prepared: Prepared,
@@ -62,9 +64,15 @@ export type AnyRoute = Route<any, any>
 
 export type HttpOptionsInput = HttpOptions.Input
 
-export type RouteModelInput = Omit<Model.Input, "provider" | "route">
+export type RouteModelInput<Options extends ProviderOptions = ProviderOptions> = Omit<
+  Model.Input<Options>,
+  "provider" | "route"
+>
 
-export type RouteRoutedModelInput = Omit<Model.Input, "route">
+export type RouteRoutedModelInput<Options extends ProviderOptions = ProviderOptions> = Omit<
+  Model.Input<Options>,
+  "route"
+>
 
 export interface RouteDefaults {
   readonly headers?: Record<string, string>
@@ -90,14 +98,19 @@ export interface RoutePatch<Body, Prepared> extends RouteDefaultsInput {
   readonly endpoint?: EndpointPatch<Body>
 }
 
-type RouteMappedModelInput = RouteModelInput | RouteRoutedModelInput
+type RouteMappedModelInput<Options extends ProviderOptions = ProviderOptions> =
+  | RouteModelInput<Options>
+  | RouteRoutedModelInput<Options>
 
-const makeRouteModel = (route: AnyRoute, mapped: RouteMappedModelInput) => {
+const makeRouteModel = <Options extends ProviderOptions = ProviderOptions>(
+  route: AnyRoute,
+  mapped: RouteMappedModelInput<Options>,
+) => {
   const provider = route.provider ?? ("provider" in mapped ? mapped.provider : undefined)
   if (!provider) throw new Error(`Route.model(${route.id}) requires a provider`)
   if (!endpointBaseURL(route.endpoint))
     throw new Error(`Route.model(${route.id}) requires an endpoint baseURL — configure it on the route first`)
-  return Model.make({
+  return Model.make<Options>({
     ...mapped,
     provider,
     route,
@@ -284,7 +297,8 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
           defaults: mergeRouteDefaults(route.defaults, defaults),
         })
       },
-      model: (input) => makeRouteModel(route, input),
+      model: <Options extends ProviderOptions = ProviderOptions>(input: RouteMappedModelInput<Options>) =>
+        makeRouteModel<Options>(route, input),
       prepareTransport: (body, request) =>
         routeInput.transport.prepare({
           body,

packages/ai/src/schema/options.ts

@@ -139,15 +139,17 @@ export class ModelDefaults extends Schema.Class<ModelDefaults>("LLM.ModelDefault
   generation: Schema.optional(GenerationOptions),
   providerOptions: Schema.optional(ProviderOptions),
   http: Schema.optional(HttpOptions),
-}) {}
+}) {
+  declare protected readonly _ModelDefaults: void
+}
 
 export namespace ModelDefaults {
-  export type Input =
+  export type Input<Options extends ProviderOptions = ProviderOptions> =
     | ModelDefaults
     | {
         readonly limits?: ModelLimits.Input
         readonly generation?: GenerationOptions.Input
-        readonly providerOptions?: ProviderOptions
+        readonly providerOptions?: Options
         readonly http?: HttpOptions.Input
       }
 
@@ -178,7 +180,8 @@ export namespace ModelCompatibility {
   export const make = (input: Input) => (input instanceof ModelCompatibility ? input : new ModelCompatibility(input))
 }
 
-export class Model {
+export class Model<Options extends ProviderOptions = ProviderOptions> {
+  declare protected readonly _ProviderOptions: Options
   readonly id: ModelID
   readonly provider: ProviderID
   readonly route: AnyRoute
@@ -193,8 +196,8 @@ export class Model {
     this.compatibility = input.compatibility
   }
 
-  static make(input: Model.Input) {
-    return new Model({
+  static make<Options extends ProviderOptions = ProviderOptions>(input: Model.Input<Options>) {
+    return new Model<Options>({
       id: ModelID.make(input.id),
       provider: ProviderID.make(input.provider),
       route: input.route,
@@ -203,7 +206,7 @@ export class Model {
     })
   }
 
-  static input(model: Model): Model.ConstructorInput {
+  static input<Options extends ProviderOptions>(model: Model<Options>): Model.ConstructorInput {
     return {
       id: model.id,
       provider: model.provider,
@@ -213,9 +216,9 @@ export class Model {
     }
   }
 
-  static update(model: Model, patch: Partial<Model.Input>) {
+  static update<Options extends ProviderOptions>(model: Model<Options>, patch: Partial<Model.Input<Options>>) {
     if (Object.keys(patch).length === 0) return model
-    return Model.make({
+    return Model.make<Options>({
       ...Model.input(model),
       ...patch,
     })
@@ -231,15 +234,20 @@ export namespace Model {
     readonly compatibility?: ModelCompatibility
   }
 
-  export type Input = Omit<ConstructorInput, "id" | "provider" | "defaults" | "compatibility"> & {
+  export type Input<Options extends ProviderOptions = ProviderOptions> = Omit<
+    ConstructorInput,
+    "id" | "provider" | "defaults" | "compatibility"
+  > & {
     readonly id: string | ModelID
     readonly provider: string | ProviderID
-    readonly defaults?: ModelDefaults.Input
+    readonly defaults?: ModelDefaults.Input<Options>
     readonly compatibility?: ModelCompatibility.Input
   }
 }
 
-export type ModelInput = Model.Input
+export type ModelInput<Options extends ProviderOptions = ProviderOptions> = Model.Input<Options>
+
+export type ModelProviderOptions<SelectedModel> = SelectedModel extends Model<infer Options> ? Options : never
 
 export const ModelSchema = Schema.declare((value): value is Model => value instanceof Model, { expected: "LLM.Model" })
 

packages/ai/test/llm-option-types.types.ts

@@ -0,0 +1,62 @@
+import { Schema } from "effect"
+import { LLM, Model, type ModelProviderOptions, type ProviderOptions } from "../src"
+import { OpenAIChat } from "../src/protocols"
+
+interface ExampleOptions {
+  readonly [key: string]: unknown
+  readonly mode?: "fast" | "thorough"
+}
+
+type ExampleProviderOptions = ProviderOptions & {
+  readonly example?: ExampleOptions
+}
+
+const model = OpenAIChat.route
+  .with({ endpoint: { baseURL: "https://example.com/v1" } })
+  .model<ExampleProviderOptions>({ id: "example" })
+
+LLM.request({ model, prompt: "Hello", providerOptions: { example: { mode: "fast" } } })
+LLM.request({ model, prompt: "Hello", providerOptions: { future: { option: true } } })
+
+LLM.request({
+  model,
+  prompt: "Hello",
+  // @ts-expect-error Known provider options preserve their value types.
+  providerOptions: { example: { mode: "slow" } },
+})
+
+LLM.generateObject({
+  model,
+  prompt: "Hello",
+  schema: Schema.Struct({ answer: Schema.String }),
+  providerOptions: { example: { mode: "thorough" } },
+})
+
+LLM.generateObject({
+  model,
+  prompt: "Hello",
+  jsonSchema: { type: "object" },
+  // @ts-expect-error Dynamic object generation uses the selected model's provider options.
+  providerOptions: { example: { mode: false } },
+})
+
+declare const generic: Model
+LLM.request({ model: generic, prompt: "Hello", providerOptions: { arbitrary: { option: true } } })
+
+const options: ModelProviderOptions<typeof model> = { example: { mode: "fast" } }
+void options
+
+model.route.model<ExampleProviderOptions>({
+  id: "example-with-defaults",
+  defaults: {
+    // @ts-expect-error Low-level model defaults preserve known provider option types.
+    providerOptions: { example: { mode: 1 } },
+  },
+})
+
+Model.update(model, {
+  defaults: {
+    // @ts-expect-error Updating a model cannot contradict its provider option type.
+    providerOptions: { example: { mode: "slow" } },
+  },
+})