Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion packages/ai/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -371,11 +371,23 @@ Request options in order of stability:

1. **`generation`** — portable knobs (`maxTokens`, `temperature`, `topP`, `topK`, penalties, seed, stop).
2. **`promptCacheKey`** — stable cache affinity lowered by every protocol that supports it.
3. **`providerOptions: { <provider>: {...} }`** — typed-at-the-facade provider-specific knobs (OpenAI `store`, Anthropic `thinking`, Gemini `thinkingConfig`, OpenRouter routing).
3. **`providerOptions: { ... }`** — flat options inferred from the selected model (OpenAI `store`, Anthropic `thinking`, Gemini `thinkingConfig`, OpenRouter routing).
4. **`http: { body, headers, query }`** — last-resort serializable overlays merged into the final HTTP request. Reach for this only when a stable typed path doesn't yet exist.

Route/provider defaults are overridden by request-level values for each axis.

The selected model supplies the provider-specific option type, so per-request overrides stay flat while the canonical runtime request remains provider-neutral:

```ts
LLM.request({
model,
prompt,
providerOptions: {
reasoningEffort: "low",
},
})
```

## Routes

Adding a new model or deployment is usually 5-15 lines using `Route.make({ protocol, endpoint, auth, framing, ... })`. The route owns endpoint/auth/framing and the protocol owns body construction plus stream parsing. Transports are reusable IO templates that receive route endpoint/auth at compile time. Capability/catalog metadata lives outside this low-level package; unsupported request shapes fail during protocol lowering. See `AGENTS.md` for the architectural detail.
Expand Down
4 changes: 2 additions & 2 deletions packages/ai/example/tutorial.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ const model = OpenAI.configure({
apiKey,
generation: { maxTokens: 160 },
providerOptions: {
openai: { store: false },
store: false,
},
}).model("gpt-4o-mini")

Expand All @@ -34,7 +34,7 @@ const model = OpenAI.configure({
// - `generation`: common controls such as max tokens, temperature, topP/topK,
// penalties, seed, and stop sequences.
// - `promptCacheKey`: stable cache affinity for protocols that support it.
// - `providerOptions`: namespaced provider-native behavior. For example,
// - `providerOptions`: model-typed provider-native behavior. For example,
// OpenAI store behavior, Anthropic thinking, Gemini thinking config, or
// OpenRouter routing/reasoning.
// - `http`: last-resort serializable overlays for final request body, headers,
Expand Down
7 changes: 2 additions & 5 deletions packages/ai/src/protocols/anthropic-messages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import {
type JsonSchema,
type LLMRequest,
type MediaPart,
type ProviderOptions,
type ProviderMetadata,
type ToolCallPart,
type ToolDefinition,
Expand Down Expand Up @@ -52,9 +51,7 @@ export interface OptionsInput {
readonly effort?: string
}

export type ProviderOptionsInput = ProviderOptions & {
readonly anthropic?: OptionsInput
}
export type ProviderOptionsInput = OptionsInput

// =============================================================================
// Request Body Schema
Expand Down Expand Up @@ -593,7 +590,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
})

const resolveOptions = Effect.fn("AnthropicMessages.resolveOptions")(function* (request: LLMRequest) {
const input = request.providerOptions?.anthropic
const input = request.providerOptions
return {
thinking: yield* resolveThinking(input?.thinking),
effort: typeof input?.effort === "string" ? input.effort : undefined,
Expand Down
7 changes: 2 additions & 5 deletions packages/ai/src/protocols/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import {
type JsonSchema,
type LLMRequest,
type MediaPart,
type ProviderOptions,
type ProviderMetadata,
type TextPart,
type ToolCallPart,
Expand Down Expand Up @@ -67,9 +66,7 @@ export interface OptionsInput {
}
}

export type ProviderOptionsInput = ProviderOptions & {
readonly gemini?: OptionsInput
}
export type ProviderOptionsInput = OptionsInput

// =============================================================================
// Request Body Schema
Expand Down Expand Up @@ -387,7 +384,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
})

const resolveOptions = (request: LLMRequest) => {
const input = request.providerOptions?.gemini
const input = request.providerOptions
const value = input?.thinkingConfig
const thinkingConfig = {
thinkingBudget:
Expand Down
2 changes: 1 addition & 1 deletion packages/ai/src/protocols/openai-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ export const route = Route.make({
endpoint,
auth,
transport,
defaults: { providerOptions: { openai: { store: false } } },
defaults: { providerOptions: { store: false } },
})

export * as OpenAIResponses from "./openai-responses.js"
4 changes: 1 addition & 3 deletions packages/ai/src/protocols/utils/open-responses-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,7 @@ export type Resolved = Omit<Options, "allowedTools"> & {
const decodeOptions = Schema.decodeUnknownOption(Options)

export const resolve = (request: LLMRequest): Resolved => {
const input = Option.getOrUndefined(
decodeOptions(request.providerOptions?.[request.model.route.providerMetadataKey ?? "openresponses"]),
)
const input = Option.getOrUndefined(decodeOptions(request.providerOptions))
if (!input) return {}
return {
...input,
Expand Down
2 changes: 1 addition & 1 deletion packages/ai/src/providers/google-vertex-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ export interface Settings extends ProviderPackage.Settings {
const route = OpenAICompatibleResponses.route.with({
id: "google-vertex-responses",
provider: id,
providerOptions: { openresponses: { store: false } },
providerOptions: { store: false },
})

export const routes = [route]
Expand Down
8 changes: 3 additions & 5 deletions packages/ai/src/providers/google-vertex.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,14 @@ import { Auth } from "../route/auth.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { Framing } from "../route/framing.js"
import { ProviderID, type LLMRequest, type ModelID, type ProviderOptions } from "../schema/index.js"
import { ProviderID, type LLMRequest, type ModelID } from "../schema/index.js"
import { GoogleVertexShared } from "./google-vertex-shared.js"

export interface GeminiOptionsInput extends Gemini.OptionsInput {
readonly labels?: Readonly<Record<string, string>>
}

export type GeminiProviderOptionsInput = ProviderOptions & {
readonly gemini?: GeminiOptionsInput
}
export type GeminiProviderOptionsInput = GeminiOptionsInput

export const id = ProviderID.make("google-vertex")

Expand All @@ -40,7 +38,7 @@ export type Settings = ProviderPackage.Settings &

const fromRequest = Effect.fn("GoogleVertex.fromRequest")(function* (request: LLMRequest) {
const body = yield* Gemini.protocol.body.from(request)
const value = request.providerOptions?.gemini?.labels
const value = request.providerOptions?.labels
const labels = ProviderShared.isRecord(value)
? Object.fromEntries(
Object.entries(value).filter((entry): entry is [string, string] => typeof entry[1] === "string"),
Expand Down
6 changes: 1 addition & 5 deletions packages/ai/src/providers/open-responses-options.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
import type { Options } from "../protocols/utils/open-responses-options.js"
import type { ProviderOptions } from "../schema/index.js"

export type OpenResponsesOptionsInput = Options & { readonly [key: string]: unknown }

export type OpenResponsesProviderOptionsInput = ProviderOptions & {
readonly openresponses?: OpenResponsesOptionsInput
}
export type OpenResponsesProviderOptionsInput = OpenResponsesOptionsInput

export * as OpenResponsesProviderOptions from "./open-responses-options.js"
13 changes: 5 additions & 8 deletions packages/ai/src/providers/openai-options.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
import type { ProviderOptions } from "../schema/index.js"
import { mergeProviderOptions } from "../schema/index.js"
import { mergeProviderOptions, type ProviderOptions } from "../schema/index.js"
import type { OpenResponsesOptionsInput } from "./open-responses-options.js"

export type { OpenAIResponseIncludable, OpenAIServiceTier } from "../protocols/utils/openai-options.js"

export type OpenAIOptionsInput = OpenResponsesOptionsInput

export type OpenAIProviderOptionsInput = ProviderOptions & {
readonly openai?: OpenAIOptionsInput
}
export type OpenAIProviderOptionsInput = OpenAIOptionsInput

const definedEntries = (input: Record<string, unknown>) =>
Object.entries(input).filter((entry) => entry[1] !== undefined)

const openAIProviderOptions = (options: OpenAIOptionsInput | undefined): ProviderOptions | undefined => {
const openai = Object.fromEntries(
const result = Object.fromEntries(
definedEntries({
store: options?.store,
reasoningEffort: options?.reasoningEffort,
Expand All @@ -24,8 +21,8 @@ const openAIProviderOptions = (options: OpenAIOptionsInput | undefined): Provide
serviceTier: options?.serviceTier,
}),
)
if (Object.keys(openai).length === 0) return undefined
return { openai }
if (Object.keys(result).length === 0) return undefined
return result
}

export const gpt5DefaultOptions = (
Expand Down
8 changes: 3 additions & 5 deletions packages/ai/src/providers/openrouter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { Endpoint } from "../route/endpoint.js"
import { Framing } from "../route/framing.js"
import { Protocol } from "../route/protocol.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { ProviderID, type CacheHint, type ModelID, type ProviderOptions } from "../schema/index.js"
import { ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
import type { ProviderPackage } from "../provider-package.js"
import * as OpenAICompatibleProfiles from "./openai-compatible-profile.js"
import * as OpenAIChat from "../protocols/openai-chat.js"
Expand Down Expand Up @@ -71,9 +71,7 @@ export interface OpenRouterOptions {
}>
}

export type OpenRouterProviderOptionsInput = ProviderOptions & {
readonly openrouter?: OpenRouterOptions
}
export type OpenRouterProviderOptionsInput = OpenRouterOptions

export type LanguageModelOptions = Omit<RouteDefaultsInput, "providerOptions"> &
ProviderAuthOption<"optional"> & {
Expand Down Expand Up @@ -120,7 +118,7 @@ export const protocol = Protocol.make({
return {
...body,
messages,
...bodyOptions(request.providerOptions?.openrouter),
...bodyOptions(request.providerOptions),
...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}),
} as OpenRouterBody
}),
Expand Down
8 changes: 3 additions & 5 deletions packages/ai/src/providers/xai.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { HttpOptions, ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js"
import { HttpOptions, ProviderID, type ModelID } from "../schema/index.js"
import * as OpenAICompatibleProfiles from "./openai-compatible-profile.js"
import * as OpenAICompatibleChat from "../protocols/openai-compatible-chat.js"
import * as OpenAIChat from "../protocols/openai-chat.js"
Expand All @@ -12,9 +12,7 @@ import type { ProviderPackage } from "../provider-package.js"

export const id = ProviderID.make("xai")

export type XAIProviderOptionsInput = ProviderOptions & {
readonly xai?: OpenAIOptionsInput
}
export type XAIProviderOptionsInput = OpenAIOptionsInput

export type LanguageModelOptions = Omit<RouteDefaultsInput, "providerOptions"> &
ProviderAuthOption<"optional"> & {
Expand All @@ -37,7 +35,7 @@ const responsesRoute = Route.make({
protocol: OpenAIResponses.protocol,
endpoint: Endpoint.path("/responses", { baseURL: OpenAICompatibleProfiles.profiles.xai.baseURL }),
transport: OpenAIResponses.httpTransport,
defaults: { providerOptions: { xai: { store: false } } },
defaults: { providerOptions: { store: false } },
})

const chatRoute = Route.make({
Expand Down
14 changes: 2 additions & 12 deletions packages/ai/src/schema/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,22 +36,12 @@ const mergeStringRecords = (
return Object.keys(result).length === 0 ? undefined : result
}

export const ProviderOptions = Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))
export const ProviderOptions = Schema.Record(Schema.String, Schema.Unknown)
export type ProviderOptions = Schema.Schema.Type<typeof ProviderOptions>

export const mergeProviderOptions = (
...items: ReadonlyArray<ProviderOptions | undefined>
): ProviderOptions | undefined => {
const result: Record<string, Record<string, unknown>> = {}
for (const item of items) {
if (!item) continue
for (const [provider, options] of Object.entries(item)) {
const merged = mergeJsonRecords(result[provider], options)
if (merged) result[provider] = merged
}
}
return Object.keys(result).length === 0 ? undefined : result
}
): ProviderOptions | undefined => mergeJsonRecords(...items)

export class HttpOptions extends Schema.Class<HttpOptions>("AI.HttpOptions")({
body: Schema.optional(JsonSchema),
Expand Down
21 changes: 11 additions & 10 deletions packages/ai/test/auth-options.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ OpenAI.configure({
}).responses("gpt-4.1-mini")
OpenAI.configure({
generation: { maxTokens: 100 },
providerOptions: { openai: { store: false } },
providerOptions: { store: false },
}).responses("gpt-4.1-mini")

// @ts-expect-error OpenAI model selectors only accept model ids.
Expand All @@ -97,7 +97,7 @@ OpenAI.configure({ bogus: true })
OpenAI.configure({ generation: { maxTokens: "many" } })

// @ts-expect-error provider-native options remain typed.
OpenAI.configure({ providerOptions: { openai: { store: "false" } } })
OpenAI.configure({ providerOptions: { store: "false" } })

// @ts-expect-error auth is an override, so OpenAI rejects apiKey with auth.
OpenAI.configure({ apiKey: "sk-test", auth: Auth.bearer("oauth-token") })
Expand Down Expand Up @@ -139,23 +139,24 @@ Anthropic.configure({ apiKey: "anthropic-key" }).model("claude-haiku")
Anthropic.configure({
apiKey: "anthropic-key",
providerOptions: {
anthropic: { thinking: { type: "enabled", budgetTokens: 1_024 }, effort: "high" },
thinking: { type: "enabled", budgetTokens: 1_024 },
effort: "high",
},
}).model("claude-haiku")
// @ts-expect-error Anthropic model selectors only accept model ids.
Anthropic.configure({ apiKey: "anthropic-key" }).model("claude-haiku", {})
// @ts-expect-error Anthropic package settings accept only one auth source.
Anthropic.model("claude-sonnet-4-6", { apiKey: "anthropic-key", authToken: "anthropic-token" })
// @ts-expect-error Enabled Anthropic thinking requires a token budget.
Anthropic.configure({ providerOptions: { anthropic: { thinking: { type: "enabled" } } } })
Anthropic.configure({ providerOptions: { thinking: { type: "enabled" } } })
// @ts-expect-error Anthropic thinking budgets must be numbers.
Anthropic.configure({ providerOptions: { anthropic: { thinking: { type: "enabled", budgetTokens: "large" } } } })
Anthropic.configure({ providerOptions: { thinking: { type: "enabled", budgetTokens: "large" } } })

AnthropicCompatible.configure({
apiKey: "messages-key",
baseURL: "https://messages.example.com/v1",
provider: "example",
providerOptions: { anthropic: { thinking: { type: "disabled" } } },
providerOptions: { thinking: { type: "disabled" } },
}).model("compatible-model")
// @ts-expect-error Anthropic-compatible providers require a base URL.
AnthropicCompatible.configure({ apiKey: "messages-key" })
Expand All @@ -171,16 +172,16 @@ AnthropicCompatible.model("compatible-model", {
Google.configure({ apiKey: "google-key" }).model("gemini-2.5-flash")
Google.configure({
apiKey: "google-key",
providerOptions: { gemini: { thinkingConfig: { thinkingBudget: 0, includeThoughts: false } } },
providerOptions: { thinkingConfig: { thinkingBudget: 0, includeThoughts: false } },
}).model("gemini-2.5-flash")
// @ts-expect-error Google model selectors only accept model ids.
Google.configure({ apiKey: "google-key" }).model("gemini-2.5-flash", {})
// @ts-expect-error Gemini thinking budgets must be numbers.
Google.configure({ providerOptions: { gemini: { thinkingConfig: { thinkingBudget: "large" } } } })
Google.configure({ providerOptions: { thinkingConfig: { thinkingBudget: "large" } } })

GoogleVertex.configure({
apiKey: "vertex-key",
providerOptions: { gemini: { thinkingConfig: { thinkingBudget: 1_024 } } },
providerOptions: { thinkingConfig: { thinkingBudget: 1_024 } },
}).model("gemini-3.5-flash")
GoogleVertex.configure({ accessToken: "vertex-token", project: "project" }).model("gemini-3.5-flash")
GoogleVertex.configure({ auth: Auth.bearer("vertex-token"), project: "project" }).model("gemini-3.5-flash")
Expand Down Expand Up @@ -230,7 +231,7 @@ GoogleVertexResponses.configure({
GoogleVertexMessages.configure({
accessToken: "vertex-token",
project: "project",
providerOptions: { anthropic: { thinking: { type: "adaptive", display: "omitted" }, effort: "low" } },
providerOptions: { thinking: { type: "adaptive", display: "omitted" }, effort: "low" },
}).model("claude-sonnet-4-6")
// @ts-expect-error Vertex Messages package settings do not accept API keys.
GoogleVertexMessages.model("claude-sonnet-4-6", { apiKey: "vertex-key", project: "project" })
Expand Down
Loading
Loading