diff --git a/packages/media-core/src/inline-image-data-url.ts b/packages/media-core/src/inline-image-data-url.ts index 496744c2e2b0..1c883385ee53 100644 --- a/packages/media-core/src/inline-image-data-url.ts +++ b/packages/media-core/src/inline-image-data-url.ts @@ -3,54 +3,32 @@ import { canonicalizeBase64 } from "./base64.js"; /** Prefix used to distinguish inline data URLs from remote/local image references. */ export const INLINE_IMAGE_DATA_URL_PREFIX = "data:"; -const IMAGE_SIGNATURES: Array<{ - mime: string; - matches: (buffer: Buffer) => boolean; -}> = [ - { - mime: "image/png", - matches: (buffer) => - buffer.length >= 8 && - buffer[0] === 0x89 && - buffer[1] === 0x50 && - buffer[2] === 0x4e && - buffer[3] === 0x47 && - buffer[4] === 0x0d && - buffer[5] === 0x0a && - buffer[6] === 0x1a && - buffer[7] === 0x0a, - }, - { - mime: "image/jpeg", - matches: (buffer) => - buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff, - }, - { - mime: "image/webp", - matches: (buffer) => - buffer.length >= 12 && - buffer.subarray(0, 4).toString("ascii") === "RIFF" && - buffer.subarray(8, 12).toString("ascii") === "WEBP", - }, - { - mime: "image/gif", - matches: (buffer) => - buffer.length >= 6 && - (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || - buffer.subarray(0, 6).toString("ascii") === "GIF89a"), - }, - { - mime: "image/bmp", - matches: (buffer) => buffer.length >= 2 && buffer[0] === 0x42 && buffer[1] === 0x4d, - }, -]; - const HEIC_BRANDS = new Set(["heic", "heix", "hevc", "hevx", "heis", "heim", "hevm", "hevs"]); const HEIF_BRANDS = new Set(["mif1", "msf1"]); const IMAGE_SIGNATURE_PREFIX_BASE64_CHARS = 128; const INLINE_IMAGE_DATA_URL_MIMES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]); -function sniffIsoBmffImageMime(buffer: Buffer): string | undefined { +/** Sniffs supported inline image formats from decoded bytes. */ +export function sniffInlineImageMime(buffer: Buffer): string | undefined { + if (buffer.length >= 8 && buffer.readBigUInt64BE(0) === 0x89504e470d0a1a0an) { + return "image/png"; + } + if (buffer.length >= 3 && buffer.readUIntBE(0, 3) === 0xffd8ff) { + return "image/jpeg"; + } + if ( + buffer.length >= 12 && + buffer.toString("ascii", 0, 4) === "RIFF" && + buffer.toString("ascii", 8, 12) === "WEBP" + ) { + return "image/webp"; + } + if (buffer.length >= 6 && /^GIF8[79]a$/.test(buffer.toString("ascii", 0, 6))) { + return "image/gif"; + } + if (buffer.length >= 2 && buffer.readUInt16BE(0) === 0x424d) { + return "image/bmp"; + } if (buffer.length < 12 || buffer.subarray(4, 8).toString("ascii") !== "ftyp") { return undefined; } @@ -67,18 +45,6 @@ function sniffIsoBmffImageMime(buffer: Buffer): string | undefined { return undefined; } -/** Sniffs supported inline image formats from decoded bytes. */ -export function sniffInlineImageMime(buffer: Buffer): string | undefined { - return ( - IMAGE_SIGNATURES.find((signature) => signature.matches(buffer))?.mime ?? - sniffIsoBmffImageMime(buffer) - ); -} - -function isImageMimeType(value: string): boolean { - return value.trim().toLowerCase().startsWith("image/"); -} - export type SanitizedInlineImageBase64 = { mimeType: string; base64: string; @@ -89,7 +55,7 @@ export function sanitizeInlineImageBase64(params: { mimeType: string; base64: string; }): SanitizedInlineImageBase64 | undefined { - if (!isImageMimeType(params.mimeType)) { + if (!params.mimeType.trim().toLowerCase().startsWith("image/")) { return undefined; } const canonicalPayload = canonicalizeBase64(params.base64); diff --git a/packages/media-generation-core/src/normalization.ts b/packages/media-generation-core/src/normalization.ts index b43e0a4d3906..184ffb3da1de 100644 --- a/packages/media-generation-core/src/normalization.ts +++ b/packages/media-generation-core/src/normalization.ts @@ -16,16 +16,3 @@ export type MediaGenerationNormalizationMetadataInput = { resolution?: MediaNormalizationEntry; durationSeconds?: MediaNormalizationEntry; }; - -/** True when a normalization entry contains any user-visible normalization metadata. */ -export function hasMediaNormalizationEntry( - entry: MediaNormalizationEntry | undefined, -): entry is MediaNormalizationEntry { - return Boolean( - entry && - (entry.requested !== undefined || - entry.applied !== undefined || - entry.derivedFrom !== undefined || - (entry.supportedValues?.length ?? 0) > 0), - ); -} diff --git a/src/image-generation/normalization.ts b/src/image-generation/normalization.ts index 1ace8febac01..ca46e2f0f3fb 100644 --- a/src/image-generation/normalization.ts +++ b/src/image-generation/normalization.ts @@ -1,6 +1,5 @@ /** Normalizes image generation request overrides against provider/model capabilities. */ import { resolveMediaGeometryOverrides } from "../media-generation/geometry-normalization.js"; -import { hasMediaNormalizationEntry } from "../media-generation/runtime-shared.js"; import type { ImageGenerationBackground, ImageGenerationIgnoredOverride, @@ -90,11 +89,6 @@ export function resolveImageGenerationOverrides(params: { outputFormat, background, ignoredOverrides, - normalization: - hasMediaNormalizationEntry(normalization.size) || - hasMediaNormalizationEntry(normalization.aspectRatio) || - hasMediaNormalizationEntry(normalization.resolution) - ? normalization - : undefined, + normalization: Object.keys(normalization).length ? normalization : undefined, }; } diff --git a/src/image-generation/openai-compatible-image-provider.ts b/src/image-generation/openai-compatible-image-provider.ts index c195b9746d7e..c6f1512113e6 100644 --- a/src/image-generation/openai-compatible-image-provider.ts +++ b/src/image-generation/openai-compatible-image-provider.ts @@ -88,50 +88,13 @@ export type OpenAiCompatibleImageProviderOptions = { }; }; -function resolveDefaultModel(model: string | undefined, fallback: string): string { - return normalizeOptionalString(model) ?? fallback; -} - -function appendImagesPath(baseUrl: string, mode: OpenAiCompatibleImageRequestMode): string { - return `${baseUrl.replace(/\/+$/u, "")}/images/${mode === "edit" ? "edits" : "generations"}`; -} - -function resolveRequestTimeoutMs(params: { - options: OpenAiCompatibleImageProviderOptions; - req: ImageGenerationRequest; - mode: OpenAiCompatibleImageRequestMode; -}): number | undefined { - if (params.options.defaultTimeoutMs === undefined) { - return params.req.timeoutMs; - } - const label = - params.mode === "edit" - ? (params.options.failureLabels?.edit ?? `${params.options.label} image edit`) - : (params.options.failureLabels?.generate ?? `${params.options.label} image generation`); - const deadline = createProviderOperationDeadline({ - timeoutMs: params.req.timeoutMs, - label, - }); - return resolveProviderOperationTimeoutMs({ - deadline, - defaultTimeoutMs: params.options.defaultTimeoutMs, - }); -} - -function resolveResponseMaxImages(params: { - count: number; - mode: OpenAiCompatibleImageRequestMode; - options: OpenAiCompatibleImageProviderOptions; -}): number { - return params.options.capabilities[params.mode].maxCount ?? params.count; -} - /** Creates an image-generation provider backed by OpenAI-style image endpoints. */ export function createOpenAiCompatibleImageGenerationProvider( options: OpenAiCompatibleImageProviderOptions, ): ImageGenerationProvider { const providerConfigKey = options.providerConfigKey ?? options.id; - const normalizeModel = options.normalizeModel ?? resolveDefaultModel; + const normalizeModel = + options.normalizeModel ?? ((model, fallback) => normalizeOptionalString(model) ?? fallback); const resolveCount = options.resolveCount ?? (({ req }) => req.count ?? 1); return { @@ -149,6 +112,7 @@ export function createOpenAiCompatibleImageGenerationProvider( // Reference images switch the request to edit mode; providers can still // disable edits or cap reference count through capabilities. const mode: OpenAiCompatibleImageRequestMode = inputImages.length > 0 ? "edit" : "generate"; + const operation = `${options.label} image ${mode === "edit" ? "edit" : "generation"}`; const maxInputImages = options.capabilities.edit.maxInputImages; if (mode === "edit" && !options.capabilities.edit.enabled) { throw new Error(`${options.label} image editing is not supported.`); @@ -212,11 +176,20 @@ export function createOpenAiCompatibleImageGenerationProvider( mode === "edit" ? options.buildEditRequest({ ...requestParams, mode }) : options.buildGenerateRequest({ ...requestParams, mode }); - const timeoutMs = resolveRequestTimeoutMs({ options, req, mode }); + const timeoutMs = + options.defaultTimeoutMs === undefined + ? req.timeoutMs + : resolveProviderOperationTimeoutMs({ + deadline: createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: options.failureLabels?.[mode] ?? operation, + }), + defaultTimeoutMs: options.defaultTimeoutMs, + }); // Multipart requests must let FormData set its own boundary header, while // JSON requests need an explicit content type after configured headers. const requestOptions = { - url: appendImagesPath(baseUrl, mode), + url: `${baseUrl.replace(/\/+$/u, "")}/images/${mode === "edit" ? "edits" : "generations"}`, headers: new Headers(headers), timeoutMs, fetchFn: fetch, @@ -238,30 +211,20 @@ export function createOpenAiCompatibleImageGenerationProvider( try { await assertOkOrThrowHttpError( response, - mode === "edit" - ? (options.failureLabels?.edit ?? `${options.label} image edit failed`) - : (options.failureLabels?.generate ?? `${options.label} image generation failed`), + options.failureLabels?.[mode] ?? `${operation} failed`, ); const payload = await readProviderJsonResponse(response, `${options.id}.image-generation`, { maxBytes: resolveInlineImageJsonResponseMaxBytes( - resolveResponseMaxImages({ count, mode, options }), + options.capabilities[mode].maxCount ?? count, resolveGeneratedMediaMaxBytes(req.cfg, "image"), ), }); const images = parseOpenAiCompatibleImageResponse(payload, { ...options.response, - malformedResponseError: - mode === "edit" - ? `${options.label} image edit response malformed` - : `${options.label} image generation response malformed`, + malformedResponseError: `${operation} response malformed`, }); if (images.length === 0) { - throw new Error( - options.emptyResponseError ?? - (mode === "edit" - ? `${options.label} image edit response missing image data` - : `${options.label} image generation response missing image data`), - ); + throw new Error(options.emptyResponseError ?? `${operation} response missing image data`); } return { images, model }; } finally { diff --git a/src/image-generation/runtime.ts b/src/image-generation/runtime.ts index cf7fdee7e372..439e5763787a 100644 --- a/src/image-generation/runtime.ts +++ b/src/image-generation/runtime.ts @@ -35,19 +35,6 @@ type ImageGenerationRuntimeDeps = { export type { GenerateImageParams, GenerateImageRuntimeResult } from "./runtime-types.js"; -function buildNoImageGenerationModelConfiguredMessage( - cfg: OpenClawConfig, - deps: ImageGenerationRuntimeDeps, -): string { - const listProviders = deps.listProviders ?? listImageGenerationProviders; - return buildNoCapabilityModelConfiguredMessage({ - capabilityLabel: "image-generation", - modelConfigKey: "mediaModels.image", - providers: listProviders(cfg), - getProviderEnvVars: deps.getProviderEnvVars, - }); -} - /** Lists image-generation providers visible for the current config. */ export function listRuntimeImageGenerationProviders( params?: { config?: OpenClawConfig }, @@ -93,7 +80,14 @@ async function runImageGeneration( autoProviderFallback: params.autoProviderFallback, }); if (candidates.length === 0) { - throw new Error(buildNoImageGenerationModelConfiguredMessage(params.cfg, deps)); + throw new Error( + buildNoCapabilityModelConfiguredMessage({ + capabilityLabel: "image-generation", + modelConfigKey: "mediaModels.image", + providers: listProviders(params.cfg), + getProviderEnvVars: deps.getProviderEnvVars, + }), + ); } return runMediaGenerationCandidates({ diff --git a/src/link-understanding/runner.ts b/src/link-understanding/runner.ts index bd188a9f351b..a6c1d337e5bc 100644 --- a/src/link-understanding/runner.ts +++ b/src/link-understanding/runner.ts @@ -3,7 +3,6 @@ import { applyTemplate } from "../auto-reply/templating.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { LinkModelConfig, LinkToolsConfig } from "../config/types.tools.js"; import { logVerbose, shouldLogVerbose } from "../globals.js"; -// Link-understanding runner fetches allowed URLs and invokes configured commands with bounded content. import { createAbortError, isAbortError } from "../infra/abort-signal.js"; import { cancelUnreadResponseBody, readResponseWithLimit } from "../infra/http-body.js"; import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../infra/net/fetch-guard.js"; @@ -40,21 +39,6 @@ function isLinkUrlTemplate(value: string): boolean { return value.includes("LinkUrl") || value.includes("LinkFinalUrl"); } -function commandName(command: string): string { - return (command.split(/[\\/]/).pop() ?? command).toLowerCase(); -} - -function isUrlFetcherCommand(command: string): boolean { - return commandName(command) === "curl" || commandName(command) === "wget"; -} - -function buildLinkCliArgs(params: { args: string[]; ctx: MsgContext }): string[] { - const templCtx = { ...params.ctx }; - return params.args - .filter((arg) => !isLinkUrlTemplate(arg)) - .map((arg) => applyTemplate(arg, templCtx)); -} - async function fetchLinkContent(params: { timeoutMs: number; url: string; @@ -108,17 +92,16 @@ async function runCliEntry(params: { } const args = params.entry.args ?? []; const timeoutMs = resolveTimeoutMsFromConfig({ config: params.config, entry: params.entry }); - if (isUrlFetcherCommand(command) && args.some(isLinkUrlTemplate)) { + const name = (command.split(/[\\/]/).pop() ?? command).toLowerCase(); + if ((name === "curl" || name === "wget") && args.some(isLinkUrlTemplate)) { // curl/wget URL templates mark the entry as a fetcher; guarded fetch already supplied content. return params.content; } + const templCtx = { ...params.ctx }; const argv = [ command, - ...buildLinkCliArgs({ - args, - ctx: params.ctx, - }), + ...args.filter((arg) => !isLinkUrlTemplate(arg)).map((arg) => applyTemplate(arg, templCtx)), ]; if (shouldLogVerbose()) { @@ -161,15 +144,7 @@ async function runLinkEntries(params: { break; } try { - const output = await runCliEntry({ - content: params.content, - entry, - finalUrl: params.finalUrl, - ctx: params.ctx, - url: params.url, - config: params.config, - signal: params.signal, - }); + const output = await runCliEntry({ ...params, entry }); if (output) { return output; } diff --git a/src/media-generation/runtime-shared.test.ts b/src/media-generation/runtime-shared.test.ts index 92b0efbde1ff..3d5807d41340 100644 --- a/src/media-generation/runtime-shared.test.ts +++ b/src/media-generation/runtime-shared.test.ts @@ -378,6 +378,37 @@ describe("media-generation runtime shared normalization", () => { ).toBeUndefined(); }); + it("keeps geometry tie-breaking independent of provider declaration order", () => { + for (const reverse of [false, true]) { + const ordered = (values: T[]) => (reverse ? values.toReversed() : values); + expect( + resolveClosestAspectRatio({ + requestedAspectRatio: "3:3", + supportedAspectRatios: ordered(["invalid", "2:2", "1:1"]), + }), + ).toBe("1:1"); + expect( + resolveClosestSize({ + requestedAspectRatio: "1:1", + supportedSizes: ordered(["invalid", "128x128", "64x64"]), + }), + ).toBe("64x64"); + expect( + resolveClosestResolution({ + requestedResolution: "480P", + supportedResolutions: ordered(["invalid", "360P", "600P"]), + }), + ).toBe("600P"); + expect( + resolveClosestResolution({ + requestedResolution: "medium", + supportedResolutions: ordered(["invalid", "small", "large"]), + order: ["small", "medium", "large"], + }), + ).toBe("small"); + } + }); + it("clamps durations to the closest supported max", () => { expect(normalizeDurationToClosestMax(12, 8)).toBe(8); expect(normalizeDurationToClosestMax(6, 8)).toBe(6); diff --git a/src/media-generation/runtime-shared.ts b/src/media-generation/runtime-shared.ts index 79ad8ee77445..ea00dd895811 100644 --- a/src/media-generation/runtime-shared.ts +++ b/src/media-generation/runtime-shared.ts @@ -1,6 +1,10 @@ import { clampTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; -import { resolveCapabilityModelRefForProviders } from "../../packages/media-generation-core/src/capability-model-ref.js"; +import { + resolveCapabilityModelRefForProviders, + type CapabilityModelRef as ParsedProviderModelRef, + type CapabilityModelProviderCandidate, +} from "../../packages/media-generation-core/src/capability-model-ref.js"; import type { MediaGenerationNormalizationMetadataInput } from "../../packages/media-generation-core/src/normalization.js"; import { DEFAULT_PROVIDER } from "../agents/defaults.js"; import { describeFailoverError, isFailoverError } from "../agents/failover-error.js"; @@ -15,13 +19,6 @@ import { formatErrorMessage, toErrorObject } from "../infra/errors.js"; import { isProviderApiKeyConfigured } from "../plugins/provider-auth-availability.js"; import { getProviderEnvVarsCore } from "../secrets/provider-env-vars.js"; -export { hasMediaNormalizationEntry } from "../../packages/media-generation-core/src/normalization.js"; - -type ParsedProviderModelRef = { - provider: string; - model: string; -}; - function buildCapabilityCandidateFailure( candidate: ParsedProviderModelRef, error: unknown, @@ -131,11 +128,7 @@ export function resolveMediaProviderRequestTimeoutMs(params: { ); } -type CapabilityProviderCandidate = { - id: string; - aliases?: readonly string[]; - defaultModel?: string | null; - models?: readonly string[]; +type CapabilityProviderCandidate = CapabilityModelProviderCandidate & { isConfigured?: (ctx: { cfg?: OpenClawConfig; agentDir?: string }) => boolean; }; @@ -303,26 +296,29 @@ export function resolveCapabilityModelCandidates(params: { } function normalizeSupportedValues(values?: readonly TValue[]): TValue[] { - return (values ?? []).flatMap((entry) => { - const normalized = normalizeOptionalString(entry); - return normalized ? [entry] : []; - }); + return (values ?? []).filter((entry) => Boolean(normalizeOptionalString(entry))); } -function compareScores( - next: { primary: number; secondary: number; tertiary: string }, - best: { primary: number; secondary: number; tertiary: string } | null, -): boolean { - if (!best) { - return true; +function selectClosestValue( + values: readonly T[], + score: (value: T) => { primary: number; secondary: number } | undefined, +): T | undefined { + let best: { value: T; primary: number; secondary: number } | undefined; + for (const value of values) { + const next = score(value); + if ( + next && + (!best || + (next.primary !== best.primary + ? next.primary < best.primary + : next.secondary !== best.secondary + ? next.secondary < best.secondary + : value.localeCompare(best.value) < 0)) + ) { + best = { value, ...next }; + } } - if (next.primary !== best.primary) { - return next.primary < best.primary; - } - if (next.secondary !== best.secondary) { - return next.secondary < best.secondary; - } - return next.tertiary.localeCompare(best.tertiary) < 0; + return best?.value; } function parsePositiveDimensionPair( @@ -414,24 +410,16 @@ export function resolveClosestAspectRatio(params: { return undefined; } - let bestValue: string | undefined; - let bestScore: { primary: number; secondary: number; tertiary: string } | null = null; - for (const candidate of supported) { + return selectClosestValue(supported, (candidate) => { const parsed = parseAspectRatioValue(candidate); if (!parsed) { - continue; + return undefined; } - const score = { + return { primary: Math.abs(Math.log(parsed.value / requested.value)), secondary: Math.abs(parsed.width * requested.height - requested.width * parsed.height), - tertiary: candidate, }; - if (compareScores(score, bestScore)) { - bestValue = candidate; - bestScore = score; - } - } - return bestValue; + }); } /** Chooses the closest supported size by aspect ratio and area. */ @@ -453,26 +441,18 @@ export function resolveClosestSize(params: { return undefined; } - let bestValue: string | undefined; - let bestScore: { primary: number; secondary: number; tertiary: string } | null = null; - for (const candidate of supported) { + return selectClosestValue(supported, (candidate) => { const parsed = parseSizeValue(candidate); if (!parsed) { - continue; + return undefined; } - const score = { + return { primary: Math.abs( Math.log(parsed.aspectRatio / (requested?.aspectRatio ?? requestedAspectRatio!.value)), ), secondary: requested ? Math.abs(Math.log(parsed.area / requested.area)) : parsed.area, - tertiary: candidate, }; - if (compareScores(score, bestScore)) { - bestValue = candidate; - bestScore = score; - } - } - return bestValue; + }); } /** Chooses the closest supported resolution by numeric rank or custom order. */ @@ -490,23 +470,16 @@ export function resolveClosestResolution(params: { } const requestedNumeric = parseResolutionRank(params.requestedResolution); if (requestedNumeric) { - let bestValue: TResolution | undefined; - let bestScore: { primary: number; secondary: number; tertiary: string } | null = null; - for (const candidate of supported) { + const bestValue = selectClosestValue(supported, (candidate) => { const candidateNumeric = parseResolutionRank(candidate); if (!candidateNumeric || candidateNumeric.unit !== requestedNumeric.unit) { - continue; + return undefined; } - const score = { + return { primary: Math.abs(candidateNumeric.value - requestedNumeric.value), secondary: candidateNumeric.value < requestedNumeric.value ? 1 : 0, - tertiary: candidate, }; - if (compareScores(score, bestScore)) { - bestValue = candidate; - bestScore = score; - } - } + }); if (bestValue) { return bestValue; } @@ -519,24 +492,12 @@ export function resolveClosestResolution(params: { return undefined; } - let bestValue: TResolution | undefined; - let bestScore: { primary: number; secondary: number; tertiary: string } | null = null; - for (const candidate of supported) { + return selectClosestValue(supported, (candidate) => { const candidateIndex = order.indexOf(candidate); - if (candidateIndex < 0) { - continue; - } - const score = { - primary: Math.abs(candidateIndex - requestedIndex), - secondary: candidateIndex, - tertiary: candidate, - }; - if (compareScores(score, bestScore)) { - bestValue = candidate; - bestScore = score; - } - } - return bestValue; + return candidateIndex < 0 + ? undefined + : { primary: Math.abs(candidateIndex - requestedIndex), secondary: candidateIndex }; + }); } function parseResolutionRank( @@ -648,33 +609,23 @@ function formatCapabilityFailureAttempts(attempts: FallbackAttempt[]): string { if (attempts.length === 0) { return "unknown"; } - - const abortedAttempts = attempts.filter(isAbortLikeFallbackAttempt); - if (abortedAttempts.length === 0) { - return attempts.map(formatCapabilityFailureAttempt).join(" | "); + const failures: string[] = []; + const aborted: string[] = []; + for (const attempt of attempts) { + const ref = `${attempt.provider}/${attempt.model}`; + const message = attempt.error.trim().toLowerCase(); + if (message.includes("operation was aborted") || message.includes("request was aborted")) { + aborted.push(ref); + } else { + failures.push(`${ref}: ${attempt.error}`); + } } - if (abortedAttempts.length === attempts.length) { - return `${abortedAttempts.length} fallback(s) aborted after the request was cancelled or timed out: ${abortedAttempts.map(formatCapabilityAttemptRef).join(", ")}`; + if (aborted.length) { + failures.push( + `${aborted.length} fallback(s) aborted after the request was cancelled or timed out: ${aborted.join(", ")}`, + ); } - - const primaryFailures = attempts.filter((attempt) => !isAbortLikeFallbackAttempt(attempt)); - return [ - primaryFailures.map(formatCapabilityFailureAttempt).join(" | "), - `${abortedAttempts.length} fallback(s) aborted after the request was cancelled or timed out: ${abortedAttempts.map(formatCapabilityAttemptRef).join(", ")}`, - ].join(" | "); -} - -function formatCapabilityFailureAttempt(attempt: FallbackAttempt): string { - return `${formatCapabilityAttemptRef(attempt)}: ${attempt.error}`; -} - -function formatCapabilityAttemptRef(attempt: FallbackAttempt): string { - return `${attempt.provider}/${attempt.model}`; -} - -function isAbortLikeFallbackAttempt(attempt: FallbackAttempt): boolean { - const message = attempt.error.trim().toLowerCase(); - return message.includes("operation was aborted") || message.includes("request was aborted"); + return failures.join(" | "); } /** Formats setup guidance when no model is configured for a media capability. */ diff --git a/src/media-understanding/fs.ts b/src/media-understanding/fs.ts deleted file mode 100644 index 559f5db2b0ce..000000000000 --- a/src/media-understanding/fs.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { pathExists } from "../infra/fs-safe.js"; - -/** Safely checks optional media file paths without throwing on empty input. */ -export async function optionalPathExists(filePath?: string | null): Promise { - return filePath ? await pathExists(filePath) : false; -} diff --git a/src/media-understanding/image.runtime-profile.test.ts b/src/media-understanding/image.runtime-profile.test.ts index bcb0c5e84bfe..ec6476718c8e 100644 --- a/src/media-understanding/image.runtime-profile.test.ts +++ b/src/media-understanding/image.runtime-profile.test.ts @@ -375,52 +375,56 @@ describe("describeImageWithModelCore", () => { expect(options.maxTokens).toBe(1024); }); - it("derives workspaceDir from agentId for image runtime resolution", async () => { - mockImageModel({ - provider: "google", - id: "gemini-2.5-flash", - api: "google-generative-ai", - }); - completeMock.mockResolvedValue( - imageCompletion("google-generative-ai", "google", "gemini-2.5-flash", "workspace ok"), - ); - const cfg = { - agents: { - list: [ - { - id: "vision-agent", - agentDir: "/tmp/openclaw-agent", - workspace: "/tmp/openclaw-workspace", - }, - ], - }, - }; + it.each([undefined, ""])( + "derives workspaceDir from agentId when workspaceDir is %j", + async (workspaceDir) => { + mockImageModel({ + provider: "google", + id: "gemini-2.5-flash", + api: "google-generative-ai", + }); + completeMock.mockResolvedValue( + imageCompletion("google-generative-ai", "google", "gemini-2.5-flash", "workspace ok"), + ); + const cfg = { + agents: { + list: [ + { + id: "vision-agent", + agentDir: "/tmp/openclaw-agent", + workspace: "/tmp/openclaw-workspace", + }, + ], + }, + }; - await describeImageWithModelCore({ - ...imageRequestDefaults(), - cfg, - agentId: "vision-agent", - provider: "google", - model: "gemini-2.5-flash", - buffer: Buffer.alloc(1), - prompt: "Describe the image.", - }); + await describeImageWithModelCore({ + ...imageRequestDefaults(), + cfg, + agentId: "vision-agent", + workspaceDir, + provider: "google", + model: "gemini-2.5-flash", + buffer: Buffer.alloc(1), + prompt: "Describe the image.", + }); - expect(acquireAgentRunPreparedModelRuntimeMock).toHaveBeenCalledWith( - expect.objectContaining({ - workspaceDir: "/tmp/openclaw-workspace", - loadRuntimePlugins: true, - }), - expect.objectContaining({ catalogMode: "static", abortSignal: expect.any(AbortSignal) }), - ); - expect(resolveModelAsyncMock).toHaveBeenCalledWith( - "google", - "gemini-2.5-flash", - "/tmp/openclaw-agent", - cfg, - expect.objectContaining({ workspaceDir: "/tmp/openclaw-workspace" }), - ); - }); + expect(acquireAgentRunPreparedModelRuntimeMock).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceDir: "/tmp/openclaw-workspace", + loadRuntimePlugins: true, + }), + expect.objectContaining({ catalogMode: "static", abortSignal: expect.any(AbortSignal) }), + ); + expect(resolveModelAsyncMock).toHaveBeenCalledWith( + "google", + "gemini-2.5-flash", + "/tmp/openclaw-agent", + cfg, + expect.objectContaining({ workspaceDir: "/tmp/openclaw-workspace" }), + ); + }, + ); it("uses one committed prepared generation for image setup and streaming", async () => { const requestedCfg: OpenClawConfig = { logging: { level: "info" } }; diff --git a/src/media-understanding/image.ts b/src/media-understanding/image.ts index b26016a1dcd1..52f472bd3386 100644 --- a/src/media-understanding/image.ts +++ b/src/media-understanding/image.ts @@ -575,28 +575,23 @@ async function describeImagesWithModelInternal( } function toImagesDescriptionRequest(params: ImageDescriptionRequest): ImagesDescriptionRequest { + const { + buffer, + fileName, + mime, + signal, + agentId, + workspaceDir, + preparedModelRuntime, + ...request + } = params; return { - images: [ - { - buffer: params.buffer, - fileName: params.fileName, - mime: params.mime, - }, - ], - model: params.model, - provider: params.provider, - prompt: params.prompt, - maxTokens: params.maxTokens, - timeoutMs: params.timeoutMs, - ...(params.signal ? { signal: params.signal } : {}), - profile: params.profile, - preferredProfile: params.preferredProfile, - authStore: params.authStore, - ...(params.agentId ? { agentId: params.agentId } : {}), - agentDir: params.agentDir, - ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), - ...(params.preparedModelRuntime ? { preparedModelRuntime: params.preparedModelRuntime } : {}), - cfg: params.cfg, + ...request, + images: [{ buffer, fileName, mime }], + ...(signal ? { signal } : {}), + ...(agentId ? { agentId } : {}), + ...(workspaceDir ? { workspaceDir } : {}), + ...(preparedModelRuntime ? { preparedModelRuntime } : {}), }; } diff --git a/src/media-understanding/local-audio.ts b/src/media-understanding/local-audio.ts index 689e839c91af..be58319849cf 100644 --- a/src/media-understanding/local-audio.ts +++ b/src/media-understanding/local-audio.ts @@ -2,10 +2,10 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { MediaUnderstandingModelConfig } from "../config/types.tools.js"; import { resolveExecutableFromPathEnv } from "../infra/executable-path.js"; +import { pathExists } from "../infra/fs-safe.js"; import { resolveEnvironmentValue } from "../infra/process-env.js"; import { runExec } from "../process/exec.js"; import { getOrCreatePromise } from "../shared/lazy-promise.js"; -import { optionalPathExists } from "./fs.js"; type LocalAudioCandidate = { id: "parakeet-mlx" | "whisper-cli" | "sherpa-onnx-offline" | "whisper"; @@ -246,7 +246,7 @@ export async function inspectLocalAudioSelection( const envModel = env.WHISPER_CPP_MODEL?.trim(); const whisperModel = whisperCommand !== null - ? envModel && (await optionalPathExists(envModel)) + ? envModel && (await pathExists(envModel)) ? envModel : await discoverWhisperCppModel(options.listDirectory ?? listDirectoryEntries) : null; @@ -272,7 +272,7 @@ export async function inspectLocalAudioSelection( const sherpaReady = sherpaCommand !== null && sherpaFiles.length === 4 && - (await Promise.all(sherpaFiles.map(optionalPathExists))).every(Boolean); + (await Promise.all(sherpaFiles.map((file) => pathExists(file)))).every(Boolean); const parakeetReady = parakeetCommand !== null && platform === "darwin" && arch === "arm64"; const parakeetArgs = [ "{{AttachmentPath}}", diff --git a/src/media-understanding/runner.entries.ts b/src/media-understanding/runner.entries.ts index 085712dc20a6..fff41c2c3759 100644 --- a/src/media-understanding/runner.entries.ts +++ b/src/media-understanding/runner.entries.ts @@ -641,12 +641,8 @@ export async function runProviderEntry(params: { if (params.secretOwnerId) { assertSecretOwnerAvailable("capability", params.secretOwnerId); } - const { maxBytes, maxChars, timeoutMs, prompt, hasConfiguredPrompt } = resolveEntryRunOptions({ - capability, - entry, - cfg, - config: params.config, - }); + const { maxBytes, maxChars, timeoutMs, prompt, hasConfiguredPrompt } = + resolveEntryRunOptions(params); if (capability === "image") { if (!params.agentDir) { @@ -740,15 +736,12 @@ export async function runProviderEntry(params: { // STT prompts are spelling/context hints; injected instructions can be echoed on silence. const audioPrompt = params.request?.prompt ?? (hasConfiguredPrompt ? prompt : undefined); const transport = resolveProviderRequestContext({ + ...params, providerId, - cfg, - entry, - config: params.config, }); const providerQuery = resolveProviderQuery({ + ...params, providerId, - config: params.config, - entry, }); const model = entry.model?.trim() || @@ -791,13 +784,9 @@ export async function runProviderEntry(params: { "audio transcription callback", ); const auth = await resolveProviderExecutionAuth({ - capability, + ...params, providerId, provider, - cfg, - entry, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, }); result = await executeProviderRequest(providerId, auth, (requestAuth) => transcribeAudio({ ...input, ...requestAuth }), @@ -833,19 +822,13 @@ export async function runProviderEntry(params: { ); } const auth = await resolveProviderExecutionAuth({ - capability, + ...params, providerId, provider, - cfg, - entry, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, }); const { baseUrl, headers, request } = resolveProviderRequestContext({ + ...params, providerId, - cfg, - entry, - config: params.config, }); const model = entry.model?.trim() || @@ -891,7 +874,7 @@ export async function runCliEntry(params: { config?: MediaUnderstandingConfig; request?: MediaRequestOverrides; }): Promise { - const { entry, capability, cfg, ctx } = params; + const { entry, capability, ctx } = params; const attachmentIndex = params.attachment.index; const cli = resolveCliModelEntry(entry); if (!cli.ok) { @@ -899,12 +882,7 @@ export async function runCliEntry(params: { } const { command, args } = cli.value; const language = params.request?.language ?? entry.language ?? params.config?.language; - const { maxBytes, maxChars, timeoutMs, prompt } = resolveEntryRunOptions({ - capability, - entry, - cfg, - config: params.config, - }); + const { maxBytes, maxChars, timeoutMs, prompt } = resolveEntryRunOptions(params); const attachmentPath = await params.cache.getPath({ attachmentIndex, maxBytes, diff --git a/src/media-understanding/runner.ts b/src/media-understanding/runner.ts index 57761d0c32e3..fc323597a9ad 100644 --- a/src/media-understanding/runner.ts +++ b/src/media-understanding/runner.ts @@ -83,6 +83,15 @@ export { export { buildMediaUnderstandingRegistry as buildProviderRegistry } from "./provider-registry.js"; type ProviderRegistry = Map; +type AutoModelSelectionParams = { + cfg: OpenClawConfig; + agentId?: string; + agentDir?: string; + workspaceDir?: string; + providerRegistry: ProviderRegistry; + capability: MediaUnderstandingCapability; + activeModel?: ActiveMediaModel; +}; /** * A provider registry, or a memoized factory that builds one on first use. * `runCapability` receives the factory form so a turn that never needs the @@ -286,25 +295,14 @@ async function resolveAutoImageModelId(params: { }); } -async function resolveKeyEntry(params: { - cfg: OpenClawConfig; - agentId?: string; - agentDir?: string; - workspaceDir?: string; - providerRegistry: ProviderRegistry; - capability: MediaUnderstandingCapability; - activeModel?: ActiveMediaModel; -}): Promise { +async function resolveKeyEntry( + params: AutoModelSelectionParams, +): Promise { const { cfg, providerRegistry, capability } = params; - const checkProvider = ( - providerId: string, - model?: string, - ): Promise => - resolveAutoProviderModelEntry(params, providerId, () => model); - const activeProvider = params.activeModel?.provider?.trim(); if (activeProvider) { - const activeEntry = await checkProvider(activeProvider, params.activeModel?.model); + const model = params.activeModel?.model; + const activeEntry = await resolveAutoProviderModelEntry(params, activeProvider, () => model); if (activeEntry) { return activeEntry; } @@ -318,7 +316,7 @@ async function resolveKeyEntry(params: { providerRegistry, }), })) { - const entry = await checkProvider(providerId, undefined); + const entry = await resolveAutoProviderModelEntry(params, providerId, () => undefined); if (entry) { return entry; } @@ -439,7 +437,7 @@ async function activeModelSupportsNativeVision(params: { } async function* resolveAutoAudioEntries( - params: Parameters[0] & { providerRegistry: ProviderRegistry }, + params: AutoModelSelectionParams, ): AsyncGenerator { const activeProvider = normalizeMediaExecutionProviderId( params.activeModel?.provider?.trim() ?? "", @@ -465,17 +463,12 @@ async function* resolveAutoAudioEntries( } } -async function resolveAutoEntries(params: { - cfg: OpenClawConfig; - agentId?: string; - agentDir?: string; - workspaceDir?: string; - providerRegistry?: ProviderRegistry; - capability: MediaUnderstandingCapability; - activeModel?: ActiveMediaModel; - nativeVisionActive: boolean; - config?: MediaUnderstandingConfig; -}): Promise { +async function resolveAutoEntries( + params: Omit & { + providerRegistry?: ProviderRegistry; + nativeVisionActive: boolean; + }, +): Promise { if (params.capability === "image" && !params.nativeVisionActive) { const imageModelEntries = resolveImageModelFromAgentDefaults(params); if (imageModelEntries.length > 0) { @@ -487,9 +480,16 @@ async function resolveAutoEntries(params: { providerRegistry: params.providerRegistry ?? buildMediaUnderstandingRegistry(undefined, params.cfg), }; - const activeEntry = await resolveActiveModelEntry(prepared); - if (activeEntry) { - return [{ entry: activeEntry }]; + const activeProvider = normalizeMediaExecutionProviderId(prepared.activeModel?.provider ?? ""); + if (activeProvider) { + const activeEntry = await resolveAutoProviderModelEntry( + prepared, + activeProvider, + () => prepared.activeModel?.model, + ); + if (activeEntry) { + return [{ entry: activeEntry }]; + } } const keys = await resolveKeyEntry(prepared); if (keys) { @@ -523,35 +523,8 @@ export async function resolveAutoImageModel(params: { return null; } -async function resolveActiveModelEntry(params: { - cfg: OpenClawConfig; - agentId?: string; - agentDir?: string; - workspaceDir?: string; - providerRegistry: ProviderRegistry; - capability: MediaUnderstandingCapability; - activeModel?: ActiveMediaModel; -}): Promise { - const activeProviderRaw = params.activeModel?.provider?.trim(); - if (!activeProviderRaw) { - return null; - } - const providerId = normalizeMediaExecutionProviderId(activeProviderRaw); - if (!providerId) { - return null; - } - return await resolveAutoProviderModelEntry(params, providerId, () => params.activeModel?.model); -} - async function resolveAutoProviderModelEntry( - params: { - cfg: OpenClawConfig; - agentId?: string; - agentDir?: string; - workspaceDir?: string; - providerRegistry: ProviderRegistry; - capability: MediaUnderstandingCapability; - }, + params: AutoModelSelectionParams, providerId: string, readModel: () => string | undefined, ): Promise { @@ -562,11 +535,8 @@ async function resolveAutoProviderModelEntry( if ( !(params.capability === "audio" && provider?.transcribeAudioWithContext) && !(await hasProviderAuthAvailable({ - capability: params.capability, + ...params, provider: providerId, - cfg: params.cfg, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, })) ) { return null; @@ -576,27 +546,21 @@ async function resolveAutoProviderModelEntry( let model: string | undefined; if (params.capability === "image") { model = await resolveAutoImageModelId({ - cfg: params.cfg, - agentId: params.agentId, + ...params, providerId, - providerRegistry: params.providerRegistry, explicitModel: readModel(), - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, }); } else if (params.capability === "audio") { model = resolveDefaultMediaModelFromRegistry({ + ...params, providerId, - capability: "audio", - providerRegistry: params.providerRegistry, }); } else { model = readModel() ?? resolveDefaultMediaModelFromRegistry({ + ...params, providerId, - capability: "video", - providerRegistry: params.providerRegistry, }); } if (params.capability === "image" && !model) { @@ -766,13 +730,7 @@ export async function runCapability(params: { // delivered image is worse than silence (#122101). let nativeVisionProbe: Promise | undefined; const resolveNativeVisionFlag = (): Promise => { - nativeVisionProbe ??= activeModelSupportsNativeVision({ - cfg, - agentId: params.agentId, - activeModel: params.activeModel, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, - }).catch((err: unknown) => { + nativeVisionProbe ??= activeModelSupportsNativeVision(params).catch((err: unknown) => { if (shouldLogVerbose()) { logVerbose(`native vision support probe failed: ${String(err)}`); } @@ -912,14 +870,8 @@ export async function runCapability(params: { let resolvedEntries: ResolvedMediaModelEntry[] = entries; if (!automaticAudio && resolvedEntries.length === 0) { resolvedEntries = await resolveAutoEntries({ - cfg, - agentId: params.agentId, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, + ...params, providerRegistry, - capability, - activeModel: params.activeModel, - config, nativeVisionActive: capability === "image" && (await resolveNativeVisionFlag()) === true, }); } @@ -942,30 +894,18 @@ export async function runCapability(params: { const attachmentDispositions = buildDispositions({ kind: "failed" }, { kind: "not-selected" }); for (const attachment of selection.selected) { const { output, attempts, processing } = await runAttachmentEntries({ - capability, - cfg, - ctx, + ...params, attachment, - agentId: params.agentId, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, providerRegistry, cache: params.attachments, entries: automaticAudio ? resolveAutoAudioEntries({ - cfg, - agentId: params.agentId, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, + ...params, providerRegistry, - capability, - activeModel: params.activeModel, - nativeVisionActive: false, }) : resolvedEntries, automaticAudio, config, - request: params.request, }); if (output) { outputs.push(output); diff --git a/src/media-understanding/runtime.ts b/src/media-understanding/runtime.ts index 0a7f7c4ea701..9710e83be3e4 100644 --- a/src/media-understanding/runtime.ts +++ b/src/media-understanding/runtime.ts @@ -244,10 +244,7 @@ export async function describeImageFile( export async function prepareImageDescriptionInput(params: PrepareImageDescriptionInputParams) { const timeoutMs = resolveMediaRuntimeTimeoutMs(params.timeoutMs); const image = await readImageDescriptionInput({ - filePath: params.filePath, - mediaUrl: params.mediaUrl, - mime: params.mime, - cfg: params.cfg, + ...params, timeoutMs, }); const normalizedImage = await normalizeImageDescriptionInput({ diff --git a/src/media/channel-inbound-roots.ts b/src/media/channel-inbound-roots.ts index bbe9064dd12a..c8686e75f1c7 100644 --- a/src/media/channel-inbound-roots.ts +++ b/src/media/channel-inbound-roots.ts @@ -27,24 +27,55 @@ type ChannelMediaContractApi = { }; type ChannelMediaRootResolver = keyof ChannelMediaContractApi; -function acceptsResolver( - loaded: ChannelMediaContractApi, - resolver: ChannelMediaRootResolver, -): boolean { - return typeof loaded[resolver] === "function"; +function resolveInstalledChannelMediaContractOwners(params: { + channelId: string; + cfg: OpenClawConfig; +}): PluginManifestRecord[] { + try { + // Read the current plugin metadata generation only. Attachment-root + // resolution must never start plugin discovery or index work of its own. + const snapshot = getCurrentPluginMetadataSnapshotRuntime({ + config: params.cfg, + allowScopedSnapshot: true, + allowWorkspaceScopedSnapshot: true, + }); + const normalizedConfig = normalizePluginsConfig(params.cfg.plugins); + // Only operator-enabled official npm installs may grant attachment-root authority. + return (snapshot?.manifestRegistry.plugins ?? []) + .filter( + (plugin) => + plugin.origin === "global" && + plugin.trustedOfficialInstall === true && + plugin.channels.some( + (channel) => normalizeOptionalLowercaseString(channel) === params.channelId, + ) && + resolveManifestOwnerBasePolicyBlock({ plugin, normalizedConfig }) === null, + ) + .toSorted((left, right) => left.id.localeCompare(right.id)); + } catch { + // Snapshot reads must never turn a missing channel artifact into a hard failure. + return []; + } } -function loadBundledChannelMediaContractApi( - channelId: string, - resolver: ChannelMediaRootResolver, -): ChannelMediaContractApi | undefined { +function findChannelMediaContractApi(params: { + channelId: string | null | undefined; + cfg: OpenClawConfig; + resolver: ChannelMediaRootResolver; +}): ChannelMediaContractApi | undefined { + const channelId = normalizeOptionalLowercaseString(params.channelId); + if (!channelId) { + return undefined; + } try { - // Media-root resolution must stay a narrow artifact load, not full channel bootstrap. + // Resolve only the narrow contract artifact, never the full channel bootstrap. const loaded = loadBundledPluginPublicArtifactModuleSync({ dirName: channelId, artifactBasename: CHANNEL_MEDIA_CONTRACT_ARTIFACT, }); - return acceptsResolver(loaded, resolver) ? loaded : undefined; + if (typeof loaded[params.resolver] === "function") { + return loaded; + } } catch (error) { if ( !( @@ -55,120 +86,25 @@ function loadBundledChannelMediaContractApi( throw error; } } - - return undefined; -} - -function declaresChannel(plugin: PluginManifestRecord, channelId: string): boolean { - return plugin.channels.some( - (ownedChannelId) => normalizeOptionalLowercaseString(ownedChannelId) === channelId, - ); -} - -type ChannelMediaContractOwner = Pick; - -/** - * Lists installed official channel plugins that may own a channel's media contract. - * - * Bundled owners are resolved from the bundled plugin surface; workspace and - * community installs never gain attachment-root authority, so only - * host-verified official npm installs qualify. Current operator policy still - * wins over install provenance: denylisted, explicitly disabled, and - * out-of-allowlist plugins lose attachment-root authority before their artifact - * executes or supplies file-access roots. - */ -function listTrustedInstalledChannelMediaContractOwners(params: { - channelId: string; - cfg: OpenClawConfig; - plugins: readonly PluginManifestRecord[]; -}): ChannelMediaContractOwner[] { - const channelId = normalizeOptionalLowercaseString(params.channelId); - if (!channelId) { - return []; - } - const normalizedConfig = normalizePluginsConfig(params.cfg.plugins); - return params.plugins - .filter( - (plugin) => - plugin.origin === "global" && - plugin.trustedOfficialInstall === true && - declaresChannel(plugin, channelId) && - resolveManifestOwnerBasePolicyBlock({ plugin, normalizedConfig }) === null, - ) - .map((plugin) => ({ id: plugin.id, rootDir: plugin.rootDir })) - .toSorted((left, right) => left.id.localeCompare(right.id)); -} - -function resolveInstalledChannelMediaContractOwners(params: { - channelId: string; - cfg: OpenClawConfig; -}): ChannelMediaContractOwner[] { - try { - // Read the current plugin metadata generation only. Attachment-root - // resolution must never start plugin discovery or index work of its own. - const snapshot = getCurrentPluginMetadataSnapshotRuntime({ - config: params.cfg, - allowScopedSnapshot: true, - allowWorkspaceScopedSnapshot: true, - }); - if (!snapshot) { - return []; - } - return listTrustedInstalledChannelMediaContractOwners({ - channelId: params.channelId, - cfg: params.cfg, - plugins: snapshot.manifestRegistry.plugins, - }); - } catch { - // Snapshot reads must never turn a missing channel artifact into a hard failure. - return []; - } -} - -function loadInstalledChannelMediaContractApi(params: { - channelId: string; - cfg: OpenClawConfig; - resolver: ChannelMediaRootResolver; -}): ChannelMediaContractApi | undefined { - for (const owner of resolveInstalledChannelMediaContractOwners({ - channelId: params.channelId, - cfg: params.cfg, - })) { + for (const owner of resolveInstalledChannelMediaContractOwners({ channelId, cfg: params.cfg })) { try { const loaded = loadPluginPublicArtifactModuleSync({ pluginRoot: owner.rootDir, artifactBasename: CHANNEL_MEDIA_CONTRACT_ARTIFACT, origin: "global", }); - if (acceptsResolver(loaded, params.resolver)) { + if (typeof loaded[params.resolver] === "function") { return loaded; } } catch (error) { - if (error instanceof MissingPublicSurfaceError) { - continue; + if (!(error instanceof MissingPublicSurfaceError)) { + throw error; } - throw error; } } return undefined; } -function findChannelMediaContractApi(params: { - channelId: string | null | undefined; - cfg: OpenClawConfig; - resolver: ChannelMediaRootResolver; -}) { - const normalized = normalizeOptionalLowercaseString(params.channelId); - if (!normalized) { - return undefined; - } - return ( - loadBundledChannelMediaContractApi(normalized, params.resolver) ?? - // External official packages expose the contract from their installed root. - loadInstalledChannelMediaContractApi({ ...params, channelId: normalized }) - ); -} - /** Resolves local inbound attachment roots from the channel named in a message context. */ export function resolveChannelInboundAttachmentRoots(params: { cfg: OpenClawConfig; diff --git a/src/media/parse-output.ts b/src/media/parse-output.ts index 196375ea4238..bcded7dc2694 100644 --- a/src/media/parse-output.ts +++ b/src/media/parse-output.ts @@ -15,12 +15,9 @@ import { findCodeRegions } from "../shared/text/code-regions.js"; import { parseInlineDirectives } from "../utils/directive-tags.js"; import { parseInboundMediaUri } from "./inbound-media-uri.js"; -/** Captures legacy MEDIA: attachment directives from model/tool output. */ -// `main`'s own pattern, backtick stripping included: one optional backtick is consumed before the -// payload is handed on, so `` MEDIA:`/tmp/a.png /tmp/b.png` `` still splits on the whitespace inside -// instead of unwrapping as the single filename `/tmp/a.png /tmp/b.png`. A quote pair is only ever given -// meaning by the code that reads references, never by the capture. -const MEDIA_TOKEN_RE = /\bMEDIA:\s*`?([^\n]+)`?/gi; +// A MEDIA directive consumes its entire line. Keep the optional leading backtick: +// removing it before parsing preserves the existing whitespace-split backtick syntax. +const MEDIA_TOKEN_RE = /\bMEDIA:\s*`?([^\n]+)`?/i; const RENDERABLE_ASSISTANT_MEDIA_PREFIX_RE = /^(?:https?:\/\/|data:(?:image|audio|video)\/|file:|~|\/|[a-z]:[\\/])/iu; @@ -213,31 +210,12 @@ function beginsIndependentMediaSource(raw: string): boolean { return MEDIA_SOURCE_ROOT_RE.test(candidate) || SCHEME_RE.test(candidate); } -// A reference that starts with a quote runs to the first quote of the same kind that is followed by -// whitespace, the end of the payload, or the comma that introduces the next quoted reference. An earlier quote is -// followed by more value, so it is part of that value rather than a delimiter: that keeps an inner quote -// (`MEDIA:'…?token=it's'`), a real filename space (`MEDIA:"/tmp/album/photo.png copy.png"`), and both at -// once (`MEDIA:'/tmp/team's.v1 final/image.png'`) inside one reference, while `MEDIA:"a" "b"` still -// separates at the whitespace between the two. Finding that quote by hand keeps the search linear: -// retrying the remaining suffix from every stray opening quote costs Θ(n²) on a payload such as -// `MEDIA:'a 'a 'a …`, where every quote is followed by a non-space character and so no quote in the -// payload ever closes (measured 0.79s / 3.01s / 11.65s for 48K / 96K / 192K characters, against 2–3ms for -// `main`). +// Scan quote boundaries once; retrying a regex at every stray quote is quadratic. const QUOTE_CHARS = new Set(['"', "'", "`"]); const MEDIA_DIRECTIVE_SPACE_RE = /\s/; -// The comma belongs to the list only when a reference follows it: an unquoted line already reads it that -// way, because `cleanCandidate` trims it from a reference's tail, so `MEDIA:/tmp/a.png, /tmp/b.png` attaches -// both. A quoted reference takes the comma as its closing delimiter only once the scan has looked past the -// comma's own whitespace for the opening quote of the next reference. Three shapes must not split here. A -// comma inside one reference (`/tmp/Hello, World.png`) sits before that reference's closing quote, so it -// never reaches this test. An apostrophe that belongs to the name (`MEDIA:'/tmp/Students', 2024/album.png'`) -// is followed by a comma and then a value character, not a quote, so the reference keeps running to its real -// closing quote. Prose after a comma (`MEDIA:"/tmp/a.png", the first one`) likewise reports no list, and the -// whole-payload reading stands. The skip costs no extra pass: a whitespace run holds no quote, so only the -// quote before that run can walk it and the runs one scan walks are disjoint, which keeps the scan linear -// (measured 0.88s / 1.95s / 3.92s for 3.2M / 6.4M / 12.9M characters of a payload that walks one run per -// apostrophe). +// A comma closes a quoted reference only when another quoted reference follows it. +// Apostrophes in filenames and prose after commas remain part of the current value. function isQuotedMediaReferenceBoundary(payload: string, afterQuote: number): boolean { if (afterQuote >= payload.length) { return true; @@ -258,8 +236,6 @@ function isQuotedMediaReferenceBoundary(payload: string, afterQuote: number): bo function findQuotedMediaReferenceEnd(payload: string, start: number, quote: string): number { for (let index = start + 1; index < payload.length; index += 1) { - // A quote closes its chunk when whitespace, nothing at all, or a comma with a quoted reference behind - // it follows. if (payload.charAt(index) === quote && isQuotedMediaReferenceBoundary(payload, index + 1)) { return index; } @@ -267,11 +243,7 @@ function findQuotedMediaReferenceEnd(payload: string, start: number, quote: stri return -1; } -// A reply that serializes its references into a JSON array states each one with the same quote pair the -// scan below already reads, so the wrapper comes off first and the members are read as a list. Only a whole -// `[…]` qualifies: an unquoted payload still reports no list, and one bracketed reference reports no list -// either, so `MEDIA:["/tmp/a.png"]` keeps `main`'s whole-payload reading and its salvage of a path followed -// by serialized JSON. +// Only a complete array wrapper participates in quoted-list parsing. function stripSerializedJsonArrayWrapper(payload: string): string { const trimmed = payload.trim(); if ( @@ -284,19 +256,8 @@ function stripSerializedJsonArrayWrapper(payload: string): string { return trimmed.slice(1, -1); } -// The references a payload lists, when every token in it is an explicitly quoted reference and there are -// at least two of them; `null` otherwise, which sends the caller back to `main`'s reading. Counting -// tokens is too weak a test for a list: `MEDIA:'/tmp/parents' photos/photo.png'` also starts and ends -// with a quote, but its pair encloses one value whose name holds that quote, so the stray tail must not -// pass for a second reference. With no list present a single quoted value still unwraps as a whole, -// including one whose own text ends with that quote (`MEDIA:"https://example.com/video.mp4?token=ends""`). -// -// One scan answers both questions the caller asks — is this a list, and which references does it hold — -// so the payload is tokenized once. A token that no quote pair bounds is a comma- or whitespace-delimited -// token like any other, and it also settles the answer: the scan stops there instead of reading the rest. -// A member of a list is a reference in its own right, so the caller validates it with the same contract a -// standalone quoted reference gets — bare filenames included, since `MEDIA:"image.png"` is accepted on its -// own — and a member the caller rejects stays out of its neighbours rather than being welded into one. +// Require at least two fully quoted tokens. A single value or an unquoted tail +// keeps whole-payload parsing, including apostrophes and filename whitespace. function readQuotedMediaReferenceList(rawPayload: string): string[] | null { const payload = stripSerializedJsonArrayWrapper(rawPayload); const tokens: string[] = []; @@ -320,10 +281,7 @@ function readQuotedMediaReferenceList(rawPayload: string): string[] | null { return tokens.length >= 2 ? tokens : null; } -// `main`'s own split, kept quote-blind on purpose: a payload that is not a list gives a quote pair no -// authority, so a quote that is text inside one path (`/tmp/album 'best' photos/image.png`) cannot block -// the join, and neither can a fragment that would be accepted as a reference on its own -// (`'best/photos'` in `/tmp/album 'best/photos' final.png`). +// Outside an explicit list, quotes inside a filename must not prevent joining its fragments. function splitMediaDirectiveParts(payload: string): string[] { const parts: string[] = []; let previousEnd = 0; @@ -623,144 +581,81 @@ export function splitMediaOutput( continue; } - const matches = Array.from(line.matchAll(MEDIA_TOKEN_RE)); - if (matches.length === 0) { + const match = MEDIA_TOKEN_RE.exec(line); + if (!match) { keptLines.push(line); pushTextSegment(line); - lineOffset += line.length + 1; // +1 for newline + lineOffset += line.length + 1; continue; } - const pieces: string[] = []; - const lineSegments: ParsedMediaOutputSegment[] = []; - let cursor = 0; - - for (const match of matches) { - const start = match.index ?? 0; - pieces.push(line.slice(cursor, start)); - - const payload = expectDefined(match[1], "parse regex capture 1"); - // A payload that lists separate quoted references keeps every reference as written, and each of - // them is admitted by the same contract a standalone quoted reference gets. Otherwise the payload - // reads the way `main` reads it: one quoted value unwraps as a whole, and anything else splits on - // whitespace. Both answers come from the one scan, so the payload is never tokenized twice. - const quotedList = readQuotedMediaReferenceList(payload); - // `main`'s whole-payload reading of this string, kept even when a list sends the references - // elsewhere: a list still carries the outer quote pair that reading takes off, and the decision - // whether an unreferenced payload is a local path to strip has to land on the string `main` - // decides on, not on the quotes still around it. - const stripped = unwrapQuoted(payload); - const unwrapped = quotedList ? undefined : stripped; - const payloadValue = unwrapped ?? payload; - const parts = quotedList ?? (unwrapped ? [unwrapped] : splitMediaDirectiveParts(payload)); - const mediaStartIndex = media.length; - let validCount = 0; - const invalidParts: string[] = []; - let hasValidMedia = false; - for (const part of parts) { - // Matched quotes delimit the reference; punctuation inside them belongs to its value. That - // holds for every reference a split payload lists, not just for a payload that unwraps as a - // single value, so a quoted part keeps its own characters instead of being cleaned. Cleaning - // one would drop the signed suffix and leave the reference short at delivery time. - const quotedPart = unwrapped === undefined ? unwrapQuoted(part) : undefined; - const candidate = unwrapped ?? quotedPart ?? cleanCandidate(part); - const allowSpaces = Boolean(unwrapped ?? quotedPart) || /\s/.test(candidate); - // A member of an explicit list is validated as the standalone reference its own quote pair makes - // it, bare filenames included: `MEDIA:"image.png"` is accepted on its own, so rejecting - // `"image.png"` here would drop a reference the payload states. Outside a list the bare-filename - // fallback stays where `main` put it, on the whole payload, so a space-separated fragment is - // never promoted to a reference of its own. - if (isValidMedia(candidate, { allowSpaces, allowBareFilename: quotedList !== null })) { - media.push(candidate); - hasValidMedia = true; - foundMediaToken = true; - validCount += 1; - } else if (!/\s/.test(part) || !hasTraversalOrUnsupportedHomeDirPrefix(candidate)) { - invalidParts.push(part); - } + const payload = expectDefined(match[1], "parse regex capture 1"); + const quotedList = readQuotedMediaReferenceList(payload); + const stripped = unwrapQuoted(payload); + const unwrapped = quotedList ? undefined : stripped; + const payloadValue = unwrapped ?? payload; + const parts = quotedList ?? (unwrapped ? [unwrapped] : splitMediaDirectiveParts(payload)); + const mediaStartIndex = media.length; + const invalidParts: string[] = []; + for (const part of parts) { + // Quoted references preserve punctuation, including signed URL suffixes. + const quotedPart = unwrapped === undefined ? unwrapQuoted(part) : undefined; + const candidate = unwrapped ?? quotedPart ?? cleanCandidate(part); + if (isValidMedia(candidate, { allowSpaces: true, allowBareFilename: quotedList !== null })) { + media.push(candidate); + } else if (!/\s/.test(part) || !hasTraversalOrUnsupportedHomeDirPrefix(candidate)) { + invalidParts.push(part); } - - const trimmedPayload = (stripped ?? payload).trim(); - const looksLikeLocalPath = - looksLikeLocalFilePath(trimmedPayload) || FILE_URL_PREFIX_RE.test(trimmedPayload); - if ( - quotedList === null && - !unwrapped && - validCount === 1 && - invalidParts.length > 0 && - !parts.slice(1).some(beginsIndependentMediaSource) && - /\s/.test(payloadValue) && - looksLikeLocalPath - ) { - // A single valid split plus invalid leftovers can be one local path containing spaces. A list is - // excluded: its quote pairs already fixed where each reference ends, so the leftovers are not - // fragments of the accepted one, and welding them on would turn `MEDIA:"/tmp/first.png" "second.png"` - // into `/tmp/first.png" "second.png` — a path that does not exist. This is the reconstruction - // `main` performs on a payload it reads as one quoted value, and it stays available for exactly - // those payloads. - const fallback = cleanCandidate(payloadValue); - if (isValidMedia(fallback, { allowSpaces: true })) { - media.splice(mediaStartIndex, media.length - mediaStartIndex, fallback); - hasValidMedia = true; - foundMediaToken = true; - invalidParts.length = 0; - } - } - - // A list gets no whole-payload reading at all: its quote pairs already fixed where every reference - // ends, so a list whose members all failed states no reference and stays the text it was. Cleaning - // the payload anyway welded the rejects into `first.png," "second.png` for - // `MEDIA:"first.png," "second.png,"` — a name no member states, which the base rejects. Payloads - // `main` reads as one value keep this step, bare-filename fallback included. - if (quotedList === null && !hasValidMedia) { - const fallback = unwrapped ?? cleanCandidate(payloadValue); - if (isValidMedia(fallback, { allowSpaces: true, allowBareFilename: true })) { - media.push(fallback); - hasValidMedia = true; - foundMediaToken = true; - invalidParts.length = 0; - } - } - - if (hasValidMedia) { - const beforeText = cleanLineText(pieces.join("")); - if (beforeText) { - lineSegments.push({ type: "text", text: beforeText }); - } - pieces.length = 0; - for (const url of media.slice(mediaStartIndex)) { - lineSegments.push({ type: "media", url }); - } - if (invalidParts.length > 0) { - pieces.push(invalidParts.join(" ")); - } - } else if (looksLikeLocalPath) { - // Strip MEDIA: lines with local paths even when invalid (e.g. absolute paths - // from internal tools like TTS). They should never leak as visible text. - foundMediaToken = true; - } else { - pieces.push(match[0]); - } - - cursor = start + match[0].length; } - pieces.push(line.slice(cursor)); + const trimmedPayload = (stripped ?? payload).trim(); + const looksLikeLocalPath = + looksLikeLocalFilePath(trimmedPayload) || FILE_URL_PREFIX_RE.test(trimmedPayload); + if ( + quotedList === null && + !unwrapped && + media.length - mediaStartIndex === 1 && + invalidParts.length > 0 && + !parts.slice(1).some(beginsIndependentMediaSource) && + /\s/.test(payloadValue) && + looksLikeLocalPath + ) { + // Unquoted fragments can belong to one local filename; explicit list members cannot. + const fallback = cleanCandidate(payloadValue); + if (isValidMedia(fallback, { allowSpaces: true })) { + media.splice(mediaStartIndex, media.length - mediaStartIndex, fallback); + invalidParts.length = 0; + } + } - const cleanedLine = cleanLineText(pieces.join("")); + // Never weld rejected list members into a synthetic whole-payload filename. + if (quotedList === null && media.length === mediaStartIndex) { + const fallback = unwrapped ?? cleanCandidate(payloadValue); + if (isValidMedia(fallback, { allowSpaces: true, allowBareFilename: true })) { + media.push(fallback); + invalidParts.length = 0; + } + } + let cleanedLine: string; + if (media.length > mediaStartIndex) { + foundMediaToken = true; + for (const url of media.slice(mediaStartIndex)) { + segments.push({ type: "media", url }); + } + cleanedLine = cleanLineText(invalidParts.join(" ")); + } else if (looksLikeLocalPath) { + // Invalid local references must not leak internal tool paths as visible text. + foundMediaToken = true; + cleanedLine = ""; + } else { + cleanedLine = cleanLineText(line); + } if (cleanedLine) { keptLines.push(cleanedLine); - lineSegments.push({ type: "text", text: cleanedLine }); + pushTextSegment(cleanedLine); } - for (const segment of lineSegments) { - if (segment.type === "text") { - pushTextSegment(segment.text); - continue; - } - segments.push(segment); - } - lineOffset += line.length + 1; // +1 for newline + lineOffset += line.length + 1; } const visibleText = keptLines.join("\n").replace(/^(?:[ \t]*\n)+/, ""); diff --git a/src/media/read-capability.ts b/src/media/read-capability.ts index daff8219b045..12655bcc6820 100644 --- a/src/media/read-capability.ts +++ b/src/media/read-capability.ts @@ -64,36 +64,6 @@ function isAgentScopedMediaReadAllowedByToolPolicy( return isToolAllowedByPolicies("read", [groupPolicy, senderPolicy]); } -/** Creates a host reader bound to the agent workspace and configured local-file safety checks. */ -function createAgentScopedHostMediaReadFile( - params: { - cfg: OpenClawConfig; - agentId?: string; - localRoots: readonly string[]; - workspaceDir?: string; - excludedLocalRoots?: readonly string[]; - workspaceOnly?: boolean; - } & OutboundHostMediaPolicyContext, -): OutboundMediaReadFile | undefined { - if ( - !resolveEffectiveToolFsRootExpansionAllowed(params) || - !isAgentScopedMediaReadAllowedByToolPolicy(params) - ) { - return undefined; - } - const inferredWorkspaceDir = - params.workspaceDir ?? - (params.agentId ? resolveAgentWorkspaceDir(params.cfg, params.agentId) : undefined); - const workspaceRoot = resolveWorkspaceRoot(inferredWorkspaceDir); - return createBoundedOutboundMediaReadFile(async (filePath, options) => { - const resolvedPath = resolvePathFromInput(filePath, workspaceRoot); - return await readLocalMediaFile(resolvedPath, params.localRoots, { - maxBytes: options?.maxBytes ?? Number.MAX_SAFE_INTEGER, - excludedRoots: params.excludedLocalRoots, - }); - }); -} - function getManagedMediaLocalRoots(mediaSources?: readonly string[]): readonly string[] { const roots = new Set([path.join(resolveConfigDir(), "media", "outbound")]); for (const source of mediaSources ?? []) { @@ -218,27 +188,17 @@ export function resolveAgentScopedOutboundMediaAccess( mediaReadAllowed && !registeredMedia ? appendWorkspaceDirToLocalRoots(baseLocalRoots, resolvedWorkspaceDir) : baseLocalRoots; - const hostReadFile = - params.mediaAccess?.readFile ?? - params.mediaReadFile ?? - createAgentScopedHostMediaReadFile({ - cfg: params.cfg, - agentId: params.agentId, - localRoots: localRoots ?? [], - workspaceDir: resolvedWorkspaceDir, - excludedLocalRoots: registeredRoots, - workspaceOnly: params.workspaceOnly, - sessionKey: params.sessionKey, - messageProvider: params.messageProvider, - groupId: params.groupId, - groupChannel: params.groupChannel, - groupSpace: params.groupSpace, - accountId: params.accountId, - requesterSenderId: params.requesterSenderId, - requesterSenderName: params.requesterSenderName, - requesterSenderUsername: params.requesterSenderUsername, - requesterSenderE164: params.requesterSenderE164, + let hostReadFile = params.mediaAccess?.readFile ?? params.mediaReadFile; + if (!hostReadFile && mediaReadAllowed && resolveEffectiveToolFsRootExpansionAllowed(params)) { + const workspaceRoot = resolveWorkspaceRoot(resolvedWorkspaceDir); + hostReadFile = createBoundedOutboundMediaReadFile(async (filePath, options) => { + const resolvedPath = resolvePathFromInput(filePath, workspaceRoot); + return await readLocalMediaFile(resolvedPath, localRoots ?? [], { + maxBytes: options?.maxBytes ?? Number.MAX_SAFE_INTEGER, + excludedRoots: registeredRoots, + }); }); + } const registeredReadFile = mediaReadAllowed && registeredMedia ? createWorkspaceAwareMediaReadFile({ diff --git a/src/music-generation/normalization.ts b/src/music-generation/normalization.ts index fc8766ef2387..85aa4ee0a156 100644 --- a/src/music-generation/normalization.ts +++ b/src/music-generation/normalization.ts @@ -1,7 +1,4 @@ -import { - hasMediaNormalizationEntry, - normalizeDurationToClosestMax, -} from "../media-generation/runtime-shared.js"; +import { normalizeDurationToClosestMax } from "../media-generation/runtime-shared.js"; import { resolveMusicGenerationModeCapabilities } from "./capabilities.js"; import type { MusicGenerationIgnoredOverride, @@ -11,12 +8,6 @@ import type { MusicGenerationSourceImage, } from "./types.js"; -/** - * Request normalization for music generation. - * - * Providers advertise per-mode and per-model support; this module removes - * unsupported caller overrides and records any duration coercion for metadata. - */ type ResolvedMusicGenerationOverrides = { lyrics?: string; instrumental?: boolean; @@ -26,15 +17,6 @@ type ResolvedMusicGenerationOverrides = { normalization?: MusicGenerationNormalization; }; -function resolveModelBooleanSupport( - model: string, - defaultSupport: boolean | undefined, - supportByModel: Readonly> | undefined, -): boolean { - // Per-model declarations override provider defaults because music models vary within a provider. - return supportByModel?.[model] ?? defaultSupport === true; -} - /** Sanitize caller overrides against provider capabilities before invoking a provider. */ export function resolveMusicGenerationOverrides(params: { provider: MusicGenerationProvider; @@ -51,10 +33,7 @@ export function resolveMusicGenerationOverrides(params: { }); const ignoredOverrides: MusicGenerationIgnoredOverride[] = []; const normalization: MusicGenerationNormalization = {}; - let lyrics = params.lyrics; - let instrumental = params.instrumental; - let durationSeconds = params.durationSeconds; - let format = params.format; + let { lyrics, instrumental, durationSeconds, format } = params; if (!caps) { return { @@ -68,7 +47,7 @@ export function resolveMusicGenerationOverrides(params: { if ( lyrics?.trim() && - !resolveModelBooleanSupport(params.model, caps.supportsLyrics, caps.supportsLyricsByModel) + !(caps.supportsLyricsByModel?.[params.model] ?? caps.supportsLyrics === true) ) { ignoredOverrides.push({ key: "lyrics", value: lyrics }); lyrics = undefined; @@ -76,11 +55,7 @@ export function resolveMusicGenerationOverrides(params: { if ( typeof instrumental === "boolean" && - !resolveModelBooleanSupport( - params.model, - caps.supportsInstrumental, - caps.supportsInstrumentalByModel, - ) + !(caps.supportsInstrumentalByModel?.[params.model] ?? caps.supportsInstrumental === true) ) { ignoredOverrides.push({ key: "instrumental", value: instrumental }); instrumental = undefined; @@ -125,8 +100,6 @@ export function resolveMusicGenerationOverrides(params: { durationSeconds, format, ignoredOverrides, - normalization: hasMediaNormalizationEntry(normalization.durationSeconds) - ? normalization - : undefined, + normalization: normalization.durationSeconds ? normalization : undefined, }; } diff --git a/src/tts/tts-payload.ts b/src/tts/tts-payload.ts index cdcb48114274..8f40524f9613 100644 --- a/src/tts/tts-payload.ts +++ b/src/tts/tts-payload.ts @@ -257,20 +257,25 @@ export async function maybeApplyTtsToPayloadCore( persistTtsAudio, ); - if (result.success && result.audioPath) { - lastTtsAttempt = { - timestamp: Date.now(), - success: true, - textLength: text.length, - summarized: wasSummarized, - provider: result.provider, - persona: result.persona, - fallbackFrom: result.fallbackFrom, - attemptedProviders: result.attemptedProviders, - attempts: result.attempts, - latencyMs: result.latencyMs, - }; + const succeeded = result.success && Boolean(result.audioPath); + lastTtsAttempt = { + timestamp: Date.now(), + success: succeeded, + textLength: text.length, + summarized: wasSummarized, + persona: result.persona, + attemptedProviders: result.attemptedProviders, + attempts: result.attempts, + ...(succeeded + ? { + provider: result.provider, + fallbackFrom: result.fallbackFrom, + latencyMs: result.latencyMs, + } + : { error: result.error }), + }; + if (result.success && result.audioPath) { const payloadWithAudio: ReplyPayload = { ...nextPayload, mediaUrl: result.audioPath, @@ -283,17 +288,6 @@ export async function maybeApplyTtsToPayloadCore( : payloadWithAudio; } - lastTtsAttempt = { - timestamp: Date.now(), - success: false, - textLength: text.length, - summarized: wasSummarized, - persona: result.persona, - attemptedProviders: result.attemptedProviders, - attempts: result.attempts, - error: result.error, - }; - const latency = Date.now() - ttsStart; logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`); return applyExplicitSpeechVisibleFallback(nextPayload, params.channel, explicitTtsText); diff --git a/src/tts/tts-synthesis-support.ts b/src/tts/tts-synthesis-support.ts index eb53924a46d5..68c1575aea24 100644 --- a/src/tts/tts-synthesis-support.ts +++ b/src/tts/tts-synthesis-support.ts @@ -497,8 +497,14 @@ export async function executeTtsProviderAttempts(params: { synthesis, latencyMs, provider, - providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), - providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), + providerModel: resolveTtsResultDetail(prepared, ["modelId", "model"]), + providerVoice: resolveTtsResultDetail(prepared, [ + "speakerVoiceId", + "speakerVoice", + "voiceId", + "voiceName", + "voice", + ]), persona: persona?.id, fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, attemptedProviders, @@ -543,34 +549,19 @@ export async function executeTtsProviderAttempts(params: { }; } -function resolveTtsResultModel( - providerConfig: SpeechProviderConfig, - providerOverrides?: SpeechProviderOverrides, +function resolveTtsResultDetail( + prepared: Pick, + keys: readonly string[], ): string | undefined { - return ( - readTtsResultString(providerOverrides?.modelId) ?? - readTtsResultString(providerOverrides?.model) ?? - readTtsResultString(providerConfig.modelId) ?? - readTtsResultString(providerConfig.model) - ); -} - -function resolveTtsResultVoice( - providerConfig: SpeechProviderConfig, - providerOverrides?: SpeechProviderOverrides, -): string | undefined { - return ( - readTtsResultString(providerOverrides?.speakerVoiceId) ?? - readTtsResultString(providerOverrides?.speakerVoice) ?? - readTtsResultString(providerOverrides?.voiceId) ?? - readTtsResultString(providerOverrides?.voiceName) ?? - readTtsResultString(providerOverrides?.voice) ?? - readTtsResultString(providerConfig.speakerVoiceId) ?? - readTtsResultString(providerConfig.speakerVoice) ?? - readTtsResultString(providerConfig.voiceId) ?? - readTtsResultString(providerConfig.voiceName) ?? - readTtsResultString(providerConfig.voice) - ); + for (const config of [prepared.providerOverrides, prepared.providerConfig]) { + for (const key of keys) { + const value = readTtsResultString(config?.[key]); + if (value !== undefined) { + return value; + } + } + } + return undefined; } function resolvePersonaBinding( diff --git a/src/video-generation/capability-overlays.ts b/src/video-generation/capability-overlays.ts index b541c024ed42..a50e03f3e41a 100644 --- a/src/video-generation/capability-overlays.ts +++ b/src/video-generation/capability-overlays.ts @@ -8,12 +8,6 @@ import type { VideoGenerationTransformCapabilities, } from "./types.js"; -function isVideoGenerationTransformCapabilities( - capabilities: VideoGenerationModeCapabilities | VideoGenerationTransformCapabilities | undefined, -): capabilities is VideoGenerationTransformCapabilities { - return Boolean(capabilities && "enabled" in capabilities); -} - export function buildVideoGenerationCapabilityFailure(params: { providerId: string; model: string; @@ -50,35 +44,21 @@ export function buildVideoGenerationCapabilityFailure(params: { : inputImageCount > 0 ? "reference image inputs" : "reference video inputs"; - if (!isVideoGenerationTransformCapabilities(capabilities) || !capabilities.enabled) { + if (!capabilities || !("enabled" in capabilities) || !capabilities.enabled) { return `${label} does not support ${visualLabel}; skipping to avoid silent reference drop`; } } - if (inputImageCount > 0) { - const maxImages = capabilities?.maxInputImages ?? provider.capabilities.maxInputImages ?? 0; - if (inputImageCount > maxImages) { - return maxImages === 0 - ? `${label} does not support reference image inputs; skipping to avoid silent image drop` - : `${label} supports at most ${maxImages} reference image(s), ${inputImageCount} requested; skipping`; - } - } - - if (inputVideoCount > 0) { - const maxVideos = capabilities?.maxInputVideos ?? provider.capabilities.maxInputVideos ?? 0; - if (inputVideoCount > maxVideos) { - return maxVideos === 0 - ? `${label} does not support reference video inputs; skipping to avoid silent video drop` - : `${label} supports at most ${maxVideos} reference video(s), ${inputVideoCount} requested; skipping`; - } - } - - if (inputAudioCount > 0) { - const maxAudio = capabilities?.maxInputAudios ?? provider.capabilities.maxInputAudios ?? 0; - if (inputAudioCount > maxAudio) { - return maxAudio === 0 - ? `${label} does not support reference audio inputs; skipping to avoid silent audio drop` - : `${label} supports at most ${maxAudio} reference audio(s), ${inputAudioCount} requested; skipping`; + for (const [kind, count, limitKey] of [ + ["image", inputImageCount, "maxInputImages"], + ["video", inputVideoCount, "maxInputVideos"], + ["audio", inputAudioCount, "maxInputAudios"], + ] as const) { + const limit = capabilities?.[limitKey] ?? provider.capabilities[limitKey] ?? 0; + if (count > 0 && count > limit) { + return limit === 0 + ? `${label} does not support reference ${kind} inputs; skipping to avoid silent ${kind} drop` + : `${label} supports at most ${limit} reference ${kind}(s), ${count} requested; skipping`; } } diff --git a/src/video-generation/normalization.ts b/src/video-generation/normalization.ts index 38bbe50bbbf6..98f47b7acaaa 100644 --- a/src/video-generation/normalization.ts +++ b/src/video-generation/normalization.ts @@ -1,5 +1,4 @@ import { resolveMediaGeometryOverrides } from "../media-generation/geometry-normalization.js"; -import { hasMediaNormalizationEntry } from "../media-generation/runtime-shared.js"; import { resolveVideoGenerationModeCapabilities } from "./capabilities.js"; import { normalizeVideoGenerationDuration, @@ -112,12 +111,6 @@ export function resolveVideoGenerationOverrides(params: { audio, watermark, ignoredOverrides, - normalization: - hasMediaNormalizationEntry(normalization.size) || - hasMediaNormalizationEntry(normalization.aspectRatio) || - hasMediaNormalizationEntry(normalization.resolution) || - hasMediaNormalizationEntry(normalization.durationSeconds) - ? normalization - : undefined, + normalization: Object.keys(normalization).length ? normalization : undefined, }; } diff --git a/src/video-generation/runtime.ts b/src/video-generation/runtime.ts index 30430950762f..57a36c87d388 100644 --- a/src/video-generation/runtime.ts +++ b/src/video-generation/runtime.ts @@ -84,19 +84,6 @@ function validateProviderOptionsAgainstDeclaration(params: { return undefined; } -function buildNoVideoGenerationModelConfiguredMessage( - cfg: OpenClawConfig, - deps: VideoGenerationRuntimeDeps, -): string { - const listProviders = deps.listProviders ?? listVideoGenerationProviders; - return buildNoCapabilityModelConfiguredMessage({ - capabilityLabel: "video-generation", - modelConfigKey: "mediaModels.video", - providers: listProviders(cfg), - getProviderEnvVars: deps.getProviderEnvVars, - }); -} - export function listRuntimeVideoGenerationProviders( params?: { config?: OpenClawConfig }, deps: VideoGenerationRuntimeDeps = {}, @@ -141,7 +128,14 @@ async function runVideoGeneration( autoProviderFallback: params.autoProviderFallback, }); if (candidates.length === 0) { - throw new Error(buildNoVideoGenerationModelConfiguredMessage(params.cfg, deps)); + throw new Error( + buildNoCapabilityModelConfiguredMessage({ + capabilityLabel: "video-generation", + modelConfigKey: "mediaModels.video", + providers: listProviders(params.cfg), + getProviderEnvVars: deps.getProviderEnvVars, + }), + ); } let skipWarnEmitted = false;