From 9b2cff2ce11cb82f71c2e2fc590bd370fe5578a3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 2 Oct 2026 18:22:21 -0700 Subject: [PATCH] refactor(media): deslop media generation and understanding (#163916) Simplify media directive parsing, geometry selection, normalization metadata, provider wrappers, understanding parameter forwarding, and speech result projection while preserving public behavior and resource ownership. Remove 550 net production lines. Preserve empty-workspace fallback and add deterministic geometry coverage. Validated with 515 focused tests, both cycle checks at zero, and independent P2 review. The PR contains the full subsystem coverage log and hosted CI evidence. --- .../media-core/src/inline-image-data-url.ts | 78 ++---- .../src/normalization.ts | 13 - src/image-generation/normalization.ts | 8 +- .../openai-compatible-image-provider.ts | 73 ++--- src/image-generation/runtime.ts | 22 +- src/link-understanding/runner.ts | 35 +-- src/media-generation/runtime-shared.test.ts | 31 +++ src/media-generation/runtime-shared.ts | 165 ++++-------- src/media-understanding/fs.ts | 6 - .../image.runtime-profile.test.ts | 92 +++---- src/media-understanding/image.ts | 37 ++- src/media-understanding/local-audio.ts | 6 +- src/media-understanding/runner.entries.ts | 40 +-- src/media-understanding/runner.ts | 142 +++------- src/media-understanding/runtime.ts | 5 +- src/media/channel-inbound-roots.ts | 156 ++++------- src/media/parse-output.ts | 249 +++++------------- src/media/read-capability.ts | 60 +---- src/music-generation/normalization.ts | 37 +-- src/tts/tts-payload.ts | 42 ++- src/tts/tts-synthesis-support.ts | 49 ++-- src/video-generation/capability-overlays.ts | 42 +-- src/video-generation/normalization.ts | 9 +- src/video-generation/runtime.ts | 22 +- 24 files changed, 452 insertions(+), 967 deletions(-) delete mode 100644 src/media-understanding/fs.ts diff --git a/packages/media-core/src/inline-image-data-url.ts b/packages/media-core/src/inline-image-data-url.ts index 496744c2e2b0..1c883385ee53 100644 --- a/packages/media-core/src/inline-image-data-url.ts +++ b/packages/media-core/src/inline-image-data-url.ts @@ -3,54 +3,32 @@ import { canonicalizeBase64 } from "./base64.js"; /** Prefix used to distinguish inline data URLs from remote/local image references. */ export const INLINE_IMAGE_DATA_URL_PREFIX = "data:"; -const IMAGE_SIGNATURES: Array<{ - mime: string; - matches: (buffer: Buffer) => boolean; -}> = [ - { - mime: "image/png", - matches: (buffer) => - buffer.length >= 8 && - buffer[0] === 0x89 && - buffer[1] === 0x50 && - buffer[2] === 0x4e && - buffer[3] === 0x47 && - buffer[4] === 0x0d && - buffer[5] === 0x0a && - buffer[6] === 0x1a && - buffer[7] === 0x0a, - }, - { - mime: "image/jpeg", - matches: (buffer) => - buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff, - }, - { - mime: "image/webp", - matches: (buffer) => - buffer.length >= 12 && - buffer.subarray(0, 4).toString("ascii") === "RIFF" && - buffer.subarray(8, 12).toString("ascii") === "WEBP", - }, - { - mime: "image/gif", - matches: (buffer) => - buffer.length >= 6 && - (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || - buffer.subarray(0, 6).toString("ascii") === "GIF89a"), - }, - { - mime: "image/bmp", - matches: (buffer) => buffer.length >= 2 && buffer[0] === 0x42 && buffer[1] === 0x4d, - }, -]; - const HEIC_BRANDS = new Set(["heic", "heix", "hevc", "hevx", "heis", "heim", "hevm", "hevs"]); const HEIF_BRANDS = new Set(["mif1", "msf1"]); const IMAGE_SIGNATURE_PREFIX_BASE64_CHARS = 128; const INLINE_IMAGE_DATA_URL_MIMES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]); -function sniffIsoBmffImageMime(buffer: Buffer): string | undefined { +/** Sniffs supported inline image formats from decoded bytes. */ +export function sniffInlineImageMime(buffer: Buffer): string | undefined { + if (buffer.length >= 8 && buffer.readBigUInt64BE(0) === 0x89504e470d0a1a0an) { + return "image/png"; + } + if (buffer.length >= 3 && buffer.readUIntBE(0, 3) === 0xffd8ff) { + return "image/jpeg"; + } + if ( + buffer.length >= 12 && + buffer.toString("ascii", 0, 4) === "RIFF" && + buffer.toString("ascii", 8, 12) === "WEBP" + ) { + return "image/webp"; + } + if (buffer.length >= 6 && /^GIF8[79]a$/.test(buffer.toString("ascii", 0, 6))) { + return "image/gif"; + } + if (buffer.length >= 2 && buffer.readUInt16BE(0) === 0x424d) { + return "image/bmp"; + } if (buffer.length < 12 || buffer.subarray(4, 8).toString("ascii") !== "ftyp") { return undefined; } @@ -67,18 +45,6 @@ function sniffIsoBmffImageMime(buffer: Buffer): string | undefined { return undefined; } -/** Sniffs supported inline image formats from decoded bytes. */ -export function sniffInlineImageMime(buffer: Buffer): string | undefined { - return ( - IMAGE_SIGNATURES.find((signature) => signature.matches(buffer))?.mime ?? - sniffIsoBmffImageMime(buffer) - ); -} - -function isImageMimeType(value: string): boolean { - return value.trim().toLowerCase().startsWith("image/"); -} - export type SanitizedInlineImageBase64 = { mimeType: string; base64: string; @@ -89,7 +55,7 @@ export function sanitizeInlineImageBase64(params: { mimeType: string; base64: string; }): SanitizedInlineImageBase64 | undefined { - if (!isImageMimeType(params.mimeType)) { + if (!params.mimeType.trim().toLowerCase().startsWith("image/")) { return undefined; } const canonicalPayload = canonicalizeBase64(params.base64); diff --git a/packages/media-generation-core/src/normalization.ts b/packages/media-generation-core/src/normalization.ts index b43e0a4d3906..184ffb3da1de 100644 --- a/packages/media-generation-core/src/normalization.ts +++ b/packages/media-generation-core/src/normalization.ts @@ -16,16 +16,3 @@ export type MediaGenerationNormalizationMetadataInput = { resolution?: MediaNormalizationEntry; durationSeconds?: MediaNormalizationEntry; }; - -/** True when a normalization entry contains any user-visible normalization metadata. */ -export function hasMediaNormalizationEntry( - entry: MediaNormalizationEntry | undefined, -): entry is MediaNormalizationEntry { - return Boolean( - entry && - (entry.requested !== undefined || - entry.applied !== undefined || - entry.derivedFrom !== undefined || - (entry.supportedValues?.length ?? 0) > 0), - ); -} diff --git a/src/image-generation/normalization.ts b/src/image-generation/normalization.ts index 1ace8febac01..ca46e2f0f3fb 100644 --- a/src/image-generation/normalization.ts +++ b/src/image-generation/normalization.ts @@ -1,6 +1,5 @@ /** Normalizes image generation request overrides against provider/model capabilities. */ import { resolveMediaGeometryOverrides } from "../media-generation/geometry-normalization.js"; -import { hasMediaNormalizationEntry } from "../media-generation/runtime-shared.js"; import type { ImageGenerationBackground, ImageGenerationIgnoredOverride, @@ -90,11 +89,6 @@ export function resolveImageGenerationOverrides(params: { outputFormat, background, ignoredOverrides, - normalization: - hasMediaNormalizationEntry(normalization.size) || - hasMediaNormalizationEntry(normalization.aspectRatio) || - hasMediaNormalizationEntry(normalization.resolution) - ? normalization - : undefined, + normalization: Object.keys(normalization).length ? normalization : undefined, }; } diff --git a/src/image-generation/openai-compatible-image-provider.ts b/src/image-generation/openai-compatible-image-provider.ts index c195b9746d7e..c6f1512113e6 100644 --- a/src/image-generation/openai-compatible-image-provider.ts +++ b/src/image-generation/openai-compatible-image-provider.ts @@ -88,50 +88,13 @@ export type OpenAiCompatibleImageProviderOptions = { }; }; -function resolveDefaultModel(model: string | undefined, fallback: string): string { - return normalizeOptionalString(model) ?? fallback; -} - -function appendImagesPath(baseUrl: string, mode: OpenAiCompatibleImageRequestMode): string { - return `${baseUrl.replace(/\/+$/u, "")}/images/${mode === "edit" ? "edits" : "generations"}`; -} - -function resolveRequestTimeoutMs(params: { - options: OpenAiCompatibleImageProviderOptions; - req: ImageGenerationRequest; - mode: OpenAiCompatibleImageRequestMode; -}): number | undefined { - if (params.options.defaultTimeoutMs === undefined) { - return params.req.timeoutMs; - } - const label = - params.mode === "edit" - ? (params.options.failureLabels?.edit ?? `${params.options.label} image edit`) - : (params.options.failureLabels?.generate ?? `${params.options.label} image generation`); - const deadline = createProviderOperationDeadline({ - timeoutMs: params.req.timeoutMs, - label, - }); - return resolveProviderOperationTimeoutMs({ - deadline, - defaultTimeoutMs: params.options.defaultTimeoutMs, - }); -} - -function resolveResponseMaxImages(params: { - count: number; - mode: OpenAiCompatibleImageRequestMode; - options: OpenAiCompatibleImageProviderOptions; -}): number { - return params.options.capabilities[params.mode].maxCount ?? params.count; -} - /** Creates an image-generation provider backed by OpenAI-style image endpoints. */ export function createOpenAiCompatibleImageGenerationProvider( options: OpenAiCompatibleImageProviderOptions, ): ImageGenerationProvider { const providerConfigKey = options.providerConfigKey ?? options.id; - const normalizeModel = options.normalizeModel ?? resolveDefaultModel; + const normalizeModel = + options.normalizeModel ?? ((model, fallback) => normalizeOptionalString(model) ?? fallback); const resolveCount = options.resolveCount ?? (({ req }) => req.count ?? 1); return { @@ -149,6 +112,7 @@ export function createOpenAiCompatibleImageGenerationProvider( // Reference images switch the request to edit mode; providers can still // disable edits or cap reference count through capabilities. const mode: OpenAiCompatibleImageRequestMode = inputImages.length > 0 ? "edit" : "generate"; + const operation = `${options.label} image ${mode === "edit" ? "edit" : "generation"}`; const maxInputImages = options.capabilities.edit.maxInputImages; if (mode === "edit" && !options.capabilities.edit.enabled) { throw new Error(`${options.label} image editing is not supported.`); @@ -212,11 +176,20 @@ export function createOpenAiCompatibleImageGenerationProvider( mode === "edit" ? options.buildEditRequest({ ...requestParams, mode }) : options.buildGenerateRequest({ ...requestParams, mode }); - const timeoutMs = resolveRequestTimeoutMs({ options, req, mode }); + const timeoutMs = + options.defaultTimeoutMs === undefined + ? req.timeoutMs + : resolveProviderOperationTimeoutMs({ + deadline: createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: options.failureLabels?.[mode] ?? operation, + }), + defaultTimeoutMs: options.defaultTimeoutMs, + }); // Multipart requests must let FormData set its own boundary header, while // JSON requests need an explicit content type after configured headers. const requestOptions = { - url: appendImagesPath(baseUrl, mode), + url: `${baseUrl.replace(/\/+$/u, "")}/images/${mode === "edit" ? "edits" : "generations"}`, headers: new Headers(headers), timeoutMs, fetchFn: fetch, @@ -238,30 +211,20 @@ export function createOpenAiCompatibleImageGenerationProvider( try { await assertOkOrThrowHttpError( response, - mode === "edit" - ? (options.failureLabels?.edit ?? `${options.label} image edit failed`) - : (options.failureLabels?.generate ?? `${options.label} image generation failed`), + options.failureLabels?.[mode] ?? `${operation} failed`, ); const payload = await readProviderJsonResponse(response, `${options.id}.image-generation`, { maxBytes: resolveInlineImageJsonResponseMaxBytes( - resolveResponseMaxImages({ count, mode, options }), + options.capabilities[mode].maxCount ?? count, resolveGeneratedMediaMaxBytes(req.cfg, "image"), ), }); const images = parseOpenAiCompatibleImageResponse(payload, { ...options.response, - malformedResponseError: - mode === "edit" - ? `${options.label} image edit response malformed` - : `${options.label} image generation response malformed`, + malformedResponseError: `${operation} response malformed`, }); if (images.length === 0) { - throw new Error( - options.emptyResponseError ?? - (mode === "edit" - ? `${options.label} image edit response missing image data` - : `${options.label} image generation response missing image data`), - ); + throw new Error(options.emptyResponseError ?? `${operation} response missing image data`); } return { images, model }; } finally { diff --git a/src/image-generation/runtime.ts b/src/image-generation/runtime.ts index cf7fdee7e372..439e5763787a 100644 --- a/src/image-generation/runtime.ts +++ b/src/image-generation/runtime.ts @@ -35,19 +35,6 @@ type ImageGenerationRuntimeDeps = { export type { GenerateImageParams, GenerateImageRuntimeResult } from "./runtime-types.js"; -function buildNoImageGenerationModelConfiguredMessage( - cfg: OpenClawConfig, - deps: ImageGenerationRuntimeDeps, -): string { - const listProviders = deps.listProviders ?? listImageGenerationProviders; - return buildNoCapabilityModelConfiguredMessage({ - capabilityLabel: "image-generation", - modelConfigKey: "mediaModels.image", - providers: listProviders(cfg), - getProviderEnvVars: deps.getProviderEnvVars, - }); -} - /** Lists image-generation providers visible for the current config. */ export function listRuntimeImageGenerationProviders( params?: { config?: OpenClawConfig }, @@ -93,7 +80,14 @@ async function runImageGeneration( autoProviderFallback: params.autoProviderFallback, }); if (candidates.length === 0) { - throw new Error(buildNoImageGenerationModelConfiguredMessage(params.cfg, deps)); + throw new Error( + buildNoCapabilityModelConfiguredMessage({ + capabilityLabel: "image-generation", + modelConfigKey: "mediaModels.image", + providers: listProviders(params.cfg), + getProviderEnvVars: deps.getProviderEnvVars, + }), + ); } return runMediaGenerationCandidates({ diff --git a/src/link-understanding/runner.ts b/src/link-understanding/runner.ts index bd188a9f351b..a6c1d337e5bc 100644 --- a/src/link-understanding/runner.ts +++ b/src/link-understanding/runner.ts @@ -3,7 +3,6 @@ import { applyTemplate } from "../auto-reply/templating.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { LinkModelConfig, LinkToolsConfig } from "../config/types.tools.js"; import { logVerbose, shouldLogVerbose } from "../globals.js"; -// Link-understanding runner fetches allowed URLs and invokes configured commands with bounded content. import { createAbortError, isAbortError } from "../infra/abort-signal.js"; import { cancelUnreadResponseBody, readResponseWithLimit } from "../infra/http-body.js"; import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../infra/net/fetch-guard.js"; @@ -40,21 +39,6 @@ function isLinkUrlTemplate(value: string): boolean { return value.includes("LinkUrl") || value.includes("LinkFinalUrl"); } -function commandName(command: string): string { - return (command.split(/[\\/]/).pop() ?? command).toLowerCase(); -} - -function isUrlFetcherCommand(command: string): boolean { - return commandName(command) === "curl" || commandName(command) === "wget"; -} - -function buildLinkCliArgs(params: { args: string[]; ctx: MsgContext }): string[] { - const templCtx = { ...params.ctx }; - return params.args - .filter((arg) => !isLinkUrlTemplate(arg)) - .map((arg) => applyTemplate(arg, templCtx)); -} - async function fetchLinkContent(params: { timeoutMs: number; url: string; @@ -108,17 +92,16 @@ async function runCliEntry(params: { } const args = params.entry.args ?? []; const timeoutMs = resolveTimeoutMsFromConfig({ config: params.config, entry: params.entry }); - if (isUrlFetcherCommand(command) && args.some(isLinkUrlTemplate)) { + const name = (command.split(/[\\/]/).pop() ?? command).toLowerCase(); + if ((name === "curl" || name === "wget") && args.some(isLinkUrlTemplate)) { // curl/wget URL templates mark the entry as a fetcher; guarded fetch already supplied content. return params.content; } + const templCtx = { ...params.ctx }; const argv = [ command, - ...buildLinkCliArgs({ - args, - ctx: params.ctx, - }), + ...args.filter((arg) => !isLinkUrlTemplate(arg)).map((arg) => applyTemplate(arg, templCtx)), ]; if (shouldLogVerbose()) { @@ -161,15 +144,7 @@ async function runLinkEntries(params: { break; } try { - const output = await runCliEntry({ - content: params.content, - entry, - finalUrl: params.finalUrl, - ctx: params.ctx, - url: params.url, - config: params.config, - signal: params.signal, - }); + const output = await runCliEntry({ ...params, entry }); if (output) { return output; } diff --git a/src/media-generation/runtime-shared.test.ts b/src/media-generation/runtime-shared.test.ts index 92b0efbde1ff..3d5807d41340 100644 --- a/src/media-generation/runtime-shared.test.ts +++ b/src/media-generation/runtime-shared.test.ts @@ -378,6 +378,37 @@ describe("media-generation runtime shared normalization", () => { ).toBeUndefined(); }); + it("keeps geometry tie-breaking independent of provider declaration order", () => { + for (const reverse of [false, true]) { + const ordered = (values: T[]) => (reverse ? values.toReversed() : values); + expect( + resolveClosestAspectRatio({ + requestedAspectRatio: "3:3", + supportedAspectRatios: ordered(["invalid", "2:2", "1:1"]), + }), + ).toBe("1:1"); + expect( + resolveClosestSize({ + requestedAspectRatio: "1:1", + supportedSizes: ordered(["invalid", "128x128", "64x64"]), + }), + ).toBe("64x64"); + expect( + resolveClosestResolution({ + requestedResolution: "480P", + supportedResolutions: ordered(["invalid", "360P", "600P"]), + }), + ).toBe("600P"); + expect( + resolveClosestResolution({ + requestedResolution: "medium", + supportedResolutions: ordered(["invalid", "small", "large"]), + order: ["small", "medium", "large"], + }), + ).toBe("small"); + } + }); + it("clamps durations to the closest supported max", () => { expect(normalizeDurationToClosestMax(12, 8)).toBe(8); expect(normalizeDurationToClosestMax(6, 8)).toBe(6); diff --git a/src/media-generation/runtime-shared.ts b/src/media-generation/runtime-shared.ts index 79ad8ee77445..ea00dd895811 100644 --- a/src/media-generation/runtime-shared.ts +++ b/src/media-generation/runtime-shared.ts @@ -1,6 +1,10 @@ import { clampTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; -import { resolveCapabilityModelRefForProviders } from "../../packages/media-generation-core/src/capability-model-ref.js"; +import { + resolveCapabilityModelRefForProviders, + type CapabilityModelRef as ParsedProviderModelRef, + type CapabilityModelProviderCandidate, +} from "../../packages/media-generation-core/src/capability-model-ref.js"; import type { MediaGenerationNormalizationMetadataInput } from "../../packages/media-generation-core/src/normalization.js"; import { DEFAULT_PROVIDER } from "../agents/defaults.js"; import { describeFailoverError, isFailoverError } from "../agents/failover-error.js"; @@ -15,13 +19,6 @@ import { formatErrorMessage, toErrorObject } from "../infra/errors.js"; import { isProviderApiKeyConfigured } from "../plugins/provider-auth-availability.js"; import { getProviderEnvVarsCore } from "../secrets/provider-env-vars.js"; -export { hasMediaNormalizationEntry } from "../../packages/media-generation-core/src/normalization.js"; - -type ParsedProviderModelRef = { - provider: string; - model: string; -}; - function buildCapabilityCandidateFailure( candidate: ParsedProviderModelRef, error: unknown, @@ -131,11 +128,7 @@ export function resolveMediaProviderRequestTimeoutMs(params: { ); } -type CapabilityProviderCandidate = { - id: string; - aliases?: readonly string[]; - defaultModel?: string | null; - models?: readonly string[]; +type CapabilityProviderCandidate = CapabilityModelProviderCandidate & { isConfigured?: (ctx: { cfg?: OpenClawConfig; agentDir?: string }) => boolean; }; @@ -303,26 +296,29 @@ export function resolveCapabilityModelCandidates(params: { } function normalizeSupportedValues(values?: readonly TValue[]): TValue[] { - return (values ?? []).flatMap((entry) => { - const normalized = normalizeOptionalString(entry); - return normalized ? [entry] : []; - }); + return (values ?? []).filter((entry) => Boolean(normalizeOptionalString(entry))); } -function compareScores( - next: { primary: number; secondary: number; tertiary: string }, - best: { primary: number; secondary: number; tertiary: string } | null, -): boolean { - if (!best) { - return true; +function selectClosestValue( + values: readonly T[], + score: (value: T) => { primary: number; secondary: number } | undefined, +): T | undefined { + let best: { value: T; primary: number; secondary: number } | undefined; + for (const value of values) { + const next = score(value); + if ( + next && + (!best || + (next.primary !== best.primary + ? next.primary < best.primary + : next.secondary !== best.secondary + ? next.secondary < best.secondary + : value.localeCompare(best.value) < 0)) + ) { + best = { value, ...next }; + } } - if (next.primary !== best.primary) { - return next.primary < best.primary; - } - if (next.secondary !== best.secondary) { - return next.secondary < best.secondary; - } - return next.tertiary.localeCompare(best.tertiary) < 0; + return best?.value; } function parsePositiveDimensionPair( @@ -414,24 +410,16 @@ export function resolveClosestAspectRatio(params: { return undefined; } - let bestValue: string | undefined; - let bestScore: { primary: number; secondary: number; tertiary: string } | null = null; - for (const candidate of supported) { + return selectClosestValue(supported, (candidate) => { const parsed = parseAspectRatioValue(candidate); if (!parsed) { - continue; + return undefined; } - const score = { + return { primary: Math.abs(Math.log(parsed.value / requested.value)), secondary: Math.abs(parsed.width * requested.height - requested.width * parsed.height), - tertiary: candidate, }; - if (compareScores(score, bestScore)) { - bestValue = candidate; - bestScore = score; - } - } - return bestValue; + }); } /** Chooses the closest supported size by aspect ratio and area. */ @@ -453,26 +441,18 @@ export function resolveClosestSize(params: { return undefined; } - let bestValue: string | undefined; - let bestScore: { primary: number; secondary: number; tertiary: string } | null = null; - for (const candidate of supported) { + return selectClosestValue(supported, (candidate) => { const parsed = parseSizeValue(candidate); if (!parsed) { - continue; + return undefined; } - const score = { + return { primary: Math.abs( Math.log(parsed.aspectRatio / (requested?.aspectRatio ?? requestedAspectRatio!.value)), ), secondary: requested ? Math.abs(Math.log(parsed.area / requested.area)) : parsed.area, - tertiary: candidate, }; - if (compareScores(score, bestScore)) { - bestValue = candidate; - bestScore = score; - } - } - return bestValue; + }); } /** Chooses the closest supported resolution by numeric rank or custom order. */ @@ -490,23 +470,16 @@ export function resolveClosestResolution(params: { } const requestedNumeric = parseResolutionRank(params.requestedResolution); if (requestedNumeric) { - let bestValue: TResolution | undefined; - let bestScore: { primary: number; secondary: number; tertiary: string } | null = null; - for (const candidate of supported) { + const bestValue = selectClosestValue(supported, (candidate) => { const candidateNumeric = parseResolutionRank(candidate); if (!candidateNumeric || candidateNumeric.unit !== requestedNumeric.unit) { - continue; + return undefined; } - const score = { + return { primary: Math.abs(candidateNumeric.value - requestedNumeric.value), secondary: candidateNumeric.value < requestedNumeric.value ? 1 : 0, - tertiary: candidate, }; - if (compareScores(score, bestScore)) { - bestValue = candidate; - bestScore = score; - } - } + }); if (bestValue) { return bestValue; } @@ -519,24 +492,12 @@ export function resolveClosestResolution(params: { return undefined; } - let bestValue: TResolution | undefined; - let bestScore: { primary: number; secondary: number; tertiary: string } | null = null; - for (const candidate of supported) { + return selectClosestValue(supported, (candidate) => { const candidateIndex = order.indexOf(candidate); - if (candidateIndex < 0) { - continue; - } - const score = { - primary: Math.abs(candidateIndex - requestedIndex), - secondary: candidateIndex, - tertiary: candidate, - }; - if (compareScores(score, bestScore)) { - bestValue = candidate; - bestScore = score; - } - } - return bestValue; + return candidateIndex < 0 + ? undefined + : { primary: Math.abs(candidateIndex - requestedIndex), secondary: candidateIndex }; + }); } function parseResolutionRank( @@ -648,33 +609,23 @@ function formatCapabilityFailureAttempts(attempts: FallbackAttempt[]): string { if (attempts.length === 0) { return "unknown"; } - - const abortedAttempts = attempts.filter(isAbortLikeFallbackAttempt); - if (abortedAttempts.length === 0) { - return attempts.map(formatCapabilityFailureAttempt).join(" | "); + const failures: string[] = []; + const aborted: string[] = []; + for (const attempt of attempts) { + const ref = `${attempt.provider}/${attempt.model}`; + const message = attempt.error.trim().toLowerCase(); + if (message.includes("operation was aborted") || message.includes("request was aborted")) { + aborted.push(ref); + } else { + failures.push(`${ref}: ${attempt.error}`); + } } - if (abortedAttempts.length === attempts.length) { - return `${abortedAttempts.length} fallback(s) aborted after the request was cancelled or timed out: ${abortedAttempts.map(formatCapabilityAttemptRef).join(", ")}`; + if (aborted.length) { + failures.push( + `${aborted.length} fallback(s) aborted after the request was cancelled or timed out: ${aborted.join(", ")}`, + ); } - - const primaryFailures = attempts.filter((attempt) => !isAbortLikeFallbackAttempt(attempt)); - return [ - primaryFailures.map(formatCapabilityFailureAttempt).join(" | "), - `${abortedAttempts.length} fallback(s) aborted after the request was cancelled or timed out: ${abortedAttempts.map(formatCapabilityAttemptRef).join(", ")}`, - ].join(" | "); -} - -function formatCapabilityFailureAttempt(attempt: FallbackAttempt): string { - return `${formatCapabilityAttemptRef(attempt)}: ${attempt.error}`; -} - -function formatCapabilityAttemptRef(attempt: FallbackAttempt): string { - return `${attempt.provider}/${attempt.model}`; -} - -function isAbortLikeFallbackAttempt(attempt: FallbackAttempt): boolean { - const message = attempt.error.trim().toLowerCase(); - return message.includes("operation was aborted") || message.includes("request was aborted"); + return failures.join(" | "); } /** Formats setup guidance when no model is configured for a media capability. */ diff --git a/src/media-understanding/fs.ts b/src/media-understanding/fs.ts deleted file mode 100644 index 559f5db2b0ce..000000000000 --- a/src/media-understanding/fs.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { pathExists } from "../infra/fs-safe.js"; - -/** Safely checks optional media file paths without throwing on empty input. */ -export async function optionalPathExists(filePath?: string | null): Promise { - return filePath ? await pathExists(filePath) : false; -} diff --git a/src/media-understanding/image.runtime-profile.test.ts b/src/media-understanding/image.runtime-profile.test.ts index bcb0c5e84bfe..ec6476718c8e 100644 --- a/src/media-understanding/image.runtime-profile.test.ts +++ b/src/media-understanding/image.runtime-profile.test.ts @@ -375,52 +375,56 @@ describe("describeImageWithModelCore", () => { expect(options.maxTokens).toBe(1024); }); - it("derives workspaceDir from agentId for image runtime resolution", async () => { - mockImageModel({ - provider: "google", - id: "gemini-2.5-flash", - api: "google-generative-ai", - }); - completeMock.mockResolvedValue( - imageCompletion("google-generative-ai", "google", "gemini-2.5-flash", "workspace ok"), - ); - const cfg = { - agents: { - list: [ - { - id: "vision-agent", - agentDir: "/tmp/openclaw-agent", - workspace: "/tmp/openclaw-workspace", - }, - ], - }, - }; + it.each([undefined, ""])( + "derives workspaceDir from agentId when workspaceDir is %j", + async (workspaceDir) => { + mockImageModel({ + provider: "google", + id: "gemini-2.5-flash", + api: "google-generative-ai", + }); + completeMock.mockResolvedValue( + imageCompletion("google-generative-ai", "google", "gemini-2.5-flash", "workspace ok"), + ); + const cfg = { + agents: { + list: [ + { + id: "vision-agent", + agentDir: "/tmp/openclaw-agent", + workspace: "/tmp/openclaw-workspace", + }, + ], + }, + }; - await describeImageWithModelCore({ - ...imageRequestDefaults(), - cfg, - agentId: "vision-agent", - provider: "google", - model: "gemini-2.5-flash", - buffer: Buffer.alloc(1), - prompt: "Describe the image.", - }); + await describeImageWithModelCore({ + ...imageRequestDefaults(), + cfg, + agentId: "vision-agent", + workspaceDir, + provider: "google", + model: "gemini-2.5-flash", + buffer: Buffer.alloc(1), + prompt: "Describe the image.", + }); - expect(acquireAgentRunPreparedModelRuntimeMock).toHaveBeenCalledWith( - expect.objectContaining({ - workspaceDir: "/tmp/openclaw-workspace", - loadRuntimePlugins: true, - }), - expect.objectContaining({ catalogMode: "static", abortSignal: expect.any(AbortSignal) }), - ); - expect(resolveModelAsyncMock).toHaveBeenCalledWith( - "google", - "gemini-2.5-flash", - "/tmp/openclaw-agent", - cfg, - expect.objectContaining({ workspaceDir: "/tmp/openclaw-workspace" }), - ); - }); + expect(acquireAgentRunPreparedModelRuntimeMock).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceDir: "/tmp/openclaw-workspace", + loadRuntimePlugins: true, + }), + expect.objectContaining({ catalogMode: "static", abortSignal: expect.any(AbortSignal) }), + ); + expect(resolveModelAsyncMock).toHaveBeenCalledWith( + "google", + "gemini-2.5-flash", + "/tmp/openclaw-agent", + cfg, + expect.objectContaining({ workspaceDir: "/tmp/openclaw-workspace" }), + ); + }, + ); it("uses one committed prepared generation for image setup and streaming", async () => { const requestedCfg: OpenClawConfig = { logging: { level: "info" } }; diff --git a/src/media-understanding/image.ts b/src/media-understanding/image.ts index b26016a1dcd1..52f472bd3386 100644 --- a/src/media-understanding/image.ts +++ b/src/media-understanding/image.ts @@ -575,28 +575,23 @@ async function describeImagesWithModelInternal( } function toImagesDescriptionRequest(params: ImageDescriptionRequest): ImagesDescriptionRequest { + const { + buffer, + fileName, + mime, + signal, + agentId, + workspaceDir, + preparedModelRuntime, + ...request + } = params; return { - images: [ - { - buffer: params.buffer, - fileName: params.fileName, - mime: params.mime, - }, - ], - model: params.model, - provider: params.provider, - prompt: params.prompt, - maxTokens: params.maxTokens, - timeoutMs: params.timeoutMs, - ...(params.signal ? { signal: params.signal } : {}), - profile: params.profile, - preferredProfile: params.preferredProfile, - authStore: params.authStore, - ...(params.agentId ? { agentId: params.agentId } : {}), - agentDir: params.agentDir, - ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), - ...(params.preparedModelRuntime ? { preparedModelRuntime: params.preparedModelRuntime } : {}), - cfg: params.cfg, + ...request, + images: [{ buffer, fileName, mime }], + ...(signal ? { signal } : {}), + ...(agentId ? { agentId } : {}), + ...(workspaceDir ? { workspaceDir } : {}), + ...(preparedModelRuntime ? { preparedModelRuntime } : {}), }; } diff --git a/src/media-understanding/local-audio.ts b/src/media-understanding/local-audio.ts index 689e839c91af..be58319849cf 100644 --- a/src/media-understanding/local-audio.ts +++ b/src/media-understanding/local-audio.ts @@ -2,10 +2,10 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { MediaUnderstandingModelConfig } from "../config/types.tools.js"; import { resolveExecutableFromPathEnv } from "../infra/executable-path.js"; +import { pathExists } from "../infra/fs-safe.js"; import { resolveEnvironmentValue } from "../infra/process-env.js"; import { runExec } from "../process/exec.js"; import { getOrCreatePromise } from "../shared/lazy-promise.js"; -import { optionalPathExists } from "./fs.js"; type LocalAudioCandidate = { id: "parakeet-mlx" | "whisper-cli" | "sherpa-onnx-offline" | "whisper"; @@ -246,7 +246,7 @@ export async function inspectLocalAudioSelection( const envModel = env.WHISPER_CPP_MODEL?.trim(); const whisperModel = whisperCommand !== null - ? envModel && (await optionalPathExists(envModel)) + ? envModel && (await pathExists(envModel)) ? envModel : await discoverWhisperCppModel(options.listDirectory ?? listDirectoryEntries) : null; @@ -272,7 +272,7 @@ export async function inspectLocalAudioSelection( const sherpaReady = sherpaCommand !== null && sherpaFiles.length === 4 && - (await Promise.all(sherpaFiles.map(optionalPathExists))).every(Boolean); + (await Promise.all(sherpaFiles.map((file) => pathExists(file)))).every(Boolean); const parakeetReady = parakeetCommand !== null && platform === "darwin" && arch === "arm64"; const parakeetArgs = [ "{{AttachmentPath}}", diff --git a/src/media-understanding/runner.entries.ts b/src/media-understanding/runner.entries.ts index 085712dc20a6..fff41c2c3759 100644 --- a/src/media-understanding/runner.entries.ts +++ b/src/media-understanding/runner.entries.ts @@ -641,12 +641,8 @@ export async function runProviderEntry(params: { if (params.secretOwnerId) { assertSecretOwnerAvailable("capability", params.secretOwnerId); } - const { maxBytes, maxChars, timeoutMs, prompt, hasConfiguredPrompt } = resolveEntryRunOptions({ - capability, - entry, - cfg, - config: params.config, - }); + const { maxBytes, maxChars, timeoutMs, prompt, hasConfiguredPrompt } = + resolveEntryRunOptions(params); if (capability === "image") { if (!params.agentDir) { @@ -740,15 +736,12 @@ export async function runProviderEntry(params: { // STT prompts are spelling/context hints; injected instructions can be echoed on silence. const audioPrompt = params.request?.prompt ?? (hasConfiguredPrompt ? prompt : undefined); const transport = resolveProviderRequestContext({ + ...params, providerId, - cfg, - entry, - config: params.config, }); const providerQuery = resolveProviderQuery({ + ...params, providerId, - config: params.config, - entry, }); const model = entry.model?.trim() || @@ -791,13 +784,9 @@ export async function runProviderEntry(params: { "audio transcription callback", ); const auth = await resolveProviderExecutionAuth({ - capability, + ...params, providerId, provider, - cfg, - entry, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, }); result = await executeProviderRequest(providerId, auth, (requestAuth) => transcribeAudio({ ...input, ...requestAuth }), @@ -833,19 +822,13 @@ export async function runProviderEntry(params: { ); } const auth = await resolveProviderExecutionAuth({ - capability, + ...params, providerId, provider, - cfg, - entry, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, }); const { baseUrl, headers, request } = resolveProviderRequestContext({ + ...params, providerId, - cfg, - entry, - config: params.config, }); const model = entry.model?.trim() || @@ -891,7 +874,7 @@ export async function runCliEntry(params: { config?: MediaUnderstandingConfig; request?: MediaRequestOverrides; }): Promise { - const { entry, capability, cfg, ctx } = params; + const { entry, capability, ctx } = params; const attachmentIndex = params.attachment.index; const cli = resolveCliModelEntry(entry); if (!cli.ok) { @@ -899,12 +882,7 @@ export async function runCliEntry(params: { } const { command, args } = cli.value; const language = params.request?.language ?? entry.language ?? params.config?.language; - const { maxBytes, maxChars, timeoutMs, prompt } = resolveEntryRunOptions({ - capability, - entry, - cfg, - config: params.config, - }); + const { maxBytes, maxChars, timeoutMs, prompt } = resolveEntryRunOptions(params); const attachmentPath = await params.cache.getPath({ attachmentIndex, maxBytes, diff --git a/src/media-understanding/runner.ts b/src/media-understanding/runner.ts index 57761d0c32e3..fc323597a9ad 100644 --- a/src/media-understanding/runner.ts +++ b/src/media-understanding/runner.ts @@ -83,6 +83,15 @@ export { export { buildMediaUnderstandingRegistry as buildProviderRegistry } from "./provider-registry.js"; type ProviderRegistry = Map; +type AutoModelSelectionParams = { + cfg: OpenClawConfig; + agentId?: string; + agentDir?: string; + workspaceDir?: string; + providerRegistry: ProviderRegistry; + capability: MediaUnderstandingCapability; + activeModel?: ActiveMediaModel; +}; /** * A provider registry, or a memoized factory that builds one on first use. * `runCapability` receives the factory form so a turn that never needs the @@ -286,25 +295,14 @@ async function resolveAutoImageModelId(params: { }); } -async function resolveKeyEntry(params: { - cfg: OpenClawConfig; - agentId?: string; - agentDir?: string; - workspaceDir?: string; - providerRegistry: ProviderRegistry; - capability: MediaUnderstandingCapability; - activeModel?: ActiveMediaModel; -}): Promise { +async function resolveKeyEntry( + params: AutoModelSelectionParams, +): Promise { const { cfg, providerRegistry, capability } = params; - const checkProvider = ( - providerId: string, - model?: string, - ): Promise => - resolveAutoProviderModelEntry(params, providerId, () => model); - const activeProvider = params.activeModel?.provider?.trim(); if (activeProvider) { - const activeEntry = await checkProvider(activeProvider, params.activeModel?.model); + const model = params.activeModel?.model; + const activeEntry = await resolveAutoProviderModelEntry(params, activeProvider, () => model); if (activeEntry) { return activeEntry; } @@ -318,7 +316,7 @@ async function resolveKeyEntry(params: { providerRegistry, }), })) { - const entry = await checkProvider(providerId, undefined); + const entry = await resolveAutoProviderModelEntry(params, providerId, () => undefined); if (entry) { return entry; } @@ -439,7 +437,7 @@ async function activeModelSupportsNativeVision(params: { } async function* resolveAutoAudioEntries( - params: Parameters[0] & { providerRegistry: ProviderRegistry }, + params: AutoModelSelectionParams, ): AsyncGenerator { const activeProvider = normalizeMediaExecutionProviderId( params.activeModel?.provider?.trim() ?? "", @@ -465,17 +463,12 @@ async function* resolveAutoAudioEntries( } } -async function resolveAutoEntries(params: { - cfg: OpenClawConfig; - agentId?: string; - agentDir?: string; - workspaceDir?: string; - providerRegistry?: ProviderRegistry; - capability: MediaUnderstandingCapability; - activeModel?: ActiveMediaModel; - nativeVisionActive: boolean; - config?: MediaUnderstandingConfig; -}): Promise { +async function resolveAutoEntries( + params: Omit & { + providerRegistry?: ProviderRegistry; + nativeVisionActive: boolean; + }, +): Promise { if (params.capability === "image" && !params.nativeVisionActive) { const imageModelEntries = resolveImageModelFromAgentDefaults(params); if (imageModelEntries.length > 0) { @@ -487,9 +480,16 @@ async function resolveAutoEntries(params: { providerRegistry: params.providerRegistry ?? buildMediaUnderstandingRegistry(undefined, params.cfg), }; - const activeEntry = await resolveActiveModelEntry(prepared); - if (activeEntry) { - return [{ entry: activeEntry }]; + const activeProvider = normalizeMediaExecutionProviderId(prepared.activeModel?.provider ?? ""); + if (activeProvider) { + const activeEntry = await resolveAutoProviderModelEntry( + prepared, + activeProvider, + () => prepared.activeModel?.model, + ); + if (activeEntry) { + return [{ entry: activeEntry }]; + } } const keys = await resolveKeyEntry(prepared); if (keys) { @@ -523,35 +523,8 @@ export async function resolveAutoImageModel(params: { return null; } -async function resolveActiveModelEntry(params: { - cfg: OpenClawConfig; - agentId?: string; - agentDir?: string; - workspaceDir?: string; - providerRegistry: ProviderRegistry; - capability: MediaUnderstandingCapability; - activeModel?: ActiveMediaModel; -}): Promise { - const activeProviderRaw = params.activeModel?.provider?.trim(); - if (!activeProviderRaw) { - return null; - } - const providerId = normalizeMediaExecutionProviderId(activeProviderRaw); - if (!providerId) { - return null; - } - return await resolveAutoProviderModelEntry(params, providerId, () => params.activeModel?.model); -} - async function resolveAutoProviderModelEntry( - params: { - cfg: OpenClawConfig; - agentId?: string; - agentDir?: string; - workspaceDir?: string; - providerRegistry: ProviderRegistry; - capability: MediaUnderstandingCapability; - }, + params: AutoModelSelectionParams, providerId: string, readModel: () => string | undefined, ): Promise { @@ -562,11 +535,8 @@ async function resolveAutoProviderModelEntry( if ( !(params.capability === "audio" && provider?.transcribeAudioWithContext) && !(await hasProviderAuthAvailable({ - capability: params.capability, + ...params, provider: providerId, - cfg: params.cfg, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, })) ) { return null; @@ -576,27 +546,21 @@ async function resolveAutoProviderModelEntry( let model: string | undefined; if (params.capability === "image") { model = await resolveAutoImageModelId({ - cfg: params.cfg, - agentId: params.agentId, + ...params, providerId, - providerRegistry: params.providerRegistry, explicitModel: readModel(), - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, }); } else if (params.capability === "audio") { model = resolveDefaultMediaModelFromRegistry({ + ...params, providerId, - capability: "audio", - providerRegistry: params.providerRegistry, }); } else { model = readModel() ?? resolveDefaultMediaModelFromRegistry({ + ...params, providerId, - capability: "video", - providerRegistry: params.providerRegistry, }); } if (params.capability === "image" && !model) { @@ -766,13 +730,7 @@ export async function runCapability(params: { // delivered image is worse than silence (#122101). let nativeVisionProbe: Promise | undefined; const resolveNativeVisionFlag = (): Promise => { - nativeVisionProbe ??= activeModelSupportsNativeVision({ - cfg, - agentId: params.agentId, - activeModel: params.activeModel, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, - }).catch((err: unknown) => { + nativeVisionProbe ??= activeModelSupportsNativeVision(params).catch((err: unknown) => { if (shouldLogVerbose()) { logVerbose(`native vision support probe failed: ${String(err)}`); } @@ -912,14 +870,8 @@ export async function runCapability(params: { let resolvedEntries: ResolvedMediaModelEntry[] = entries; if (!automaticAudio && resolvedEntries.length === 0) { resolvedEntries = await resolveAutoEntries({ - cfg, - agentId: params.agentId, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, + ...params, providerRegistry, - capability, - activeModel: params.activeModel, - config, nativeVisionActive: capability === "image" && (await resolveNativeVisionFlag()) === true, }); } @@ -942,30 +894,18 @@ export async function runCapability(params: { const attachmentDispositions = buildDispositions({ kind: "failed" }, { kind: "not-selected" }); for (const attachment of selection.selected) { const { output, attempts, processing } = await runAttachmentEntries({ - capability, - cfg, - ctx, + ...params, attachment, - agentId: params.agentId, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, providerRegistry, cache: params.attachments, entries: automaticAudio ? resolveAutoAudioEntries({ - cfg, - agentId: params.agentId, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, + ...params, providerRegistry, - capability, - activeModel: params.activeModel, - nativeVisionActive: false, }) : resolvedEntries, automaticAudio, config, - request: params.request, }); if (output) { outputs.push(output); diff --git a/src/media-understanding/runtime.ts b/src/media-understanding/runtime.ts index 0a7f7c4ea701..9710e83be3e4 100644 --- a/src/media-understanding/runtime.ts +++ b/src/media-understanding/runtime.ts @@ -244,10 +244,7 @@ export async function describeImageFile( export async function prepareImageDescriptionInput(params: PrepareImageDescriptionInputParams) { const timeoutMs = resolveMediaRuntimeTimeoutMs(params.timeoutMs); const image = await readImageDescriptionInput({ - filePath: params.filePath, - mediaUrl: params.mediaUrl, - mime: params.mime, - cfg: params.cfg, + ...params, timeoutMs, }); const normalizedImage = await normalizeImageDescriptionInput({ diff --git a/src/media/channel-inbound-roots.ts b/src/media/channel-inbound-roots.ts index bbe9064dd12a..c8686e75f1c7 100644 --- a/src/media/channel-inbound-roots.ts +++ b/src/media/channel-inbound-roots.ts @@ -27,24 +27,55 @@ type ChannelMediaContractApi = { }; type ChannelMediaRootResolver = keyof ChannelMediaContractApi; -function acceptsResolver( - loaded: ChannelMediaContractApi, - resolver: ChannelMediaRootResolver, -): boolean { - return typeof loaded[resolver] === "function"; +function resolveInstalledChannelMediaContractOwners(params: { + channelId: string; + cfg: OpenClawConfig; +}): PluginManifestRecord[] { + try { + // Read the current plugin metadata generation only. Attachment-root + // resolution must never start plugin discovery or index work of its own. + const snapshot = getCurrentPluginMetadataSnapshotRuntime({ + config: params.cfg, + allowScopedSnapshot: true, + allowWorkspaceScopedSnapshot: true, + }); + const normalizedConfig = normalizePluginsConfig(params.cfg.plugins); + // Only operator-enabled official npm installs may grant attachment-root authority. + return (snapshot?.manifestRegistry.plugins ?? []) + .filter( + (plugin) => + plugin.origin === "global" && + plugin.trustedOfficialInstall === true && + plugin.channels.some( + (channel) => normalizeOptionalLowercaseString(channel) === params.channelId, + ) && + resolveManifestOwnerBasePolicyBlock({ plugin, normalizedConfig }) === null, + ) + .toSorted((left, right) => left.id.localeCompare(right.id)); + } catch { + // Snapshot reads must never turn a missing channel artifact into a hard failure. + return []; + } } -function loadBundledChannelMediaContractApi( - channelId: string, - resolver: ChannelMediaRootResolver, -): ChannelMediaContractApi | undefined { +function findChannelMediaContractApi(params: { + channelId: string | null | undefined; + cfg: OpenClawConfig; + resolver: ChannelMediaRootResolver; +}): ChannelMediaContractApi | undefined { + const channelId = normalizeOptionalLowercaseString(params.channelId); + if (!channelId) { + return undefined; + } try { - // Media-root resolution must stay a narrow artifact load, not full channel bootstrap. + // Resolve only the narrow contract artifact, never the full channel bootstrap. const loaded = loadBundledPluginPublicArtifactModuleSync({ dirName: channelId, artifactBasename: CHANNEL_MEDIA_CONTRACT_ARTIFACT, }); - return acceptsResolver(loaded, resolver) ? loaded : undefined; + if (typeof loaded[params.resolver] === "function") { + return loaded; + } } catch (error) { if ( !( @@ -55,120 +86,25 @@ function loadBundledChannelMediaContractApi( throw error; } } - - return undefined; -} - -function declaresChannel(plugin: PluginManifestRecord, channelId: string): boolean { - return plugin.channels.some( - (ownedChannelId) => normalizeOptionalLowercaseString(ownedChannelId) === channelId, - ); -} - -type ChannelMediaContractOwner = Pick; - -/** - * Lists installed official channel plugins that may own a channel's media contract. - * - * Bundled owners are resolved from the bundled plugin surface; workspace and - * community installs never gain attachment-root authority, so only - * host-verified official npm installs qualify. Current operator policy still - * wins over install provenance: denylisted, explicitly disabled, and - * out-of-allowlist plugins lose attachment-root authority before their artifact - * executes or supplies file-access roots. - */ -function listTrustedInstalledChannelMediaContractOwners(params: { - channelId: string; - cfg: OpenClawConfig; - plugins: readonly PluginManifestRecord[]; -}): ChannelMediaContractOwner[] { - const channelId = normalizeOptionalLowercaseString(params.channelId); - if (!channelId) { - return []; - } - const normalizedConfig = normalizePluginsConfig(params.cfg.plugins); - return params.plugins - .filter( - (plugin) => - plugin.origin === "global" && - plugin.trustedOfficialInstall === true && - declaresChannel(plugin, channelId) && - resolveManifestOwnerBasePolicyBlock({ plugin, normalizedConfig }) === null, - ) - .map((plugin) => ({ id: plugin.id, rootDir: plugin.rootDir })) - .toSorted((left, right) => left.id.localeCompare(right.id)); -} - -function resolveInstalledChannelMediaContractOwners(params: { - channelId: string; - cfg: OpenClawConfig; -}): ChannelMediaContractOwner[] { - try { - // Read the current plugin metadata generation only. Attachment-root - // resolution must never start plugin discovery or index work of its own. - const snapshot = getCurrentPluginMetadataSnapshotRuntime({ - config: params.cfg, - allowScopedSnapshot: true, - allowWorkspaceScopedSnapshot: true, - }); - if (!snapshot) { - return []; - } - return listTrustedInstalledChannelMediaContractOwners({ - channelId: params.channelId, - cfg: params.cfg, - plugins: snapshot.manifestRegistry.plugins, - }); - } catch { - // Snapshot reads must never turn a missing channel artifact into a hard failure. - return []; - } -} - -function loadInstalledChannelMediaContractApi(params: { - channelId: string; - cfg: OpenClawConfig; - resolver: ChannelMediaRootResolver; -}): ChannelMediaContractApi | undefined { - for (const owner of resolveInstalledChannelMediaContractOwners({ - channelId: params.channelId, - cfg: params.cfg, - })) { + for (const owner of resolveInstalledChannelMediaContractOwners({ channelId, cfg: params.cfg })) { try { const loaded = loadPluginPublicArtifactModuleSync({ pluginRoot: owner.rootDir, artifactBasename: CHANNEL_MEDIA_CONTRACT_ARTIFACT, origin: "global", }); - if (acceptsResolver(loaded, params.resolver)) { + if (typeof loaded[params.resolver] === "function") { return loaded; } } catch (error) { - if (error instanceof MissingPublicSurfaceError) { - continue; + if (!(error instanceof MissingPublicSurfaceError)) { + throw error; } - throw error; } } return undefined; } -function findChannelMediaContractApi(params: { - channelId: string | null | undefined; - cfg: OpenClawConfig; - resolver: ChannelMediaRootResolver; -}) { - const normalized = normalizeOptionalLowercaseString(params.channelId); - if (!normalized) { - return undefined; - } - return ( - loadBundledChannelMediaContractApi(normalized, params.resolver) ?? - // External official packages expose the contract from their installed root. - loadInstalledChannelMediaContractApi({ ...params, channelId: normalized }) - ); -} - /** Resolves local inbound attachment roots from the channel named in a message context. */ export function resolveChannelInboundAttachmentRoots(params: { cfg: OpenClawConfig; diff --git a/src/media/parse-output.ts b/src/media/parse-output.ts index 196375ea4238..bcded7dc2694 100644 --- a/src/media/parse-output.ts +++ b/src/media/parse-output.ts @@ -15,12 +15,9 @@ import { findCodeRegions } from "../shared/text/code-regions.js"; import { parseInlineDirectives } from "../utils/directive-tags.js"; import { parseInboundMediaUri } from "./inbound-media-uri.js"; -/** Captures legacy MEDIA: attachment directives from model/tool output. */ -// `main`'s own pattern, backtick stripping included: one optional backtick is consumed before the -// payload is handed on, so `` MEDIA:`/tmp/a.png /tmp/b.png` `` still splits on the whitespace inside -// instead of unwrapping as the single filename `/tmp/a.png /tmp/b.png`. A quote pair is only ever given -// meaning by the code that reads references, never by the capture. -const MEDIA_TOKEN_RE = /\bMEDIA:\s*`?([^\n]+)`?/gi; +// A MEDIA directive consumes its entire line. Keep the optional leading backtick: +// removing it before parsing preserves the existing whitespace-split backtick syntax. +const MEDIA_TOKEN_RE = /\bMEDIA:\s*`?([^\n]+)`?/i; const RENDERABLE_ASSISTANT_MEDIA_PREFIX_RE = /^(?:https?:\/\/|data:(?:image|audio|video)\/|file:|~|\/|[a-z]:[\\/])/iu; @@ -213,31 +210,12 @@ function beginsIndependentMediaSource(raw: string): boolean { return MEDIA_SOURCE_ROOT_RE.test(candidate) || SCHEME_RE.test(candidate); } -// A reference that starts with a quote runs to the first quote of the same kind that is followed by -// whitespace, the end of the payload, or the comma that introduces the next quoted reference. An earlier quote is -// followed by more value, so it is part of that value rather than a delimiter: that keeps an inner quote -// (`MEDIA:'…?token=it's'`), a real filename space (`MEDIA:"/tmp/album/photo.png copy.png"`), and both at -// once (`MEDIA:'/tmp/team's.v1 final/image.png'`) inside one reference, while `MEDIA:"a" "b"` still -// separates at the whitespace between the two. Finding that quote by hand keeps the search linear: -// retrying the remaining suffix from every stray opening quote costs Θ(n²) on a payload such as -// `MEDIA:'a 'a 'a …`, where every quote is followed by a non-space character and so no quote in the -// payload ever closes (measured 0.79s / 3.01s / 11.65s for 48K / 96K / 192K characters, against 2–3ms for -// `main`). +// Scan quote boundaries once; retrying a regex at every stray quote is quadratic. const QUOTE_CHARS = new Set(['"', "'", "`"]); const MEDIA_DIRECTIVE_SPACE_RE = /\s/; -// The comma belongs to the list only when a reference follows it: an unquoted line already reads it that -// way, because `cleanCandidate` trims it from a reference's tail, so `MEDIA:/tmp/a.png, /tmp/b.png` attaches -// both. A quoted reference takes the comma as its closing delimiter only once the scan has looked past the -// comma's own whitespace for the opening quote of the next reference. Three shapes must not split here. A -// comma inside one reference (`/tmp/Hello, World.png`) sits before that reference's closing quote, so it -// never reaches this test. An apostrophe that belongs to the name (`MEDIA:'/tmp/Students', 2024/album.png'`) -// is followed by a comma and then a value character, not a quote, so the reference keeps running to its real -// closing quote. Prose after a comma (`MEDIA:"/tmp/a.png", the first one`) likewise reports no list, and the -// whole-payload reading stands. The skip costs no extra pass: a whitespace run holds no quote, so only the -// quote before that run can walk it and the runs one scan walks are disjoint, which keeps the scan linear -// (measured 0.88s / 1.95s / 3.92s for 3.2M / 6.4M / 12.9M characters of a payload that walks one run per -// apostrophe). +// A comma closes a quoted reference only when another quoted reference follows it. +// Apostrophes in filenames and prose after commas remain part of the current value. function isQuotedMediaReferenceBoundary(payload: string, afterQuote: number): boolean { if (afterQuote >= payload.length) { return true; @@ -258,8 +236,6 @@ function isQuotedMediaReferenceBoundary(payload: string, afterQuote: number): bo function findQuotedMediaReferenceEnd(payload: string, start: number, quote: string): number { for (let index = start + 1; index < payload.length; index += 1) { - // A quote closes its chunk when whitespace, nothing at all, or a comma with a quoted reference behind - // it follows. if (payload.charAt(index) === quote && isQuotedMediaReferenceBoundary(payload, index + 1)) { return index; } @@ -267,11 +243,7 @@ function findQuotedMediaReferenceEnd(payload: string, start: number, quote: stri return -1; } -// A reply that serializes its references into a JSON array states each one with the same quote pair the -// scan below already reads, so the wrapper comes off first and the members are read as a list. Only a whole -// `[…]` qualifies: an unquoted payload still reports no list, and one bracketed reference reports no list -// either, so `MEDIA:["/tmp/a.png"]` keeps `main`'s whole-payload reading and its salvage of a path followed -// by serialized JSON. +// Only a complete array wrapper participates in quoted-list parsing. function stripSerializedJsonArrayWrapper(payload: string): string { const trimmed = payload.trim(); if ( @@ -284,19 +256,8 @@ function stripSerializedJsonArrayWrapper(payload: string): string { return trimmed.slice(1, -1); } -// The references a payload lists, when every token in it is an explicitly quoted reference and there are -// at least two of them; `null` otherwise, which sends the caller back to `main`'s reading. Counting -// tokens is too weak a test for a list: `MEDIA:'/tmp/parents' photos/photo.png'` also starts and ends -// with a quote, but its pair encloses one value whose name holds that quote, so the stray tail must not -// pass for a second reference. With no list present a single quoted value still unwraps as a whole, -// including one whose own text ends with that quote (`MEDIA:"https://example.com/video.mp4?token=ends""`). -// -// One scan answers both questions the caller asks — is this a list, and which references does it hold — -// so the payload is tokenized once. A token that no quote pair bounds is a comma- or whitespace-delimited -// token like any other, and it also settles the answer: the scan stops there instead of reading the rest. -// A member of a list is a reference in its own right, so the caller validates it with the same contract a -// standalone quoted reference gets — bare filenames included, since `MEDIA:"image.png"` is accepted on its -// own — and a member the caller rejects stays out of its neighbours rather than being welded into one. +// Require at least two fully quoted tokens. A single value or an unquoted tail +// keeps whole-payload parsing, including apostrophes and filename whitespace. function readQuotedMediaReferenceList(rawPayload: string): string[] | null { const payload = stripSerializedJsonArrayWrapper(rawPayload); const tokens: string[] = []; @@ -320,10 +281,7 @@ function readQuotedMediaReferenceList(rawPayload: string): string[] | null { return tokens.length >= 2 ? tokens : null; } -// `main`'s own split, kept quote-blind on purpose: a payload that is not a list gives a quote pair no -// authority, so a quote that is text inside one path (`/tmp/album 'best' photos/image.png`) cannot block -// the join, and neither can a fragment that would be accepted as a reference on its own -// (`'best/photos'` in `/tmp/album 'best/photos' final.png`). +// Outside an explicit list, quotes inside a filename must not prevent joining its fragments. function splitMediaDirectiveParts(payload: string): string[] { const parts: string[] = []; let previousEnd = 0; @@ -623,144 +581,81 @@ export function splitMediaOutput( continue; } - const matches = Array.from(line.matchAll(MEDIA_TOKEN_RE)); - if (matches.length === 0) { + const match = MEDIA_TOKEN_RE.exec(line); + if (!match) { keptLines.push(line); pushTextSegment(line); - lineOffset += line.length + 1; // +1 for newline + lineOffset += line.length + 1; continue; } - const pieces: string[] = []; - const lineSegments: ParsedMediaOutputSegment[] = []; - let cursor = 0; - - for (const match of matches) { - const start = match.index ?? 0; - pieces.push(line.slice(cursor, start)); - - const payload = expectDefined(match[1], "parse regex capture 1"); - // A payload that lists separate quoted references keeps every reference as written, and each of - // them is admitted by the same contract a standalone quoted reference gets. Otherwise the payload - // reads the way `main` reads it: one quoted value unwraps as a whole, and anything else splits on - // whitespace. Both answers come from the one scan, so the payload is never tokenized twice. - const quotedList = readQuotedMediaReferenceList(payload); - // `main`'s whole-payload reading of this string, kept even when a list sends the references - // elsewhere: a list still carries the outer quote pair that reading takes off, and the decision - // whether an unreferenced payload is a local path to strip has to land on the string `main` - // decides on, not on the quotes still around it. - const stripped = unwrapQuoted(payload); - const unwrapped = quotedList ? undefined : stripped; - const payloadValue = unwrapped ?? payload; - const parts = quotedList ?? (unwrapped ? [unwrapped] : splitMediaDirectiveParts(payload)); - const mediaStartIndex = media.length; - let validCount = 0; - const invalidParts: string[] = []; - let hasValidMedia = false; - for (const part of parts) { - // Matched quotes delimit the reference; punctuation inside them belongs to its value. That - // holds for every reference a split payload lists, not just for a payload that unwraps as a - // single value, so a quoted part keeps its own characters instead of being cleaned. Cleaning - // one would drop the signed suffix and leave the reference short at delivery time. - const quotedPart = unwrapped === undefined ? unwrapQuoted(part) : undefined; - const candidate = unwrapped ?? quotedPart ?? cleanCandidate(part); - const allowSpaces = Boolean(unwrapped ?? quotedPart) || /\s/.test(candidate); - // A member of an explicit list is validated as the standalone reference its own quote pair makes - // it, bare filenames included: `MEDIA:"image.png"` is accepted on its own, so rejecting - // `"image.png"` here would drop a reference the payload states. Outside a list the bare-filename - // fallback stays where `main` put it, on the whole payload, so a space-separated fragment is - // never promoted to a reference of its own. - if (isValidMedia(candidate, { allowSpaces, allowBareFilename: quotedList !== null })) { - media.push(candidate); - hasValidMedia = true; - foundMediaToken = true; - validCount += 1; - } else if (!/\s/.test(part) || !hasTraversalOrUnsupportedHomeDirPrefix(candidate)) { - invalidParts.push(part); - } + const payload = expectDefined(match[1], "parse regex capture 1"); + const quotedList = readQuotedMediaReferenceList(payload); + const stripped = unwrapQuoted(payload); + const unwrapped = quotedList ? undefined : stripped; + const payloadValue = unwrapped ?? payload; + const parts = quotedList ?? (unwrapped ? [unwrapped] : splitMediaDirectiveParts(payload)); + const mediaStartIndex = media.length; + const invalidParts: string[] = []; + for (const part of parts) { + // Quoted references preserve punctuation, including signed URL suffixes. + const quotedPart = unwrapped === undefined ? unwrapQuoted(part) : undefined; + const candidate = unwrapped ?? quotedPart ?? cleanCandidate(part); + if (isValidMedia(candidate, { allowSpaces: true, allowBareFilename: quotedList !== null })) { + media.push(candidate); + } else if (!/\s/.test(part) || !hasTraversalOrUnsupportedHomeDirPrefix(candidate)) { + invalidParts.push(part); } - - const trimmedPayload = (stripped ?? payload).trim(); - const looksLikeLocalPath = - looksLikeLocalFilePath(trimmedPayload) || FILE_URL_PREFIX_RE.test(trimmedPayload); - if ( - quotedList === null && - !unwrapped && - validCount === 1 && - invalidParts.length > 0 && - !parts.slice(1).some(beginsIndependentMediaSource) && - /\s/.test(payloadValue) && - looksLikeLocalPath - ) { - // A single valid split plus invalid leftovers can be one local path containing spaces. A list is - // excluded: its quote pairs already fixed where each reference ends, so the leftovers are not - // fragments of the accepted one, and welding them on would turn `MEDIA:"/tmp/first.png" "second.png"` - // into `/tmp/first.png" "second.png` — a path that does not exist. This is the reconstruction - // `main` performs on a payload it reads as one quoted value, and it stays available for exactly - // those payloads. - const fallback = cleanCandidate(payloadValue); - if (isValidMedia(fallback, { allowSpaces: true })) { - media.splice(mediaStartIndex, media.length - mediaStartIndex, fallback); - hasValidMedia = true; - foundMediaToken = true; - invalidParts.length = 0; - } - } - - // A list gets no whole-payload reading at all: its quote pairs already fixed where every reference - // ends, so a list whose members all failed states no reference and stays the text it was. Cleaning - // the payload anyway welded the rejects into `first.png," "second.png` for - // `MEDIA:"first.png," "second.png,"` — a name no member states, which the base rejects. Payloads - // `main` reads as one value keep this step, bare-filename fallback included. - if (quotedList === null && !hasValidMedia) { - const fallback = unwrapped ?? cleanCandidate(payloadValue); - if (isValidMedia(fallback, { allowSpaces: true, allowBareFilename: true })) { - media.push(fallback); - hasValidMedia = true; - foundMediaToken = true; - invalidParts.length = 0; - } - } - - if (hasValidMedia) { - const beforeText = cleanLineText(pieces.join("")); - if (beforeText) { - lineSegments.push({ type: "text", text: beforeText }); - } - pieces.length = 0; - for (const url of media.slice(mediaStartIndex)) { - lineSegments.push({ type: "media", url }); - } - if (invalidParts.length > 0) { - pieces.push(invalidParts.join(" ")); - } - } else if (looksLikeLocalPath) { - // Strip MEDIA: lines with local paths even when invalid (e.g. absolute paths - // from internal tools like TTS). They should never leak as visible text. - foundMediaToken = true; - } else { - pieces.push(match[0]); - } - - cursor = start + match[0].length; } - pieces.push(line.slice(cursor)); + const trimmedPayload = (stripped ?? payload).trim(); + const looksLikeLocalPath = + looksLikeLocalFilePath(trimmedPayload) || FILE_URL_PREFIX_RE.test(trimmedPayload); + if ( + quotedList === null && + !unwrapped && + media.length - mediaStartIndex === 1 && + invalidParts.length > 0 && + !parts.slice(1).some(beginsIndependentMediaSource) && + /\s/.test(payloadValue) && + looksLikeLocalPath + ) { + // Unquoted fragments can belong to one local filename; explicit list members cannot. + const fallback = cleanCandidate(payloadValue); + if (isValidMedia(fallback, { allowSpaces: true })) { + media.splice(mediaStartIndex, media.length - mediaStartIndex, fallback); + invalidParts.length = 0; + } + } - const cleanedLine = cleanLineText(pieces.join("")); + // Never weld rejected list members into a synthetic whole-payload filename. + if (quotedList === null && media.length === mediaStartIndex) { + const fallback = unwrapped ?? cleanCandidate(payloadValue); + if (isValidMedia(fallback, { allowSpaces: true, allowBareFilename: true })) { + media.push(fallback); + invalidParts.length = 0; + } + } + let cleanedLine: string; + if (media.length > mediaStartIndex) { + foundMediaToken = true; + for (const url of media.slice(mediaStartIndex)) { + segments.push({ type: "media", url }); + } + cleanedLine = cleanLineText(invalidParts.join(" ")); + } else if (looksLikeLocalPath) { + // Invalid local references must not leak internal tool paths as visible text. + foundMediaToken = true; + cleanedLine = ""; + } else { + cleanedLine = cleanLineText(line); + } if (cleanedLine) { keptLines.push(cleanedLine); - lineSegments.push({ type: "text", text: cleanedLine }); + pushTextSegment(cleanedLine); } - for (const segment of lineSegments) { - if (segment.type === "text") { - pushTextSegment(segment.text); - continue; - } - segments.push(segment); - } - lineOffset += line.length + 1; // +1 for newline + lineOffset += line.length + 1; } const visibleText = keptLines.join("\n").replace(/^(?:[ \t]*\n)+/, ""); diff --git a/src/media/read-capability.ts b/src/media/read-capability.ts index daff8219b045..12655bcc6820 100644 --- a/src/media/read-capability.ts +++ b/src/media/read-capability.ts @@ -64,36 +64,6 @@ function isAgentScopedMediaReadAllowedByToolPolicy( return isToolAllowedByPolicies("read", [groupPolicy, senderPolicy]); } -/** Creates a host reader bound to the agent workspace and configured local-file safety checks. */ -function createAgentScopedHostMediaReadFile( - params: { - cfg: OpenClawConfig; - agentId?: string; - localRoots: readonly string[]; - workspaceDir?: string; - excludedLocalRoots?: readonly string[]; - workspaceOnly?: boolean; - } & OutboundHostMediaPolicyContext, -): OutboundMediaReadFile | undefined { - if ( - !resolveEffectiveToolFsRootExpansionAllowed(params) || - !isAgentScopedMediaReadAllowedByToolPolicy(params) - ) { - return undefined; - } - const inferredWorkspaceDir = - params.workspaceDir ?? - (params.agentId ? resolveAgentWorkspaceDir(params.cfg, params.agentId) : undefined); - const workspaceRoot = resolveWorkspaceRoot(inferredWorkspaceDir); - return createBoundedOutboundMediaReadFile(async (filePath, options) => { - const resolvedPath = resolvePathFromInput(filePath, workspaceRoot); - return await readLocalMediaFile(resolvedPath, params.localRoots, { - maxBytes: options?.maxBytes ?? Number.MAX_SAFE_INTEGER, - excludedRoots: params.excludedLocalRoots, - }); - }); -} - function getManagedMediaLocalRoots(mediaSources?: readonly string[]): readonly string[] { const roots = new Set([path.join(resolveConfigDir(), "media", "outbound")]); for (const source of mediaSources ?? []) { @@ -218,27 +188,17 @@ export function resolveAgentScopedOutboundMediaAccess( mediaReadAllowed && !registeredMedia ? appendWorkspaceDirToLocalRoots(baseLocalRoots, resolvedWorkspaceDir) : baseLocalRoots; - const hostReadFile = - params.mediaAccess?.readFile ?? - params.mediaReadFile ?? - createAgentScopedHostMediaReadFile({ - cfg: params.cfg, - agentId: params.agentId, - localRoots: localRoots ?? [], - workspaceDir: resolvedWorkspaceDir, - excludedLocalRoots: registeredRoots, - workspaceOnly: params.workspaceOnly, - sessionKey: params.sessionKey, - messageProvider: params.messageProvider, - groupId: params.groupId, - groupChannel: params.groupChannel, - groupSpace: params.groupSpace, - accountId: params.accountId, - requesterSenderId: params.requesterSenderId, - requesterSenderName: params.requesterSenderName, - requesterSenderUsername: params.requesterSenderUsername, - requesterSenderE164: params.requesterSenderE164, + let hostReadFile = params.mediaAccess?.readFile ?? params.mediaReadFile; + if (!hostReadFile && mediaReadAllowed && resolveEffectiveToolFsRootExpansionAllowed(params)) { + const workspaceRoot = resolveWorkspaceRoot(resolvedWorkspaceDir); + hostReadFile = createBoundedOutboundMediaReadFile(async (filePath, options) => { + const resolvedPath = resolvePathFromInput(filePath, workspaceRoot); + return await readLocalMediaFile(resolvedPath, localRoots ?? [], { + maxBytes: options?.maxBytes ?? Number.MAX_SAFE_INTEGER, + excludedRoots: registeredRoots, + }); }); + } const registeredReadFile = mediaReadAllowed && registeredMedia ? createWorkspaceAwareMediaReadFile({ diff --git a/src/music-generation/normalization.ts b/src/music-generation/normalization.ts index fc8766ef2387..85aa4ee0a156 100644 --- a/src/music-generation/normalization.ts +++ b/src/music-generation/normalization.ts @@ -1,7 +1,4 @@ -import { - hasMediaNormalizationEntry, - normalizeDurationToClosestMax, -} from "../media-generation/runtime-shared.js"; +import { normalizeDurationToClosestMax } from "../media-generation/runtime-shared.js"; import { resolveMusicGenerationModeCapabilities } from "./capabilities.js"; import type { MusicGenerationIgnoredOverride, @@ -11,12 +8,6 @@ import type { MusicGenerationSourceImage, } from "./types.js"; -/** - * Request normalization for music generation. - * - * Providers advertise per-mode and per-model support; this module removes - * unsupported caller overrides and records any duration coercion for metadata. - */ type ResolvedMusicGenerationOverrides = { lyrics?: string; instrumental?: boolean; @@ -26,15 +17,6 @@ type ResolvedMusicGenerationOverrides = { normalization?: MusicGenerationNormalization; }; -function resolveModelBooleanSupport( - model: string, - defaultSupport: boolean | undefined, - supportByModel: Readonly> | undefined, -): boolean { - // Per-model declarations override provider defaults because music models vary within a provider. - return supportByModel?.[model] ?? defaultSupport === true; -} - /** Sanitize caller overrides against provider capabilities before invoking a provider. */ export function resolveMusicGenerationOverrides(params: { provider: MusicGenerationProvider; @@ -51,10 +33,7 @@ export function resolveMusicGenerationOverrides(params: { }); const ignoredOverrides: MusicGenerationIgnoredOverride[] = []; const normalization: MusicGenerationNormalization = {}; - let lyrics = params.lyrics; - let instrumental = params.instrumental; - let durationSeconds = params.durationSeconds; - let format = params.format; + let { lyrics, instrumental, durationSeconds, format } = params; if (!caps) { return { @@ -68,7 +47,7 @@ export function resolveMusicGenerationOverrides(params: { if ( lyrics?.trim() && - !resolveModelBooleanSupport(params.model, caps.supportsLyrics, caps.supportsLyricsByModel) + !(caps.supportsLyricsByModel?.[params.model] ?? caps.supportsLyrics === true) ) { ignoredOverrides.push({ key: "lyrics", value: lyrics }); lyrics = undefined; @@ -76,11 +55,7 @@ export function resolveMusicGenerationOverrides(params: { if ( typeof instrumental === "boolean" && - !resolveModelBooleanSupport( - params.model, - caps.supportsInstrumental, - caps.supportsInstrumentalByModel, - ) + !(caps.supportsInstrumentalByModel?.[params.model] ?? caps.supportsInstrumental === true) ) { ignoredOverrides.push({ key: "instrumental", value: instrumental }); instrumental = undefined; @@ -125,8 +100,6 @@ export function resolveMusicGenerationOverrides(params: { durationSeconds, format, ignoredOverrides, - normalization: hasMediaNormalizationEntry(normalization.durationSeconds) - ? normalization - : undefined, + normalization: normalization.durationSeconds ? normalization : undefined, }; } diff --git a/src/tts/tts-payload.ts b/src/tts/tts-payload.ts index cdcb48114274..8f40524f9613 100644 --- a/src/tts/tts-payload.ts +++ b/src/tts/tts-payload.ts @@ -257,20 +257,25 @@ export async function maybeApplyTtsToPayloadCore( persistTtsAudio, ); - if (result.success && result.audioPath) { - lastTtsAttempt = { - timestamp: Date.now(), - success: true, - textLength: text.length, - summarized: wasSummarized, - provider: result.provider, - persona: result.persona, - fallbackFrom: result.fallbackFrom, - attemptedProviders: result.attemptedProviders, - attempts: result.attempts, - latencyMs: result.latencyMs, - }; + const succeeded = result.success && Boolean(result.audioPath); + lastTtsAttempt = { + timestamp: Date.now(), + success: succeeded, + textLength: text.length, + summarized: wasSummarized, + persona: result.persona, + attemptedProviders: result.attemptedProviders, + attempts: result.attempts, + ...(succeeded + ? { + provider: result.provider, + fallbackFrom: result.fallbackFrom, + latencyMs: result.latencyMs, + } + : { error: result.error }), + }; + if (result.success && result.audioPath) { const payloadWithAudio: ReplyPayload = { ...nextPayload, mediaUrl: result.audioPath, @@ -283,17 +288,6 @@ export async function maybeApplyTtsToPayloadCore( : payloadWithAudio; } - lastTtsAttempt = { - timestamp: Date.now(), - success: false, - textLength: text.length, - summarized: wasSummarized, - persona: result.persona, - attemptedProviders: result.attemptedProviders, - attempts: result.attempts, - error: result.error, - }; - const latency = Date.now() - ttsStart; logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`); return applyExplicitSpeechVisibleFallback(nextPayload, params.channel, explicitTtsText); diff --git a/src/tts/tts-synthesis-support.ts b/src/tts/tts-synthesis-support.ts index eb53924a46d5..68c1575aea24 100644 --- a/src/tts/tts-synthesis-support.ts +++ b/src/tts/tts-synthesis-support.ts @@ -497,8 +497,14 @@ export async function executeTtsProviderAttempts(params: { synthesis, latencyMs, provider, - providerModel: resolveTtsResultModel(prepared.providerConfig, prepared.providerOverrides), - providerVoice: resolveTtsResultVoice(prepared.providerConfig, prepared.providerOverrides), + providerModel: resolveTtsResultDetail(prepared, ["modelId", "model"]), + providerVoice: resolveTtsResultDetail(prepared, [ + "speakerVoiceId", + "speakerVoice", + "voiceId", + "voiceName", + "voice", + ]), persona: persona?.id, fallbackFrom: provider !== primaryProvider ? primaryProvider : undefined, attemptedProviders, @@ -543,34 +549,19 @@ export async function executeTtsProviderAttempts(params: { }; } -function resolveTtsResultModel( - providerConfig: SpeechProviderConfig, - providerOverrides?: SpeechProviderOverrides, +function resolveTtsResultDetail( + prepared: Pick, + keys: readonly string[], ): string | undefined { - return ( - readTtsResultString(providerOverrides?.modelId) ?? - readTtsResultString(providerOverrides?.model) ?? - readTtsResultString(providerConfig.modelId) ?? - readTtsResultString(providerConfig.model) - ); -} - -function resolveTtsResultVoice( - providerConfig: SpeechProviderConfig, - providerOverrides?: SpeechProviderOverrides, -): string | undefined { - return ( - readTtsResultString(providerOverrides?.speakerVoiceId) ?? - readTtsResultString(providerOverrides?.speakerVoice) ?? - readTtsResultString(providerOverrides?.voiceId) ?? - readTtsResultString(providerOverrides?.voiceName) ?? - readTtsResultString(providerOverrides?.voice) ?? - readTtsResultString(providerConfig.speakerVoiceId) ?? - readTtsResultString(providerConfig.speakerVoice) ?? - readTtsResultString(providerConfig.voiceId) ?? - readTtsResultString(providerConfig.voiceName) ?? - readTtsResultString(providerConfig.voice) - ); + for (const config of [prepared.providerOverrides, prepared.providerConfig]) { + for (const key of keys) { + const value = readTtsResultString(config?.[key]); + if (value !== undefined) { + return value; + } + } + } + return undefined; } function resolvePersonaBinding( diff --git a/src/video-generation/capability-overlays.ts b/src/video-generation/capability-overlays.ts index b541c024ed42..a50e03f3e41a 100644 --- a/src/video-generation/capability-overlays.ts +++ b/src/video-generation/capability-overlays.ts @@ -8,12 +8,6 @@ import type { VideoGenerationTransformCapabilities, } from "./types.js"; -function isVideoGenerationTransformCapabilities( - capabilities: VideoGenerationModeCapabilities | VideoGenerationTransformCapabilities | undefined, -): capabilities is VideoGenerationTransformCapabilities { - return Boolean(capabilities && "enabled" in capabilities); -} - export function buildVideoGenerationCapabilityFailure(params: { providerId: string; model: string; @@ -50,35 +44,21 @@ export function buildVideoGenerationCapabilityFailure(params: { : inputImageCount > 0 ? "reference image inputs" : "reference video inputs"; - if (!isVideoGenerationTransformCapabilities(capabilities) || !capabilities.enabled) { + if (!capabilities || !("enabled" in capabilities) || !capabilities.enabled) { return `${label} does not support ${visualLabel}; skipping to avoid silent reference drop`; } } - if (inputImageCount > 0) { - const maxImages = capabilities?.maxInputImages ?? provider.capabilities.maxInputImages ?? 0; - if (inputImageCount > maxImages) { - return maxImages === 0 - ? `${label} does not support reference image inputs; skipping to avoid silent image drop` - : `${label} supports at most ${maxImages} reference image(s), ${inputImageCount} requested; skipping`; - } - } - - if (inputVideoCount > 0) { - const maxVideos = capabilities?.maxInputVideos ?? provider.capabilities.maxInputVideos ?? 0; - if (inputVideoCount > maxVideos) { - return maxVideos === 0 - ? `${label} does not support reference video inputs; skipping to avoid silent video drop` - : `${label} supports at most ${maxVideos} reference video(s), ${inputVideoCount} requested; skipping`; - } - } - - if (inputAudioCount > 0) { - const maxAudio = capabilities?.maxInputAudios ?? provider.capabilities.maxInputAudios ?? 0; - if (inputAudioCount > maxAudio) { - return maxAudio === 0 - ? `${label} does not support reference audio inputs; skipping to avoid silent audio drop` - : `${label} supports at most ${maxAudio} reference audio(s), ${inputAudioCount} requested; skipping`; + for (const [kind, count, limitKey] of [ + ["image", inputImageCount, "maxInputImages"], + ["video", inputVideoCount, "maxInputVideos"], + ["audio", inputAudioCount, "maxInputAudios"], + ] as const) { + const limit = capabilities?.[limitKey] ?? provider.capabilities[limitKey] ?? 0; + if (count > 0 && count > limit) { + return limit === 0 + ? `${label} does not support reference ${kind} inputs; skipping to avoid silent ${kind} drop` + : `${label} supports at most ${limit} reference ${kind}(s), ${count} requested; skipping`; } } diff --git a/src/video-generation/normalization.ts b/src/video-generation/normalization.ts index 38bbe50bbbf6..98f47b7acaaa 100644 --- a/src/video-generation/normalization.ts +++ b/src/video-generation/normalization.ts @@ -1,5 +1,4 @@ import { resolveMediaGeometryOverrides } from "../media-generation/geometry-normalization.js"; -import { hasMediaNormalizationEntry } from "../media-generation/runtime-shared.js"; import { resolveVideoGenerationModeCapabilities } from "./capabilities.js"; import { normalizeVideoGenerationDuration, @@ -112,12 +111,6 @@ export function resolveVideoGenerationOverrides(params: { audio, watermark, ignoredOverrides, - normalization: - hasMediaNormalizationEntry(normalization.size) || - hasMediaNormalizationEntry(normalization.aspectRatio) || - hasMediaNormalizationEntry(normalization.resolution) || - hasMediaNormalizationEntry(normalization.durationSeconds) - ? normalization - : undefined, + normalization: Object.keys(normalization).length ? normalization : undefined, }; } diff --git a/src/video-generation/runtime.ts b/src/video-generation/runtime.ts index 30430950762f..57a36c87d388 100644 --- a/src/video-generation/runtime.ts +++ b/src/video-generation/runtime.ts @@ -84,19 +84,6 @@ function validateProviderOptionsAgainstDeclaration(params: { return undefined; } -function buildNoVideoGenerationModelConfiguredMessage( - cfg: OpenClawConfig, - deps: VideoGenerationRuntimeDeps, -): string { - const listProviders = deps.listProviders ?? listVideoGenerationProviders; - return buildNoCapabilityModelConfiguredMessage({ - capabilityLabel: "video-generation", - modelConfigKey: "mediaModels.video", - providers: listProviders(cfg), - getProviderEnvVars: deps.getProviderEnvVars, - }); -} - export function listRuntimeVideoGenerationProviders( params?: { config?: OpenClawConfig }, deps: VideoGenerationRuntimeDeps = {}, @@ -141,7 +128,14 @@ async function runVideoGeneration( autoProviderFallback: params.autoProviderFallback, }); if (candidates.length === 0) { - throw new Error(buildNoVideoGenerationModelConfiguredMessage(params.cfg, deps)); + throw new Error( + buildNoCapabilityModelConfiguredMessage({ + capabilityLabel: "video-generation", + modelConfigKey: "mediaModels.video", + providers: listProviders(params.cfg), + getProviderEnvVars: deps.getProviderEnvVars, + }), + ); } let skipWarnEmitted = false;