diff --git a/config/assertion-safety-baseline.txt b/config/assertion-safety-baseline.txt index 0c1077f9111b..3d12d2b7423e 100644 --- a/config/assertion-safety-baseline.txt +++ b/config/assertion-safety-baseline.txt @@ -341,8 +341,7 @@ extensions/discord/src/voice/segment.ts 1 extensions/duckduckgo/src/config.ts 1 extensions/duckduckgo/src/ddg-search-provider.ts 1 extensions/elevenlabs/config-compat.ts 2 -extensions/exa/src/exa-web-search-provider.runtime.ts 5 -extensions/fal/image-generation-provider.ts 1 +extensions/exa/src/exa-web-search-provider.runtime.ts 1 extensions/fal/video-generation-provider.ts 3 extensions/feishu/src/accounts.ts 4 extensions/feishu/src/bitable.ts 1 @@ -618,7 +617,7 @@ extensions/memory-core/src/standing-intents-tool.ts 1 extensions/memory-core/src/standing-intents.ts 2 extensions/memory-core/src/tools.ts 2 extensions/memory-lancedb/doctor-contract-api.ts 1 -extensions/memory-lancedb/embeddings.ts 2 +extensions/memory-lancedb/embeddings.ts 1 extensions/memory-lancedb/index.ts 8 extensions/memory-lancedb/lancedb-store.ts 12 extensions/memory-lancedb/memory-cli.ts 7 @@ -716,7 +715,6 @@ extensions/ollama/src/setup.runtime.ts 1 extensions/ollama/src/stream-compat.ts 1 extensions/ollama/src/stream.runtime.ts 6 extensions/onepassword/index.ts 2 -extensions/onepassword/src/op-client.ts 1 extensions/openai/embedding-batch.ts 2 extensions/openai/index.ts 2 extensions/openai/native-web-search.ts 1 @@ -734,7 +732,7 @@ extensions/openrouter/provider-routing.ts 2 extensions/openrouter/stream.ts 4 extensions/openrouter/video-generation-provider.ts 1 extensions/openrouter/video-model-catalog.ts 1 -extensions/openshell/src/backend.ts 2 +extensions/openshell/src/backend.ts 1 extensions/parallel/src/parallel-free-web-search-provider.runtime.ts 1 extensions/parallel/src/parallel-web-search-provider.runtime.ts 1 extensions/pixverse/video-generation-provider.ts 1 @@ -857,7 +855,6 @@ extensions/qa-lab/web/src/ui-render-utils.ts 1 extensions/qwen/models.ts 1 extensions/qwen/stream.ts 3 extensions/raft/src/accounts.ts 1 -extensions/raft/src/gateway.ts 2 extensions/raft/src/inbound.ts 1 extensions/raft/src/setup.ts 2 extensions/reef/doctor-contract-api.ts 1 @@ -1042,7 +1039,6 @@ extensions/twitch/src/config.ts 6 extensions/twitch/src/outbound.ts 2 extensions/twitch/src/resolver.ts 1 extensions/twitch/src/setup-surface.ts 1 -extensions/twitch/src/status.ts 2 extensions/twitch/src/token.ts 3 extensions/twitch/src/twitch-ingress.ts 4 extensions/venice/models.ts 1 diff --git a/extensions/admin-http-rpc/src/handler.ts b/extensions/admin-http-rpc/src/handler.ts index 71ed1da2c5c5..d5e4eb155c25 100644 --- a/extensions/admin-http-rpc/src/handler.ts +++ b/extensions/admin-http-rpc/src/handler.ts @@ -13,15 +13,6 @@ import { } from "openclaw/plugin-sdk/webhook-request-guards"; import { isAdminHttpRpcAllowedMethod, listAdminHttpRpcAllowedMethods } from "./methods.js"; -const ErrorCodes = { - AGENT_TIMEOUT: "AGENT_TIMEOUT", - APPROVAL_NOT_FOUND: "APPROVAL_NOT_FOUND", - INVALID_REQUEST: "INVALID_REQUEST", - NOT_LINKED: "NOT_LINKED", - NOT_PAIRED: "NOT_PAIRED", - UNAVAILABLE: "UNAVAILABLE", -} as const; - type RpcResponse = | { id: string; ok: true; payload: unknown; meta?: Record } | { id: string; ok: false; error: GatewayMethodDispatchError; meta?: Record }; @@ -32,11 +23,6 @@ type ParsedRequest = { params?: unknown; }; -type RequestBodyLimitFailureCode = - | "PAYLOAD_TOO_LARGE" - | "REQUEST_BODY_TIMEOUT" - | "CONNECTION_CLOSED"; - type ReadJsonBodyResult = | { ok: true; value: unknown } | { @@ -46,25 +32,21 @@ type ReadJsonBodyResult = closeAfterResponse?: boolean; }; -function createError(code: string, message: string): GatewayMethodDispatchError { - return { code, message }; -} - function rpcHttpStatus(response: RpcResponse): number { if (response.ok) { return 200; } switch (response.error.code) { - case ErrorCodes.INVALID_REQUEST: + case "INVALID_REQUEST": return 400; - case ErrorCodes.APPROVAL_NOT_FOUND: + case "APPROVAL_NOT_FOUND": return 404; - case ErrorCodes.UNAVAILABLE: + case "UNAVAILABLE": return 503; - case ErrorCodes.AGENT_TIMEOUT: + case "AGENT_TIMEOUT": return 504; - case ErrorCodes.NOT_LINKED: - case ErrorCodes.NOT_PAIRED: + case "NOT_LINKED": + case "NOT_PAIRED": return 409; default: return 500; @@ -82,18 +64,6 @@ function sendError(res: ServerResponse, status: number, error: { type: string; m sendJson(res, status, { ok: false, error }); } -function statusForBodyErrorCode(code: RequestBodyLimitFailureCode): number { - switch (code) { - case "PAYLOAD_TOO_LARGE": - return 413; - case "REQUEST_BODY_TIMEOUT": - return 408; - case "CONNECTION_CLOSED": - return 400; - } - return 400; -} - async function readAdminJsonBody(req: IncomingMessage): Promise { const body = await readJsonBodyWithLimit(req, { // Admin responses are part of the client contract. The response-first profile @@ -116,7 +86,8 @@ async function readAdminJsonBody(req: IncomingMessage): Promise { + const unavailable: GatewayMethodDispatchError = { + code: "UNAVAILABLE", + message: "gateway method failed before returning a response", + }; try { const response = await dispatchGatewayMethod(request.method, request.params); if (response.ok) { @@ -180,19 +134,14 @@ async function dispatchAdminRpc(request: ParsedRequest): Promise { return { id: request.id, ok: false, - error: - response.error ?? - createError(ErrorCodes.UNAVAILABLE, "gateway method failed before returning a response"), + error: response.error ?? unavailable, ...(response.meta ? { meta: response.meta } : {}), }; } catch { return { id: request.id, ok: false, - error: createError( - ErrorCodes.UNAVAILABLE, - "gateway method failed before returning a response", - ), + error: unavailable, }; } } @@ -243,14 +192,23 @@ export async function handleAdminHttpRpcRequest( } if (!isAdminHttpRpcAllowedMethod(parsed.request.method)) { - const response = methodNotAllowed(parsed.request.id, parsed.request.method); - sendJson(res, rpcHttpStatus(response), response); + sendJson(res, 400, { + id: parsed.request.id, + ok: false, + error: { + code: "INVALID_REQUEST", + message: `admin HTTP RPC method is not supported: ${parsed.request.method}`, + }, + }); return true; } if (parsed.request.method === "commands.list") { - const response = commandsList(parsed.request.id); - sendJson(res, 200, response); + sendJson(res, 200, { + id: parsed.request.id, + ok: true, + payload: { methods: listAdminHttpRpcAllowedMethods() }, + }); return true; } diff --git a/extensions/azure-speech/speech-provider.ts b/extensions/azure-speech/speech-provider.ts index 2f4887209669..2bbbd82003e2 100644 --- a/extensions/azure-speech/speech-provider.ts +++ b/extensions/azure-speech/speech-provider.ts @@ -38,12 +38,6 @@ type AzureSpeechProviderConfig = { timeoutMs?: number; }; -type AzureSpeechProviderOverrides = { - voice?: string; - lang?: string; - outputFormat?: string; -}; - function readAzureSpeechEnvApiKey(): string | undefined { return ( trimToUndefined(process.env.AZURE_SPEECH_KEY) ?? @@ -122,9 +116,7 @@ function readAzureSpeechProviderConfig(config: SpeechProviderConfig): AzureSpeec }; } -function readAzureSpeechOverrides( - overrides: SpeechProviderOverrides | undefined, -): AzureSpeechProviderOverrides { +function readAzureSpeechOverrides(overrides: SpeechProviderOverrides | undefined) { return { voice: trimToUndefined(overrides?.voice ?? overrides?.voiceId), lang: trimToUndefined(overrides?.lang ?? overrides?.languageCode), diff --git a/extensions/canvas/scripts/bundle-a2ui.mjs b/extensions/canvas/scripts/bundle-a2ui.mjs index ac477c6cff0a..2f239366ded9 100644 --- a/extensions/canvas/scripts/bundle-a2ui.mjs +++ b/extensions/canvas/scripts/bundle-a2ui.mjs @@ -171,7 +171,7 @@ async function main() { hasA2uiPackage = false; } if (!hasA2uiPackage || !hasAppDir) { - if (hasOutputFile) { + if (hasOutputFile && hasV09OutputFile) { console.log("A2UI package missing; keeping prebuilt bundle."); return; } @@ -181,7 +181,9 @@ async function main() { ); return; } - fail(`A2UI package missing and no prebuilt bundle found at: ${outputFile}`); + fail( + `A2UI package missing and no complete prebuilt bundle found at: ${outputFile}, ${outputV09File}`, + ); } const currentHash = await computeHash(); diff --git a/extensions/canvas/scripts/bundle-a2ui.test.ts b/extensions/canvas/scripts/bundle-a2ui.test.ts index 1c6fb3680989..fde80ea0e1c5 100644 --- a/extensions/canvas/scripts/bundle-a2ui.test.ts +++ b/extensions/canvas/scripts/bundle-a2ui.test.ts @@ -1,5 +1,5 @@ // Canvas tests cover bundle a2ui plugin behavior. -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path"; @@ -7,6 +7,42 @@ import { describe, expect, it, vi } from "vitest"; import { compareNormalizedPaths, listTrackedInputFiles } from "./bundle-a2ui.mjs"; describe("scripts/bundle-a2ui.mjs", () => { + it("requires both prebuilt dialects when the A2UI build dependencies are absent", async () => { + await withTempWorkspace( + { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "openclaw-a2ui-prebuilt-" }, + async ({ dir }) => { + const scriptsDir = path.join(dir, "extensions", "canvas", "scripts"); + await fs.mkdir(scriptsDir, { recursive: true }); + for (const name of ["bundle-a2ui.mjs", "pnpm-runner.mjs"]) { + await fs.copyFile(new URL(`./${name}`, import.meta.url), path.join(scriptsDir, name)); + } + const outputFile = path.join(dir, "a2ui.bundle.js"); + await fs.writeFile(outputFile, "// v0.8 prebuilt renderer\n"); + const run = () => + spawnSync(process.execPath, [path.join(scriptsDir, "bundle-a2ui.mjs")], { + cwd: dir, + encoding: "utf8", + env: { + ...process.env, + OPENCLAW_A2UI_BUNDLE_OUT: outputFile, + OPENCLAW_SPARSE_PROFILE: "", + OPENCLAW_A2UI_SKIP_MISSING: "0", + }, + }); + + const incomplete = run(); + expect(incomplete.status).toBe(1); + expect(incomplete.stderr).toContain("no complete prebuilt bundle"); + expect(incomplete.stderr).toContain(`${outputFile}.v0.9.js`); + + await fs.writeFile(`${outputFile}.v0.9.js`, "// v0.9 prebuilt renderer\n"); + const complete = run(); + expect(complete.status).toBe(0); + expect(complete.stdout).toContain("keeping prebuilt bundle"); + }, + ); + }); + it("sorts hash inputs without locale-dependent collation", () => { const paths = ["repo/Z.ts", "repo/a.ts", "repo/ä.ts", "repo/A.ts"]; diff --git a/extensions/canvas/src/config.ts b/extensions/canvas/src/config.ts index 33111dcae2f0..1fea5ea36088 100644 --- a/extensions/canvas/src/config.ts +++ b/extensions/canvas/src/config.ts @@ -1,43 +1,24 @@ -/** - * Canvas plugin config parsing, enablement, and schema metadata. - */ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { resolvePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime"; import { isTruthyEnvValue } from "openclaw/plugin-sdk/runtime-env"; import { asBoolean as readBoolean, isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; -/** Enablement for Canvas-owned document and renderer routes. */ export type CanvasHostConfig = { enabled?: boolean; }; -/** Canvas plugin configuration shape. */ export type CanvasPluginConfig = { host?: CanvasHostConfig; }; -type CanvasPluginConfigSchema = { - parse: (value: unknown) => CanvasPluginConfig; -}; - -function parseCanvasHostConfig(value: unknown): CanvasHostConfig | undefined { - if (!isRecord(value)) { - return undefined; - } - const enabled = readBoolean(value.enabled); - return enabled === undefined ? {} : { enabled }; -} - -/** Parses raw Canvas plugin config into a typed, normalized shape. */ export function parseCanvasPluginConfig(value: unknown): CanvasPluginConfig { - if (!isRecord(value)) { + if (!isRecord(value) || !isRecord(value.host)) { return {}; } - const host = parseCanvasHostConfig(value.host); - return host ? { host } : {}; + const enabled = readBoolean(value.host.enabled); + return { host: enabled === undefined ? {} : { enabled } }; } -/** Resolves Canvas route configuration from plugin-owned config. */ export function resolveCanvasHostConfig(params: { config?: OpenClawConfig; pluginConfig?: Record; @@ -48,7 +29,6 @@ export function resolveCanvasHostConfig(params: { return parsedPluginConfig.host ?? {}; } -/** Returns whether Canvas-owned document and renderer routes should be active. */ export function isCanvasHostEnabled(config?: OpenClawConfig): boolean { if (isTruthyEnvValue(process.env.OPENCLAW_SKIP_CANVAS_HOST)) { return false; @@ -56,7 +36,6 @@ export function isCanvasHostEnabled(config?: OpenClawConfig): boolean { return resolveCanvasHostConfig({ config }).enabled !== false; } -/** Runtime config parser for Canvas plugin settings. */ -export const canvasConfigSchema: CanvasPluginConfigSchema = { +export const canvasConfigSchema = { parse: parseCanvasPluginConfig, }; diff --git a/extensions/canvas/src/tool.ts b/extensions/canvas/src/tool.ts index c3e4cbd5ab21..094ab507be68 100644 --- a/extensions/canvas/src/tool.ts +++ b/extensions/canvas/src/tool.ts @@ -43,7 +43,7 @@ export function createCanvasTool(options?: CanvasToolOptions): AnyAgentTool { // Preserve the node lookup budget while letting Gateway outlive node execution. const transportTimeoutMs = addTimerTimeoutGraceMs(timeoutMs, CANVAS_NODE_INVOKE_TRANSPORT_GRACE_MS) ?? timeoutMs; - const result = await callGatewayTool( + await callGatewayTool( "node.invoke", { ...gatewayOpts, timeoutMs: transportTimeoutMs }, { @@ -55,7 +55,7 @@ export function createCanvasTool(options?: CanvasToolOptions): AnyAgentTool { ...(options?.agentSessionKey ? { sessionKey: options.agentSessionKey } : {}), }, ); - return { node: nodeId, result }; + return nodeId; }; switch (action) { @@ -76,18 +76,18 @@ export function createCanvasTool(options?: CanvasToolOptions): AnyAgentTool { if (Object.values(placement).some(Number.isFinite)) { invokeParams.placement = placement; } - const { node } = await invoke(CANVAS_PRESENT_COMMAND, invokeParams); + const node = await invoke(CANVAS_PRESENT_COMMAND, invokeParams); return jsonResult({ ok: true, node, ...(presentTarget ? { url: presentTarget } : {}) }); } case "hide": { - const { node } = await invoke("canvas.hide", undefined); + const node = await invoke("canvas.hide"); return jsonResult({ ok: true, node }); } case "navigate": { const url = readStringParam(params, "url", { trim: true }) ?? readStringParam(params, "target", { required: true, trim: true, label: "url" }); - const { node } = await invoke("canvas.navigate", { url }); + const node = await invoke("canvas.navigate", { url }); return jsonResult({ ok: true, node, url }); } default: diff --git a/extensions/comfy/workflow-runtime.ts b/extensions/comfy/workflow-runtime.ts index 869217106425..1d8b0a79a21c 100644 --- a/extensions/comfy/workflow-runtime.ts +++ b/extensions/comfy/workflow-runtime.ts @@ -81,9 +81,6 @@ type ComfyStatusResponse = { message?: string; error?: string; }; -type ComfyNetworkPolicy = { - apiPolicy?: SsrFPolicy; -}; type ComfyApiKeyResolution = | { status: "available"; @@ -283,29 +280,28 @@ function resolveComfyNetworkPolicy(params: { allowPrivateNetwork: boolean; explicitAllowPrivateNetwork: boolean; mode: ComfyMode; -}): ComfyNetworkPolicy { +}): SsrFPolicy | undefined { let parsed: URL; try { parsed = new URL(params.baseUrl); } catch { - return {}; + return undefined; } const hostname = normalizeOptionalLowercaseString(parsed.hostname) ?? ""; if (!hostname) { - return {}; + return undefined; } const localHostnamePolicy: SsrFPolicy | undefined = params.mode === "local" ? { hostnameAllowlist: [hostname] } : undefined; - const hostnameOnlyPolicy = localHostnamePolicy ? { apiPolicy: localHostnamePolicy } : {}; if (!params.allowPrivateNetwork) { - return hostnameOnlyPolicy; + return localHostnamePolicy; } // Local mode auto-trusts loopback/IP targets and Compose-style single-label // service names; public-looking FQDNs require the operator's explicit // allowPrivateNetwork opt-in. if (!params.explicitAllowPrivateNetwork && params.mode !== "local") { - return {}; + return undefined; } if ( !params.explicitAllowPrivateNetwork && @@ -313,18 +309,17 @@ function resolveComfyNetworkPolicy(params: { !isPrivateOrLoopbackHost(hostname) && !isSingleLabelServiceHostname(hostname) ) { - return hostnameOnlyPolicy; + return localHostnamePolicy; } const originPolicy = ssrfPolicyFromHttpBaseUrlAllowedOrigin(params.baseUrl); if (!originPolicy) { - return hostnameOnlyPolicy; + return localHostnamePolicy; } - return { - apiPolicy: - params.mode === "local" ? mergeSsrFPolicies(originPolicy, localHostnamePolicy) : originPolicy, - }; + return params.mode === "local" + ? mergeSsrFPolicies(originPolicy, localHostnamePolicy) + : originPolicy; } function isSingleLabelServiceHostname(hostname: string): boolean { @@ -357,22 +352,6 @@ async function readJsonResponse(params: { } } -function resolveFileExtension(params: { fileName?: string; mimeType?: string }): string { - const extension = extensionForMime(params.mimeType); - if (extension) { - return extension.slice(1); - } - const fileName = params.fileName?.trim(); - if (!fileName) { - return "bin"; - } - const dotIndex = fileName.lastIndexOf("."); - if (dotIndex < 0 || dotIndex === fileName.length - 1) { - return "bin"; - } - return fileName.slice(dotIndex + 1); -} - async function uploadInputImage(params: { baseUrl: string; headers: Headers; @@ -388,7 +367,7 @@ async function uploadInputImage(params: { "image", new Blob([bufferToBlobPart(params.image.buffer)], { type: params.image.mimeType }), normalizeOptionalString(params.image.fileName) || - `input.${resolveFileExtension({ mimeType: params.image.mimeType })}`, + `input.${extensionForMime(params.image.mimeType)?.slice(1) || "bin"}`, ); form.set("type", "input"); form.set("overwrite", "true"); @@ -562,7 +541,7 @@ async function downloadOutputFile(params: { mode: ComfyMode; capability: ComfyCapability; maxBytes: number; -}): Promise<{ buffer: Buffer; mimeType: string }> { +}): Promise<{ buffer: Buffer; mimeType: string; fileName: string }> { const fileName = normalizeOptionalString(params.file.filename) || normalizeOptionalString(params.file.name); if (!fileName) { @@ -609,7 +588,7 @@ async function downloadOutputFile(params: { new Error(`${downloadLabel} stalled after ${chunkTimeoutMs}ms`), }, ); - return { buffer, mimeType }; + return { buffer, mimeType, fileName }; } finally { await firstResponse.release(); } @@ -791,7 +770,7 @@ export async function runComfyWorkflow(params: { baseUrl: normalizedBaseUrl, headers: new Headers(headers), timeoutMs, - policy: networkPolicy.apiPolicy, + policy: networkPolicy, dispatcherPolicy, image: params.inputImage, mode, @@ -820,7 +799,7 @@ export async function runComfyWorkflow(params: { body: JSON.stringify(submitPayload), }, timeoutMs, - policy: networkPolicy.apiPolicy, + policy: networkPolicy, dispatcherPolicy, auditContext: `comfy-${params.capability}-generate`, errorPrefix: "Comfy workflow submit failed", @@ -837,7 +816,7 @@ export async function runComfyWorkflow(params: { headers: new Headers(headers), timeoutMs, pollIntervalMs, - policy: networkPolicy.apiPolicy, + policy: networkPolicy, dispatcherPolicy, mode, }); @@ -860,28 +839,20 @@ export async function runComfyWorkflow(params: { const assets: ComfyGeneratedAsset[] = []; const outputKind = params.capability === "music" ? "audio" : params.capability; const maxOutputBytes = resolveGeneratedMediaMaxBytes(params.cfg, outputKind); - let assetIndex = 0; for (const output of outputFiles) { const downloaded = await downloadOutputFile({ baseUrl: normalizedBaseUrl, headers: new Headers(headers), timeoutMs, - policy: networkPolicy.apiPolicy, + policy: networkPolicy, dispatcherPolicy, file: output.file, mode, capability: params.capability, maxBytes: maxOutputBytes, }); - assetIndex += 1; - const originalName = - normalizeOptionalString(output.file.filename) || normalizeOptionalString(output.file.name); assets.push({ - buffer: downloaded.buffer, - mimeType: downloaded.mimeType, - fileName: - originalName || - `${params.capability}-${assetIndex}.${resolveFileExtension({ mimeType: downloaded.mimeType })}`, + ...downloaded, metadata: { nodeId: output.nodeId, promptId }, }); } diff --git a/extensions/cua-computer/src/browser-actions.ts b/extensions/cua-computer/src/browser-actions.ts index 0551db2f1f51..ecbf21293b2a 100644 --- a/extensions/cua-computer/src/browser-actions.ts +++ b/extensions/cua-computer/src/browser-actions.ts @@ -10,7 +10,6 @@ import { } from "./driver-result.js"; import type { CuaExecutionResources } from "./execution-resources.js"; import { - clearDialogRef, invalidateBrowserObservation, resolveBrowserObservation, resolveDialogRef, @@ -163,7 +162,7 @@ export async function handleBrowserAct( signal, ); if (input.dialogAction !== "inspect") { - clearDialogRef(state); + state.dialog = undefined; } return JSON.stringify(browserDialogEnvelope(result, state, target)); } diff --git a/extensions/cua-computer/src/driver-result.ts b/extensions/cua-computer/src/driver-result.ts index 828e4b22cb07..b58c605f762d 100644 --- a/extensions/cua-computer/src/driver-result.ts +++ b/extensions/cua-computer/src/driver-result.ts @@ -9,7 +9,6 @@ import { z } from "zod"; import type { CuaDriverSession, CuaToolResult } from "./driver-client.js"; import { adoptGeneration, - clearDialogRef, invalidateBrowserReferences, issueBrowserElementRef, issueBrowserObservation, @@ -649,7 +648,7 @@ export function browserDialogEnvelope( const structured = projectedToolDetails(result, "browser_dialog"); const present = structured.present === true; if (!present) { - clearDialogRef(state); + state.dialog = undefined; } const details: Record = { present }; if (typeof structured.kind === "string") { diff --git a/extensions/cua-computer/src/frame.ts b/extensions/cua-computer/src/frame.ts index 3d5796a3e0d6..614467ec9099 100644 --- a/extensions/cua-computer/src/frame.ts +++ b/extensions/cua-computer/src/frame.ts @@ -73,15 +73,11 @@ type CuaPageTarget = { tabId: string; }; -type CuaBrowserElementTarget = { - nativeRef: string; -}; - type CuaBrowserObservationState = { id: string; browserRef: string; pageRef: string; - elements: Map; + elements: Map; }; type CuaDialogState = { @@ -105,6 +101,24 @@ function opaqueRef( return `cua:v2:${kind}:${randomUUID()}`; } +function issueRef( + kind: Parameters[0], + targets: Map, + target: T, + matches?: (current: T) => boolean, +): string { + if (matches) { + for (const [ref, current] of targets) { + if (matches(current)) { + return ref; + } + } + } + const ref = opaqueRef(kind); + targets.set(ref, target); + return ref; +} + export function adoptGeneration(state: CuaFrameState, generation: string): void { // Native session replacement invalidates every authority-bearing reference, // even when the same window ids and display geometry reappear. @@ -129,36 +143,30 @@ export function verifyGeneration(state: CuaFrameState, generation: string): void } export function issueAppRef(state: CuaFrameState, target: CuaAppTarget): string { - state.apps ??= new Map(); - const ref = opaqueRef("app"); - state.apps.set(ref, target); - return ref; -} - -export function resolveAppRef(state: CuaFrameState, ref: string): CuaAppTarget | undefined { - return state.apps?.get(ref); + return issueRef("app", (state.apps ??= new Map()), target); } export function issueWindowRef(state: CuaFrameState, target: CuaWindowTarget): string { - state.windows ??= new Map(); - for (const [ref, current] of state.windows) { - if (current.pid === target.pid && current.windowId === target.windowId) { - return ref; - } - } - const ref = opaqueRef("window"); - state.windows.set(ref, target); - return ref; + return issueRef( + "window", + (state.windows ??= new Map()), + target, + (current) => current.pid === target.pid && current.windowId === target.windowId, + ); } -export function resolveWindowRef(state: CuaFrameState, ref: string): CuaWindowTarget { - const target = state.windows?.get(ref); - if (!target) { +function requireRef(targets: Map | undefined, ref: string): T { + const target = targets?.get(ref); + if (target === undefined) { throw staleObservation(); } return target; } +export function resolveWindowRef(state: CuaFrameState, ref: string): CuaWindowTarget { + return requireRef(state.windows, ref); +} + export function issueObservation( state: CuaFrameState, windowRef: string, @@ -180,9 +188,7 @@ export function issueElementRef( observation: CuaObservationState, target: CuaElementTarget, ): string { - const ref = opaqueRef("element"); - observation.elements.set(ref, target); - return ref; + return issueRef("element", observation.elements, target); } export function resolveObservation( @@ -201,43 +207,29 @@ export function resolveElementRef( observation: CuaObservationState, elementRef: string, ): CuaElementTarget { - const target = observation.elements.get(elementRef); - if (!target) { - throw staleObservation(); - } - return target; + return requireRef(observation.elements, elementRef); } export function issueBrowserRef(state: CuaFrameState, target: CuaBrowserTarget): string { - state.browsers ??= new Map(); - for (const [ref, current] of state.browsers) { - if (current.targetId === target.targetId && current.windowRef === target.windowRef) { - return ref; - } - } - const ref = opaqueRef("browser"); - state.browsers.set(ref, target); - return ref; + return issueRef( + "browser", + (state.browsers ??= new Map()), + target, + (current) => current.targetId === target.targetId && current.windowRef === target.windowRef, + ); } export function resolveBrowserRef(state: CuaFrameState, ref: string): CuaBrowserTarget { - const target = state.browsers?.get(ref); - if (!target) { - throw staleObservation(); - } - return target; + return requireRef(state.browsers, ref); } export function issuePageRef(state: CuaFrameState, browserRef: string, tabId: string): string { - state.pages ??= new Map(); - for (const [ref, current] of state.pages) { - if (current.browserRef === browserRef && current.tabId === tabId) { - return ref; - } - } - const ref = opaqueRef("page"); - state.pages.set(ref, { browserRef, tabId }); - return ref; + return issueRef( + "page", + (state.pages ??= new Map()), + { browserRef, tabId }, + (current) => current.browserRef === browserRef && current.tabId === tabId, + ); } export function resolvePageRef( @@ -274,9 +266,7 @@ export function issueBrowserElementRef( observation: CuaBrowserObservationState, nativeRef: string, ): string { - const ref = opaqueRef("element"); - observation.elements.set(ref, { nativeRef }); - return ref; + return issueRef("element", observation.elements, nativeRef); } export function resolveBrowserObservation( @@ -301,11 +291,7 @@ export function resolveBrowserElementRef( observation: CuaBrowserObservationState, elementRef: string, ): string { - const target = observation.elements.get(elementRef); - if (!target) { - throw staleObservation(); - } - return target.nativeRef; + return requireRef(observation.elements, elementRef); } export function invalidateBrowserObservation(state: CuaFrameState): void { @@ -348,10 +334,6 @@ export function resolveDialogRef( return dialog.nativeId; } -export function clearDialogRef(state: CuaFrameState): void { - state.dialog = undefined; -} - /** * CUA Driver exposes only the primary-display label, not a stable display ID. * Bind authorization to connection generation plus the complete live geometry. diff --git a/extensions/cua-computer/src/window-actions.ts b/extensions/cua-computer/src/window-actions.ts index 91c248ca331f..87caf06884b0 100644 --- a/extensions/cua-computer/src/window-actions.ts +++ b/extensions/cua-computer/src/window-actions.ts @@ -24,7 +24,6 @@ import { import type { CuaExecutionState } from "./execution-state.js"; import { adoptGeneration, - resolveAppRef, resolveObservation, resolveWindowRef, verifyGeneration, @@ -312,7 +311,7 @@ export async function handleWindowAct( case "launch_app": { verifyGeneration(state, driver.generation); const appName = input.app!; - const app = resolveAppRef(state, appName); + const app = state.apps?.get(appName); if (!app) { throw new Error("COMPUTER_STALE_OBSERVATION: refresh list_apps and retry"); } @@ -339,7 +338,7 @@ export async function handleWindowAct( case "kill_app": { verifyGeneration(state, driver.generation); const appName = input.app!; - const app = resolveAppRef(state, appName); + const app = state.apps?.get(appName); if (!app?.pid) { throw new Error( "COMPUTER_INVALID_REQUEST: kill_app requires a running app reference from list_apps", diff --git a/extensions/deepgram/realtime-transcription-provider-factory.ts b/extensions/deepgram/realtime-transcription-provider-factory.ts index 6e598c3f8832..9425126aaec9 100644 --- a/extensions/deepgram/realtime-transcription-provider-factory.ts +++ b/extensions/deepgram/realtime-transcription-provider-factory.ts @@ -253,18 +253,13 @@ function createDeepgramRealtimeTranscriptionSession( if (!text) { return; } - if (event.is_final) { - const nextFinalized = joinTranscript(finalizedTranscript, text); - if (!updateTurn(nextFinalized, "", transport)) { - return; - } - config.onPartial?.(nextFinalized); - } else { - if (!updateTurn(finalizedTranscript, text, transport)) { - return; - } - config.onPartial?.(joinTranscript(finalizedTranscript, text)); + const nextFinalized = event.is_final + ? joinTranscript(finalizedTranscript, text) + : finalizedTranscript; + if (!updateTurn(nextFinalized, event.is_final ? "" : text, transport)) { + return; } + config.onPartial?.(event.is_final ? nextFinalized : joinTranscript(nextFinalized, text)); return; } case "SpeechStarted": diff --git a/extensions/device-pair/index.ts b/extensions/device-pair/index.ts index e9fbba9b1a85..d106f2fe9355 100644 --- a/extensions/device-pair/index.ts +++ b/extensions/device-pair/index.ts @@ -8,7 +8,6 @@ import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { buildDevicePairPairingQrChannelData } from "./pairing-qr-channel-data.js"; type NotifyModule = typeof import("./notify.js"); const loadDevicePairApiModule = createLazyRuntimeModule(() => import("./api.js")); @@ -450,12 +449,8 @@ export default definePluginEntry({ const gatewayClientScopes = Array.isArray(ctx.gatewayClientScopes) ? ctx.gatewayClientScopes : undefined; - const { - buildMissingPairingScopeReply, - buildMissingSetupHandoffScopeReply, - resolveAuthLabel, - resolvePairingCommandAuthState, - } = await loadPairCommandAuthModule(); + const { resolveAuthLabel, resolvePairingCommandAuthState } = + await loadPairCommandAuthModule(); const authState = resolvePairingCommandAuthState({ channel: ctx.channel, gatewayClientScopes, @@ -471,7 +466,7 @@ export default definePluginEntry({ ); if (authState.isMissingPairingPrivilege) { - return buildMissingPairingScopeReply(); + return { text: "⚠️ This command requires operator.pairing." }; } assertOwnerCurrent?.(); @@ -531,7 +526,9 @@ export default definePluginEntry({ } if (authState.isMissingSetupHandoffPrivilege) { - return buildMissingSetupHandoffScopeReply(); + return { + text: "⚠️ Setup code handoff includes Talk secrets and requires operator.talk.secrets.", + }; } const authLabelResult = resolveAuthLabel(api.config); @@ -659,10 +656,9 @@ export default definePluginEntry({ markdown: true, }).join("\n"), ].join("\n"), - channelData: buildDevicePairPairingQrChannelData({ - setupCode, - expiresAtMs: payload.expiresAtMs, - }), + channelData: { + openclawPairingQr: { setupCode, expiresAtMs: payload.expiresAtMs }, + }, sensitiveMedia: true, }; } diff --git a/extensions/device-pair/pair-command-approve.ts b/extensions/device-pair/pair-command-approve.ts index 1f6782018bfd..affee59986cd 100644 --- a/extensions/device-pair/pair-command-approve.ts +++ b/extensions/device-pair/pair-command-approve.ts @@ -9,7 +9,6 @@ import { formatPendingRequests } from "./notify.js"; type PendingPairingEntry = Awaited>["pending"][number]; type ApprovePairingResult = Awaited>; type ApprovedPairingEntry = Exclude; -type ForbiddenPairingEntry = Extract; function buildMultiplePendingApprovalReply(pending: PendingPairingEntry[]): { text: string } { return { @@ -60,10 +59,6 @@ function formatApprovedPairingReply(approved: ApprovedPairingEntry): { text: str return { text: `✅ Paired ${label}${platformLabel}.` }; } -function formatForbiddenPairingRequirement(approved: ForbiddenPairingEntry): string { - return approved.scope ?? approved.role ?? "additional approval"; -} - export async function approvePendingPairingRequest(params: { requestId: string; callerScopes?: readonly string[]; @@ -88,7 +83,7 @@ export async function approvePendingPairingRequest(params: { } if (approved.status === "forbidden") { return { - text: `⚠️ This command requires ${formatForbiddenPairingRequirement(approved)} to approve this pairing request.`, + text: `⚠️ This command requires ${approved.scope ?? approved.role ?? "additional approval"} to approve this pairing request.`, }; } return formatApprovedPairingReply(approved); diff --git a/extensions/device-pair/pair-command-auth.ts b/extensions/device-pair/pair-command-auth.ts index e0f3d69b14c8..77fc05be41ec 100644 --- a/extensions/device-pair/pair-command-auth.ts +++ b/extensions/device-pair/pair-command-auth.ts @@ -106,15 +106,3 @@ export function resolvePairingCommandAuthState( approvalCallerScopes: undefined, }; } - -export function buildMissingPairingScopeReply(): { text: string } { - return { - text: "⚠️ This command requires operator.pairing.", - }; -} - -export function buildMissingSetupHandoffScopeReply(): { text: string } { - return { - text: "⚠️ Setup code handoff includes Talk secrets and requires operator.talk.secrets.", - }; -} diff --git a/extensions/device-pair/pairing-qr-channel-data.ts b/extensions/device-pair/pairing-qr-channel-data.ts deleted file mode 100644 index 9ad2cfcac9f2..000000000000 --- a/extensions/device-pair/pairing-qr-channel-data.ts +++ /dev/null @@ -1,19 +0,0 @@ -// Private device-pair -> Gateway live-display envelope. -// Keep this local so pairing QR metadata does not become public Plugin SDK API. -const DEVICE_PAIR_PAIRING_QR_CHANNEL_DATA_KEY = "openclawPairingQr"; - -type DevicePairPairingQrChannelData = { - setupCode: string; - expiresAtMs: number; -}; - -export function buildDevicePairPairingQrChannelData( - params: DevicePairPairingQrChannelData, -): Record { - return { - [DEVICE_PAIR_PAIRING_QR_CHANNEL_DATA_KEY]: { - setupCode: params.setupCode, - expiresAtMs: params.expiresAtMs, - }, - }; -} diff --git a/extensions/diffs/src/tool.ts b/extensions/diffs/src/tool.ts index 45f4ddd41449..16a67f070aea 100644 --- a/extensions/diffs/src/tool.ts +++ b/extensions/diffs/src/tool.ts @@ -9,6 +9,7 @@ import { asNonArrayRecord, normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { textResult } from "openclaw/plugin-sdk/tool-results"; import { Type } from "typebox"; import type { Static } from "typebox"; import type { DiffScreenshotter } from "./browser.runtime.js"; @@ -144,18 +145,10 @@ export function createDiffsTool(params: { const artifactContext = buildArtifactContext(params.context); const input = normalizeDiffInput(toolParams); if (input.kind === "before_after" && input.before === input.after) { - return { - content: [ - { - type: "text", - text: "Before and after are identical — no changes to render.", - }, - ], - details: { - changed: false, - ...(artifactContext ? { context: artifactContext } : {}), - }, - }; + return textResult("Before and after are identical — no changes to render.", { + changed: false, + ...(artifactContext ? { context: artifactContext } : {}), + }); } const mode = DIFF_MODES.find((value) => value === toolParams.mode) ?? params.defaults.mode; const theme = @@ -231,15 +224,7 @@ export function createDiffsTool(params: { : undefined; if (mode === "view") { - return { - content: [ - { - type: "text", - text: `Diff viewer ready.\n${viewerUrl}`, - }, - ], - details: viewerDetails, - }; + return textResult(`Diff viewer ready.\n${viewerUrl}`, viewerDetails); } try { @@ -254,19 +239,14 @@ export function createDiffsTool(params: { context: artifactContext, }); - return { - content: [ - { - type: "text", - text: buildFileArtifactMessage({ - format: image.format, - filePath: artifactFile.path, - viewerUrl, - }), - }, - ], - details: buildArtifactDetails({ - baseDetails: viewerDetails ?? { + return textResult( + buildFileArtifactMessage({ + format: image.format, + filePath: artifactFile.path, + viewerUrl, + }), + { + ...(viewerDetails ?? { changed: true, ...(artifactFile.artifactId ? { artifactId: artifactFile.artifactId } : {}), ...(artifactFile.expiresAt ? { expiresAt: artifactFile.expiresAt } : {}), @@ -275,26 +255,27 @@ export function createDiffsTool(params: { fileCount: rendered.fileCount, mode, ...(artifactContext ? { context: artifactContext } : {}), - }, - artifactFile, - image, - }), - }; + }), + filePath: artifactFile.path, + // `path` mirrors filePath so the message tool can send the artifact directly. + path: artifactFile.path, + fileBytes: artifactFile.bytes, + fileFormat: image.format, + fileQuality: image.qualityPreset, + fileScale: image.scale, + fileMaxWidth: image.maxWidth, + }, + ); } catch (error) { if (mode === "both") { const errorMessage = formatErrorMessage(error); - return { - content: [ - { - type: "text", - text: `Diff viewer ready.\n${viewerUrl}\nFile rendering failed: ${errorMessage}`, - }, - ], - details: { + return textResult( + `Diff viewer ready.\n${viewerUrl}\nFile rendering failed: ${errorMessage}`, + { ...viewerDetails, fileError: errorMessage, }, - }; + ); } throw error; } @@ -323,24 +304,6 @@ function requireRenderedHtml(html: string | undefined, target: DiffRenderTarget) throw new Error(`Missing ${target} render output.`); } -function buildArtifactDetails(params: { - baseDetails: Record; - artifactFile: { path: string; bytes: number }; - image: DiffRenderOptions["image"]; -}) { - return { - ...params.baseDetails, - filePath: params.artifactFile.path, - // `path` mirrors filePath so the message tool can send the artifact directly. - path: params.artifactFile.path, - fileBytes: params.artifactFile.bytes, - fileFormat: params.image.format, - fileQuality: params.image.qualityPreset, - fileScale: params.image.scale, - fileMaxWidth: params.image.maxWidth, - }; -} - function buildFileArtifactMessage(params: { format: DiffOutputFormat; filePath: string; diff --git a/extensions/exa/src/exa-web-search-provider.runtime.ts b/extensions/exa/src/exa-web-search-provider.runtime.ts index 1768214c58c5..c9f903988c2d 100644 --- a/extensions/exa/src/exa-web-search-provider.runtime.ts +++ b/extensions/exa/src/exa-web-search-provider.runtime.ts @@ -88,12 +88,7 @@ async function readExaSearchResults(response: Response): Promise freshness === trimmed); } function resolveExaApiKey(exa?: ExaConfig): string | undefined { @@ -142,18 +137,12 @@ function resolveExaSearchEndpoint( } function resolveExaDescription(result: ExaSearchResult): string { - const highlights = result.highlights; - if (Array.isArray(highlights)) { - const highlightText = normalizeTrimmedStringList(highlights).join("\n"); - if (highlightText) { - return highlightText; - } - } - const summary = normalizeOptionalString(result.summary); - if (summary) { - return summary; - } - return normalizeOptionalString(result.text) ?? ""; + return ( + normalizeTrimmedStringList(result.highlights).join("\n") || + normalizeOptionalString(result.summary) || + normalizeOptionalString(result.text) || + "" + ); } function parsePositiveInteger(value: unknown): number | undefined { @@ -390,9 +379,7 @@ export async function executeExaWebSearchProviderTool( const query = readStringParam(args, "query", { required: true }); const rawType = readStringParam(args, "type"); - const type: ExaSearchType = EXA_SEARCH_TYPES.includes(rawType as ExaSearchType) - ? (rawType as ExaSearchType) - : "auto"; + const type = EXA_SEARCH_TYPES.find((candidate) => candidate === rawType) ?? "auto"; const count = readPositiveIntegerParam(args, "count", { max: EXA_MAX_SEARCH_COUNT, diff --git a/extensions/fal/image-generation-provider.ts b/extensions/fal/image-generation-provider.ts index f4cdf6bf1bc9..eeb71aabed8f 100644 --- a/extensions/fal/image-generation-provider.ts +++ b/extensions/fal/image-generation-provider.ts @@ -147,20 +147,10 @@ function parseFalImageGenerationResponse(payload: unknown): { if (!isRecord(payload)) { throw new Error(FAL_IMAGE_MALFORMED_RESPONSE); } - const rawImages = payload.images; - if (rawImages === undefined || rawImages === null) { - return { images: [], prompt: normalizeOptionalString(payload.prompt) }; - } - if (!Array.isArray(rawImages)) { + const images = payload.images ?? []; + if (!Array.isArray(images) || !images.every(isRecord)) { throw new Error(FAL_IMAGE_MALFORMED_RESPONSE); } - const images: Record[] = []; - for (const entry of rawImages) { - if (!isRecord(entry)) { - throw new Error(FAL_IMAGE_MALFORMED_RESPONSE); - } - images.push(entry); - } return { images, prompt: normalizeOptionalString(payload.prompt) }; } @@ -289,7 +279,7 @@ function parseSize(raw: string | undefined): { width: number; height: number } | } const width = Number.parseInt(match[1] ?? "", 10); const height = Number.parseInt(match[2] ?? "", 10); - if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + if (width <= 0 || height <= 0) { return null; } return { width, height }; @@ -426,14 +416,10 @@ function resolveClosestFalAspectRatioForSize( }, undefined); } -function resolveKreaCreativity(raw: string | undefined): string { - const normalized = normalizeLowercaseStringOrEmpty(raw); - return (KREA_CREATIVITY_LEVELS as readonly string[]).includes(normalized) ? normalized : "medium"; -} - function resolveFalCreativityOption(providerOptions: Record | undefined): string { const falOptions = isRecord(providerOptions?.fal) ? providerOptions.fal : undefined; - return typeof falOptions?.creativity === "string" ? falOptions.creativity : ""; + const normalized = normalizeLowercaseStringOrEmpty(falOptions?.creativity); + return KREA_CREATIVITY_LEVELS.some((level) => level === normalized) ? normalized : "medium"; } function resolveNativeFalAspectRatio(params: { @@ -478,14 +464,7 @@ function applyFalImageGeometry(params: { params.requestBody.aspect_ratio = nativeAspectRatio; } if (params.resolution && params.schema.referenceImages === "image_urls") { - // Schemas may opt in to resolution validation by declaring `resolutions`. - // - `resolutions: undefined` (default, e.g. Nano Banana 2): forward the - // uppercase value unchanged, matching legacy behaviour. - // - `resolutions: ["1K", "2K"]` with `resolutionCase: "lower"` (Grok - // Imagine): validate against the allowlist and lowercase before - // sending. - // - `resolutions: []` (Nano Banana 2 Lite): reject overrides when the - // published endpoint schema has no resolution field. + // An absent allowlist forwards resolutions; an empty one rejects all overrides. const allowedResolutions = params.schema.resolutions; if (allowedResolutions === undefined) { params.requestBody.resolution = params.resolution; @@ -516,7 +495,7 @@ function applyFalReferenceImages(params: { schema: FalImageModelSchema; inputImages: ImageGenerationSourceImage[]; }) { - const encoded = params.inputImages.map((img) => toImageDataUrl(img)); + const encoded = params.inputImages.map(toImageDataUrl); if (params.schema.referenceImages === "image_urls") { params.requestBody.image_urls = encoded; return; @@ -708,10 +687,8 @@ export function buildFalImageGenerationProvider(): ImageGenerationProvider { } // Flux/custom edit endpoints use the singular image_url contract. - if (hasInputImages && schema.referenceImages === "image_url") { - if (req.aspectRatio) { - throw new Error("fal flux image edit endpoint does not support aspectRatio overrides"); - } + if (hasInputImages && schema.referenceImages === "image_url" && req.aspectRatio) { + throw new Error("fal flux image edit endpoint does not support aspectRatio overrides"); } if (!schema.supportsCount && (req.count ?? 1) > 1) { throw new Error(`fal ${requestedModel} supports one output image per request`); @@ -734,9 +711,7 @@ export function buildFalImageGenerationProvider(): ImageGenerationProvider { ...(isGptImage25 && req.background ? { background: req.background } : {}), }; if (schema.referenceImages === "image_style_references") { - requestBody.creativity = resolveKreaCreativity( - resolveFalCreativityOption(req.providerOptions), - ); + requestBody.creativity = resolveFalCreativityOption(req.providerOptions); } applyFalImageGeometry({ requestBody, @@ -789,9 +764,7 @@ export function buildFalImageGenerationProvider(): ImageGenerationProvider { images.push({ buffer: downloaded.buffer, mimeType: downloaded.mimeType, - fileName: `image-${imageIndex}.${imageFileExtensionForMimeType( - downloaded.mimeType || normalizeOptionalString(entry.content_type), - )}`, + fileName: `image-${imageIndex}.${imageFileExtensionForMimeType(downloaded.mimeType)}`, }); } diff --git a/extensions/fal/music-generation-provider.ts b/extensions/fal/music-generation-provider.ts index d967f56aa958..fd54f09e36bf 100644 --- a/extensions/fal/music-generation-provider.ts +++ b/extensions/fal/music-generation-provider.ts @@ -25,8 +25,29 @@ const FAL_MUSIC_MODELS = [ FAL_STABLE_AUDIO_MODEL, ] as const; -function buildFalMinimaxBody(req: MusicGenerationRequest): Record { +function buildFalMusicRequestBody( + req: MusicGenerationRequest, + model: string, +): Record { const lyrics = normalizeOptionalString(req.lyrics); + if (model === FAL_ACE_STEP_MODEL || model === FAL_STABLE_AUDIO_MODEL) { + const isStableAudio = model === FAL_STABLE_AUDIO_MODEL; + if (lyrics) { + throw new Error( + `fal ${isStableAudio ? "Stable Audio" : "ACE-Step"} music generation does not support explicit lyrics.`, + ); + } + if (isStableAudio && req.instrumental === true) { + throw new Error("fal Stable Audio music generation does not support instrumental mode."); + } + return { + prompt: req.prompt, + ...(req.instrumental === true ? { instrumental: true } : {}), + ...(typeof req.durationSeconds === "number" + ? { [isStableAudio ? "seconds_total" : "duration"]: req.durationSeconds } + : {}), + }; + } if (lyrics && req.instrumental === true) { throw new Error("fal MiniMax music generation cannot use lyrics when instrumental=true."); } @@ -44,43 +65,6 @@ function buildFalMinimaxBody(req: MusicGenerationRequest): Record { - if (normalizeOptionalString(req.lyrics)) { - throw new Error("fal ACE-Step music generation does not support explicit lyrics."); - } - return { - prompt: req.prompt, - ...(req.instrumental === true ? { instrumental: true } : {}), - ...(typeof req.durationSeconds === "number" ? { duration: req.durationSeconds } : {}), - }; -} - -function buildFalStableAudioBody(req: MusicGenerationRequest): Record { - if (normalizeOptionalString(req.lyrics)) { - throw new Error("fal Stable Audio music generation does not support explicit lyrics."); - } - if (req.instrumental === true) { - throw new Error("fal Stable Audio music generation does not support instrumental mode."); - } - return { - prompt: req.prompt, - ...(typeof req.durationSeconds === "number" ? { seconds_total: req.durationSeconds } : {}), - }; -} - -function buildFalMusicRequestBody( - req: MusicGenerationRequest, - model: string, -): Record { - if (model === FAL_ACE_STEP_MODEL) { - return buildFalAceStepBody(req); - } - if (model === FAL_STABLE_AUDIO_MODEL) { - return buildFalStableAudioBody(req); - } - return buildFalMinimaxBody(req); -} - function resolveFalMusicMetadata(payload: unknown): Record | undefined { if (!isRecord(payload)) { return undefined; diff --git a/extensions/fal/video-generation-provider.ts b/extensions/fal/video-generation-provider.ts index b1124e560b48..a28d65cc48da 100644 --- a/extensions/fal/video-generation-provider.ts +++ b/extensions/fal/video-generation-provider.ts @@ -84,7 +84,6 @@ type FalQueueResponse = { request_id?: string; response_url?: string; status_url?: string; - cancel_url?: string; detail?: string; response?: FalVideoResponse; prompt?: string; @@ -149,7 +148,6 @@ function readFalQueueResponse(payload: unknown): FalQueueResponse { request_id: normalizeOptionalString(payload.request_id), response_url: normalizeOptionalString(payload.response_url), status_url: normalizeOptionalString(payload.status_url), - cancel_url: normalizeOptionalString(payload.cancel_url), detail: normalizeOptionalString(payload.detail), response: payload.response === undefined ? undefined : readFalVideoPayload(payload.response), prompt: normalizeOptionalString(payload.prompt), @@ -177,10 +175,10 @@ function toDataUrl(buffer: Buffer, mimeType: string): string { } function extractFalVideoEntry(payload: FalVideoResponse) { - if (normalizeOptionalString(payload.video?.url)) { + if (payload.video?.url) { return payload.video; } - return payload.videos?.find((entry) => normalizeOptionalString(entry.url)); + return payload.videos?.find((entry) => entry.url); } async function downloadFalVideo(url: string, maxBytes: number): Promise { @@ -323,29 +321,15 @@ function buildFalVideoRequestBody(params: { }; if (isFalSeedance2ReferenceModel(params.model)) { - const imageUrls = resolveFalReferenceUrls( - params.req.inputImages, - "image/png", - "reference image", - ); - const videoUrls = resolveFalReferenceUrls( - params.req.inputVideos, - "video/mp4", - "reference video", - ); - const audioUrls = resolveFalReferenceUrls( - params.req.inputAudios, - "audio/mpeg", - "reference audio", - ); - if (imageUrls.length > 0) { - requestBody.image_urls = imageUrls; - } - if (videoUrls.length > 0) { - requestBody.video_urls = videoUrls; - } - if (audioUrls.length > 0) { - requestBody.audio_urls = audioUrls; + for (const [field, assets, mimeType, label] of [ + ["image_urls", params.req.inputImages, "image/png", "reference image"], + ["video_urls", params.req.inputVideos, "video/mp4", "reference video"], + ["audio_urls", params.req.inputAudios, "audio/mpeg", "reference audio"], + ] as const) { + const urls = resolveFalReferenceUrls(assets, mimeType, label); + if (urls.length > 0) { + requestBody[field] = urls; + } } applyFalSeedanceControls({ req: params.req, model: params.model, body: requestBody }); return requestBody; @@ -353,11 +337,11 @@ function buildFalVideoRequestBody(params: { const input = params.req.inputImages?.[0]; if (input) { - requestBody.image_url = normalizeOptionalString(input.url) - ? normalizeOptionalString(input.url) - : input.buffer + requestBody.image_url = + normalizeOptionalString(input.url) ?? + (input.buffer ? toDataUrl(input.buffer, normalizeOptionalString(input.mimeType) ?? "image/png") - : undefined; + : undefined); } // MiniMax Live on fal currently documents prompt + optional image_url only. // Keep the default model conservative so queue requests do not hang behind @@ -377,26 +361,15 @@ function validateFalVideoReferenceInputs(params: { const videoCount = params.req.inputVideos?.length ?? 0; const audioCount = params.req.inputAudios?.length ?? 0; if (isFalSeedance2ReferenceModel(params.model)) { - if (imageCount > SEEDANCE_REFERENCE_MAX_IMAGES) { - throw new Error( - `fal Seedance reference-to-video supports at most ${SEEDANCE_REFERENCE_MAX_IMAGES} reference images.`, - ); - } - if (videoCount > SEEDANCE_REFERENCE_MAX_VIDEOS) { - throw new Error( - `fal Seedance reference-to-video supports at most ${SEEDANCE_REFERENCE_MAX_VIDEOS} reference videos.`, - ); - } - if (audioCount > SEEDANCE_REFERENCE_MAX_AUDIOS) { - throw new Error( - `fal Seedance reference-to-video supports at most ${SEEDANCE_REFERENCE_MAX_AUDIOS} reference audios.`, - ); - } - const totalFiles = imageCount + videoCount + audioCount; - if (totalFiles > SEEDANCE_REFERENCE_MAX_FILES) { - throw new Error( - `fal Seedance reference-to-video supports at most ${SEEDANCE_REFERENCE_MAX_FILES} total reference files.`, - ); + for (const [count, max, label] of [ + [imageCount, SEEDANCE_REFERENCE_MAX_IMAGES, "reference images"], + [videoCount, SEEDANCE_REFERENCE_MAX_VIDEOS, "reference videos"], + [audioCount, SEEDANCE_REFERENCE_MAX_AUDIOS, "reference audios"], + [imageCount + videoCount + audioCount, SEEDANCE_REFERENCE_MAX_FILES, "total reference files"], + ] as const) { + if (count > max) { + throw new Error(`fal Seedance reference-to-video supports at most ${max} ${label}.`); + } } if (audioCount > 0 && imageCount === 0 && videoCount === 0) { throw new Error( @@ -474,7 +447,7 @@ async function waitForFalQueueResult(params: { errorContext: "fal video status request failed", }), ); - const status = normalizeOptionalString(payload.status)?.toUpperCase(); + const status = payload.status?.toUpperCase(); if (!status) { throw new Error(FAL_VIDEO_MALFORMED_RESPONSE); } @@ -500,8 +473,8 @@ async function waitForFalQueueResult(params: { } if (status === "FAILED" || status === "CANCELLED") { throw new Error( - normalizeOptionalString(payload.detail) || - normalizeOptionalString(payload.error?.message) || + payload.detail || + payload.error?.message || `fal video generation ${normalizeLowercaseStringOrEmpty(status)}`, ); } @@ -607,8 +580,8 @@ export function buildFalVideoGenerationProvider(): VideoGenerationProvider { errorContext: "fal video generation failed", }), ); - const statusUrl = normalizeOptionalString(submitted.status_url); - const responseUrl = normalizeOptionalString(submitted.response_url); + const statusUrl = submitted.status_url; + const responseUrl = submitted.response_url; if (!statusUrl || !responseUrl) { throw new Error("fal video generation response missing queue URLs"); } @@ -629,7 +602,7 @@ export function buildFalVideoGenerationProvider(): VideoGenerationProvider { }); const videoPayload = extractFalVideoPayload(payload); const entry = extractFalVideoEntry(videoPayload); - const url = normalizeOptionalString(entry?.url); + const url = entry?.url; if (!url) { throw new Error("fal video generation response missing output URL"); } @@ -638,9 +611,7 @@ export function buildFalVideoGenerationProvider(): VideoGenerationProvider { videos: [video], model, metadata: { - ...(normalizeOptionalString(submitted.request_id) - ? { requestId: normalizeOptionalString(submitted.request_id) } - : {}), + ...(submitted.request_id ? { requestId: submitted.request_id } : {}), ...(videoPayload.prompt ? { prompt: videoPayload.prompt } : {}), ...(typeof videoPayload.seed === "number" ? { seed: videoPayload.seed } : {}), }, diff --git a/extensions/firecrawl/package.json b/extensions/firecrawl/package.json index 8723aa914f2d..849fdf7f04ee 100644 --- a/extensions/firecrawl/package.json +++ b/extensions/firecrawl/package.json @@ -8,8 +8,7 @@ }, "type": "module", "dependencies": { - "typebox": "1.3.34", - "zod": "4.6.5" + "typebox": "1.3.34" }, "devDependencies": { "@openclaw/plugin-sdk": "workspace:*" diff --git a/extensions/firecrawl/src/firecrawl-client.test.ts b/extensions/firecrawl/src/firecrawl-client.test.ts index 9846a77b97fe..1ed893364323 100644 --- a/extensions/firecrawl/src/firecrawl-client.test.ts +++ b/extensions/firecrawl/src/firecrawl-client.test.ts @@ -54,7 +54,9 @@ describe("Firecrawl search payloads", () => { { data: [ { + url: 42, sourceURL: "https://www.example.com/source", + title: { text: "Ignore non-string title" }, metadata: { title: "Fallback title" }, description: "Fallback description", markdown: "Body", diff --git a/extensions/firecrawl/src/firecrawl-client.ts b/extensions/firecrawl/src/firecrawl-client.ts index 0b83d507b714..84aa05da165e 100644 --- a/extensions/firecrawl/src/firecrawl-client.ts +++ b/extensions/firecrawl/src/firecrawl-client.ts @@ -39,8 +39,8 @@ import { asOptionalObjectRecord, asOptionalRecord, normalizeOptionalString, + readStringField, } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { z } from "zod"; import { DEFAULT_FIRECRAWL_BASE_URL, resolveFirecrawlApiKey, @@ -200,9 +200,7 @@ async function resolveEndpoint( } async function postFirecrawlJson( - params: { - url: string; - mode?: FirecrawlEndpointMode; + params: FirecrawlResolvedEndpoint & { timeoutSeconds: number; apiKey?: string; body: Record; @@ -212,9 +210,8 @@ async function postFirecrawlJson( parse: (response: Response) => Promise, ): Promise { const apiKey = normalizeSecretInput(params.apiKey); - const mode = params.mode ?? (await validateFirecrawlBaseUrl(params.url)); const withEndpoint = - mode === "selfHosted" ? withSelfHostedWebToolsEndpoint : withStrictWebToolsEndpoint; + params.mode === "selfHosted" ? withSelfHostedWebToolsEndpoint : withStrictWebToolsEndpoint; const result = await withEndpoint( { url: params.url, @@ -308,32 +305,6 @@ function isValidFirecrawlPublishedDate(value: string): boolean { return timestamp !== undefined && new Date(timestamp).toISOString().startsWith(calendarDate); } -const optionalFirecrawlStringSchema = z.string().optional().catch(undefined); -const firecrawlSearchMetadataSchema = z - .object({ - sourceURL: optionalFirecrawlStringSchema, - title: optionalFirecrawlStringSchema, - publishedTime: optionalFirecrawlStringSchema, - publishedDate: optionalFirecrawlStringSchema, - }) - .optional() - .catch(undefined); -const firecrawlSearchItemSchema = z.object({ - url: optionalFirecrawlStringSchema, - sourceURL: optionalFirecrawlStringSchema, - sourceUrl: optionalFirecrawlStringSchema, - title: optionalFirecrawlStringSchema, - description: optionalFirecrawlStringSchema, - snippet: optionalFirecrawlStringSchema, - summary: optionalFirecrawlStringSchema, - markdown: optionalFirecrawlStringSchema, - content: optionalFirecrawlStringSchema, - text: optionalFirecrawlStringSchema, - publishedDate: optionalFirecrawlStringSchema, - published: optionalFirecrawlStringSchema, - metadata: firecrawlSearchMetadataSchema, -}); - function resolveSearchItems( payload: Record, count: number, @@ -357,27 +328,39 @@ function resolveSearchItems( if (inspectedObjects >= FIRECRAWL_SEARCH_MAX_RESULTS || items.length >= count) { break; } - const record = asOptionalRecord(rawItem); - if (!record) { + const entry = asOptionalRecord(rawItem); + if (!entry) { continue; } // The scan cap counts objects, including invalid URLs, after discarding non-object rows. inspectedObjects += 1; - const entry = firecrawlSearchItemSchema.parse(record); - const metadata = entry.metadata; - const rawUrl = entry.url || entry.sourceURL || entry.sourceUrl || metadata?.sourceURL || ""; + const metadata = asOptionalRecord(entry.metadata); + const rawUrl = + readStringField(entry, "url") || + readStringField(entry, "sourceURL") || + readStringField(entry, "sourceUrl") || + readStringField(metadata, "sourceURL") || + ""; const url = normalizeFirecrawlResultUrl(rawUrl); if (!url) { continue; } - const title = entry.title || metadata?.title || ""; - const description = entry.description || entry.snippet || entry.summary || undefined; - const content = entry.markdown || entry.content || entry.text || undefined; + const title = readStringField(entry, "title") || readStringField(metadata, "title") || ""; + const description = + readStringField(entry, "description") || + readStringField(entry, "snippet") || + readStringField(entry, "summary") || + undefined; + const content = + readStringField(entry, "markdown") || + readStringField(entry, "content") || + readStringField(entry, "text") || + undefined; const rawPublished = - entry.publishedDate || - entry.published || - metadata?.publishedTime || - metadata?.publishedDate || + readStringField(entry, "publishedDate") || + readStringField(entry, "published") || + readStringField(metadata, "publishedTime") || + readStringField(metadata, "publishedDate") || undefined; const published = rawPublished && isValidFirecrawlPublishedDate(rawPublished) ? rawPublished : undefined; @@ -493,33 +476,15 @@ export async function runFirecrawlSearch( const body: Record = { query: params.query, limit: count, + ...(sources.length > 0 ? { sources } : {}), + ...(categories.length > 0 ? { categories } : {}), + ...(includeDomains.length > 0 ? { includeDomains } : {}), + ...(excludeDomains.length > 0 ? { excludeDomains } : {}), + ...(tbs ? { tbs } : {}), + ...(location ? { location } : {}), + ...(country ? { country } : {}), + ...(scrapeResults ? { scrapeOptions: { formats: ["markdown"] } } : {}), }; - if (sources.length > 0) { - body.sources = sources; - } - if (categories.length > 0) { - body.categories = categories; - } - if (includeDomains.length > 0) { - body.includeDomains = includeDomains; - } - if (excludeDomains.length > 0) { - body.excludeDomains = excludeDomains; - } - if (tbs) { - body.tbs = tbs; - } - if (location) { - body.location = location; - } - if (country) { - body.country = country; - } - if (scrapeResults) { - body.scrapeOptions = { - formats: ["markdown"], - }; - } const start = Date.now(); const endpoint = await resolveEndpoint(baseUrl, "/v2/search"); @@ -711,7 +676,6 @@ export async function runFirecrawlScrape( export const testing = { assertFirecrawlScrapeTargetAllowed, parseFirecrawlScrapePayload, - postFirecrawlJson, resolveEndpoint, resolveSearchItems, }; diff --git a/extensions/firecrawl/src/firecrawl-tools.test.ts b/extensions/firecrawl/src/firecrawl-tools.test.ts index 32119db38883..ddc1831e2f5c 100644 --- a/extensions/firecrawl/src/firecrawl-tools.test.ts +++ b/extensions/firecrawl/src/firecrawl-tools.test.ts @@ -149,16 +149,11 @@ describe("firecrawl tools", () => { ), ) as typeof fetch; - const failure = firecrawlClientTesting.postFirecrawlJson( - { - url: "https://api.firecrawl.dev/v2/search", - timeoutSeconds: 5, - apiKey: "firecrawl-key", - body: { query: "openclaw" }, - errorLabel: "Firecrawl search", - }, - async () => "ok", - ); + const failure = runActualFirecrawlSearch({ + cfg: firecrawlConfig({ webSearch: { apiKey: "firecrawl-key" } }), + query: "bounded Firecrawl HTTP failure", + timeoutSeconds: 5, + }); await expect(failure).rejects.toMatchObject({ status: 400, statusCode: 400 }); await expect(failure).rejects.toSatisfy( (error: unknown) => @@ -433,16 +428,11 @@ describe("firecrawl tools", () => { }); global.fetch = fetchSpy as typeof fetch; - await firecrawlClientTesting.postFirecrawlJson( - { - url: "https://api.firecrawl.dev/v2/search", - timeoutSeconds: 5, - apiKey: "firecrawl-test-\r\nkey", - body: { query: "openclaw" }, - errorLabel: "Firecrawl search", - }, - async () => "ok", - ); + await runActualFirecrawlSearch({ + cfg: firecrawlConfig({ webSearch: { apiKey: "firecrawl-test-\r\nkey" } }), + query: "normalized Firecrawl authorization", + timeoutSeconds: 5, + }); const authHeader = new Headers(capturedInit?.headers).get("Authorization"); expect(authHeader).toBe("Bearer firecrawl-test-key"); @@ -1160,19 +1150,16 @@ describe("firecrawl tools", () => { const fetchSpy = vi.fn(async () => Response.json({ success: true, data: [] })); global.fetch = fetchSpy as typeof fetch; - const result = await firecrawlClientTesting.postFirecrawlJson( - { - url: "http://127.0.0.1:8787/v2/search", - timeoutSeconds: 5, - apiKey: "firecrawl-key", - body: { query: "openclaw" }, - errorLabel: "Firecrawl Search", - }, - async (response) => (await response.json()) as Record, - ); + const result = await runActualFirecrawlSearch({ + cfg: firecrawlConfig({ + webSearch: { apiKey: "firecrawl-key", baseUrl: "http://127.0.0.1:8787" }, + }), + query: "self-hosted Firecrawl search", + timeoutSeconds: 5, + }); expect(fetchSpy).toHaveBeenCalledTimes(1); - expect(result.success).toBe(true); + expect(result).toMatchObject({ provider: "firecrawl", results: [] }); }); describe.each(["search", "scrape"] as const)("malformed Firecrawl %s responses", (operation) => { diff --git a/extensions/irc/src/client.ts b/extensions/irc/src/client.ts index d2bf843e521d..b441178f5b78 100644 --- a/extensions/irc/src/client.ts +++ b/extensions/irc/src/client.ts @@ -1,5 +1,6 @@ import net from "node:net"; import tls from "node:tls"; +import { createDeferred } from "openclaw/plugin-sdk/extension-shared"; import { withTimeout } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime"; import { @@ -162,20 +163,13 @@ export async function connectIrcClient(options: IrcClientOptions): Promise void) | null = null; - let rejectReady: ((error: Error) => void) | null = null; - const readyPromise = new Promise((resolve, reject) => { - resolveReady = resolve; - rejectReady = reject; - }); + const readyDeferred = createDeferred(); const fail = (err: unknown) => { const error = toIrcError(err); options.onError?.(error); - if (!ready && rejectReady) { - rejectReady(error); - rejectReady = null; - resolveReady = null; + if (!ready) { + readyDeferred.reject(error); } }; @@ -355,9 +349,7 @@ export async function connectIrcClient(options: IrcClientOptions): Promise ({ + post: vi.fn<() => Promise<{ data: Array<{ embedding: unknown }> }>>(), +})); + +vi.mock("openai", () => ({ + default: class { + post = post; + }, +})); + +vi.mock("openclaw/plugin-sdk/runtime-env", () => ({ + ensureGlobalUndiciEnvProxyDispatcher: vi.fn(), +})); + +async function embedResponse(value: unknown): Promise { + post.mockResolvedValue({ data: [{ embedding: value }] }); + const embeddings = createEmbeddings(createTestPluginApi()); + try { + return await embeddings.embed("main", "embedding fixture", { + provider: "openai", + apiKey: "fixture-key", + model: "test-model", + }); + } finally { + await embeddings.close?.(); + } +} + +describe("memory-lancedb embedding responses", () => { + test("accepts float arrays and base64 float32 responses", async () => { + await expect(embedResponse([0.1, 0.2, 0.3])).resolves.toEqual([0.1, 0.2, 0.3]); + + const bytes = Buffer.alloc(2 * Float32Array.BYTES_PER_ELEMENT); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + view.setFloat32(0, 1.25, true); + view.setFloat32(Float32Array.BYTES_PER_ELEMENT, -2.5, true); + + const decoded = await embedResponse(bytes.toString("base64")); + expect(decoded[0]).toBeCloseTo(1.25); + expect(decoded[1]).toBeCloseTo(-2.5); + }); + + test.each( + [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY].flatMap((coordinate) => [ + { encoding: "float array", coordinate }, + { encoding: "base64", coordinate }, + ]), + )("rejects nonfinite $coordinate in $encoding embeddings", async ({ encoding, coordinate }) => { + const bytes = Buffer.alloc(Float32Array.BYTES_PER_ELEMENT); + bytes.writeFloatLE(coordinate); + const vector = encoding === "base64" ? bytes.toString("base64") : [coordinate]; + await expect(embedResponse(vector)).rejects.toThrow( + "Embedding response contains non-numeric values", + ); + }); + + test("rejects malformed embedding payloads", async () => { + await expect(embedResponse("abc")).rejects.toThrow( + "Base64 embedding response has invalid byte length", + ); + await expect(embedResponse("!!!!")).rejects.toThrow("Base64 embedding response is malformed"); + await expect(embedResponse("ZE==")).rejects.toThrow("Base64 embedding response is malformed"); + await expect(embedResponse("AQIDBE==")).rejects.toThrow( + "Base64 embedding response is malformed", + ); + await expect(embedResponse(undefined)).rejects.toThrow( + "Embedding response is missing a vector", + ); + }); +}); diff --git a/extensions/memory-lancedb/embeddings.ts b/extensions/memory-lancedb/embeddings.ts index cc3da305c49d..eb79b2178a14 100644 --- a/extensions/memory-lancedb/embeddings.ts +++ b/extensions/memory-lancedb/embeddings.ts @@ -1,5 +1,6 @@ import { Buffer } from "node:buffer"; import { resolve as resolveFilePath } from "node:path"; +import type OpenAI from "openai"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage, toErrorObject } from "openclaw/plugin-sdk/error-runtime"; import { resolveGlobalSingleton } from "openclaw/plugin-sdk/global-singleton"; @@ -14,12 +15,6 @@ import { asOptionalRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; import { textResult, type AgentToolResult } from "openclaw/plugin-sdk/tool-results"; import type { MemoryConfig } from "./config.js"; -type OpenAiEmbeddingClient = { - post( - path: string, - options: { body: unknown; timeout?: number; maxRetries?: number }, - ): Promise; -}; const loadOpenAiModule = createLazyRuntimeModule(() => import("openai")); const loadMemoryEmbeddingProviderModule = createLazyRuntimeModule( () => import("openclaw/plugin-sdk/memory-core-host-engine-embeddings"), @@ -91,7 +86,7 @@ function embeddingConfigFingerprint(embedding: EmbeddingConfig): string { } class OpenAiCompatibleEmbeddings { - private clientPromise: Promise; + private clientPromise: Promise; constructor( apiKey: string, @@ -100,7 +95,7 @@ class OpenAiCompatibleEmbeddings { private dimensions?: number, ) { this.clientPromise = loadOpenAiModule().then( - ({ default: OpenAI }) => new OpenAI({ apiKey, baseURL: baseUrl }) as OpenAiEmbeddingClient, + ({ default: OpenAI }) => new OpenAI({ apiKey, baseURL: baseUrl }), ); } @@ -554,7 +549,7 @@ type EmbeddingCreateResponse = { }>; }; -export function normalizeEmbeddingVector(value: unknown): number[] { +function normalizeEmbeddingVector(value: unknown): number[] { if (typeof value === "string") { const canonicalEmbedding = canonicalizeBase64(value); if (!canonicalEmbedding) { diff --git a/extensions/memory-lancedb/index.test.ts b/extensions/memory-lancedb/index.test.ts index b184e05f61ab..90dfb0fc55a5 100644 --- a/extensions/memory-lancedb/index.test.ts +++ b/extensions/memory-lancedb/index.test.ts @@ -8,7 +8,6 @@ * - Auto-capture filtering */ -import { Buffer } from "node:buffer"; import fs from "node:fs/promises"; import path from "node:path"; import { expectDefined } from "@openclaw/normalization-core"; @@ -26,23 +25,19 @@ import { registerMemoryCapability, type MemoryPluginCapability, } from "openclaw/plugin-sdk/memory-host-core"; -import { MESSAGE_TOOL_DELIVERY_HINTS } from "openclaw/plugin-sdk/message-tool-delivery-hints"; import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; import { afterEach, describe, test, expect, vi } from "vitest"; import { createEmbeddings, isMemoryRecallTimeoutError, runWithTimeout } from "./embeddings.js"; -import memoryPlugin, { +import memoryPlugin from "./index.js"; +import { looksLikeEnvelopeSludge } from "./memory-capture-sanitization.js"; +import { detectCategory, - escapeMemoryForPrompt, formatRelevantMemoriesContext, - looksLikeEnvelopeSludge, looksLikePromptInjection, - normalizeEmbeddingVector, normalizeRecallQuery, - parseMemoryCliFilter, - sanitizeForMemoryCapture, shouldCapture, -} from "./index.js"; +} from "./memory-policy.js"; import { installTmpDirHarness } from "./test-helpers.js"; const moduleMocks = vi.hoisted(() => ({ @@ -150,31 +145,6 @@ function resetMemoryModuleMocks(): void { type MockCallSource = { mock: { calls: Array> } }; -describe("memory CLI filters", () => { - test("parses one typed comparison", () => { - expect(parseMemoryCliFilter("category = 'preference'")).toEqual({ - column: "category", - operator: "=", - value: "preference", - }); - expect(parseMemoryCliFilter("importance >= 0.8")).toEqual({ - column: "importance", - operator: ">=", - value: 0.8, - }); - }); - - test.each([ - "agentId = 'beta'", - "category = 'preference' OR agentId = 'beta'", - "category = 'preference') OR (1 = 1", - "category IN ('preference', 'fact')", - "importance = 'high'", - ])("rejects a filter that could escape the owner predicate: %s", (filter) => { - expect(() => parseMemoryCliFilter(filter)).toThrow(); - }); -}); - function registerTestPlugin(plugin: { register: (api: never) => void }, api: unknown): void { plugin.register(api as never); } @@ -3347,51 +3317,6 @@ describe("memory plugin e2e", () => { expect(normalizeRecallQuery(`look up ${"x".repeat(2000)}`, Number.NaN)).toHaveLength(1000); }); - test("normalizeEmbeddingVector accepts float arrays and base64 float32 responses", () => { - expect(normalizeEmbeddingVector([0.1, 0.2, 0.3])).toEqual([0.1, 0.2, 0.3]); - - const bytes = Buffer.alloc(2 * Float32Array.BYTES_PER_ELEMENT); - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - view.setFloat32(0, 1.25, true); - view.setFloat32(Float32Array.BYTES_PER_ELEMENT, -2.5, true); - - const decoded = normalizeEmbeddingVector(bytes.toString("base64")); - expect(decoded[0]).toBeCloseTo(1.25); - expect(decoded[1]).toBeCloseTo(-2.5); - }); - - test.each( - [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY].flatMap((coordinate) => [ - { encoding: "float array", coordinate }, - { encoding: "base64", coordinate }, - ]), - )("rejects nonfinite $coordinate in $encoding embeddings", ({ encoding, coordinate }) => { - const bytes = Buffer.alloc(Float32Array.BYTES_PER_ELEMENT); - bytes.writeFloatLE(coordinate); - const vector = encoding === "base64" ? bytes.toString("base64") : [coordinate]; - expect(() => normalizeEmbeddingVector(vector)).toThrow( - "Embedding response contains non-numeric values", - ); - }); - - test("normalizeEmbeddingVector rejects malformed embedding payloads", () => { - expect(() => normalizeEmbeddingVector("abc")).toThrow( - "Base64 embedding response has invalid byte length", - ); - expect(() => normalizeEmbeddingVector("!!!!")).toThrow( - "Base64 embedding response is malformed", - ); - expect(() => normalizeEmbeddingVector("ZE==")).toThrow( - "Base64 embedding response is malformed", - ); - expect(() => normalizeEmbeddingVector("AQIDBE==")).toThrow( - "Base64 embedding response is malformed", - ); - expect(() => normalizeEmbeddingVector(undefined)).toThrow( - "Embedding response is missing a vector", - ); - }); - test.each([ { name: "unknown dimensions parameter", @@ -4116,587 +4041,6 @@ describe("memory plugin e2e", () => { expect(looksLikeEnvelopeSludge("[telegram] foo")).toBe(false); }); - test("sanitizeForMemoryCapture strips structurally marker-free channel envelope prefix", () => { - // Mirror the looksLikeEnvelopeSludge marker-free coverage so the full - // capture flow (sanitize -> shouldCapture) also handles the shape. - expect(sanitizeForMemoryCapture("[telegram Alice] Alice: I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect(sanitizeForMemoryCapture("[telegram Alice id:123] Alice: I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect(sanitizeForMemoryCapture("[LINE user:U123] (sender): I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect(sanitizeForMemoryCapture("[discord #general user] user: ping")).toBe("ping"); - expect(sanitizeForMemoryCapture("[Google Chat Room] Room: I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect(sanitizeForMemoryCapture("[Nextcloud Talk Board] Board: I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect(sanitizeForMemoryCapture("[Teams General] General: I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect(sanitizeForMemoryCapture("[Signal Hill] is my favorite hike")).toBe( - "[Signal Hill] is my favorite hike", - ); - // Group-chat sender-prefix on the body is also stripped when the bracket is - // recognized as an envelope. - expect(sanitizeForMemoryCapture("[slack #general user] user: hello")).toBe("hello"); - }); - - test("sanitizeForMemoryCapture leaves markdown links and unknown labels alone", () => { - expect(sanitizeForMemoryCapture("[click here](https://example.com)")).toBe( - "[click here](https://example.com)", - ); - expect(sanitizeForMemoryCapture("[note] my thoughts")).toBe("[note] my thoughts"); - }); - - test("sanitizeForMemoryCapture strips formatInboundEnvelope direct-message prefix", () => { - expect(sanitizeForMemoryCapture("[Telegram Alice +5m] I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect( - sanitizeForMemoryCapture("[Telegram Alice +5m Mon 2026-05-17 14:30 EDT] I prefer dark mode"), - ).toBe("I prefer dark mode"); - }); - - test("sanitizeForMemoryCapture strips group-chat envelope prefix AND sender label", () => { - expect( - sanitizeForMemoryCapture( - "[Telegram Group id:123 Alice +5m Mon 2026-05-17 14:30 EDT] Alice: I prefer dark mode", - ), - ).toBe("I prefer dark mode"); - }); - - test("sanitizeForMemoryCapture strips sender label from real room-label envelope shapes", () => { - // Real group/channel callers pass the room/conversation as `from` and the - // sender separately; the sender is not necessarily present in the header. - expect(sanitizeForMemoryCapture("[Telegram group:123] Alice: I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect(sanitizeForMemoryCapture("[Slack #general] Alice: I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect( - sanitizeForMemoryCapture( - "[Discord OpenClaw #dev channel id:456 +5m] Alice: I prefer dark mode", - ), - ).toBe("I prefer dark mode"); - expect(sanitizeForMemoryCapture("[Telegram OpenClaw id:-100] Alice: I prefer dark mode")).toBe( - "I prefer dark mode", - ); - expect(sanitizeForMemoryCapture("[Signal Signal Group id:123] Bob (42): ping")).toBe("ping"); - }); - - test("sanitizeForMemoryCapture preserves user labels in generic room envelopes", () => { - expect( - sanitizeForMemoryCapture( - "[Nextcloud Talk room:ops Mon 2026-05-17 14:30 UTC] TODO: keep this", - ), - ).toBe("TODO: keep this"); - expect(sanitizeForMemoryCapture("[Slack #general] TODO: keep this")).toBe("TODO: keep this"); - expect(sanitizeForMemoryCapture("[WhatsApp Family Chat +5m] Alice: hello")).toBe( - "Alice: hello", - ); - expect(sanitizeForMemoryCapture("[Telegram Alice +5m] Bob (42): I prefer dark mode")).toBe( - "Bob (42): I prefer dark mode", - ); - }); - - test("sanitizeForMemoryCapture leaves text with no envelope prefix alone", () => { - // No bracket envelope: the `Name: ` sender-stripper must NOT fire on - // user-typed text that happens to look like `Name: body`. - expect(sanitizeForMemoryCapture("Alice: I prefer dark mode")).toBe("Alice: I prefer dark mode"); - }); - - test("sanitizeForMemoryCapture preserves DM body that starts with `TODO:` / `FIXME:`", () => { - // Direct-message envelope: per the formatter contract there is no sender - // prefix on the body. A user-typed `TODO: ...` or `FIXME: ...` must not - // be truncated to `...`. The leading label does not match any token in - // the envelope header, so the gated strip leaves it alone. - expect(sanitizeForMemoryCapture("[telegram alice +5m] TODO: fix this")).toBe("TODO: fix this"); - expect(sanitizeForMemoryCapture("[Telegram Alice +5m] FIXME: clean up sanitizer")).toBe( - "FIXME: clean up sanitizer", - ); - }); - - test("sanitizeForMemoryCapture preserves group body whose `Name: ` does not match envelope", () => { - // Group envelope `[discord alice]` with body `Bob: hello` (Alice is - // quoting Bob). `Bob` is not a token in the envelope header, so the - // formatter could not have emitted it; the gated strip leaves it alone. - expect(sanitizeForMemoryCapture("[discord alice +5m] Bob: hello there")).toBe( - "Bob: hello there", - ); - }); - - test("sanitizeForMemoryCapture strips `(self):` body prefix from direct fromMe envelope", () => { - // Direct chat + fromMe contract: body is `(self): `. The literal - // `(self)` sentinel is always safe to strip after an envelope bracket. - expect(sanitizeForMemoryCapture("[telegram alice] (self): typed this")).toBe("typed this"); - expect(sanitizeForMemoryCapture("[Telegram Alice +5m] (self): note to self")).toBe( - "note to self", - ); - }); - - test("shouldCapture rejects formatInboundEnvelope-prefixed messages", () => { - // The agent_end hook still receives envelope-prefixed user content, so the - // capture gate must reject these without relying on prior sanitize. - expect(shouldCapture("[Telegram Alice +5m] I prefer dark mode")).toBe(false); - expect( - shouldCapture( - "[Telegram Group id:123 Alice +5m Mon 2026-05-17 14:30 EDT] Alice: I prefer dark mode", - ), - ).toBe(false); - }); - - test("sanitize-then-shouldCapture preserves clean body from envelope-wrapped input", () => { - // End-to-end shape of the real auto-capture flow: sanitize first, then - // shouldCapture decides on the body-only text. A genuine memory like - // "I prefer dark mode" wrapped in envelope metadata must survive the - // sanitize step as bare body and pass the gate. - const wrapped = "[Telegram Alice +5m] I prefer dark mode"; - const sanitized = sanitizeForMemoryCapture(wrapped); - expect(sanitized).toBe("I prefer dark mode"); - expect(shouldCapture(sanitized)).toBe(true); - }); - - test("shouldCapture rejects envelope sludge", () => { - expect( - shouldCapture( - `${ctxHeader("Conversation info:")}\n\`\`\`json\n{"id":"123"}\n\`\`\`\nI always prefer dark mode`, - ), - ).toBe(false); - }); - - test("sanitizeForMemoryCapture strips timestamp prefix", () => { - expect(sanitizeForMemoryCapture("[Mon 2026-04-14 12:34 EDT] I prefer dark mode")).toBe( - "I prefer dark mode", - ); - }); - - test("sanitizeForMemoryCapture strips inbound metadata blocks", () => { - const input = [ - ctxHeader("Sender:"), - "```json", - '{"name": "Alex"}', - "```", - "", - "I always prefer verbose output", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I always prefer verbose output"); - }); - - test("sanitizeForMemoryCapture strips known current inbound metadata blocks", () => { - const locationInput = [ - ctxHeader("Location:"), - "```json", - '{"lat": 48.2, "lng": 16.3}', - "```", - "", - "I always prefer dark mode", - ].join("\n"); - expect(sanitizeForMemoryCapture(locationInput)).toBe("I always prefer dark mode"); - - const replyChainInput = [ - ctxHeader("Reply chain of current user message (nearest first):"), - "```json", - '[{"body":"quoted context"}]', - "```", - "", - "I always prefer concise replies", - ].join("\n"); - expect(sanitizeForMemoryCapture(replyChainInput)).toBe("I always prefer concise replies"); - }); - - test("sanitizeForMemoryCapture drops presentation-only media-note lines", () => { - const input = [ - "[media attached: /tmp/photo.jpg (image/jpeg)]", - "Check this and remember it", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("Check this and remember it"); - - const bracketedFilename = - "[media attached: /tmp/foo] I always prefer dark-mode.png (image/png)]"; - expect(sanitizeForMemoryCapture(bracketedFilename)).toBe(""); - expect(sanitizeForMemoryCapture(`${bracketedFilename}\nI prefer concise captions`)).toBe( - "I prefer concise captions", - ); - }); - - test("sanitizeForMemoryCapture preserves captions after inline legacy media text", () => { - const input = "[media attached: stale.png] I always prefer dark mode"; - expect(sanitizeForMemoryCapture(input)).toBe(input); - expect(shouldCapture(input)).toBe(true); - - const prose = "[media attached files are how I always prefer to receive reports]"; - expect(sanitizeForMemoryCapture(prose)).toBe(prose); - expect(shouldCapture(prose)).toBe(true); - - const numberedCaption = "[media attached 1/1: stale.png] I always prefer dark mode"; - expect(sanitizeForMemoryCapture(numberedCaption)).toBe(numberedCaption); - }); - - test("sanitizeForMemoryCapture strips active_memory_plugin blocks", () => { - const input = - "some plugin data\nI prefer concise replies"; - expect(sanitizeForMemoryCapture(input)).toBe("I prefer concise replies"); - }); - - test("sanitizeForMemoryCapture strips active memory prefix before user text", () => { - const input = [ - "Context:", - "recall context", - "", - "I prefer dark mode", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I prefer dark mode"); - }); - - test("sanitizeForMemoryCapture strips marked context header and trailing content", () => { - const input = `I prefer dark mode\n${ctxHeader("Context:")}\nsome trailing metadata`; - expect(sanitizeForMemoryCapture(input)).toBe("I prefer dark mode"); - }); - - test("sanitizeForMemoryCapture preserves a bare context header and trailing content", () => { - const input = "I prefer dark mode\nContext:\nsome user-authored text"; - expect(sanitizeForMemoryCapture(input)).toBe(input); - }); - - test("sanitizeForMemoryCapture does not strip a context label mid-line", () => { - const input = "The user mentioned Context: in their question about security"; - expect(sanitizeForMemoryCapture(input)).toBe( - "The user mentioned Context: in their question about security", - ); - }); - - test("sanitizeForMemoryCapture preserves a near-miss context header with trailing text", () => { - const input = "Context: I prefer dark mode at work\nplease remember that"; - expect(sanitizeForMemoryCapture(input)).toBe(input); - }); - - test("sanitizeForMemoryCapture pre-truncates very large inputs", () => { - const padding = "x".repeat(11_000); - const input = `${padding}\nI always prefer dark mode`; - const result = sanitizeForMemoryCapture(input); - expect(result).not.toContain("I always prefer dark mode"); - expect(result.length).toBeLessThanOrEqual(10_000); - }); - - test("sanitizeForMemoryCapture returns empty string for pure metadata", () => { - const input = [ - ctxHeader("Conversation info:"), - "```json", - '{"id": "chat-123", "title": "Test"}', - "```", - ctxHeader("Sender:"), - "```json", - '{"name": "Alex"}', - "```", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe(""); - }); - - test("sanitizeForMemoryCapture handles combined contamination", () => { - const input = [ - `[Sun 2026-04-13 09:15 EDT] ${ctxHeader("Conversation info:")}`, - "```json", - '{"id": "chat-456"}', - "```", - ctxHeader("Sender:"), - "```json", - '{"name": "Alex"}', - "```", - "", - "[media attached: /tmp/screenshot.png (image/png)]", - "I always prefer TypeScript over JavaScript", - "", - "recall context", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I always prefer TypeScript over JavaScript"); - }); - - test("sanitizeForMemoryCapture truncates chat-history plain-text body so MEMORY_TRIGGER words inside are not captured", () => { - // The "Chat history since last reply" sentinel is followed by a plain-text - // transcript rather than a ```json``` fence. The body must be truncated so - // that MEMORY_TRIGGER phrases inside quoted bot replies are never vectorized - // as long-term memories. - const input = [ - "I always prefer dark mode", - ctxHeader("Chat history since last reply:"), - "User: what do you recommend?", - "Bot: I always recommend TypeScript for large projects", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I always prefer dark mode"); - }); - - test("sanitizeForMemoryCapture drops leading plain-text metadata bodies without a current boundary", () => { - const input = [ - ctxHeader("Chat history since last reply:"), - "User: what do you recommend?", - "Bot: I always recommend TypeScript for large projects", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe(""); - }); - - test("sanitizeForMemoryCapture keeps current marker content after leading plain-text metadata", () => { - const input = [ - ctxHeader("Chat history since last reply:"), - "[Telegram Bob] Bob: I always recommend historical wrong value", - "", - "[Current message - respond to this]", - "[Telegram group:-100] obviyus: I prefer dark mode", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I prefer dark mode"); - }); - - test("sanitizeForMemoryCapture truncates thread-starter plain-text body", () => { - // Same fix for "Thread starter:" which also carries - // a plain-text body instead of a JSON code fence. - const input = [ - "I always use ESLint in every project", - ctxHeader("Thread starter:"), - "Original message: I always want verbose logging enabled", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I always use ESLint in every project"); - }); - - test("sanitizeForMemoryCapture truncates at earliest sentinel across multiple inbound-meta blocks", () => { - // Regression guard for the per-sentinel loop ordering bug: when a body - // contains two different sentinels the sanitizer must truncate at the - // EARLIEST position, regardless of INBOUND_META_SENTINELS declaration - // order. Here `Chat history since last reply` appears BEFORE - // `Conversation info`; the iteration-order-dependent code would - // truncate at `Conversation info` (declared first) and preserve the - // plain-text history that followed `Chat history`. - const input = [ - "I always prefer dark mode", - ctxHeader("Chat history since last reply:"), - "User: hi", - "Bot: I always say hello back", - ctxHeader("Conversation info:"), - "irrelevant trailing metadata", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I always prefer dark mode"); - }); - - test("sanitizeForMemoryCapture strips current context before envelope prefixes", () => { - const input = [ - ctxHeader("Conversation info:"), - "```json", - '{"channel":"slack"}', - "```", - "", - ctxHeader("Conversation context (chronological, selected for current message):"), - "[Slack #general Alice] Alice: I always prefer dark mode", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I always prefer dark mode"); - }); - - test("sanitizeForMemoryCapture does not capture stale chronological history envelopes", () => { - const input = [ - ctxHeader("Conversation context (chronological, selected for current message):"), - "Bob: [telegram bob] I always prefer stale context", - "[Telegram Alice] I always prefer dark mode", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe(""); - }); - - test("sanitizeForMemoryCapture preserves prompt after plain chronological context", () => { - const input = [ - ctxHeader("Conversation context (chronological, selected for current message):"), - "#35674 Other: stale context", - "", - "I always prefer dark mode", - ].join("\n"); - const sanitized = sanitizeForMemoryCapture(input); - expect(sanitized).toBe("I always prefer dark mode"); - expect(shouldCapture(sanitized)).toBe(true); - }); - - test("sanitizeForMemoryCapture keeps inline envelope after current-message prefix", () => { - const input = [ - ctxHeader("Conversation context (chronological, selected for current message):"), - "#34974 obviyus: [Telegram group:-100] obviyus: I prefer dark mode", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I prefer dark mode"); - }); - - test("sanitizeForMemoryCapture strips envelopes after JSON-only metadata", () => { - const input = [ - ctxHeader("Conversation info:"), - "```json", - '{"channel":"telegram"}', - "```", - "", - "[Telegram Alice] I prefer dark mode", - ].join("\n"); - expect(sanitizeForMemoryCapture(input)).toBe("I prefer dark mode"); - }); - - test("sanitizeForMemoryCapture preserves an unknown structured-context label as user content", () => { - // An arbitrary `