diff --git a/CHANGELOG.md b/CHANGELOG.md index ab46752efc2..14a5072aeca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Docs: https://docs.openclaw.ai - **Google Chat request deadlines:** bound control calls to 30 seconds while giving media transfers size-aware total budgets and a separate 30-second stalled-body guard, preventing hung Chat API requests without breaking large attachment uploads. (#102227) Thanks @hugenshen. - **Google Gemini prefixed model IDs:** recognize `google/gemini-*` and `models/gemini-*` when selecting multimodal function-response behavior, preserving the Gemini 2 image fallback without regressing Gemini 3 inline image responses. (#102382) Thanks @LiLan0125. - **Generated provider model catalogs:** keep MiniMax and NVIDIA catalog rows when they advertise audio or video metadata while projecting runtime model inputs to text/image, preventing configured multimodal primaries from being dropped and falling back. (#97858, #97048) Thanks @ly-wang19 and @zackchiutw. +- **CLI audio transcript files:** treat inferred Whisper and Parakeet text files as authoritative so empty or missing output cannot expose progress/status stdout as user speech. (#87393, #87384) Thanks @kesslerio. - **Browser actions on Node 24:** keep browser request cancellation bound to the client and response lifetime instead of Node 24.16+'s prematurely aborted body-stream signal, preventing valid POST actions from failing after JSON parsing. Thanks @obviyus and @vincentkoc. - **SecretRef model credentials:** keep resolved provider secrets behind process-local sentinels through auth storage, stream setup, SDK configuration, and managed local-provider probing, then inject plaintext only at the final network or provider-plugin boundary while retaining exact-value log redaction. (#102008, #102009) - **Lean local model shell access:** keep `exec` directly visible beside the default structured Tool Search controls so coding-tuned local models can use their shell fallback instead of searching for missing domain tools. (#87587) Thanks @vincentkoc. diff --git a/docs/nodes/audio.md b/docs/nodes/audio.md index baa6e6a1be8..e816a30c51d 100644 --- a/docs/nodes/audio.md +++ b/docs/nodes/audio.md @@ -196,7 +196,8 @@ On channels that support audio preflight, OpenClaw transcribes audio **before** - Scope rules use first-match-wins; `chatType` is normalized to `direct`, `group`, or `channel`. - Ensure your CLI exits 0 and prints plain text; JSON output needs to be massaged via `jq -r .text`. -- For `parakeet-mlx`, if you pass `--output-dir`, OpenClaw reads `/.txt` when `--output-format` is `txt` (or omitted); non-`txt` output formats fall back to stdout parsing. +- Known file-output modes are authoritative: an empty or missing inferred transcript file produces no transcript instead of falling back to CLI progress output. +- For `parakeet-mlx`, use `--output-format txt` (or `all`) with `--output-dir` and the default `{filename}` output template. The upstream `PARAKEET_OUTPUT_FORMAT` and `PARAKEET_OUTPUT_TEMPLATE` environment variables are also honored. OpenClaw reads `/.txt`; the default `srt` format, other formats, and custom output templates continue to use stdout. - Keep timeouts reasonable (`timeoutSeconds`, default 60s) to avoid blocking the reply queue. - Preflight transcription only processes the **first** audio attachment for mention detection. Additional audio attachments are processed during the main media-understanding phase. diff --git a/docs/nodes/media-understanding.md b/docs/nodes/media-understanding.md index 60c634b7f6d..4dfcde4c495 100644 --- a/docs/nodes/media-understanding.md +++ b/docs/nodes/media-understanding.md @@ -243,7 +243,8 @@ For CLI entries, set `capabilities` explicitly to avoid surprising matches; if o - For tool-enabled agents handling untrusted inputs, avoid older/weaker media models. - Keep at least one fallback per capability for availability (quality model + faster/cheaper model). - CLI fallbacks (`whisper-cli`, `whisper`, `gemini`) help when provider APIs are unavailable. -- `parakeet-mlx`: with `--output-dir`, OpenClaw reads `/.txt` when the output format is `txt` or unspecified; other formats fall back to stdout. +- Known file-output modes are authoritative: an empty or missing inferred transcript file produces no transcript instead of falling back to CLI progress output. +- `parakeet-mlx`: use `--output-format txt` (or `all`) with `--output-dir` and the default `{filename}` output template. The upstream `PARAKEET_OUTPUT_FORMAT` and `PARAKEET_OUTPUT_TEMPLATE` environment variables are also honored. OpenClaw reads `/.txt`; the default `srt` format, other formats, and custom output templates continue to use stdout. ## Attachment policy diff --git a/src/media-understanding/apply.test.ts b/src/media-understanding/apply.test.ts index 00a9189ee7d..e61b5984a4a 100644 --- a/src/media-understanding/apply.test.ts +++ b/src/media-understanding/apply.test.ts @@ -225,7 +225,7 @@ async function createAudioCtx(params?: { } satisfies MsgContext; } -async function setupAudioAutoDetectCase(stdout: string): Promise<{ +async function setupAudioAutoDetectCase(stdout?: string): Promise<{ ctx: MsgContext; cfg: OpenClawConfig; }> { @@ -235,13 +235,27 @@ async function setupAudioAutoDetectCase(stdout: string): Promise<{ content: createSafeAudioFixtureBuffer(2048), }); const cfg: OpenClawConfig = { tools: { media: { audio: {} } } }; - mockedRunExec.mockResolvedValueOnce({ - stdout, - stderr: "", - }); + if (stdout !== undefined) { + mockedRunExec.mockResolvedValueOnce({ + stdout, + stderr: "", + }); + } return { ctx, cfg }; } +function mockWhisperCliTranscript(transcript: string) { + mockedRunExec.mockImplementationOnce(async (_command, args) => { + const outputBaseIndex = args.indexOf("-of"); + const outputBase = outputBaseIndex >= 0 ? args[outputBaseIndex + 1] : undefined; + if (typeof outputBase !== "string") { + throw new Error("missing whisper-cli output base"); + } + await fs.writeFile(`${outputBase}.txt`, transcript); + return { stdout: "Transcribing with Whisper...\n", stderr: "" }; + }); +} + async function applyWithDisabledMedia(params: { body: string; mediaPath: string; @@ -840,7 +854,8 @@ describe("applyMediaUnderstanding", () => { const modelPath = path.join(modelDir, "tiny.bin"); await fs.writeFile(modelPath, "model"); - const { ctx, cfg } = await setupAudioAutoDetectCase("whisper cpp ok\n"); + const { ctx, cfg } = await setupAudioAutoDetectCase(); + mockWhisperCliTranscript("whisper cpp ok\n"); await withMediaAutoDetectEnv( { @@ -889,10 +904,7 @@ describe("applyMediaUnderstanding", () => { await fs.writeFile(wavPath, Buffer.from("RIFF")); return ""; }); - mockedRunExec.mockResolvedValueOnce({ - stdout: "whisper cpp ogg ok\n", - stderr: "", - }); + mockWhisperCliTranscript("whisper cpp ogg ok\n"); await withMediaAutoDetectEnv( { diff --git a/src/media-understanding/runner.cli-audio.test.ts b/src/media-understanding/runner.cli-audio.test.ts index 0e823d05e0d..2a1c9548efd 100644 --- a/src/media-understanding/runner.cli-audio.test.ts +++ b/src/media-understanding/runner.cli-audio.test.ts @@ -1,8 +1,10 @@ // CLI audio runner tests cover prompt/language templating and command execution // options for local transcription binaries. import fs from "node:fs/promises"; +import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/types.js"; +import { withEnvAsync } from "../test-utils/env.js"; import { CLI_OUTPUT_MAX_BUFFER } from "./defaults.constants.js"; import { withAudioFixture } from "./runner.test-utils.js"; @@ -14,6 +16,65 @@ vi.mock("../process/exec.js", () => ({ let runCliEntry: typeof import("./runner.entries.js").runCliEntry; +type TranscriptFileCase = { + name: string; + command: string; + args: string[]; + resolvePath: (args: string[]) => string; +}; + +const transcriptFileCases: TranscriptFileCase[] = [ + { + name: "whisper.cpp short flags", + command: "whisper-cli", + args: ["-otxt", "-of", "{{OutputBase}}", "{{MediaPath}}"], + resolvePath: (args) => `${args[2]}.txt`, + }, + { + name: "whisper.cpp long flags", + command: "whisper-cli", + args: ["--output-txt", "--output-file={{OutputBase}}", "{{MediaPath}}"], + resolvePath: (args) => `${args[1]?.slice("--output-file=".length)}.txt`, + }, + { + name: "OpenAI Whisper explicit txt", + command: "whisper", + args: ["{{MediaPath}}", "--output_format=txt", "--output_dir={{OutputDir}}"], + resolvePath: (args) => + path.join( + args[2]?.slice("--output_dir=".length) ?? "", + `${path.parse(args[0] ?? "").name}.txt`, + ), + }, + { + name: "OpenAI Whisper default all output", + command: "whisper", + args: ["-o", "{{OutputDir}}", "{{MediaPath}}"], + resolvePath: (args) => path.join(args[1] ?? "", `${path.parse(args[2] ?? "").name}.txt`), + }, + { + name: "parakeet txt output", + command: "parakeet-mlx", + args: ["{{MediaPath}}", "--output-format", "txt", "--output-dir", "{{OutputDir}}"], + resolvePath: (args) => path.join(args[4] ?? "", `${path.parse(args[0] ?? "").name}.txt`), + }, + { + name: "parakeet all output with default template", + command: "parakeet-mlx", + args: [ + "{{MediaPath}}", + "--output-format=all", + "--output-dir={{OutputDir}}", + "--output-template={filename}", + ], + resolvePath: (args) => + path.join( + args[2]?.slice("--output-dir=".length) ?? "", + `${path.parse(args[0] ?? "").name}.txt`, + ), + }, +]; + function requireFirstRunExecCall(): unknown[] { const [call] = runExecMock.mock.calls; if (!call) { @@ -22,6 +83,25 @@ function requireFirstRunExecCall(): unknown[] { return call; } +async function runAudioEntry(params: { + command: string; + args: string[]; +}): Promise>> { + let result: Awaited> = null; + await withAudioFixture(`openclaw-cli-${params.command}`, async ({ ctx, cache }) => { + result = await runCliEntry({ + capability: "audio", + entry: { type: "cli", command: params.command, args: params.args }, + cfg: { tools: { media: { audio: {} } } } as OpenClawConfig, + ctx, + attachmentIndex: 0, + cache, + config: {} as never, + }); + }); + return result; +} + describe("media-understanding CLI audio entry", () => { beforeAll(async () => { ({ runCliEntry } = await import("./runner.entries.js")); @@ -84,6 +164,100 @@ describe("media-understanding CLI audio entry", () => { }); }); + it.each(transcriptFileCases)("reads $name transcript output", async (testCase) => { + runExecMock.mockImplementationOnce(async (_command, args: string[]) => { + await fs.writeFile(testCase.resolvePath(args), "file transcript\n"); + return { stdout: "Transcribing...\n", stderr: "" }; + }); + + const result = await runAudioEntry(testCase); + + expect(result?.text).toBe("file transcript"); + }); + + it("reads parakeet txt output selected through its upstream environment", async () => { + const testCase: TranscriptFileCase = { + name: "parakeet environment output", + command: "parakeet-mlx", + args: ["{{MediaPath}}", "--output-dir", "{{OutputDir}}"], + resolvePath: (args) => path.join(args[2] ?? "", `${path.parse(args[0] ?? "").name}.txt`), + }; + runExecMock.mockImplementationOnce(async (_command, args: string[]) => { + await fs.writeFile(testCase.resolvePath(args), "environment transcript\n"); + return { stdout: "Transcribing...\n", stderr: "" }; + }); + + const result = await withEnvAsync( + { PARAKEET_OUTPUT_FORMAT: "txt", PARAKEET_OUTPUT_TEMPLATE: undefined }, + async () => await runAudioEntry(testCase), + ); + + expect(result?.text).toBe("environment transcript"); + }); + + it.each( + transcriptFileCases.flatMap((testCase) => + (["empty", "missing"] as const).map((fileState) => Object.assign({ fileState }, testCase)), + ), + )("treats $fileState $name transcript output as empty", async (testCase) => { + runExecMock.mockImplementationOnce(async (_command, args: string[]) => { + if (testCase.fileState === "empty") { + await fs.writeFile(testCase.resolvePath(args), " \n"); + } + return { stdout: "Transcribing with Whisper...\n", stderr: "" }; + }); + + await expect(runAudioEntry(testCase)).resolves.toBeNull(); + }); + + it.each([ + { + name: "generic Node wrapper", + command: "node", + args: [ + "./skills/local-whisper/transcribe.js", + "{{MediaPath}}", + "--output-dir", + "{{OutputDir}}", + ], + }, + { + name: "parakeet default srt output", + command: "parakeet-mlx", + args: ["{{MediaPath}}", "--output-dir", "{{OutputDir}}"], + }, + { + name: "parakeet custom output template", + command: "parakeet-mlx", + args: [ + "{{MediaPath}}", + "--output-format", + "txt", + "--output-dir", + "{{OutputDir}}", + "--output-template", + "custom-{filename}", + ], + }, + ])("preserves stdout for $name without an inferred file contract", async (testCase) => { + const result = await runAudioEntry(testCase); + + expect(result?.text).toBe("cli transcript"); + }); + + it("surfaces unexpected transcript file read errors", async () => { + const testCase = transcriptFileCases[0]; + if (!testCase) { + throw new Error("missing transcript file test case"); + } + runExecMock.mockImplementationOnce(async (_command, args: string[]) => { + await fs.mkdir(testCase.resolvePath(args)); + return { stdout: "Transcribing...\n", stderr: "" }; + }); + + await expect(runAudioEntry(testCase)).rejects.toMatchObject({ code: "EISDIR" }); + }); + it("treats sherpa structured JSON with empty text as empty output", async () => { runExecMock.mockResolvedValueOnce({ stdout: diff --git a/src/media-understanding/runner.entries.ts b/src/media-understanding/runner.entries.ts index 0bf640a9faa..1e92a95d2f8 100644 --- a/src/media-understanding/runner.entries.ts +++ b/src/media-understanding/runner.entries.ts @@ -33,6 +33,7 @@ import type { MediaUnderstandingModelConfig, } from "../config/types.tools.js"; import { logVerbose, shouldLogVerbose } from "../globals.js"; +import { hasErrnoCode } from "../infra/errors.js"; import { writeExternalFileWithinRoot } from "../infra/fs-safe.js"; import { resolveProxyFetchFromEnv } from "../infra/net/proxy-fetch.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; @@ -51,7 +52,6 @@ import { DEFAULT_TIMEOUT_SECONDS, MIN_AUDIO_FILE_BYTES, } from "./defaults.constants.js"; -import { fileExists } from "./fs.js"; import { normalizeImageDescriptionInput } from "./image-input-normalize.js"; import { describeImageWithModel } from "./image-runtime.js"; import { resolveOpenAiAudioAuthModelApi } from "./openai-audio-api.js"; @@ -155,13 +155,22 @@ function isAntigravityCliCommand(command: string): boolean { } function findArgValue(args: string[], keys: string[]): string | undefined { - for (let i = 0; i < args.length; i += 1) { - if (keys.includes(args[i] ?? "")) { - const value = args[i + 1]; + for (const [index, arg] of args.entries()) { + if (keys.includes(arg)) { + const value = args[index + 1]; if (value) { return value; } } + for (const key of keys) { + const prefix = `${key}=`; + if (arg.startsWith(prefix)) { + const value = arg.slice(prefix.length); + if (value) { + return value; + } + } + } } return undefined; } @@ -172,16 +181,14 @@ function hasArg(args: string[], keys: string[]): boolean { function resolveWhisperOutputPath(args: string[], mediaPath: string): string | null { const outputDir = findArgValue(args, ["--output_dir", "-o"]); - const outputFormat = findArgValue(args, ["--output_format"]); - if (!outputDir || !outputFormat) { + if (!outputDir) { return null; } - const formats = outputFormat.split(",").map((value) => value.trim()); - if (!formats.includes("txt")) { + const outputFormat = findArgValue(args, ["--output_format", "-f"]) ?? "all"; + if (outputFormat !== "txt" && outputFormat !== "all") { return null; } - const base = path.parse(mediaPath).name; - return path.join(outputDir, `${base}.txt`); + return path.join(outputDir, `${path.parse(mediaPath).name}.txt`); } function resolveWhisperCppOutputPath(args: string[]): string | null { @@ -197,15 +204,30 @@ function resolveWhisperCppOutputPath(args: string[]): string | null { function resolveParakeetOutputPath(args: string[], mediaPath: string): string | null { const outputDir = findArgValue(args, ["--output-dir"]); - const outputFormat = findArgValue(args, ["--output-format"]); - if (!outputDir) { + const outputFormat = + findArgValue(args, ["--output-format"]) ?? (process.env.PARAKEET_OUTPUT_FORMAT || "srt"); + const outputTemplate = + findArgValue(args, ["--output-template"]) ?? + (process.env.PARAKEET_OUTPUT_TEMPLATE || "{filename}"); + if ( + !outputDir || + (outputFormat !== "txt" && outputFormat !== "all") || + outputTemplate !== "{filename}" + ) { return null; } - if (outputFormat && outputFormat !== "txt") { - return null; + return path.join(outputDir, `${path.parse(mediaPath).name}.txt`); +} + +async function readCliTranscriptFile(filePath: string): Promise { + try { + return (await fs.readFile(filePath, "utf8")).trim(); + } catch (error) { + if (hasErrnoCode(error, "ENOENT")) { + return ""; + } + throw error; } - const base = path.parse(mediaPath).name; - return path.join(outputDir, `${base}.txt`); } async function resolveCliOutput(params: { @@ -223,13 +245,10 @@ async function resolveCliOutput(params: { : commandId === "parakeet-mlx" ? resolveParakeetOutputPath(params.args, params.mediaPath) : null; - if (fileOutput && (await fileExists(fileOutput))) { - try { - const content = await fs.readFile(fileOutput, "utf8"); - if (content.trim()) { - return content.trim(); - } - } catch {} + if (fileOutput) { + // A known file-output contract is authoritative: falling back would expose + // progress/status stdout as user speech when transcription is empty or missing. + return await readCliTranscriptFile(fileOutput); } if (commandId === "gemini") {