From 4994ec450134ed826e6e4fb63f6a277aee53d4a1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 27 Sep 2026 15:52:14 -0700 Subject: [PATCH] fix(openai): retire the Sora video provider after the API shutdown (#159543) OpenAI shut down its Sora video API: sora-2 and sora-2-pro report shutdown_date 2026-09-24 and POST/GET /v1/videos now return HTTP 404 for every project. Every video_generate call routed to openai/* failed with an opaque "OpenAI video generation failed (HTTP 404)", and because the bundled openai plugin still advertised a configured video provider, agents on OpenAI-only installs kept selecting it. Remove the OpenAI video-generation provider, its manifest contract and metadata, live-test defaults and workflow filters, and the docs that advertised Sora. Stale openai/sora-* refs need no migration: the media runtime already skips them with "No video-generation provider registered for openai" and continues to configured fallbacks or auto-detected providers. --- .../openclaw-live-and-e2e-checks-reusable.yml | 2 +- config/assertion-safety-baseline.txt | 1 - docs/.i18n/glossary.zh-CN.json | 4 +- docs/help/testing-live/media-providers.md | 12 +- .../architecture-internals/new-capability.md | 10 +- docs/plugins/architecture.md | 2 +- docs/plugins/reference/openai.md | 2 +- docs/providers/openai.md | 26 +- docs/providers/openai/coverage-and-cost.md | 1 - docs/providers/openai/image-and-video.md | 47 +- docs/tools/video-generation.md | 12 +- .../deepinfra/video-generation-provider.ts | 2 +- extensions/openai/README.md | 2 +- extensions/openai/binary-transport.test.ts | 394 +++------ extensions/openai/index.ts | 4 - extensions/openai/openclaw.plugin.json | 10 - .../openai/video-generation-provider.test.ts | 772 ------------------ .../openai/video-generation-provider.ts | 392 --------- .../video-generation-providers.live.test.ts | 24 +- src/agents/tools/video-generate-tool.test.ts | 10 +- .../plugin-registration-contract-cases.ts | 1 - .../live-test-helpers.test.ts | 15 +- src/video-generation/live-test-helpers.ts | 3 +- taxonomy.yaml | 10 +- .../media/hosted-media-provider-live.test.ts | 13 +- .../media/hosted-media-provider-live.ts | 6 +- .../package-acceptance-workflow.test.ts | 2 +- 27 files changed, 178 insertions(+), 1601 deletions(-) delete mode 100644 extensions/openai/video-generation-provider.test.ts delete mode 100644 extensions/openai/video-generation-provider.ts diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 77599dc0d573..6c7b8ab87c43 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -4790,7 +4790,7 @@ jobs: - suite_id: native-live-extensions-media-video-c suite_group: native-live-extensions-media-video label: Native live media video plugins C - command: OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=openai,openrouter,xai node .release-harness/scripts/test-live-shard.mjs native-live-extensions-media-video + command: OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=openrouter,xai node .release-harness/scripts/test-live-shard.mjs native-live-extensions-media-video timeout_minutes: 30 profile_env_only: false profiles: full diff --git a/config/assertion-safety-baseline.txt b/config/assertion-safety-baseline.txt index 608b700a926d..f40768e41300 100644 --- a/config/assertion-safety-baseline.txt +++ b/config/assertion-safety-baseline.txt @@ -768,7 +768,6 @@ extensions/openai/openai-chatgpt-oauth-token.runtime.ts 1 extensions/openai/provider-policy-api.ts 2 extensions/openai/realtime-voice-bridge.ts 3 extensions/openai/realtime-voice-session-policy.ts 4 -extensions/openai/video-generation-provider.ts 1 extensions/opencode-go/reasoning-sanitizer.ts 2 extensions/opencode-go/stream-termination.ts 8 extensions/opencode/provider-catalog.ts 2 diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index 515319e29b9a..a70c2de4a53a 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -3800,8 +3800,8 @@ "target": "OpenAI 覆盖范围与费用" }, { - "source": "OpenAI image and video generation", - "target": "OpenAI 图像与视频生成" + "source": "OpenAI image generation", + "target": "OpenAI 图像生成" }, { "source": "OpenAI voice and speech", diff --git a/docs/help/testing-live/media-providers.md b/docs/help/testing-live/media-providers.md index a877577cf328..aea6ebf1871f 100644 --- a/docs/help/testing-live/media-providers.md +++ b/docs/help/testing-live/media-providers.md @@ -100,7 +100,7 @@ request. Plugin dependencies are expected to be present before runtime load. - Enable: `OPENCLAW_LIVE_TEST=1 pnpm test:live -- extensions/video-generation-providers.live.test.ts` - Harness: `pnpm test:live:media video` - Scope: - - Exercises the shared bundled video-generation provider path across `alibaba`, `byteplus`, `deepinfra`, `fal`, `google`, `minimax`, `openai`, `openrouter`, `pixverse`, `qwen`, `runway`, `together`, `vydra`, `xai` + - Exercises the shared bundled video-generation provider path across `alibaba`, `byteplus`, `deepinfra`, `fal`, `google`, `minimax`, `openrouter`, `pixverse`, `qwen`, `runway`, `together`, `vydra`, `xai` - Defaults to the release-safe smoke path: one text-to-video request per provider, one-second lobster prompt, and a per-provider operation cap from `OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS` (`180000` by default) - Skips FAL by default because provider-side queue latency can dominate release time; pass `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="fal"` (or clear the skip list) to run it explicitly - Uses already-exported provider env vars before probing @@ -123,10 +123,10 @@ request. Plugin dependencies are expected to be present before runtime load. - Current `videoToVideo` live coverage: - `runway` only when the selected model resolves to `gen4_aleph` - Current declared-but-skipped `videoToVideo` providers in the shared sweep: - - `alibaba`, `google`, `openai`, `qwen`, `xai` because those paths currently require remote `http(s)` reference URLs rather than buffer-backed local input + - `alibaba`, `google`, `qwen`, `xai` because those paths currently require remote `http(s)` reference URLs rather than buffer-backed local input - Optional narrowing: - - `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="deepinfra,google,openai,runway"` - - `OPENCLAW_LIVE_VIDEO_GENERATION_MODELS="google/veo-3.1-fast-generate-preview,openai/sora-2,runway/gen4_aleph"` + - `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="deepinfra,google,runway,xai"` + - `OPENCLAW_LIVE_VIDEO_GENERATION_MODELS="google/veo-3.1-fast-generate-preview,xai/grok-imagine-video,runway/gen4_aleph"` - `OPENCLAW_LIVE_VIDEO_GENERATION_SKIP_PROVIDERS=""` to include every provider in the default sweep, including FAL - `OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS=60000` to reduce each provider operation cap for an aggressive smoke run - Optional auth behavior: @@ -147,6 +147,6 @@ request. Plugin dependencies are expected to be present before runtime load. - `--quiet` / `--no-quiet` passed through to `test:live` - Examples: - `pnpm test:live:media` - - `pnpm test:live:media image video --providers openai,google,minimax` - - `pnpm test:live:media video --video-providers openai,runway --all-providers` + - `pnpm test:live:media image video --providers google,minimax,xai` + - `pnpm test:live:media video --video-providers runway,xai --all-providers` - `pnpm test:live:media music --quiet` diff --git a/docs/plugins/architecture-internals/new-capability.md b/docs/plugins/architecture-internals/new-capability.md index ff52e81d7450..5fd656b268cf 100644 --- a/docs/plugins/architecture-internals/new-capability.md +++ b/docs/plugins/architecture-internals/new-capability.md @@ -69,11 +69,11 @@ export type VideoGenerationProviderPlugin = { // plugin API api.registerVideoGenerationProvider({ - id: "openai", - label: "OpenAI", + id: "xai", + label: "xAI", async generateVideo(req) { - // generateOpenAiVideo is a placeholder for your own vendor call. - return await generateOpenAiVideo(req); + // generateXaiVideo is a placeholder for your own vendor call. + return await generateXaiVideo(req); }, }); @@ -89,7 +89,7 @@ lookups such as `providerContractPluginIds`; tests assert a plugin's `contracts.videoGenerationProviders` list matches what it actually registers): ```ts -expect(pluginManifest.contracts?.videoGenerationProviders).toEqual(["openai"]); +expect(pluginManifest.contracts?.videoGenerationProviders).toEqual(["xai"]); ``` That keeps the rule simple: diff --git a/docs/plugins/architecture.md b/docs/plugins/architecture.md index b997e299fc49..8141b15b6bc0 100644 --- a/docs/plugins/architecture.md +++ b/docs/plugins/architecture.md @@ -600,7 +600,7 @@ That means: - `google` owns text inference, CLI backend, embeddings, speech, realtime voice, media understanding, image/music/video generation, and web search. `openai` owns text inference, embeddings, speech, realtime transcription, realtime voice, media understanding, image/video generation. `minimax` owns text inference plus media understanding, speech, image/music/video generation, and web search. + `google` owns text inference, CLI backend, embeddings, speech, realtime voice, media understanding, image/music/video generation, and web search. `openai` owns text inference, embeddings, speech, realtime transcription, realtime voice, media understanding, image generation. `minimax` owns text inference plus media understanding, speech, image/music/video generation, and web search. `arcee` and `chutes` own text inference only; `microsoft` owns speech only. A vendor plugin can stay this narrow until it needs to cover more of that vendor's surface. diff --git a/docs/plugins/reference/openai.md b/docs/plugins/reference/openai.md index 6bdbbb63bf31..b7d0f6685c89 100644 --- a/docs/plugins/reference/openai.md +++ b/docs/plugins/reference/openai.md @@ -20,7 +20,7 @@ Adds OpenAI model provider support to OpenClaw. ## Surface - Providers: `openai` -- Contracts: `embeddingProviders`, `imageGenerationProviders`, `mediaUnderstandingProviders`, `realtimeTranscriptionProviders`, `realtimeVoiceProviders`, `speechProviders`, `usageProviders`, `videoGenerationProviders` +- Contracts: `embeddingProviders`, `imageGenerationProviders`, `mediaUnderstandingProviders`, `realtimeTranscriptionProviders`, `realtimeVoiceProviders`, `speechProviders`, `usageProviders` ## Related docs diff --git a/docs/providers/openai.md b/docs/providers/openai.md index 40e63f282855..50a7af5aad4a 100644 --- a/docs/providers/openai.md +++ b/docs/providers/openai.md @@ -30,17 +30,17 @@ workflows like OpenClaw. This page is an index. OpenAI is documented on nine pages, one per reader job. Open the page that matches your task. -| Page | Read it when | -| ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| [OpenAI setup](/providers/openai/setup) | You are connecting an account: the API-key and Codex subscription paths, route summaries, OAuth recovery, and the long-context opt-in. | -| [OpenAI authentication](/providers/openai/authentication) | Choose Codex login, an API key, or SIWC based on model access, plugins, usage tracking, and permissions. | -| [OpenAI models](/providers/openai/models) | You are choosing a model ref: the quick-choice table, GPT-6 Astra, Sol, Luna, and the GPT-5.6 tiers. | -| [OpenAI runtimes and Codex auth](/providers/openai/runtimes) | You need to know which runtime runs an `openai/*` turn, and how native Codex resolves its account. | -| [OpenAI coverage and cost](/providers/openai/coverage-and-cost) | You want the capability matrix, memory embeddings, or how subscription quota and Platform billing are reported. | -| [OpenAI image and video generation](/providers/openai/image-and-video) | You are generating or editing images and video through the bundled `openai` plugin. | -| [OpenAI voice and speech](/providers/openai/voice-and-speech) | You are configuring text-to-speech, transcription, or realtime voice, including per-route auth order. | -| [Azure OpenAI endpoints](/providers/openai/azure) | You are pointing the bundled `openai` provider at an Azure OpenAI resource. | -| [OpenAI advanced configuration](/providers/openai/advanced) | You are tuning prompt contribution, transport, Fast mode, compaction, strict-agentic mode, or proxy compat. | +| Page | Read it when | +| --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| [OpenAI setup](/providers/openai/setup) | You are connecting an account: the API-key and Codex subscription paths, route summaries, OAuth recovery, and the long-context opt-in. | +| [OpenAI authentication](/providers/openai/authentication) | Choose Codex login, an API key, or SIWC based on model access, plugins, usage tracking, and permissions. | +| [OpenAI models](/providers/openai/models) | You are choosing a model ref: the quick-choice table, GPT-6 Astra, Sol, Luna, and the GPT-5.6 tiers. | +| [OpenAI runtimes and Codex auth](/providers/openai/runtimes) | You need to know which runtime runs an `openai/*` turn, and how native Codex resolves its account. | +| [OpenAI coverage and cost](/providers/openai/coverage-and-cost) | You want the capability matrix, memory embeddings, or how subscription quota and Platform billing are reported. | +| [OpenAI image generation](/providers/openai/image-and-video) | You are generating or editing images through the bundled `openai` plugin. | +| [OpenAI voice and speech](/providers/openai/voice-and-speech) | You are configuring text-to-speech, transcription, or realtime voice, including per-route auth order. | +| [Azure OpenAI endpoints](/providers/openai/azure) | You are pointing the bundled `openai` provider at an Azure OpenAI resource. | +| [OpenAI advanced configuration](/providers/openai/advanced) | You are tuning prompt contribution, transport, Fast mode, compaction, strict-agentic mode, or proxy compat. | ## Where each section moved @@ -91,10 +91,10 @@ working. Each entry points at the page that now holds the content. - [OpenClaw feature coverage](/providers/openai/coverage-and-cost#openclaw-feature-coverage) - [Memory embeddings](/providers/openai/coverage-and-cost#memory-embeddings) -**[OpenAI image and video generation](/providers/openai/image-and-video)** +**[OpenAI image generation](/providers/openai/image-and-video)** - [Image generation](/providers/openai/image-and-video#image-generation) -- [Video generation](/providers/openai/image-and-video#video-generation) +- [Supported video providers](/tools/video-generation) **[OpenAI voice and speech](/providers/openai/voice-and-speech)** diff --git a/docs/providers/openai/coverage-and-cost.md b/docs/providers/openai/coverage-and-cost.md index 127daf7f941d..05788a4e7a71 100644 --- a/docs/providers/openai/coverage-and-cost.md +++ b/docs/providers/openai/coverage-and-cost.md @@ -35,7 +35,6 @@ changing config. | Codex app-server harness | Codex-compatible HTTPS route with runtime unset/`auto`, or explicit `agentRuntime.id: codex` | Yes | | Server-side web search | Native OpenAI Responses tool | Yes, when web search is enabled and no other provider is pinned | | Images | `image_generate` | Yes | -| Videos | `video_generate` | Yes | | Text-to-speech | `tts.provider: "openai"` / `tts` | Yes | | Batch speech-to-text | `tools.media.audio` / media understanding | Yes | | Streaming speech-to-text | Voice Call `streaming.provider: "openai"` | Yes | diff --git a/docs/providers/openai/image-and-video.md b/docs/providers/openai/image-and-video.md index 1425d7e21fb7..35bc2e915b34 100644 --- a/docs/providers/openai/image-and-video.md +++ b/docs/providers/openai/image-and-video.md @@ -1,11 +1,10 @@ --- -summary: "Generate and edit images with gpt-image, and generate video with Sora" +summary: "Generate and edit images with OpenAI gpt-image" read_when: - You are generating or editing images through the openai provider - You need transparent-background image output - - You are generating video with the video_generate tool -title: "OpenAI image and video generation" -sidebarTitle: "Image and video" +title: "OpenAI image generation" +sidebarTitle: "Image generation" --- ## Image generation @@ -152,42 +151,4 @@ Edit: /tool image_generate model=openai/gpt-image-2 prompt="Preserve the object shape, change the material to translucent glass" image=/path/to/reference.png size=1024x1536 ``` -## Video generation - -The bundled `openai` plugin registers video generation through the -`video_generate` tool. - -| Capability | Value | -| ---------------- | ---------------------------------------------------------------------------------- | -| Default model | `openai/sora-2` | -| Modes | Text-to-video, image-to-video, single-video edit | -| Reference inputs | 1 image or 1 video | -| Size overrides | Supported for text-to-video and image-to-video | -| Aspect ratio | Converted to the closest supported size, not forwarded raw | -| Other overrides | `resolution`, `audio`, `watermark` are unsupported and dropped with a tool warning | - -OpenAI image-to-video requests use `POST /v1/videos` with an image -`input_reference`. Single-video edits use `POST /v1/videos/edits` with the -uploaded video in the `video` field. - -```json5 -{ - agents: { - defaults: { - mediaModels: { video: { primary: "openai/sora-2" } }, - }, - }, -} -``` - - -See [Video Generation](/tools/video-generation) for shared tool parameters, -provider selection, and failover behavior. - -The OpenAI provider declares `supportsSize` but not `supportsAspectRatio` or -`supportsResolution`. OpenClaw's shared normalization layer converts a -requested `aspectRatio` into the closest matching OpenAI `size` before the -request reaches the provider, so aspect-ratio requests generally still work. -`resolution` has no size fallback and is dropped, surfaced to the caller as -`Ignored unsupported overrides for openai/: resolution=`. - +OpenAI retired its Sora video API on 2026-09-24; see [Video generation](/tools/video-generation) for supported providers. diff --git a/docs/tools/video-generation.md b/docs/tools/video-generation.md index 00fce19e2aaa..08bbc52ec9e2 100644 --- a/docs/tools/video-generation.md +++ b/docs/tools/video-generation.md @@ -1,5 +1,5 @@ --- -summary: "Generate videos via video_generate from text, image, or video references across 16 provider backends" +summary: "Generate videos via video_generate from text, image, or video references across 15 provider backends" read_when: - Generating videos via the agent - Configuring video-generation providers and models @@ -9,7 +9,7 @@ sidebarTitle: "Video generation" --- OpenClaw agents generate videos from text prompts, reference images, or -existing videos through `video_generate`. Sixteen provider backends are +existing videos through `video_generate`. Fifteen provider backends are supported; the agent picks the right one automatically based on config and available API keys. @@ -104,7 +104,6 @@ of failing the task if local persistence rejects an oversized file. | fal | `fal-ai/minimax/video-01-live` | ✓ | 1 image; up to 9 with Seedance reference-to-video | Up to 3 videos with Seedance reference-to-video | `FAL_KEY` | | Google | `veo-3.1-fast-generate-preview` | ✓ | 1 image | 1 video | `GEMINI_API_KEY` | | MiniMax | `MiniMax-Hailuo-2.3` | ✓ | 1 image | - | `MINIMAX_API_KEY` or MiniMax OAuth | -| OpenAI | `sora-2` | ✓ | 1 image | 1 video | `OPENAI_API_KEY` | | OpenRouter | `google/veo-3.1-fast` | ✓ | Up to 4 images (first/last frame or references) | - | `OPENROUTER_API_KEY` | | Qwen | `wan2.6-t2v` | ✓ | Yes (remote URL) | Yes (remote URL) | `QWEN_API_KEY` | | Runway | `gen4.5` | ✓ | 1 image | 1 video | `RUNWAYML_API_SECRET` | @@ -132,7 +131,6 @@ the shared live sweep: | fal | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` only when using Seedance reference-to-video | | Google | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; shared `videoToVideo` skipped because the current buffer-backed Gemini/Veo sweep does not accept that input | | MiniMax | ✓ | ✓ | - | `generate`, `imageToVideo` | -| OpenAI | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; shared `videoToVideo` skipped because this org/input path needs provider-side video edit access | | OpenRouter | ✓ | ✓ | - | `generate`, `imageToVideo` | | Qwen | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider needs remote `http(s)` video URLs | | Runway | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` runs only when the selected model is `runway/gen4_aleph` | @@ -386,11 +384,6 @@ OpenClaw does not append auto-detected providers. resolutions; requests such as `720P` are normalized to the closest supported value before submission. - - Only `size` override is forwarded. Other style overrides - (`aspectRatio`, `resolution`, `audio`, `watermark`) are ignored with - a warning. - Uses OpenRouter's asynchronous `/videos` API. OpenClaw submits the job, polls `polling_url`, and downloads either `unsigned_urls` or the @@ -547,7 +540,6 @@ openclaw config set agents.defaults.mediaModels.video.primary "qwen/wan2.6-t2v" - [Google (Gemini)](/providers/google) - [MiniMax](/providers/minimax) - [Models](/concepts/models) -- [OpenAI](/providers/openai) - [OpenRouter](/providers/openrouter) - [Qwen](/providers/qwen) - [Runway](/providers/runway) diff --git a/extensions/deepinfra/video-generation-provider.ts b/extensions/deepinfra/video-generation-provider.ts index a4818cf3b7f2..5b582dae7307 100644 --- a/extensions/deepinfra/video-generation-provider.ts +++ b/extensions/deepinfra/video-generation-provider.ts @@ -45,7 +45,7 @@ const POLL_INTERVAL_MS = 5_000; const MAX_POLL_ATTEMPTS = 120; // /v1/openai/videos is async: POST returns a job, GET /{id} polls until the -// job leaves the queue. Mirrors the OpenAI Sora surface (extensions/openai). +// job succeeds or fails, then the result contains downloadable video URLs. type DeepInfraVideoStatus = "queued" | "processing" | "succeeded" | "failed"; type DeepInfraVideoJob = { diff --git a/extensions/openai/README.md b/extensions/openai/README.md index 53a861fb80e5..2c6ed8eb13af 100644 --- a/extensions/openai/README.md +++ b/extensions/openai/README.md @@ -1,7 +1,7 @@ # OpenAI Connect OpenAI models to OpenClaw. The plugin also provides embeddings, media -understanding, image and video generation, speech output, realtime transcription, +understanding, image generation, speech output, realtime transcription, and realtime voice. ## Get started diff --git a/extensions/openai/binary-transport.test.ts b/extensions/openai/binary-transport.test.ts index b1ac5bb52d01..173cc2f06c3b 100644 --- a/extensions/openai/binary-transport.test.ts +++ b/extensions/openai/binary-transport.test.ts @@ -1,5 +1,3 @@ -import { createCapturedPluginRegistration } from "openclaw/plugin-sdk/plugin-test-runtime"; -import * as providerHttp from "openclaw/plugin-sdk/provider-http"; import { createDebugProxyCaptureReaderAsync, finalizeDebugProxyCaptureAsync, @@ -11,314 +9,138 @@ import { createOpenClawTestState } from "openclaw/plugin-sdk/test-state"; import { describe, expect, it, vi } from "vitest"; import { installDebugProxyTestResetHooks } from "../test-support/debug-proxy-env-test-helpers.js"; import { buildOpenAISpeechProvider } from "./speech-provider.js"; -import { buildOpenAIVideoGenerationProvider } from "./video-generation-provider.js"; const proxyReset = installDebugProxyTestResetHooks(); -const modelAuth = createCapturedPluginRegistration().api.runtime.modelAuth; -async function requestMedia( +async function requestAudio( baseUrl: string, - kind: "audio" | "video", - options: { timeoutMs?: number; mediaMaxMb?: number; reference?: "image" | "video" } = {}, + options: { timeoutMs?: number; mediaMaxMb?: number } = {}, ) { const budget = options.mediaMaxMb === undefined ? {} : { agents: { defaults: { mediaMaxMb: options.mediaMaxMb } } }; - if (kind === "audio") { - const result = await buildOpenAISpeechProvider().synthesize({ - text: "local binary acceptance", - cfg: budget, - providerConfig: { - apiKey: "local-test-key", - baseUrl: `${baseUrl}/v1`, - model: "tts-1", - voice: "alloy", - }, - target: "audio-file", - timeoutMs: options.timeoutMs ?? 5_000, - }); - return result.audioBuffer; - } - const result = await buildOpenAIVideoGenerationProvider(modelAuth).generateVideo({ - provider: "openai", - model: "sora-2", - prompt: "local binary acceptance", - cfg: { - ...budget, - models: { - providers: { - openai: { - apiKey: "local-test-key", - baseUrl: `${baseUrl}/v1`, - models: [], - request: { allowPrivateNetwork: true }, - }, - }, - }, + const result = await buildOpenAISpeechProvider().synthesize({ + text: "local binary acceptance", + cfg: budget, + providerConfig: { + apiKey: "local-test-key", + baseUrl: `${baseUrl}/v1`, + model: "tts-1", + voice: "alloy", }, + target: "audio-file", timeoutMs: options.timeoutMs ?? 5_000, - ...(options.reference === "image" - ? { inputImages: [{ buffer: Buffer.from("image"), mimeType: "image/png" }] } - : {}), - ...(options.reference === "video" - ? { inputVideos: [{ buffer: Buffer.from("video"), mimeType: "video/mp4" }] } - : {}), }); - return result.videos[0]?.buffer; + return result.audioBuffer; } describe("production OpenAI binary transport", () => { - it.each(["audio", "video"] as const)( - "rejects invalid %s over TCP and preserves valid codec parameters", - async (kind) => { - const type = kind === "audio" ? "audio/ogg" : "video/mp4"; - const cases = [ - { header: "image/png", body: "wrong family", valid: false }, - { header: "", body: "empty header", valid: false }, - { header: `${type}; charset=utf-8, text/html`, body: "hidden error", valid: false }, - { header: [type, "application/json"], body: "repeated header", valid: false }, - { header: type, body: "", valid: false }, - { header: "application/ogg", body: "Ogg container bytes", valid: kind === "audio" }, - { header: "application/octet-stream", body: "opaque bytes", valid: true }, - { header: "binary/octet-stream", body: "opaque alias bytes", valid: true }, - { header: undefined, body: "missing header bytes", valid: true }, - { header: `${type}; codecs="one, two"`, body: "codec bytes", valid: true }, - { header: `${type};; codecs="one, two";`, body: "empty parameter slots", valid: true }, - ]; - for (const fixture of cases) { - await withServer( - (request, response) => { - request.resume(); - if (request.url === "/v1/videos") { - response.setHeader("Content-Type", "application/json"); - response.end(JSON.stringify({ id: "local-video", status: "completed" })); - } else { - if (fixture.header !== undefined) { - response.setHeader("Content-Type", fixture.header); - } - response.end(fixture.body); - } - }, - async (baseUrl) => { - const result = requestMedia(baseUrl, kind); - if (fixture.valid) { - await expect(result).resolves.toEqual(Buffer.from(fixture.body)); - } else { - await expect(result).rejects.toThrow(`malformed ${kind} response`); - } - }, - ); - } - }, - ); - - it.each(["audio", "video"] as const)( - "preserves %s byte limits and transport timeouts over TCP", - async (kind) => { - for (const stalled of [false, true]) { - let closed = false; - await withServer( - (request, response) => { - request.resume(); - if (request.url === "/v1/videos") { - response.writeHead(200, { "Content-Type": "application/json" }); - response.end(JSON.stringify({ id: "budget-video", status: "completed" })); - } else { - request.socket.once("close", () => { - closed = true; - }); - response.writeHead(200, { - "Content-Type": kind === "audio" ? "audio/mpeg" : "video/mp4", - }); - response.write(stalled ? Buffer.from([1]) : Buffer.alloc(4096)); - } - }, - async (baseUrl) => { - const result = requestMedia(baseUrl, kind, { - mediaMaxMb: 0.001, - timeoutMs: stalled ? 300 : 5_000, - }); - if (stalled) { - await expect(result).rejects.toThrow(/timed out|aborted/i); - } else { - await expect(result).rejects.toThrow( - kind === "audio" - ? "OpenAI TTS audio response exceeds" - : "OpenAI generated video download exceeds", - ); - } - await vi.waitFor(() => expect(closed).toBe(true)); - }, - ); - } - }, - ); - - it.each([ - { status: "queued", reference: "image" }, - { status: "completed", reference: "video" }, - ] as const)( - "releases $status $reference submission before real follow-up transport", - async ({ status, reference }) => { - const originalPost = providerHttp.postMultipartRequest; - let releases = 0; - let released = false; - let clone: Response | undefined; - const paths: string[] = []; - const releasedAtFollowUp: boolean[] = []; - const post = vi - .spyOn(providerHttp, "postMultipartRequest") - .mockImplementation(async (params) => { - const handle = await originalPost(params); - clone = handle.response.clone(); - return { - ...handle, - release: async () => { - releases += 1; - await handle.release(); - released = true; - }, - }; - }); - try { - await withServer( - (request, response) => { - request.resume(); - paths.push(request.url ?? ""); - if (request.method === "GET") { - releasedAtFollowUp.push(released); - } - if (request.url?.includes("/content")) { - response.setHeader("Content-Type", "video/mp4"); - response.end("rendered-video"); - } else { - response.setHeader("Content-Type", "application/json"); - response.end( - JSON.stringify({ - id: "release-video", - status: request.method === "POST" ? status : "completed", - }), - ); - } - }, - async (baseUrl) => { - const result = await requestMedia(baseUrl, "video", { reference }); - expect(result).toEqual(Buffer.from("rendered-video")); - expect(paths[0]).toBe(reference === "video" ? "/v1/videos/edits" : "/v1/videos"); - expect(paths).toHaveLength(status === "queued" ? 3 : 2); - expect(releases).toBe(1); - expect(releasedAtFollowUp).toEqual(status === "queued" ? [true, true] : [true]); - }, - ); - } finally { - post.mockRestore(); - void clone?.body?.cancel().catch(() => undefined); - } - }, - ); - - it.each([ - { - label: "HTTP failure", - status: 400, - body: '{"error":{"message":"submission refused"}}', - error: "submission refused", - }, - { label: "malformed JSON", status: 200, body: "{", error: "malformed JSON response" }, - { label: "missing id", status: 200, body: '{"status":"queued"}', error: "missing video id" }, - { - label: "failed job", - status: 200, - body: '{"status":"failed","error":{"message":"job refused"}}', - error: "job refused", - }, - ])("releases failed submission exactly once: $label", async ({ status, body, error }) => { - const originalPost = providerHttp.postMultipartRequest; - let releases = 0; - let requests = 0; - const post = vi - .spyOn(providerHttp, "postMultipartRequest") - .mockImplementation(async (params) => { - const handle = await originalPost(params); - return { - ...handle, - release: async () => { - releases += 1; - await handle.release(); - }, - }; - }); - try { + it("rejects invalid audio over TCP and preserves valid codec parameters", async () => { + const type = "audio/ogg"; + const cases = [ + { header: "image/png", body: "wrong family", valid: false }, + { header: "", body: "empty header", valid: false }, + { header: `${type}; charset=utf-8, text/html`, body: "hidden error", valid: false }, + { header: [type, "application/json"], body: "repeated header", valid: false }, + { header: type, body: "", valid: false }, + { header: "application/ogg", body: "Ogg container bytes", valid: true }, + { header: "application/octet-stream", body: "opaque bytes", valid: true }, + { header: "binary/octet-stream", body: "opaque alias bytes", valid: true }, + { header: undefined, body: "missing header bytes", valid: true }, + { header: `${type}; codecs="one, two"`, body: "codec bytes", valid: true }, + { header: `${type};; codecs="one, two";`, body: "empty parameter slots", valid: true }, + ]; + for (const fixture of cases) { await withServer( (request, response) => { - requests += 1; request.resume(); - response.writeHead(status, { "Content-Type": "application/json" }); - response.end(body); + if (fixture.header !== undefined) { + response.setHeader("Content-Type", fixture.header); + } + response.end(fixture.body); }, async (baseUrl) => { - await expect(requestMedia(baseUrl, "video")).rejects.toThrow(error); - expect(releases).toBe(1); - expect(requests).toBe(1); + const result = requestAudio(baseUrl); + if (fixture.valid) { + await expect(result).resolves.toEqual(Buffer.from(fixture.body)); + } else { + await expect(result).rejects.toThrow("malformed audio response"); + } }, ); - } finally { - post.mockRestore(); } }); - it.each(["audio", "video"] as const)( - "persists %s capture through finalization and closes the rejected upstream socket", - async (kind) => { - proxyReset.captureProxyEnv(); - const state = await createOpenClawTestState({ layout: "state-only", prefix: "binary-tcp-" }); - vi.stubEnv("OPENCLAW_DEBUG_PROXY_ENABLED", "1"); - vi.stubEnv("OPENCLAW_DEBUG_PROXY_SESSION_ID", `binary-${kind}`); + it("preserves audio byte limits and transport timeouts over TCP", async () => { + for (const stalled of [false, true]) { let closed = false; - let mediaResponses = 0; - try { - await initializeDebugProxyCaptureAsync("test"); - await withServer( - (request, response) => { - request.resume(); - if (request.url === "/v1/videos") { - response.setHeader("Content-Type", "application/json"); - response.end(JSON.stringify({ id: "local-video", status: "completed" })); - } else if (mediaResponses++ === 0) { - response.writeHead(200, { - "Content-Type": kind === "audio" ? "audio/ogg" : "video/mp4", - }); - response.end("captured valid media"); - } else { - request.socket.once("close", () => { - closed = true; - }); - response.writeHead(200, { "Content-Type": "image/png" }); - response.write("not the requested media"); - // Leave the body open: capture must not own the caller's completion. - } - }, - async (baseUrl) => { - await expect(requestMedia(baseUrl, kind)).resolves.toEqual( - Buffer.from("captured valid media"), - ); - await expect(requestMedia(baseUrl, kind)).rejects.toThrow(`malformed ${kind} response`); - await vi.waitFor(() => expect(closed).toBe(true)); - await finalizeDebugProxyCaptureAsync(); - await closeOpenClawStateDatabaseAsync(); - const reopened = createDebugProxyCaptureReaderAsync({ env: process.env }); - const persisted = await reopened.getSessionEvents(`binary-${kind}`, 20); - expect(persisted.some((event) => event.kind === "request")).toBe(true); - expect(persisted.some((event) => event.kind === "response")).toBe(true); - expect(persisted).toHaveLength(kind === "audio" ? 4 : 8); - }, - ); - } finally { - await finalizeDebugProxyCaptureAsync(); - vi.unstubAllEnvs(); - await state.cleanup(); - } - }, - ); + await withServer( + (request, response) => { + request.resume(); + request.socket.once("close", () => { + closed = true; + }); + response.writeHead(200, { "Content-Type": "audio/mpeg" }); + response.write(stalled ? Buffer.from([1]) : Buffer.alloc(4096)); + }, + async (baseUrl) => { + const result = requestAudio(baseUrl, { + mediaMaxMb: 0.001, + timeoutMs: stalled ? 300 : 5_000, + }); + if (stalled) { + await expect(result).rejects.toThrow(/timed out|aborted/i); + } else { + await expect(result).rejects.toThrow("OpenAI TTS audio response exceeds"); + } + await vi.waitFor(() => expect(closed).toBe(true)); + }, + ); + } + }); + + it("persists audio capture through finalization and closes the rejected upstream socket", async () => { + proxyReset.captureProxyEnv(); + const state = await createOpenClawTestState({ layout: "state-only", prefix: "binary-tcp-" }); + vi.stubEnv("OPENCLAW_DEBUG_PROXY_ENABLED", "1"); + vi.stubEnv("OPENCLAW_DEBUG_PROXY_SESSION_ID", "binary-audio"); + let closed = false; + let mediaResponses = 0; + try { + await initializeDebugProxyCaptureAsync("test"); + await withServer( + (request, response) => { + request.resume(); + if (mediaResponses++ === 0) { + response.writeHead(200, { "Content-Type": "audio/ogg" }); + response.end("captured valid media"); + } else { + request.socket.once("close", () => { + closed = true; + }); + response.writeHead(200, { "Content-Type": "image/png" }); + response.write("not the requested media"); + // Leave the body open: capture must not own the caller's completion. + } + }, + async (baseUrl) => { + await expect(requestAudio(baseUrl)).resolves.toEqual(Buffer.from("captured valid media")); + await expect(requestAudio(baseUrl)).rejects.toThrow("malformed audio response"); + await vi.waitFor(() => expect(closed).toBe(true)); + await finalizeDebugProxyCaptureAsync(); + await closeOpenClawStateDatabaseAsync(); + const reopened = createDebugProxyCaptureReaderAsync({ env: process.env }); + const persisted = await reopened.getSessionEvents("binary-audio", 20); + expect(persisted.some((event) => event.kind === "request")).toBe(true); + expect(persisted.some((event) => event.kind === "response")).toBe(true); + expect(persisted).toHaveLength(4); + }, + ); + } finally { + await finalizeDebugProxyCaptureAsync(); + vi.unstubAllEnvs(); + await state.cleanup(); + } + }); }); diff --git a/extensions/openai/index.ts b/extensions/openai/index.ts index a43ecd587d59..fad253339586 100644 --- a/extensions/openai/index.ts +++ b/extensions/openai/index.ts @@ -18,7 +18,6 @@ import { OPENAI_QUICKSILVER_OFFER_PATH } from "./realtime-quicksilver-session.js import { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider-factory.js"; import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider-factory.js"; import { buildOpenAISpeechProvider } from "./speech-provider.js"; -import { buildOpenAIVideoGenerationProvider } from "./video-generation-provider.js"; export default definePluginEntry({ id: "openai", @@ -92,8 +91,5 @@ export default definePluginEntry({ }); api.registerSpeechProvider(buildOpenAISpeechProvider()); api.registerMediaUnderstandingProvider(openaiMediaUnderstandingProvider); - api.registerVideoGenerationProvider( - buildOpenAIVideoGenerationProvider({ isProviderApiKeyConfigured }), - ); }, }); diff --git a/extensions/openai/openclaw.plugin.json b/extensions/openai/openclaw.plugin.json index da744cf58566..43f796d57b31 100644 --- a/extensions/openai/openclaw.plugin.json +++ b/extensions/openai/openclaw.plugin.json @@ -377,7 +377,6 @@ "embeddingProviders": ["openai"], "mediaUnderstandingProviders": ["openai"], "imageGenerationProviders": ["openai"], - "videoGenerationProviders": ["openai"], "usageProviders": ["openai"] }, "imageGenerationProviderMetadata": { @@ -389,15 +388,6 @@ ] } }, - "videoGenerationProviderMetadata": { - "openai": { - "authSignals": [ - { - "provider": "openai" - } - ] - } - }, "mediaUnderstandingProviderMetadata": { "openai": { "capabilities": ["image", "audio"], diff --git a/extensions/openai/video-generation-provider.test.ts b/extensions/openai/video-generation-provider.test.ts deleted file mode 100644 index 78c053f39783..000000000000 --- a/extensions/openai/video-generation-provider.test.ts +++ /dev/null @@ -1,772 +0,0 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { - clearRuntimeAuthProfileStoreSnapshots, - saveAuthProfileStore, -} from "openclaw/plugin-sdk/agent-runtime"; -import { createCapturedPluginRegistration } from "openclaw/plugin-sdk/plugin-test-runtime"; -import { - getProviderHttpMocks, - installProviderHttpMockCleanup, -} from "openclaw/plugin-sdk/provider-http-test-mocks"; -import { expectExplicitVideoGenerationCapabilities } from "openclaw/plugin-sdk/provider-test-contracts"; -import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; -import { withServer } from "openclaw/plugin-sdk/test-env"; -import type { VideoGenerationRequest } from "openclaw/plugin-sdk/video-generation"; -import { beforeAll, describe, expect, it, vi } from "vitest"; - -const { - resolveApiKeyForProviderMock, - postJsonRequestMock, - postMultipartRequestMock, - fetchWithTimeoutMock, - fetchWithTimeoutGuardedMock, - pollProviderOperationJsonMock, - assertOkOrThrowHttpErrorMock, - executeProviderOperationWithRetryMock, - resolveProviderHttpRequestConfigMock, - sanitizeConfiguredModelProviderRequestMock, -} = getProviderHttpMocks(); - -let buildOpenAIVideoGenerationProvider: typeof import("./video-generation-provider.js").buildOpenAIVideoGenerationProvider; - -let modelAuth: Parameters[0]; - -beforeAll(async () => { - modelAuth = createCapturedPluginRegistration().api.runtime.modelAuth; - ({ buildOpenAIVideoGenerationProvider } = await import("./video-generation-provider.js")); -}); - -installProviderHttpMockCleanup(); - -function generateVideo( - overrides: Partial & Pick, -) { - return buildOpenAIVideoGenerationProvider(modelAuth).generateVideo({ - provider: "openai", - model: "sora-2", - cfg: {}, - ...overrides, - }); -} - -function localVideoConfig(allowPrivateNetwork?: boolean, baseUrl = "http://127.0.0.1:44080/v1") { - return { - models: { - providers: { - openai: { - baseUrl, - ...(allowPrivateNetwork === undefined ? {} : { request: { allowPrivateNetwork } }), - models: [], - }, - }, - }, - }; -} - -function videoJob(id: string, status: string, fields: { seconds?: string; size?: string } = {}) { - return { id, model: "sora-2", status, ...fields }; -} - -function releasedJson(value: unknown, release = vi.fn(async () => {})) { - return { response: Response.json(value), release }; -} - -function postMultipartRequest(index = 0): Record { - const request = postMultipartRequestMock.mock.calls[index]?.[0] as - | Record - | undefined; - if (!request) { - throw new Error(`expected postMultipartRequest call ${index}`); - } - return request; -} - -function fetchWithTimeoutCall(index: number): [string, RequestInit | undefined, number, unknown] { - const call = fetchWithTimeoutMock.mock.calls[index] as - | [string, RequestInit | undefined, number, unknown] - | undefined; - if (!call) { - throw new Error(`expected fetchWithTimeout call ${index}`); - } - return call; -} - -function fetchWithTimeoutGuardedCall( - index = 0, -): [string, RequestInit | undefined, number, unknown, Record | undefined] { - const call = fetchWithTimeoutGuardedMock.mock.calls[index] as - | [string, RequestInit | undefined, number, unknown, Record | undefined] - | undefined; - if (!call) { - throw new Error(`expected fetchWithTimeoutGuarded call ${index}`); - } - return call; -} - -function pollProviderOperationRequest(index = 0): Record { - const request = pollProviderOperationJsonMock.mock.calls[index]?.[0] as - | Record - | undefined; - if (!request) { - throw new Error(`expected pollProviderOperationJson call ${index}`); - } - return request; -} - -function providerHttpConfigRequest(): Record { - const [call] = resolveProviderHttpRequestConfigMock.mock.calls; - if (!call) { - throw new Error("expected provider HTTP config request"); - } - const [request] = call; - if (!request || typeof request !== "object" || Array.isArray(request)) { - throw new Error("expected provider HTTP config request"); - } - return request as Record; -} - -function streamedVideoResponse(bytes: string): Response { - return new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode(bytes)); - controller.close(); - }, - }), - { headers: { "content-type": "video/mp4" } }, - ); -} - -describe("openai video generation provider", () => { - it("declares explicit mode capabilities", () => { - expectExplicitVideoGenerationCapabilities(buildOpenAIVideoGenerationProvider(modelAuth)); - }); - - it("does not claim size or duration controls for OpenAI video edits", () => { - const provider = buildOpenAIVideoGenerationProvider(modelAuth); - - expect(provider.capabilities.videoToVideo).toEqual({ - enabled: true, - maxVideos: 1, - maxInputVideos: 1, - }); - }); - - it("advertises OpenAI video for an actual config-only API key", () => { - expect( - buildOpenAIVideoGenerationProvider(modelAuth).isConfigured?.({ - cfg: { - models: { - providers: { - openai: { - apiKey: "openai-video-config-key", - baseUrl: "https://api.openai.com/v1", - models: [], - }, - }, - }, - }, - }), - ).toBe(true); - }); - - it("does not advertise video generation for OAuth-only OpenAI profiles", () => { - const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-openai-video-auth-")); - const previousOpenAIKey = process.env.OPENAI_API_KEY; - delete process.env.OPENAI_API_KEY; - try { - saveAuthProfileStore( - { - version: 1, - profiles: { - "openai:chatgpt": { - type: "oauth", - provider: "openai", - access: "chatgpt-oauth-token", - refresh: "refresh-token", - expires: Date.now() + 60_000, - }, - }, - }, - agentDir, - { filterExternalAuthProfiles: false, syncExternalCli: false }, - ); - - expect(buildOpenAIVideoGenerationProvider(modelAuth).isConfigured?.({ agentDir })).toBe( - false, - ); - } finally { - clearRuntimeAuthProfileStoreSnapshots(); - if (previousOpenAIKey === undefined) { - delete process.env.OPENAI_API_KEY; - } else { - process.env.OPENAI_API_KEY = previousOpenAIKey; - } - // Saving the profile store opens the per-agent database under the temporary agent - // dir, and clearing the snapshots does not release it, so Windows fails the removal - // with EBUSY unless the cached handles are closed first. - closeOpenClawAgentDatabasesForTest(); - fs.rmSync(agentDir, { recursive: true, force: true }); - } - }); - - it("requires an OpenAI API key credential for direct video generation", async () => { - resolveApiKeyForProviderMock.mockResolvedValueOnce({ - apiKey: "chatgpt-oauth-token", - mode: "oauth", - } as never); - - await expect( - generateVideo({ - prompt: "A paper airplane gliding through golden hour light", - }), - ).rejects.toThrow("OpenAI API key missing"); - - expect(resolveApiKeyForProviderMock).toHaveBeenCalledWith( - expect.objectContaining({ - provider: "openai", - modelApi: "openai-responses", - }), - ); - expect(postJsonRequestMock).not.toHaveBeenCalled(); - }); - - it("uses SDK-compatible multipart for text-only Sora requests", async () => { - const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now()); - try { - postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_123", "queued"))); - fetchWithTimeoutMock - .mockResolvedValueOnce( - Response.json(videoJob("vid_123", "completed", { seconds: "4", size: "720x1280" })), - ) - .mockResolvedValueOnce( - new Response(Buffer.from("webm-bytes"), { - headers: new Headers({ "content-type": "video/webm" }), - }), - ); - - const result = await generateVideo({ - prompt: "A paper airplane gliding through golden hour light", - durationSeconds: 4, - }); - - const createRequest = postMultipartRequest(); - expect(createRequest.url).toBe("https://api.openai.com/v1/videos"); - const form = createRequest.body as FormData; - expect(form.get("prompt")).toBe("A paper airplane gliding through golden hour light"); - expect(form.get("model")).toBe("sora-2"); - expect(form.get("seconds")).toBe("4"); - expect(form.get("input_reference")).toBeNull(); - const [pollUrl, pollInit, pollTimeout, pollFetch] = fetchWithTimeoutCall(0); - expect(pollUrl).toBe("https://api.openai.com/v1/videos/vid_123"); - expect(pollInit?.method).toBe("GET"); - expect(pollTimeout).toBe(120000); - expect(pollFetch).toBe(fetch); - expect(result.videos).toHaveLength(1); - expect(result.videos[0]?.mimeType).toBe("video/webm"); - expect(result.videos[0]?.fileName).toBe("video-1.webm"); - expect(result.metadata?.videoId).toBe("vid_123"); - expect(result.metadata?.status).toBe("completed"); - } finally { - clock.mockRestore(); - } - }); - - it.each([undefined])( - "surfaces an immediately failed OpenAI submission before polling or validating id (%s)", - async (videoId) => { - const release = vi.fn(async () => {}); - postMultipartRequestMock.mockResolvedValueOnce( - releasedJson( - { - ...(videoId ? { id: videoId } : {}), - status: "failed", - error: { message: "OpenAI video generation was rejected" }, - }, - release, - ), - ); - - await expect( - generateVideo({ - prompt: "A scene that cannot be generated", - }), - ).rejects.toThrow("OpenAI video generation was rejected"); - - expect(pollProviderOperationJsonMock).not.toHaveBeenCalled(); - expect(fetchWithTimeoutMock).not.toHaveBeenCalled(); - expect(release).toHaveBeenCalledOnce(); - }, - ); - - it("downloads an immediately completed OpenAI submission without polling it again", async () => { - const release = vi.fn(async () => {}); - const cancel = vi.fn(); - postMultipartRequestMock.mockResolvedValueOnce( - releasedJson( - videoJob("vid_completed", "completed", { seconds: "4", size: "720x1280" }), - release, - ), - ); - fetchWithTimeoutMock.mockResolvedValueOnce( - new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode("completed-video")); - controller.close(); - }, - cancel, - }), - { headers: { "content-type": "video/mp4" } }, - ), - ); - - const result = await generateVideo({ - prompt: "A scene already generated", - }); - - expect(pollProviderOperationJsonMock).not.toHaveBeenCalled(); - expect(fetchWithTimeoutMock).toHaveBeenCalledOnce(); - expect(result).toMatchObject({ - model: "sora-2", - metadata: { seconds: "4", size: "720x1280", status: "completed", videoId: "vid_completed" }, - }); - expect(cancel).not.toHaveBeenCalled(); - expect(release).toHaveBeenCalledOnce(); - }); - - it.each([ - { - label: "JSON error", - contentType: "application/json", - body: JSON.stringify({ error: "not a rendered video" }), - }, - { - label: "problem JSON error", - contentType: "application/problem+json", - body: JSON.stringify({ detail: "render failed" }), - }, - { label: "plain-text error", contentType: "text/plain", body: "render failed" }, - { label: "empty video", contentType: "video/mp4", body: "" }, - ])( - "rejects a successful $label download and releases both requests", - async ({ contentType, body }) => { - const submissionRelease = vi.fn(async () => {}); - const downloadRelease = vi.fn(async () => {}); - postMultipartRequestMock.mockResolvedValueOnce( - releasedJson(videoJob("vid_malformed", "completed"), submissionRelease), - ); - fetchWithTimeoutGuardedMock.mockResolvedValueOnce({ - response: new Response(body, { headers: { "content-type": contentType } }), - finalUrl: "http://127.0.0.1:44080/v1/videos/vid_malformed/content?variant=video", - release: downloadRelease, - }); - - await expect( - generateVideo({ - prompt: "Reject an invalid generated video", - cfg: localVideoConfig(true), - }), - ).rejects.toThrow("OpenAI generated video download: malformed video response"); - - expect(pollProviderOperationJsonMock).not.toHaveBeenCalled(); - expect(submissionRelease).toHaveBeenCalledOnce(); - expect(downloadRelease).toHaveBeenCalledOnce(); - }, - ); - - it.each([ - { mode: "public", allowPrivateNetwork: false }, - { mode: "guarded private", allowPrivateNetwork: true }, - ])( - "cancels unread malformed $mode video responses and closes their upstream socket", - async ({ allowPrivateNetwork }) => { - let notifySocketClosed: ((closed: boolean) => void) | undefined; - const socketClosed = new Promise((resolve) => { - notifySocketClosed = resolve; - }); - await withServer( - (request, response) => { - request.socket.once("close", () => notifySocketClosed?.(true)); - response.writeHead(200, { "content-type": "application/json" }); - response.write('{"error":"still streaming'); - }, - async (baseUrl) => { - postMultipartRequestMock.mockResolvedValueOnce( - releasedJson({ id: "vid_unread", status: "completed" }), - ); - const upstreamUrl = `${baseUrl}/videos/vid_unread/content`; - const downloadRelease = vi.fn(async () => {}); - if (allowPrivateNetwork) { - fetchWithTimeoutGuardedMock.mockImplementationOnce(async () => ({ - response: await fetch(upstreamUrl), - finalUrl: upstreamUrl, - release: downloadRelease, - })); - } else { - fetchWithTimeoutMock.mockImplementationOnce(async () => await fetch(upstreamUrl)); - } - - await expect( - generateVideo({ - prompt: "Reject an unending public video error response", - cfg: allowPrivateNetwork ? localVideoConfig(true, `${baseUrl}/v1`) : {}, - }), - ).rejects.toThrow("OpenAI generated video download: malformed video response"); - - await expect( - Promise.race([ - socketClosed, - new Promise((resolve) => { - setTimeout(() => resolve(false), 250); - }), - ]), - ).resolves.toBe(true); - if (allowPrivateNetwork) { - expect(fetchWithTimeoutGuardedMock).toHaveBeenCalledOnce(); - expect(downloadRelease).toHaveBeenCalledOnce(); - } else { - expect(fetchWithTimeoutGuardedMock).not.toHaveBeenCalled(); - } - }, - ); - }, - ); - - it.each([ - { mode: "public", allowPrivateNetwork: false }, - { mode: "guarded private", allowPrivateNetwork: true }, - ])( - "rejects cloned endless $mode video errors without waiting for capture cancellation", - async ({ allowPrivateNetwork }) => { - const response = new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode('{"error":"still streaming')); - }, - }), - { headers: { "content-type": "application/json" } }, - ); - const captureClone = response.clone(); - const submissionRelease = vi.fn(async () => {}); - const downloadRelease = vi.fn(async () => {}); - postMultipartRequestMock.mockResolvedValueOnce( - releasedJson({ id: "vid_cloned", status: "completed" }, submissionRelease), - ); - if (allowPrivateNetwork) { - fetchWithTimeoutGuardedMock.mockResolvedValueOnce({ - response, - finalUrl: "http://127.0.0.1:44080/v1/videos/vid_cloned/content", - release: downloadRelease, - }); - } else { - fetchWithTimeoutMock.mockResolvedValueOnce(response); - } - - const generation = generateVideo({ - prompt: "Reject a cloned, unending video error", - cfg: allowPrivateNetwork ? localVideoConfig(true) : {}, - }); - const captureCancellationPending = Symbol("capture cancellation pending"); - - try { - const result = await Promise.race([ - generation.then( - () => undefined, - (error: unknown) => error, - ), - new Promise((resolve) => { - setImmediate(() => resolve(captureCancellationPending)); - }), - ]); - - expect(result).not.toBe(captureCancellationPending); - expect(result).toMatchObject({ - message: "OpenAI generated video download: malformed video response", - }); - expect(submissionRelease).toHaveBeenCalledOnce(); - if (allowPrivateNetwork) { - expect(downloadRelease).toHaveBeenCalledOnce(); - } - } finally { - void captureClone.body?.cancel().catch(() => undefined); - await generation.catch(() => undefined); - } - }, - ); - - it.each(["application/json"])( - "keeps the malformed public video error when %s body cancellation fails", - async (contentType) => { - const cancel = vi.fn(async () => { - throw new Error("upstream cancellation failed"); - }); - const submissionRelease = vi.fn(async () => {}); - postMultipartRequestMock.mockResolvedValueOnce( - releasedJson({ id: "vid_cancel_failed", status: "completed" }, submissionRelease), - ); - fetchWithTimeoutMock.mockResolvedValueOnce( - new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode("still streaming")); - }, - cancel, - }), - { headers: { "content-type": contentType } }, - ), - ); - - await expect( - generateVideo({ - prompt: "Preserve the malformed public video response error", - }), - ).rejects.toThrow("OpenAI generated video download: malformed video response"); - - expect(cancel).toHaveBeenCalledOnce(); - expect(submissionRelease).toHaveBeenCalledOnce(); - expect(fetchWithTimeoutGuardedMock).not.toHaveBeenCalled(); - }, - ); - - it("rejects generated video downloads that exceed the configured media cap", async () => { - postMultipartRequestMock.mockResolvedValueOnce( - releasedJson(videoJob("vid_too_large", "queued")), - ); - fetchWithTimeoutMock - .mockResolvedValueOnce(Response.json(videoJob("vid_too_large", "completed"))) - .mockResolvedValueOnce(streamedVideoResponse("too-large")); - - await expect( - generateVideo({ - prompt: "short video", - cfg: { agents: { defaults: { mediaMaxMb: 0.000001 } } }, - }), - ).rejects.toThrow("OpenAI generated video download exceeds 1 bytes"); - }); - - it("uploads the SDK-compatible image reference in a multipart video request", async () => { - const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now()); - try { - postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_456", "queued"))); - fetchWithTimeoutMock - .mockResolvedValueOnce(Response.json(videoJob("vid_456", "completed"))) - .mockResolvedValueOnce( - new Response(Buffer.from("mp4-bytes"), { - headers: new Headers({ "content-type": "video/mp4" }), - }), - ); - - const input = Buffer.from("!png-bytes?").subarray(1, -1); - await generateVideo({ - prompt: "Animate this frame", - inputImages: [{ buffer: input, mimeType: "image/png" }], - }); - input.fill(0); - - const createRequest = postMultipartRequest(); - expect(createRequest.url).toBe("https://api.openai.com/v1/videos"); - const form = createRequest.body as FormData; - const reference = form.get("input_reference"); - expect(reference).toBeInstanceOf(File); - const referenceFile = reference as File; - expect(referenceFile.name).toBe("reference-image.png"); - expect(referenceFile.type).toBe("image/png"); - expect(Buffer.from(await referenceFile.arrayBuffer())).toEqual(Buffer.from("png-bytes")); - const [pollUrl, pollInit, pollTimeout, pollFetch] = fetchWithTimeoutCall(0); - expect(pollUrl).toBe("https://api.openai.com/v1/videos/vid_456"); - expect(pollInit?.method).toBe("GET"); - expect(pollTimeout).toBe(120000); - expect(pollFetch).toBe(fetch); - } finally { - clock.mockRestore(); - } - }); - - it("keeps configured local baseUrl private-network blocked unless explicitly enabled", async () => { - postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_local", "queued"))); - fetchWithTimeoutMock - .mockResolvedValueOnce(Response.json(videoJob("vid_local", "completed"))) - .mockResolvedValueOnce( - new Response(Buffer.from("mp4-bytes"), { - headers: new Headers({ "content-type": "video/mp4" }), - }), - ); - - await generateVideo({ - prompt: "Render via local relay", - cfg: localVideoConfig(), - }); - - expect(providerHttpConfigRequest().baseUrl).toBe("http://127.0.0.1:44080/v1"); - expect(providerHttpConfigRequest().request).toBeUndefined(); - const createRequest = postMultipartRequest(); - expect(createRequest.url).toBe("http://127.0.0.1:44080/v1/videos"); - expect(createRequest.allowPrivateNetwork).toBe(false); - }); - - it("honors configured request allowPrivateNetwork for local video providers", async () => { - postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_local", "queued"))); - fetchWithTimeoutMock - .mockResolvedValueOnce(Response.json(videoJob("vid_local", "completed"))) - .mockResolvedValueOnce({ - headers: new Headers({ "content-type": "video/mp4" }), - arrayBuffer: async () => Buffer.from("mp4-bytes"), - }); - - await generateVideo({ - prompt: "Render via local relay", - cfg: localVideoConfig(true), - }); - - expect(sanitizeConfiguredModelProviderRequestMock).toHaveBeenCalledWith({ - allowPrivateNetwork: true, - }); - expect(providerHttpConfigRequest().baseUrl).toBe("http://127.0.0.1:44080/v1"); - expect(providerHttpConfigRequest().request).toEqual({ allowPrivateNetwork: true }); - const createRequest = postMultipartRequest(); - expect(createRequest.url).toBe("http://127.0.0.1:44080/v1/videos"); - expect(createRequest.allowPrivateNetwork).toBe(true); - const statusRequest = pollProviderOperationRequest(); - expect(statusRequest.url).toBe("http://127.0.0.1:44080/v1/videos/vid_local"); - expect(statusRequest.allowPrivateNetwork).toBe(true); - expect(statusRequest.auditContext).toBe("openai-video-status"); - const [downloadUrl, downloadInit, downloadTimeout, downloadFetch, downloadOptions] = - fetchWithTimeoutGuardedCall(); - expect(downloadUrl).toBe("http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video"); - expect(downloadInit?.method).toBe("GET"); - // Download shares the generation deadline, so earlier phases consume part of this budget. - expect(downloadTimeout).toBeGreaterThan(0); - expect(downloadTimeout).toBeLessThanOrEqual(120_000); - expect(downloadFetch).toBe(fetch); - expect(downloadOptions).toEqual({ - ssrfPolicy: { allowPrivateNetwork: true }, - auditContext: "openai-video-download", - }); - }); - - it("retries guarded local video downloads after transient HTTP errors", async () => { - const firstRelease = vi.fn(async () => {}); - const secondRelease = vi.fn(async () => {}); - assertOkOrThrowHttpErrorMock - .mockImplementationOnce(async () => {}) - .mockImplementationOnce(async (_response, label) => { - throw Object.assign(new Error(label), { status: _response.status }); - }) - .mockImplementationOnce(async () => {}); - postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_local", "queued"))); - fetchWithTimeoutMock.mockResolvedValueOnce(Response.json(videoJob("vid_local", "completed"))); - fetchWithTimeoutGuardedMock - .mockResolvedValueOnce({ - response: new Response("busy", { status: 503, statusText: "Service Unavailable" }), - finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video", - release: firstRelease, - }) - .mockResolvedValueOnce({ - response: { - headers: new Headers({ "content-type": "video/mp4" }), - arrayBuffer: async () => Buffer.from("mp4-bytes"), - }, - finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video", - release: secondRelease, - }); - - const result = await generateVideo({ - prompt: "Render via local relay", - cfg: localVideoConfig(true), - }); - - expect(result.videos[0]?.buffer?.toString()).toBe("mp4-bytes"); - expect(executeProviderOperationWithRetryMock).toHaveBeenCalledWith( - expect.objectContaining({ provider: "openai", stage: "download" }), - ); - expect(postMultipartRequestMock).toHaveBeenCalledOnce(); - expect(fetchWithTimeoutGuardedMock).toHaveBeenCalledTimes(2); - expect(firstRelease).toHaveBeenCalledTimes(1); - expect(secondRelease).toHaveBeenCalledTimes(1); - }); - - it("releases guarded local video download requests when HTTP errors throw", async () => { - const firstRelease = vi.fn(async () => {}); - const secondRelease = vi.fn(async () => {}); - assertOkOrThrowHttpErrorMock - .mockImplementationOnce(async () => {}) - .mockImplementationOnce(async (_response, label) => { - throw Object.assign(new Error(label), { status: _response.status }); - }) - .mockImplementationOnce(async (_response, label) => { - throw Object.assign(new Error(label), { status: _response.status }); - }); - postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_local", "queued"))); - fetchWithTimeoutMock.mockResolvedValueOnce(Response.json(videoJob("vid_local", "completed"))); - fetchWithTimeoutGuardedMock - .mockResolvedValueOnce({ - response: new Response("busy", { status: 503, statusText: "Service Unavailable" }), - finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video", - release: firstRelease, - }) - .mockResolvedValueOnce({ - response: new Response("busy", { status: 503, statusText: "Service Unavailable" }), - finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video", - release: secondRelease, - }); - - await expect( - generateVideo({ - prompt: "Render via local relay", - cfg: localVideoConfig(true), - }), - ).rejects.toThrow("OpenAI video download failed"); - - expect(postMultipartRequestMock).toHaveBeenCalledOnce(); - expect(fetchWithTimeoutGuardedMock).toHaveBeenCalledTimes(2); - expect(firstRelease).toHaveBeenCalledTimes(1); - expect(secondRelease).toHaveBeenCalledTimes(1); - }); - - it("uses the video edits endpoint for video-to-video uploads", async () => { - fetchWithTimeoutMock - .mockResolvedValueOnce(Response.json(videoJob("vid_789", "queued"))) - .mockResolvedValueOnce(Response.json(videoJob("vid_789", "completed"))) - .mockResolvedValueOnce( - new Response(Buffer.from("mp4-bytes"), { - headers: new Headers({ "content-type": "video/mp4" }), - }), - ); - - const input = Buffer.from("!mp4-bytes?").subarray(1, -1); - await generateVideo({ - prompt: "Remix this clip", - inputVideos: [{ buffer: input, mimeType: "video/mp4" }], - }); - input.fill(0); - - expect(postJsonRequestMock).not.toHaveBeenCalled(); - const createRequest = postMultipartRequest(); - expect(createRequest.url).toBe("https://api.openai.com/v1/videos/edits"); - expect(createRequest.body).toBeInstanceOf(FormData); - const form = createRequest.body as FormData; - expect(form.get("prompt")).toBe("Remix this clip"); - expect(form.get("model")).toBeNull(); - expect(form.get("video")).toBeInstanceOf(File); - expect(Buffer.from(await (form.get("video") as File).arrayBuffer())).toEqual( - Buffer.from("mp4-bytes"), - ); - expect(form.get("input_reference")).toBeNull(); - expect(createRequest.timeoutMs).toBe(120000); - expect(createRequest.fetchFn).toBe(fetch); - expect(createRequest.allowPrivateNetwork).toBe(false); - }); - - it("rejects multiple reference assets", async () => { - await expect( - generateVideo({ - prompt: "Animate these", - inputImages: [{ buffer: Buffer.from("a"), mimeType: "image/png" }], - inputVideos: [{ buffer: Buffer.from("b"), mimeType: "video/mp4" }], - }), - ).rejects.toThrow("OpenAI video generation supports at most one reference image or video."); - }); -}); diff --git a/extensions/openai/video-generation-provider.ts b/extensions/openai/video-generation-provider.ts deleted file mode 100644 index 8426d0762a9f..000000000000 --- a/extensions/openai/video-generation-provider.ts +++ /dev/null @@ -1,392 +0,0 @@ -import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime"; -import { extensionForMime, type MediaKind } from "openclaw/plugin-sdk/media-mime"; -import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; -import type { - createProviderOperationDeadline, - postMultipartRequest, -} from "openclaw/plugin-sdk/provider-http"; -import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; -import type { - VideoGenerationProvider, - VideoGenerationRequest, -} from "openclaw/plugin-sdk/video-generation"; -import { resolveConfiguredOpenAIBaseUrl } from "./shared.js"; - -const DEFAULT_OPENAI_VIDEO_BASE_URL = "https://api.openai.com/v1"; -const DEFAULT_OPENAI_VIDEO_MODEL = "sora-2"; -const DEFAULT_TIMEOUT_MS = 120_000; -const POLL_INTERVAL_MS = 2_500; -const MAX_POLL_ATTEMPTS = 120; -const OPENAI_VIDEO_SECONDS = [4, 8, 12] as const; -const OPENAI_VIDEO_SIZES = ["720x1280", "1280x720", "1024x1792", "1792x1024"] as const; - -type OpenAIVideoRequestPolicy = { - allowPrivateNetwork: boolean; - dispatcherPolicy?: Parameters[0]["dispatcherPolicy"]; -}; - -type OpenAIVideoStatus = "queued" | "in_progress" | "completed" | "failed"; - -type OpenAIReferenceAsset = { - kind: Extract; - file: File; -}; - -type OpenAIVideoResponse = { - id?: string; - model?: string; - status?: OpenAIVideoStatus; - prompt?: string | null; - seconds?: string; - size?: string; - error?: { - code?: string; - message?: string; - } | null; -}; - -function readOpenAIVideoFailureMessage(payload: OpenAIVideoResponse): string | undefined { - return payload.status === "failed" - ? (normalizeOptionalString(payload.error?.message) ?? "OpenAI video generation failed") - : undefined; -} - -function resolveDurationSeconds(durationSeconds: number | undefined): "4" | "8" | "12" | undefined { - if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) { - return undefined; - } - const rounded = Math.max(OPENAI_VIDEO_SECONDS[0], Math.round(durationSeconds)); - const nearest = OPENAI_VIDEO_SECONDS.reduce((best, current) => - Math.abs(current - rounded) < Math.abs(best - rounded) ? current : best, - ); - return String(nearest) as "4" | "8" | "12"; -} - -function resolveSize(params: { - size?: string; - aspectRatio?: string; - resolution?: string; -}): (typeof OPENAI_VIDEO_SIZES)[number] | undefined { - const explicitSize = normalizeOptionalString(params.size); - const supportedSize = OPENAI_VIDEO_SIZES.find((size) => size === explicitSize); - if (supportedSize) { - return supportedSize; - } - switch (normalizeOptionalString(params.aspectRatio)) { - case "9:16": - return "720x1280"; - case "16:9": - return "1280x720"; - case "4:7": - return "1024x1792"; - case "7:4": - return "1792x1024"; - default: - break; - } - if (params.resolution === "1080P") { - return "1792x1024"; - } - return undefined; -} - -function resolveReferenceAsset(req: VideoGenerationRequest): OpenAIReferenceAsset | null { - const allAssets = [...(req.inputImages ?? []), ...(req.inputVideos ?? [])]; - if (allAssets.length === 0) { - return null; - } - if (allAssets.length > 1) { - throw new Error("OpenAI video generation supports at most one reference image or video."); - } - const [asset] = allAssets; - if (!asset?.buffer) { - throw new Error( - "OpenAI video generation currently requires local image/video uploads for reference assets.", - ); - } - const kind = (req.inputVideos?.length ?? 0) > 0 ? "video" : "image"; - const mimeType = - normalizeOptionalString(asset.mimeType) || (kind === "video" ? "video/mp4" : "image/png"); - const extension = - extensionForMime(mimeType)?.slice(1) ?? (mimeType.startsWith("video/") ? "mp4" : "png"); - const fileName = normalizeOptionalString(asset.fileName) || `reference-${kind}.${extension}`; - return { - kind, - file: new File([bufferToBlobPart(asset.buffer)], fileName, { type: mimeType }), - }; -} - -async function fetchOpenAIVideoDownload( - params: { - url: string; - init: RequestInit; - deadline: ReturnType; - fetchFn: typeof fetch; - } & OpenAIVideoRequestPolicy, -) { - const { - assertOkOrThrowHttpError, - createProviderOperationTimeoutResolver, - executeProviderOperationWithRetry, - fetchProviderDownloadResponse, - fetchWithTimeoutGuarded, - } = await import("openclaw/plugin-sdk/provider-http"); - const timeoutMs = createProviderOperationTimeoutResolver({ - deadline: params.deadline, - defaultTimeoutMs: params.deadline.timeoutMs ?? DEFAULT_TIMEOUT_MS, - }); - if (!params.allowPrivateNetwork && !params.dispatcherPolicy) { - const response = await fetchProviderDownloadResponse({ - url: params.url, - init: params.init, - deadline: params.deadline, - fetchFn: params.fetchFn, - provider: "openai", - requestFailedMessage: "OpenAI video download failed", - }); - return { - response, - release: async () => {}, - }; - } - - return await executeProviderOperationWithRetry({ - provider: "openai", - stage: "download", - operation: async () => { - const result = await fetchWithTimeoutGuarded( - params.url, - params.init, - timeoutMs(), - params.fetchFn, - { - ...(params.allowPrivateNetwork ? { ssrfPolicy: { allowPrivateNetwork: true } } : {}), - ...(params.dispatcherPolicy ? { dispatcherPolicy: params.dispatcherPolicy } : {}), - auditContext: "openai-video-download", - }, - ); - try { - await assertOkOrThrowHttpError(result.response, "OpenAI video download failed"); - return result; - } catch (error) { - await result.release(); - throw error; - } - }, - }); -} - -export function buildOpenAIVideoGenerationProvider({ - isProviderApiKeyConfigured, -}: Pick< - OpenClawPluginApi["runtime"]["modelAuth"], - "isProviderApiKeyConfigured" ->): VideoGenerationProvider { - return { - id: "openai", - label: "OpenAI", - defaultModel: DEFAULT_OPENAI_VIDEO_MODEL, - models: [DEFAULT_OPENAI_VIDEO_MODEL, "sora-2-pro"], - isConfigured: (ctx) => - isProviderApiKeyConfigured({ - provider: "openai", - ...ctx, - profileTypes: ["api_key"], - }), - capabilities: { - generate: { - maxVideos: 1, - maxDurationSeconds: 12, - supportedDurationSeconds: OPENAI_VIDEO_SECONDS, - supportsSize: true, - sizes: OPENAI_VIDEO_SIZES, - }, - imageToVideo: { - enabled: true, - maxVideos: 1, - maxInputImages: 1, - maxDurationSeconds: 12, - supportedDurationSeconds: OPENAI_VIDEO_SECONDS, - supportsSize: true, - sizes: OPENAI_VIDEO_SIZES, - }, - videoToVideo: { - enabled: true, - maxVideos: 1, - maxInputVideos: 1, - }, - }, - async generateVideo(req) { - const { resolveApiKeyForProvider } = - await import("openclaw/plugin-sdk/provider-auth-runtime"); - const auth = await resolveApiKeyForProvider({ - provider: "openai", - cfg: req.cfg, - agentDir: req.agentDir, - store: req.authStore, - modelApi: "openai-responses", - }); - if (!auth.apiKey || (auth.mode !== undefined && auth.mode !== "api-key")) { - throw new Error("OpenAI API key missing"); - } - - const [ - { - assertOkOrThrowHttpError, - createProviderOperationDeadline, - createProviderOperationTimeoutResolver, - pollProviderOperationJson, - postMultipartRequest, - readProviderJsonResponse, - resolveProviderOperationTimeoutMs, - resolveProviderHttpRequestConfig, - sanitizeConfiguredModelProviderRequest, - }, - { downloadGeneratedVideoAsset, resolveGeneratedMediaMaxBytes }, - ] = await Promise.all([ - import("openclaw/plugin-sdk/provider-http"), - import("openclaw/plugin-sdk/media-generation-runtime"), - ]); - const fetchFn = fetch; - const deadline = createProviderOperationDeadline({ - timeoutMs: req.timeoutMs, - label: "OpenAI video generation", - }); - const providerConfig = req.cfg.models?.providers?.openai; - const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = - resolveProviderHttpRequestConfig({ - baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg), - defaultBaseUrl: DEFAULT_OPENAI_VIDEO_BASE_URL, - request: sanitizeConfiguredModelProviderRequest(providerConfig?.request), - defaultHeaders: { - Authorization: `Bearer ${auth.apiKey}`, - }, - provider: "openai", - capability: "video", - transport: "http", - }); - - const model = normalizeOptionalString(req.model) ?? DEFAULT_OPENAI_VIDEO_MODEL; - const seconds = resolveDurationSeconds(req.durationSeconds); - const size = resolveSize(req); - const referenceAsset = resolveReferenceAsset(req); - const isVideoEdit = referenceAsset?.kind === "video"; - const form = new FormData(); - form.set("prompt", req.prompt); - if (isVideoEdit) { - form.set("video", referenceAsset.file); - } else { - form.set("model", model); - if (seconds) { - form.set("seconds", seconds); - } - if (size) { - form.set("size", size); - } - if (referenceAsset) { - form.set("input_reference", referenceAsset.file); - } - } - const multipartHeaders = new Headers(headers); - multipartHeaders.delete("Content-Type"); - const { response, release } = await postMultipartRequest({ - url: `${baseUrl}/videos${isVideoEdit ? "/edits" : ""}`, - headers: multipartHeaders, - body: form, - timeoutMs: resolveProviderOperationTimeoutMs({ - deadline, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, - }), - fetchFn, - allowPrivateNetwork, - dispatcherPolicy, - }); - - let submitted: OpenAIVideoResponse; - try { - await assertOkOrThrowHttpError(response, "OpenAI video generation failed"); - submitted = await readProviderJsonResponse( - response, - "OpenAI video generation failed", - ); - } finally { - // A consumed submission no longer owns transport during polling or download. - await release(); - } - const failureMessage = readOpenAIVideoFailureMessage(submitted); - if (failureMessage) { - throw new Error(failureMessage); - } - const videoId = normalizeOptionalString(submitted.id); - if (!videoId) { - throw new Error("OpenAI video generation response missing video id"); - } - const completed = - submitted.status === "completed" - ? submitted - : await pollProviderOperationJson({ - url: `${baseUrl}/videos/${videoId}`, - headers, - deadline: createProviderOperationDeadline({ - timeoutMs: resolveProviderOperationTimeoutMs({ - deadline, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, - }), - label: `OpenAI video generation task ${videoId}`, - }), - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, - fetchFn, - maxAttempts: MAX_POLL_ATTEMPTS, - pollIntervalMs: POLL_INTERVAL_MS, - requestFailedMessage: "OpenAI video status request failed", - timeoutMessage: `OpenAI video generation task ${videoId} did not finish in time`, - allowPrivateNetwork, - dispatcherPolicy, - auditContext: "openai-video-status", - isComplete: (payload) => payload.status === "completed", - getFailureMessage: readOpenAIVideoFailureMessage, - }); - const url = new URL(`${baseUrl}/videos/${videoId}/content`); - url.searchParams.set("variant", "video"); - const video = await downloadGeneratedVideoAsset({ - url: url.toString(), - timeoutMs: createProviderOperationTimeoutResolver({ - deadline, - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, - }), - defaultTimeoutMs: DEFAULT_TIMEOUT_MS, - fetchFn, - provider: "openai", - label: "OpenAI generated video download", - requestFailedMessage: "OpenAI video download failed", - maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "video"), - validateBinaryResponse: true, - fetchResponse: async ({ deadline: downloadDeadline }) => - await fetchOpenAIVideoDownload({ - url: url.toString(), - init: { - method: "GET", - headers: new Headers({ - ...Object.fromEntries(headers.entries()), - Accept: "application/binary", - }), - }, - deadline: downloadDeadline, - fetchFn, - allowPrivateNetwork, - dispatcherPolicy, - }), - }); - return { - videos: [video], - model: completed.model ?? submitted.model ?? model, - metadata: { - videoId, - status: completed.status, - seconds: completed.seconds ?? submitted.seconds, - size: completed.size ?? submitted.size, - }, - }; - }, - }; -} diff --git a/extensions/video-generation-providers.live.test.ts b/extensions/video-generation-providers.live.test.ts index 84cb729d26b6..f3acbc7032d7 100644 --- a/extensions/video-generation-providers.live.test.ts +++ b/extensions/video-generation-providers.live.test.ts @@ -50,7 +50,6 @@ import deepinfraPlugin from "./deepinfra/index.js"; import falPlugin from "./fal/index.js"; import googlePlugin from "./google/index.js"; import minimaxPlugin from "./minimax/index.js"; -import openaiPlugin from "./openai/index.js"; import openrouterPlugin from "./openrouter/index.js"; import pixversePlugin from "./pixverse/index.js"; import qwenPlugin from "./qwen/index.js"; @@ -123,7 +122,6 @@ const CASES: LiveProviderCase[] = [ pluginName: "MiniMax Provider", providerId: "minimax", }, - { plugin: openaiPlugin, pluginId: "openai", pluginName: "OpenAI Provider", providerId: "openai" }, { plugin: openrouterPlugin, pluginId: "openrouter", @@ -168,9 +166,9 @@ function withPluginsEnabled(cfg: OpenClawConfig): OpenClawConfig { }; } -function createEditReferencePng(params?: { width?: number; height?: number }): Buffer { - const width = params?.width ?? 384; - const height = params?.height ?? 384; +function createEditReferencePng(): Buffer { + const width = 384; + const height = 384; const buf = Buffer.alloc(width * height * 4, 255); for (let y = 0; y < height; y += 1) { @@ -229,11 +227,9 @@ function expectGeneratedVideo(video: GeneratedVideoAsset | undefined): LiveGener function buildLiveCapabilityOverrides(params: { caps: VideoGenerationModeCapabilities | undefined; liveResolution: VideoGenerationRequest["resolution"]; - liveSize: string | undefined; -}): Pick { - const { caps, liveResolution, liveSize } = params; +}): Pick { + const { caps, liveResolution } = params; return { - ...(caps?.supportsSize && liveSize ? { size: liveSize } : undefined), ...(caps?.supportsAspectRatio ? { aspectRatio: "16:9" } : undefined), ...(caps?.supportsResolution ? { resolution: liveResolution } : undefined), ...(caps?.supportsAudio ? { audio: false } : undefined), @@ -465,7 +461,6 @@ async function runLiveVideoProviderCase( providerId: testCase.providerId, modelRef, }); - const liveSize = testCase.providerId === "openai" ? "1280x720" : undefined; const logPrefix = `[live:video-generation] provider=${testCase.providerId} model=${providerModel}`; const generateAttempt = await runLiveVideoAttempt({ @@ -486,7 +481,7 @@ async function runLiveVideoProviderCase( authStore, timeoutMs: LIVE_VIDEO_OPERATION_TIMEOUT_MS, durationSeconds, - ...buildLiveCapabilityOverrides({ caps: generateCaps, liveResolution, liveSize }), + ...buildLiveCapabilityOverrides({ caps: generateCaps, liveResolution }), }, skipped, }); @@ -516,10 +511,7 @@ async function runLiveVideoProviderCase( return; } - const referenceImage = - testCase.providerId === "openai" - ? createEditReferencePng({ width: 1280, height: 720 }) - : createEditReferencePng(); + const referenceImage = createEditReferencePng(); const imageAttempt = await runLiveVideoAttempt({ authLabel, attempted, @@ -552,7 +544,6 @@ async function runLiveVideoProviderCase( ...buildLiveCapabilityOverrides({ caps: imageToVideoCaps, liveResolution, - liveSize, }), }, skipped, @@ -608,7 +599,6 @@ async function runLiveVideoProviderCase( ...buildLiveCapabilityOverrides({ caps: videoToVideoCaps, liveResolution, - liveSize: undefined, }), }, skipped, diff --git a/src/agents/tools/video-generate-tool.test.ts b/src/agents/tools/video-generate-tool.test.ts index f71bf03015fb..5851819963c5 100644 --- a/src/agents/tools/video-generate-tool.test.ts +++ b/src/agents/tools/video-generate-tool.test.ts @@ -309,13 +309,13 @@ describe("createVideoGenerateTool", () => { expect(emptyConfigTool).toBeNull(); }); - it("treats legacy OpenAI-Codex auth profiles as canonical OpenAI video auth", () => { + it("exposes video generation for an auth-backed video provider", () => { vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([]); expectVideoGenerateTool( createVideoGenerateTool({ config: asConfig({}), - authProfileStore: createAuthStore(["openai"]), + authProfileStore: createAuthStore(["runway"]), }), ); }); @@ -341,7 +341,7 @@ describe("createVideoGenerateTool", () => { const properties = toolParameterProperties( createVideoGenerateTool({ config: configWithDefaults({ - videoGenerationModel: { primary: "openai/sora-2" }, + videoGenerationModel: { primary: "runway/gen4.5" }, }), }), ); @@ -428,7 +428,7 @@ describe("createVideoGenerateTool", () => { const properties = toolParameterProperties( createVideoGenerateTool({ config: configWithDefaults({ - videoGenerationModel: { primary: "openai/sora-2" }, + videoGenerationModel: { primary: "runway/gen4.5" }, }), }), ); @@ -449,7 +449,7 @@ describe("createVideoGenerateTool", () => { }, agents: { defaults: { - videoGenerationModel: { primary: "openai/sora-2" }, + videoGenerationModel: { primary: "runway/gen4.5" }, }, }, }), diff --git a/src/plugin-sdk/test-helpers/plugin-registration-contract-cases.ts b/src/plugin-sdk/test-helpers/plugin-registration-contract-cases.ts index 00d7514f7339..de98c0072463 100644 --- a/src/plugin-sdk/test-helpers/plugin-registration-contract-cases.ts +++ b/src/plugin-sdk/test-helpers/plugin-registration-contract-cases.ts @@ -138,7 +138,6 @@ export const pluginRegistrationContractCases = { realtimeVoiceProviderIds: ["openai"], mediaUnderstandingProviderIds: ["openai"], imageGenerationProviderIds: ["openai"], - videoGenerationProviderIds: ["openai"], }, "opencode-go": { pluginId: "opencode-go", diff --git a/src/video-generation/live-test-helpers.test.ts b/src/video-generation/live-test-helpers.test.ts index ef1a034a5bf0..3e0872b79e27 100644 --- a/src/video-generation/live-test-helpers.test.ts +++ b/src/video-generation/live-test-helpers.test.ts @@ -23,16 +23,18 @@ describe("video-generation live-test helpers", () => { it("parses provider filters and treats empty/all as unfiltered", () => { expect(parseVideoProviderFilter()).toBeNull(); expect(parseVideoProviderFilter("all")).toBeNull(); - expect(parseVideoProviderFilter(" google , openai ")).toEqual(new Set(["google", "openai"])); + expect(parseVideoProviderFilter(" google , xai ")).toEqual(new Set(["google", "xai"])); }); it("parses provider model overrides by provider id", () => { expect( - parseProviderModelMap("google/veo-3.1-fast-generate-preview, openai/sora-2, invalid"), + parseProviderModelMap( + "google/veo-3.1-fast-generate-preview, xai/grok-imagine-video, invalid", + ), ).toEqual( new Map([ ["google", "google/veo-3.1-fast-generate-preview"], - ["openai", "openai/sora-2"], + ["xai", "xai/grok-imagine-video"], ]), ); }); @@ -44,7 +46,7 @@ describe("video-generation live-test helpers", () => { mediaModels: { video: { primary: "google/veo-3.1-fast-generate-preview", - fallbacks: ["openai/sora-2", "invalid"], + fallbacks: ["xai/grok-imagine-video", "invalid"], }, }, }, @@ -54,7 +56,7 @@ describe("video-generation live-test helpers", () => { expect(resolveConfiguredLiveVideoModels(cfg)).toEqual( new Map([ ["google", "google/veo-3.1-fast-generate-preview"], - ["openai", "openai/sora-2"], + ["xai", "xai/grok-imagine-video"], ]), ); }); @@ -62,7 +64,6 @@ describe("video-generation live-test helpers", () => { it("runs buffer-backed video-to-video only for supported providers/models", () => { for (const [providerId, modelRef, expected] of [ ["google", "google/veo-3.1-fast-generate-preview", false], - ["openai", "openai/sora-2", false], ["runway", "runway/gen4_aleph", true], ["runway", "runway/gen4.5", false], ["alibaba", "alibaba/wan2.6-r2v", false], @@ -77,7 +78,7 @@ describe("video-generation live-test helpers", () => { it("runs buffer-backed image-to-video only for providers that accept bundled image inputs", () => { for (const [providerId, modelRef, expected] of [ - ["openai", "openai/sora-2", true], + ["xai", "xai/grok-imagine-video", true], ["vydra", "vydra/veo3", false], ["together", "together/Wan-AI/Wan2.2-T2V-A14B", false], ["together", "together/Wan-AI/Wan2.2-I2V-A14B", true], diff --git a/src/video-generation/live-test-helpers.ts b/src/video-generation/live-test-helpers.ts index d2736fee56a8..40684d8ea966 100644 --- a/src/video-generation/live-test-helpers.ts +++ b/src/video-generation/live-test-helpers.ts @@ -19,7 +19,6 @@ export const DEFAULT_LIVE_VIDEO_MODELS: Record = { fal: "fal/fal-ai/minimax/video-01-live", google: "google/veo-3.1-fast-generate-preview", minimax: "minimax/MiniMax-Hailuo-2.3", - openai: "openai/sora-2", openrouter: "openrouter/google/veo-3.1-fast", pixverse: "pixverse/v6", qwen: "qwen/wan2.6-t2v", @@ -29,7 +28,7 @@ export const DEFAULT_LIVE_VIDEO_MODELS: Record = { xai: "xai/grok-imagine-video", }; -const REMOTE_URL_VIDEO_TO_VIDEO_PROVIDERS = new Set(["alibaba", "google", "openai", "qwen", "xai"]); +const REMOTE_URL_VIDEO_TO_VIDEO_PROVIDERS = new Set(["alibaba", "google", "qwen", "xai"]); const BUFFER_BACKED_IMAGE_TO_VIDEO_UNSUPPORTED_PROVIDERS = new Set(["vydra"]); const TOGETHER_BUFFER_BACKED_IMAGE_TO_VIDEO_MODEL = "Wan-AI/Wan2.2-I2V-A14B"; diff --git a/taxonomy.yaml b/taxonomy.yaml index e61d9afd1966..7c106e27e673 100644 --- a/taxonomy.yaml +++ b/taxonomy.yaml @@ -11600,19 +11600,19 @@ surfaces: description: "Covers typed providerOptions across video generation request normalization before provider execution: `generate`, `imageToVideo`, and `videoToVideo` modes, reference media typing and roles, and related video generation modes behavior." - name: queue-backed jobs coverageIds: [media-generation.queue-backed-jobs] - description: "Covers queue-backed jobs across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior." + description: "Covers queue-backed jobs across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior." - name: polling/timeout handling coverageIds: [media-generation.polling-timeout-handling] - description: "Covers polling/timeout handling across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior." + description: "Covers polling/timeout handling across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior." - name: Hosted URL download coverageIds: [media-generation.hosted-url-download] - description: "Covers hosted URL download across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior." + description: "Covers hosted URL download across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior." - name: provider skip explanations coverageIds: [media-generation.provider-skip-explanations] - description: "Covers provider skip explanations across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior." + description: "Covers provider skip explanations across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior." - name: returned asset metadata coverageIds: [media-generation.returned-asset-metadata] - description: "Covers returned asset metadata across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior." + description: "Covers returned asset metadata across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior." docs: - docs/tools/video-generation.md - docs/providers/runway.md diff --git a/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts b/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts index e1b1024ffc83..000fffaae6c1 100644 --- a/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts +++ b/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts @@ -172,7 +172,7 @@ describe("hosted media provider live CLI", () => { providers: [], skippedReason: "no providers selected", }, - { suite: MEDIA_SUITES.video, providers: ["openai"] }, + { suite: MEDIA_SUITES.video, providers: [], skippedReason: "no providers selected" }, ]); expect(skipped).toEqual([]); @@ -228,17 +228,12 @@ describe("hosted media provider live CLI", () => { "vydra", ]); expect(requirePlanEntry(plan, "music").providers).toEqual(["fal", "google", "minimax"]); - expect(requirePlanEntry(plan, "video").providers).toEqual([ - "google", - "minimax", - "openai", - "vydra", - ]); + expect(requirePlanEntry(plan, "video").providers).toEqual(["google", "minimax", "vydra"]); }); it("supports suite-specific provider filters without auth narrowing", async () => { const plan = await buildRunPlan( - parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]), + parseArgs(["video", "--video-providers", "fal,google,runway", "--all-providers"]), { collectProviderApiKeysImpl: collectProviderApiKeysMock, getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], @@ -249,7 +244,7 @@ describe("hosted media provider live CLI", () => { expect(plan).toHaveLength(1); const [entry] = plan; expect(entry?.suite.id).toBe("video"); - expect(entry?.providers).toEqual(["fal", "openai", "runway"]); + expect(entry?.providers).toEqual(["fal", "google", "runway"]); }); it("forwards quiet flags separately from passthrough args", () => { diff --git a/test/e2e/qa-lab/media/hosted-media-provider-live.ts b/test/e2e/qa-lab/media/hosted-media-provider-live.ts index f4b3374c2589..cfa471e241fa 100644 --- a/test/e2e/qa-lab/media/hosted-media-provider-live.ts +++ b/test/e2e/qa-lab/media/hosted-media-provider-live.ts @@ -52,7 +52,6 @@ export const MEDIA_SUITES: Record = { "fal", "google", "minimax", - "openai", "openrouter", "qwen", "runway", @@ -66,7 +65,6 @@ export const MEDIA_SUITES: Record = { "deepinfra", "google", "minimax", - "openai", "openrouter", "qwen", "runway", @@ -476,8 +474,8 @@ function formatHelp(): string { Usage: pnpm test:live:media pnpm test:live:media image - pnpm test:live:media image video --providers openai,google,minimax - pnpm test:live:media video --video-providers openai,runway --all-providers + pnpm test:live:media image video --providers google,minimax,xai + pnpm test:live:media video --video-providers runway,xai --all-providers QA evidence mode: node --import tsx ${SOURCE_PATH} --qa-evidence --suite image --artifact-base diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index d240c25b2e9a..a6be566da14e 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -10318,7 +10318,7 @@ describe("package artifact reuse", () => { expect(workflow).toContain("suite_id: native-live-extensions-media-video"); expect(workflow).toContain("suite_group: native-live-extensions-media-video"); expect(workflow).toContain("OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=google,minimax"); - expect(workflow).toContain("OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=openai,openrouter,xai"); + expect(workflow).toContain("OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=openrouter,xai"); expect(workflow).toContain( "inputs.live_suite_filter == 'native-live-src-gateway-profiles-anthropic'", );