mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
fix(openai): retire the Sora video provider after the API shutdown (#159543)
OpenAI shut down its Sora video API: sora-2 and sora-2-pro report shutdown_date 2026-09-24 and POST/GET /v1/videos now return HTTP 404 for every project. Every video_generate call routed to openai/* failed with an opaque "OpenAI video generation failed (HTTP 404)", and because the bundled openai plugin still advertised a configured video provider, agents on OpenAI-only installs kept selecting it. Remove the OpenAI video-generation provider, its manifest contract and metadata, live-test defaults and workflow filters, and the docs that advertised Sora. Stale openai/sora-* refs need no migration: the media runtime already skips them with "No video-generation provider registered for openai" and continues to configured fallbacks or auto-detected providers.
This commit is contained in:
Peter Steinberger
GitHub
parent
4d38e23cb6
commit
4994ec4501
27 changed files with 178 additions and 1601 deletions
|
|
@ -4790,7 +4790,7 @@ jobs:
|
|||
- suite_id: native-live-extensions-media-video-c
|
||||
suite_group: native-live-extensions-media-video
|
||||
label: Native live media video plugins C
|
||||
command: OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=openai,openrouter,xai node .release-harness/scripts/test-live-shard.mjs native-live-extensions-media-video
|
||||
command: OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=openrouter,xai node .release-harness/scripts/test-live-shard.mjs native-live-extensions-media-video
|
||||
timeout_minutes: 30
|
||||
profile_env_only: false
|
||||
profiles: full
|
||||
|
|
|
|||
|
|
@ -768,7 +768,6 @@ extensions/openai/openai-chatgpt-oauth-token.runtime.ts 1
|
|||
extensions/openai/provider-policy-api.ts 2
|
||||
extensions/openai/realtime-voice-bridge.ts 3
|
||||
extensions/openai/realtime-voice-session-policy.ts 4
|
||||
extensions/openai/video-generation-provider.ts 1
|
||||
extensions/opencode-go/reasoning-sanitizer.ts 2
|
||||
extensions/opencode-go/stream-termination.ts 8
|
||||
extensions/opencode/provider-catalog.ts 2
|
||||
|
|
|
|||
|
|
@ -3800,8 +3800,8 @@
|
|||
"target": "OpenAI 覆盖范围与费用"
|
||||
},
|
||||
{
|
||||
"source": "OpenAI image and video generation",
|
||||
"target": "OpenAI 图像与视频生成"
|
||||
"source": "OpenAI image generation",
|
||||
"target": "OpenAI 图像生成"
|
||||
},
|
||||
{
|
||||
"source": "OpenAI voice and speech",
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ request. Plugin dependencies are expected to be present before runtime load.
|
|||
- Enable: `OPENCLAW_LIVE_TEST=1 pnpm test:live -- extensions/video-generation-providers.live.test.ts`
|
||||
- Harness: `pnpm test:live:media video`
|
||||
- Scope:
|
||||
- Exercises the shared bundled video-generation provider path across `alibaba`, `byteplus`, `deepinfra`, `fal`, `google`, `minimax`, `openai`, `openrouter`, `pixverse`, `qwen`, `runway`, `together`, `vydra`, `xai`
|
||||
- Exercises the shared bundled video-generation provider path across `alibaba`, `byteplus`, `deepinfra`, `fal`, `google`, `minimax`, `openrouter`, `pixverse`, `qwen`, `runway`, `together`, `vydra`, `xai`
|
||||
- Defaults to the release-safe smoke path: one text-to-video request per provider, one-second lobster prompt, and a per-provider operation cap from `OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS` (`180000` by default)
|
||||
- Skips FAL by default because provider-side queue latency can dominate release time; pass `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="fal"` (or clear the skip list) to run it explicitly
|
||||
- Uses already-exported provider env vars before probing
|
||||
|
|
@ -123,10 +123,10 @@ request. Plugin dependencies are expected to be present before runtime load.
|
|||
- Current `videoToVideo` live coverage:
|
||||
- `runway` only when the selected model resolves to `gen4_aleph`
|
||||
- Current declared-but-skipped `videoToVideo` providers in the shared sweep:
|
||||
- `alibaba`, `google`, `openai`, `qwen`, `xai` because those paths currently require remote `http(s)` reference URLs rather than buffer-backed local input
|
||||
- `alibaba`, `google`, `qwen`, `xai` because those paths currently require remote `http(s)` reference URLs rather than buffer-backed local input
|
||||
- Optional narrowing:
|
||||
- `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="deepinfra,google,openai,runway"`
|
||||
- `OPENCLAW_LIVE_VIDEO_GENERATION_MODELS="google/veo-3.1-fast-generate-preview,openai/sora-2,runway/gen4_aleph"`
|
||||
- `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="deepinfra,google,runway,xai"`
|
||||
- `OPENCLAW_LIVE_VIDEO_GENERATION_MODELS="google/veo-3.1-fast-generate-preview,xai/grok-imagine-video,runway/gen4_aleph"`
|
||||
- `OPENCLAW_LIVE_VIDEO_GENERATION_SKIP_PROVIDERS=""` to include every provider in the default sweep, including FAL
|
||||
- `OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS=60000` to reduce each provider operation cap for an aggressive smoke run
|
||||
- Optional auth behavior:
|
||||
|
|
@ -147,6 +147,6 @@ request. Plugin dependencies are expected to be present before runtime load.
|
|||
- `--quiet` / `--no-quiet` passed through to `test:live`
|
||||
- Examples:
|
||||
- `pnpm test:live:media`
|
||||
- `pnpm test:live:media image video --providers openai,google,minimax`
|
||||
- `pnpm test:live:media video --video-providers openai,runway --all-providers`
|
||||
- `pnpm test:live:media image video --providers google,minimax,xai`
|
||||
- `pnpm test:live:media video --video-providers runway,xai --all-providers`
|
||||
- `pnpm test:live:media music --quiet`
|
||||
|
|
|
|||
|
|
@ -69,11 +69,11 @@ export type VideoGenerationProviderPlugin = {
|
|||
|
||||
// plugin API
|
||||
api.registerVideoGenerationProvider({
|
||||
id: "openai",
|
||||
label: "OpenAI",
|
||||
id: "xai",
|
||||
label: "xAI",
|
||||
async generateVideo(req) {
|
||||
// generateOpenAiVideo is a placeholder for your own vendor call.
|
||||
return await generateOpenAiVideo(req);
|
||||
// generateXaiVideo is a placeholder for your own vendor call.
|
||||
return await generateXaiVideo(req);
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ lookups such as `providerContractPluginIds`; tests assert a plugin's
|
|||
`contracts.videoGenerationProviders` list matches what it actually registers):
|
||||
|
||||
```ts
|
||||
expect(pluginManifest.contracts?.videoGenerationProviders).toEqual(["openai"]);
|
||||
expect(pluginManifest.contracts?.videoGenerationProviders).toEqual(["xai"]);
|
||||
```
|
||||
|
||||
That keeps the rule simple:
|
||||
|
|
|
|||
|
|
@ -600,7 +600,7 @@ That means:
|
|||
|
||||
<AccordionGroup>
|
||||
<Accordion title="Vendor multi-capability">
|
||||
`google` owns text inference, CLI backend, embeddings, speech, realtime voice, media understanding, image/music/video generation, and web search. `openai` owns text inference, embeddings, speech, realtime transcription, realtime voice, media understanding, image/video generation. `minimax` owns text inference plus media understanding, speech, image/music/video generation, and web search.
|
||||
`google` owns text inference, CLI backend, embeddings, speech, realtime voice, media understanding, image/music/video generation, and web search. `openai` owns text inference, embeddings, speech, realtime transcription, realtime voice, media understanding, image generation. `minimax` owns text inference plus media understanding, speech, image/music/video generation, and web search.
|
||||
</Accordion>
|
||||
<Accordion title="Vendor single-capability">
|
||||
`arcee` and `chutes` own text inference only; `microsoft` owns speech only. A vendor plugin can stay this narrow until it needs to cover more of that vendor's surface.
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ Adds OpenAI model provider support to OpenClaw.
|
|||
## Surface
|
||||
|
||||
- Providers: `openai`
|
||||
- Contracts: `embeddingProviders`, `imageGenerationProviders`, `mediaUnderstandingProviders`, `realtimeTranscriptionProviders`, `realtimeVoiceProviders`, `speechProviders`, `usageProviders`, `videoGenerationProviders`
|
||||
- Contracts: `embeddingProviders`, `imageGenerationProviders`, `mediaUnderstandingProviders`, `realtimeTranscriptionProviders`, `realtimeVoiceProviders`, `speechProviders`, `usageProviders`
|
||||
|
||||
## Related docs
|
||||
|
||||
|
|
|
|||
|
|
@ -30,17 +30,17 @@ workflows like OpenClaw.
|
|||
This page is an index. OpenAI is documented on nine pages, one per reader
|
||||
job. Open the page that matches your task.
|
||||
|
||||
| Page | Read it when |
|
||||
| ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| [OpenAI setup](/providers/openai/setup) | You are connecting an account: the API-key and Codex subscription paths, route summaries, OAuth recovery, and the long-context opt-in. |
|
||||
| [OpenAI authentication](/providers/openai/authentication) | Choose Codex login, an API key, or SIWC based on model access, plugins, usage tracking, and permissions. |
|
||||
| [OpenAI models](/providers/openai/models) | You are choosing a model ref: the quick-choice table, GPT-6 Astra, Sol, Luna, and the GPT-5.6 tiers. |
|
||||
| [OpenAI runtimes and Codex auth](/providers/openai/runtimes) | You need to know which runtime runs an `openai/*` turn, and how native Codex resolves its account. |
|
||||
| [OpenAI coverage and cost](/providers/openai/coverage-and-cost) | You want the capability matrix, memory embeddings, or how subscription quota and Platform billing are reported. |
|
||||
| [OpenAI image and video generation](/providers/openai/image-and-video) | You are generating or editing images and video through the bundled `openai` plugin. |
|
||||
| [OpenAI voice and speech](/providers/openai/voice-and-speech) | You are configuring text-to-speech, transcription, or realtime voice, including per-route auth order. |
|
||||
| [Azure OpenAI endpoints](/providers/openai/azure) | You are pointing the bundled `openai` provider at an Azure OpenAI resource. |
|
||||
| [OpenAI advanced configuration](/providers/openai/advanced) | You are tuning prompt contribution, transport, Fast mode, compaction, strict-agentic mode, or proxy compat. |
|
||||
| Page | Read it when |
|
||||
| --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| [OpenAI setup](/providers/openai/setup) | You are connecting an account: the API-key and Codex subscription paths, route summaries, OAuth recovery, and the long-context opt-in. |
|
||||
| [OpenAI authentication](/providers/openai/authentication) | Choose Codex login, an API key, or SIWC based on model access, plugins, usage tracking, and permissions. |
|
||||
| [OpenAI models](/providers/openai/models) | You are choosing a model ref: the quick-choice table, GPT-6 Astra, Sol, Luna, and the GPT-5.6 tiers. |
|
||||
| [OpenAI runtimes and Codex auth](/providers/openai/runtimes) | You need to know which runtime runs an `openai/*` turn, and how native Codex resolves its account. |
|
||||
| [OpenAI coverage and cost](/providers/openai/coverage-and-cost) | You want the capability matrix, memory embeddings, or how subscription quota and Platform billing are reported. |
|
||||
| [OpenAI image generation](/providers/openai/image-and-video) | You are generating or editing images through the bundled `openai` plugin. |
|
||||
| [OpenAI voice and speech](/providers/openai/voice-and-speech) | You are configuring text-to-speech, transcription, or realtime voice, including per-route auth order. |
|
||||
| [Azure OpenAI endpoints](/providers/openai/azure) | You are pointing the bundled `openai` provider at an Azure OpenAI resource. |
|
||||
| [OpenAI advanced configuration](/providers/openai/advanced) | You are tuning prompt contribution, transport, Fast mode, compaction, strict-agentic mode, or proxy compat. |
|
||||
|
||||
## Where each section moved
|
||||
|
||||
|
|
@ -91,10 +91,10 @@ working. Each entry points at the page that now holds the content.
|
|||
- <a id="openclaw-feature-coverage" />[OpenClaw feature coverage](/providers/openai/coverage-and-cost#openclaw-feature-coverage)
|
||||
- <a id="memory-embeddings" />[Memory embeddings](/providers/openai/coverage-and-cost#memory-embeddings)
|
||||
|
||||
**[OpenAI image and video generation](/providers/openai/image-and-video)**
|
||||
**[OpenAI image generation](/providers/openai/image-and-video)**
|
||||
|
||||
- <a id="image-generation" />[Image generation](/providers/openai/image-and-video#image-generation)
|
||||
- <a id="video-generation" />[Video generation](/providers/openai/image-and-video#video-generation)
|
||||
- <a id="video-generation" />[Supported video providers](/tools/video-generation)
|
||||
|
||||
**[OpenAI voice and speech](/providers/openai/voice-and-speech)**
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ changing config.
|
|||
| Codex app-server harness | Codex-compatible HTTPS route with runtime unset/`auto`, or explicit `agentRuntime.id: codex` | Yes |
|
||||
| Server-side web search | Native OpenAI Responses tool | Yes, when web search is enabled and no other provider is pinned |
|
||||
| Images | `image_generate` | Yes |
|
||||
| Videos | `video_generate` | Yes |
|
||||
| Text-to-speech | `tts.provider: "openai"` / `tts` | Yes |
|
||||
| Batch speech-to-text | `tools.media.audio` / media understanding | Yes |
|
||||
| Streaming speech-to-text | Voice Call `streaming.provider: "openai"` | Yes |
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
---
|
||||
summary: "Generate and edit images with gpt-image, and generate video with Sora"
|
||||
summary: "Generate and edit images with OpenAI gpt-image"
|
||||
read_when:
|
||||
- You are generating or editing images through the openai provider
|
||||
- You need transparent-background image output
|
||||
- You are generating video with the video_generate tool
|
||||
title: "OpenAI image and video generation"
|
||||
sidebarTitle: "Image and video"
|
||||
title: "OpenAI image generation"
|
||||
sidebarTitle: "Image generation"
|
||||
---
|
||||
|
||||
## Image generation
|
||||
|
|
@ -152,42 +151,4 @@ Edit:
|
|||
/tool image_generate model=openai/gpt-image-2 prompt="Preserve the object shape, change the material to translucent glass" image=/path/to/reference.png size=1024x1536
|
||||
```
|
||||
|
||||
## Video generation
|
||||
|
||||
The bundled `openai` plugin registers video generation through the
|
||||
`video_generate` tool.
|
||||
|
||||
| Capability | Value |
|
||||
| ---------------- | ---------------------------------------------------------------------------------- |
|
||||
| Default model | `openai/sora-2` |
|
||||
| Modes | Text-to-video, image-to-video, single-video edit |
|
||||
| Reference inputs | 1 image or 1 video |
|
||||
| Size overrides | Supported for text-to-video and image-to-video |
|
||||
| Aspect ratio | Converted to the closest supported size, not forwarded raw |
|
||||
| Other overrides | `resolution`, `audio`, `watermark` are unsupported and dropped with a tool warning |
|
||||
|
||||
OpenAI image-to-video requests use `POST /v1/videos` with an image
|
||||
`input_reference`. Single-video edits use `POST /v1/videos/edits` with the
|
||||
uploaded video in the `video` field.
|
||||
|
||||
```json5
|
||||
{
|
||||
agents: {
|
||||
defaults: {
|
||||
mediaModels: { video: { primary: "openai/sora-2" } },
|
||||
},
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
<Note>
|
||||
See [Video Generation](/tools/video-generation) for shared tool parameters,
|
||||
provider selection, and failover behavior.
|
||||
|
||||
The OpenAI provider declares `supportsSize` but not `supportsAspectRatio` or
|
||||
`supportsResolution`. OpenClaw's shared normalization layer converts a
|
||||
requested `aspectRatio` into the closest matching OpenAI `size` before the
|
||||
request reaches the provider, so aspect-ratio requests generally still work.
|
||||
`resolution` has no size fallback and is dropped, surfaced to the caller as
|
||||
`Ignored unsupported overrides for openai/<model>: resolution=<value>`.
|
||||
</Note>
|
||||
OpenAI retired its Sora video API on 2026-09-24; see [Video generation](/tools/video-generation) for supported providers.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
---
|
||||
summary: "Generate videos via video_generate from text, image, or video references across 16 provider backends"
|
||||
summary: "Generate videos via video_generate from text, image, or video references across 15 provider backends"
|
||||
read_when:
|
||||
- Generating videos via the agent
|
||||
- Configuring video-generation providers and models
|
||||
|
|
@ -9,7 +9,7 @@ sidebarTitle: "Video generation"
|
|||
---
|
||||
|
||||
OpenClaw agents generate videos from text prompts, reference images, or
|
||||
existing videos through `video_generate`. Sixteen provider backends are
|
||||
existing videos through `video_generate`. Fifteen provider backends are
|
||||
supported; the agent picks the right one automatically based on config and
|
||||
available API keys.
|
||||
|
||||
|
|
@ -104,7 +104,6 @@ of failing the task if local persistence rejects an oversized file.
|
|||
| fal | `fal-ai/minimax/video-01-live` | ✓ | 1 image; up to 9 with Seedance reference-to-video | Up to 3 videos with Seedance reference-to-video | `FAL_KEY` |
|
||||
| Google | `veo-3.1-fast-generate-preview` | ✓ | 1 image | 1 video | `GEMINI_API_KEY` |
|
||||
| MiniMax | `MiniMax-Hailuo-2.3` | ✓ | 1 image | - | `MINIMAX_API_KEY` or MiniMax OAuth |
|
||||
| OpenAI | `sora-2` | ✓ | 1 image | 1 video | `OPENAI_API_KEY` |
|
||||
| OpenRouter | `google/veo-3.1-fast` | ✓ | Up to 4 images (first/last frame or references) | - | `OPENROUTER_API_KEY` |
|
||||
| Qwen | `wan2.6-t2v` | ✓ | Yes (remote URL) | Yes (remote URL) | `QWEN_API_KEY` |
|
||||
| Runway | `gen4.5` | ✓ | 1 image | 1 video | `RUNWAYML_API_SECRET` |
|
||||
|
|
@ -132,7 +131,6 @@ the shared live sweep:
|
|||
| fal | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` only when using Seedance reference-to-video |
|
||||
| Google | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; shared `videoToVideo` skipped because the current buffer-backed Gemini/Veo sweep does not accept that input |
|
||||
| MiniMax | ✓ | ✓ | - | `generate`, `imageToVideo` |
|
||||
| OpenAI | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; shared `videoToVideo` skipped because this org/input path needs provider-side video edit access |
|
||||
| OpenRouter | ✓ | ✓ | - | `generate`, `imageToVideo` |
|
||||
| Qwen | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider needs remote `http(s)` video URLs |
|
||||
| Runway | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` runs only when the selected model is `runway/gen4_aleph` |
|
||||
|
|
@ -386,11 +384,6 @@ OpenClaw does not append auto-detected providers.
|
|||
resolutions; requests such as `720P` are normalized to the closest
|
||||
supported value before submission.
|
||||
</Accordion>
|
||||
<Accordion title="OpenAI">
|
||||
Only `size` override is forwarded. Other style overrides
|
||||
(`aspectRatio`, `resolution`, `audio`, `watermark`) are ignored with
|
||||
a warning.
|
||||
</Accordion>
|
||||
<Accordion title="OpenRouter">
|
||||
Uses OpenRouter's asynchronous `/videos` API. OpenClaw submits the
|
||||
job, polls `polling_url`, and downloads either `unsigned_urls` or the
|
||||
|
|
@ -547,7 +540,6 @@ openclaw config set agents.defaults.mediaModels.video.primary "qwen/wan2.6-t2v"
|
|||
- [Google (Gemini)](/providers/google)
|
||||
- [MiniMax](/providers/minimax)
|
||||
- [Models](/concepts/models)
|
||||
- [OpenAI](/providers/openai)
|
||||
- [OpenRouter](/providers/openrouter)
|
||||
- [Qwen](/providers/qwen)
|
||||
- [Runway](/providers/runway)
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ const POLL_INTERVAL_MS = 5_000;
|
|||
const MAX_POLL_ATTEMPTS = 120;
|
||||
|
||||
// /v1/openai/videos is async: POST returns a job, GET /{id} polls until the
|
||||
// job leaves the queue. Mirrors the OpenAI Sora surface (extensions/openai).
|
||||
// job succeeds or fails, then the result contains downloadable video URLs.
|
||||
type DeepInfraVideoStatus = "queued" | "processing" | "succeeded" | "failed";
|
||||
|
||||
type DeepInfraVideoJob = {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# OpenAI
|
||||
|
||||
Connect OpenAI models to OpenClaw. The plugin also provides embeddings, media
|
||||
understanding, image and video generation, speech output, realtime transcription,
|
||||
understanding, image generation, speech output, realtime transcription,
|
||||
and realtime voice.
|
||||
|
||||
## Get started
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
import { createCapturedPluginRegistration } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import * as providerHttp from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
createDebugProxyCaptureReaderAsync,
|
||||
finalizeDebugProxyCaptureAsync,
|
||||
|
|
@ -11,314 +9,138 @@ import { createOpenClawTestState } from "openclaw/plugin-sdk/test-state";
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { installDebugProxyTestResetHooks } from "../test-support/debug-proxy-env-test-helpers.js";
|
||||
import { buildOpenAISpeechProvider } from "./speech-provider.js";
|
||||
import { buildOpenAIVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
|
||||
const proxyReset = installDebugProxyTestResetHooks();
|
||||
const modelAuth = createCapturedPluginRegistration().api.runtime.modelAuth;
|
||||
|
||||
async function requestMedia(
|
||||
async function requestAudio(
|
||||
baseUrl: string,
|
||||
kind: "audio" | "video",
|
||||
options: { timeoutMs?: number; mediaMaxMb?: number; reference?: "image" | "video" } = {},
|
||||
options: { timeoutMs?: number; mediaMaxMb?: number } = {},
|
||||
) {
|
||||
const budget =
|
||||
options.mediaMaxMb === undefined
|
||||
? {}
|
||||
: { agents: { defaults: { mediaMaxMb: options.mediaMaxMb } } };
|
||||
if (kind === "audio") {
|
||||
const result = await buildOpenAISpeechProvider().synthesize({
|
||||
text: "local binary acceptance",
|
||||
cfg: budget,
|
||||
providerConfig: {
|
||||
apiKey: "local-test-key",
|
||||
baseUrl: `${baseUrl}/v1`,
|
||||
model: "tts-1",
|
||||
voice: "alloy",
|
||||
},
|
||||
target: "audio-file",
|
||||
timeoutMs: options.timeoutMs ?? 5_000,
|
||||
});
|
||||
return result.audioBuffer;
|
||||
}
|
||||
const result = await buildOpenAIVideoGenerationProvider(modelAuth).generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
prompt: "local binary acceptance",
|
||||
cfg: {
|
||||
...budget,
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: "local-test-key",
|
||||
baseUrl: `${baseUrl}/v1`,
|
||||
models: [],
|
||||
request: { allowPrivateNetwork: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
const result = await buildOpenAISpeechProvider().synthesize({
|
||||
text: "local binary acceptance",
|
||||
cfg: budget,
|
||||
providerConfig: {
|
||||
apiKey: "local-test-key",
|
||||
baseUrl: `${baseUrl}/v1`,
|
||||
model: "tts-1",
|
||||
voice: "alloy",
|
||||
},
|
||||
target: "audio-file",
|
||||
timeoutMs: options.timeoutMs ?? 5_000,
|
||||
...(options.reference === "image"
|
||||
? { inputImages: [{ buffer: Buffer.from("image"), mimeType: "image/png" }] }
|
||||
: {}),
|
||||
...(options.reference === "video"
|
||||
? { inputVideos: [{ buffer: Buffer.from("video"), mimeType: "video/mp4" }] }
|
||||
: {}),
|
||||
});
|
||||
return result.videos[0]?.buffer;
|
||||
return result.audioBuffer;
|
||||
}
|
||||
|
||||
describe("production OpenAI binary transport", () => {
|
||||
it.each(["audio", "video"] as const)(
|
||||
"rejects invalid %s over TCP and preserves valid codec parameters",
|
||||
async (kind) => {
|
||||
const type = kind === "audio" ? "audio/ogg" : "video/mp4";
|
||||
const cases = [
|
||||
{ header: "image/png", body: "wrong family", valid: false },
|
||||
{ header: "", body: "empty header", valid: false },
|
||||
{ header: `${type}; charset=utf-8, text/html`, body: "hidden error", valid: false },
|
||||
{ header: [type, "application/json"], body: "repeated header", valid: false },
|
||||
{ header: type, body: "", valid: false },
|
||||
{ header: "application/ogg", body: "Ogg container bytes", valid: kind === "audio" },
|
||||
{ header: "application/octet-stream", body: "opaque bytes", valid: true },
|
||||
{ header: "binary/octet-stream", body: "opaque alias bytes", valid: true },
|
||||
{ header: undefined, body: "missing header bytes", valid: true },
|
||||
{ header: `${type}; codecs="one, two"`, body: "codec bytes", valid: true },
|
||||
{ header: `${type};; codecs="one, two";`, body: "empty parameter slots", valid: true },
|
||||
];
|
||||
for (const fixture of cases) {
|
||||
await withServer(
|
||||
(request, response) => {
|
||||
request.resume();
|
||||
if (request.url === "/v1/videos") {
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(JSON.stringify({ id: "local-video", status: "completed" }));
|
||||
} else {
|
||||
if (fixture.header !== undefined) {
|
||||
response.setHeader("Content-Type", fixture.header);
|
||||
}
|
||||
response.end(fixture.body);
|
||||
}
|
||||
},
|
||||
async (baseUrl) => {
|
||||
const result = requestMedia(baseUrl, kind);
|
||||
if (fixture.valid) {
|
||||
await expect(result).resolves.toEqual(Buffer.from(fixture.body));
|
||||
} else {
|
||||
await expect(result).rejects.toThrow(`malformed ${kind} response`);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["audio", "video"] as const)(
|
||||
"preserves %s byte limits and transport timeouts over TCP",
|
||||
async (kind) => {
|
||||
for (const stalled of [false, true]) {
|
||||
let closed = false;
|
||||
await withServer(
|
||||
(request, response) => {
|
||||
request.resume();
|
||||
if (request.url === "/v1/videos") {
|
||||
response.writeHead(200, { "Content-Type": "application/json" });
|
||||
response.end(JSON.stringify({ id: "budget-video", status: "completed" }));
|
||||
} else {
|
||||
request.socket.once("close", () => {
|
||||
closed = true;
|
||||
});
|
||||
response.writeHead(200, {
|
||||
"Content-Type": kind === "audio" ? "audio/mpeg" : "video/mp4",
|
||||
});
|
||||
response.write(stalled ? Buffer.from([1]) : Buffer.alloc(4096));
|
||||
}
|
||||
},
|
||||
async (baseUrl) => {
|
||||
const result = requestMedia(baseUrl, kind, {
|
||||
mediaMaxMb: 0.001,
|
||||
timeoutMs: stalled ? 300 : 5_000,
|
||||
});
|
||||
if (stalled) {
|
||||
await expect(result).rejects.toThrow(/timed out|aborted/i);
|
||||
} else {
|
||||
await expect(result).rejects.toThrow(
|
||||
kind === "audio"
|
||||
? "OpenAI TTS audio response exceeds"
|
||||
: "OpenAI generated video download exceeds",
|
||||
);
|
||||
}
|
||||
await vi.waitFor(() => expect(closed).toBe(true));
|
||||
},
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ status: "queued", reference: "image" },
|
||||
{ status: "completed", reference: "video" },
|
||||
] as const)(
|
||||
"releases $status $reference submission before real follow-up transport",
|
||||
async ({ status, reference }) => {
|
||||
const originalPost = providerHttp.postMultipartRequest;
|
||||
let releases = 0;
|
||||
let released = false;
|
||||
let clone: Response | undefined;
|
||||
const paths: string[] = [];
|
||||
const releasedAtFollowUp: boolean[] = [];
|
||||
const post = vi
|
||||
.spyOn(providerHttp, "postMultipartRequest")
|
||||
.mockImplementation(async (params) => {
|
||||
const handle = await originalPost(params);
|
||||
clone = handle.response.clone();
|
||||
return {
|
||||
...handle,
|
||||
release: async () => {
|
||||
releases += 1;
|
||||
await handle.release();
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
});
|
||||
try {
|
||||
await withServer(
|
||||
(request, response) => {
|
||||
request.resume();
|
||||
paths.push(request.url ?? "");
|
||||
if (request.method === "GET") {
|
||||
releasedAtFollowUp.push(released);
|
||||
}
|
||||
if (request.url?.includes("/content")) {
|
||||
response.setHeader("Content-Type", "video/mp4");
|
||||
response.end("rendered-video");
|
||||
} else {
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
id: "release-video",
|
||||
status: request.method === "POST" ? status : "completed",
|
||||
}),
|
||||
);
|
||||
}
|
||||
},
|
||||
async (baseUrl) => {
|
||||
const result = await requestMedia(baseUrl, "video", { reference });
|
||||
expect(result).toEqual(Buffer.from("rendered-video"));
|
||||
expect(paths[0]).toBe(reference === "video" ? "/v1/videos/edits" : "/v1/videos");
|
||||
expect(paths).toHaveLength(status === "queued" ? 3 : 2);
|
||||
expect(releases).toBe(1);
|
||||
expect(releasedAtFollowUp).toEqual(status === "queued" ? [true, true] : [true]);
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
post.mockRestore();
|
||||
void clone?.body?.cancel().catch(() => undefined);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "HTTP failure",
|
||||
status: 400,
|
||||
body: '{"error":{"message":"submission refused"}}',
|
||||
error: "submission refused",
|
||||
},
|
||||
{ label: "malformed JSON", status: 200, body: "{", error: "malformed JSON response" },
|
||||
{ label: "missing id", status: 200, body: '{"status":"queued"}', error: "missing video id" },
|
||||
{
|
||||
label: "failed job",
|
||||
status: 200,
|
||||
body: '{"status":"failed","error":{"message":"job refused"}}',
|
||||
error: "job refused",
|
||||
},
|
||||
])("releases failed submission exactly once: $label", async ({ status, body, error }) => {
|
||||
const originalPost = providerHttp.postMultipartRequest;
|
||||
let releases = 0;
|
||||
let requests = 0;
|
||||
const post = vi
|
||||
.spyOn(providerHttp, "postMultipartRequest")
|
||||
.mockImplementation(async (params) => {
|
||||
const handle = await originalPost(params);
|
||||
return {
|
||||
...handle,
|
||||
release: async () => {
|
||||
releases += 1;
|
||||
await handle.release();
|
||||
},
|
||||
};
|
||||
});
|
||||
try {
|
||||
it("rejects invalid audio over TCP and preserves valid codec parameters", async () => {
|
||||
const type = "audio/ogg";
|
||||
const cases = [
|
||||
{ header: "image/png", body: "wrong family", valid: false },
|
||||
{ header: "", body: "empty header", valid: false },
|
||||
{ header: `${type}; charset=utf-8, text/html`, body: "hidden error", valid: false },
|
||||
{ header: [type, "application/json"], body: "repeated header", valid: false },
|
||||
{ header: type, body: "", valid: false },
|
||||
{ header: "application/ogg", body: "Ogg container bytes", valid: true },
|
||||
{ header: "application/octet-stream", body: "opaque bytes", valid: true },
|
||||
{ header: "binary/octet-stream", body: "opaque alias bytes", valid: true },
|
||||
{ header: undefined, body: "missing header bytes", valid: true },
|
||||
{ header: `${type}; codecs="one, two"`, body: "codec bytes", valid: true },
|
||||
{ header: `${type};; codecs="one, two";`, body: "empty parameter slots", valid: true },
|
||||
];
|
||||
for (const fixture of cases) {
|
||||
await withServer(
|
||||
(request, response) => {
|
||||
requests += 1;
|
||||
request.resume();
|
||||
response.writeHead(status, { "Content-Type": "application/json" });
|
||||
response.end(body);
|
||||
if (fixture.header !== undefined) {
|
||||
response.setHeader("Content-Type", fixture.header);
|
||||
}
|
||||
response.end(fixture.body);
|
||||
},
|
||||
async (baseUrl) => {
|
||||
await expect(requestMedia(baseUrl, "video")).rejects.toThrow(error);
|
||||
expect(releases).toBe(1);
|
||||
expect(requests).toBe(1);
|
||||
const result = requestAudio(baseUrl);
|
||||
if (fixture.valid) {
|
||||
await expect(result).resolves.toEqual(Buffer.from(fixture.body));
|
||||
} else {
|
||||
await expect(result).rejects.toThrow("malformed audio response");
|
||||
}
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
post.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["audio", "video"] as const)(
|
||||
"persists %s capture through finalization and closes the rejected upstream socket",
|
||||
async (kind) => {
|
||||
proxyReset.captureProxyEnv();
|
||||
const state = await createOpenClawTestState({ layout: "state-only", prefix: "binary-tcp-" });
|
||||
vi.stubEnv("OPENCLAW_DEBUG_PROXY_ENABLED", "1");
|
||||
vi.stubEnv("OPENCLAW_DEBUG_PROXY_SESSION_ID", `binary-${kind}`);
|
||||
it("preserves audio byte limits and transport timeouts over TCP", async () => {
|
||||
for (const stalled of [false, true]) {
|
||||
let closed = false;
|
||||
let mediaResponses = 0;
|
||||
try {
|
||||
await initializeDebugProxyCaptureAsync("test");
|
||||
await withServer(
|
||||
(request, response) => {
|
||||
request.resume();
|
||||
if (request.url === "/v1/videos") {
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(JSON.stringify({ id: "local-video", status: "completed" }));
|
||||
} else if (mediaResponses++ === 0) {
|
||||
response.writeHead(200, {
|
||||
"Content-Type": kind === "audio" ? "audio/ogg" : "video/mp4",
|
||||
});
|
||||
response.end("captured valid media");
|
||||
} else {
|
||||
request.socket.once("close", () => {
|
||||
closed = true;
|
||||
});
|
||||
response.writeHead(200, { "Content-Type": "image/png" });
|
||||
response.write("not the requested media");
|
||||
// Leave the body open: capture must not own the caller's completion.
|
||||
}
|
||||
},
|
||||
async (baseUrl) => {
|
||||
await expect(requestMedia(baseUrl, kind)).resolves.toEqual(
|
||||
Buffer.from("captured valid media"),
|
||||
);
|
||||
await expect(requestMedia(baseUrl, kind)).rejects.toThrow(`malformed ${kind} response`);
|
||||
await vi.waitFor(() => expect(closed).toBe(true));
|
||||
await finalizeDebugProxyCaptureAsync();
|
||||
await closeOpenClawStateDatabaseAsync();
|
||||
const reopened = createDebugProxyCaptureReaderAsync({ env: process.env });
|
||||
const persisted = await reopened.getSessionEvents(`binary-${kind}`, 20);
|
||||
expect(persisted.some((event) => event.kind === "request")).toBe(true);
|
||||
expect(persisted.some((event) => event.kind === "response")).toBe(true);
|
||||
expect(persisted).toHaveLength(kind === "audio" ? 4 : 8);
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await finalizeDebugProxyCaptureAsync();
|
||||
vi.unstubAllEnvs();
|
||||
await state.cleanup();
|
||||
}
|
||||
},
|
||||
);
|
||||
await withServer(
|
||||
(request, response) => {
|
||||
request.resume();
|
||||
request.socket.once("close", () => {
|
||||
closed = true;
|
||||
});
|
||||
response.writeHead(200, { "Content-Type": "audio/mpeg" });
|
||||
response.write(stalled ? Buffer.from([1]) : Buffer.alloc(4096));
|
||||
},
|
||||
async (baseUrl) => {
|
||||
const result = requestAudio(baseUrl, {
|
||||
mediaMaxMb: 0.001,
|
||||
timeoutMs: stalled ? 300 : 5_000,
|
||||
});
|
||||
if (stalled) {
|
||||
await expect(result).rejects.toThrow(/timed out|aborted/i);
|
||||
} else {
|
||||
await expect(result).rejects.toThrow("OpenAI TTS audio response exceeds");
|
||||
}
|
||||
await vi.waitFor(() => expect(closed).toBe(true));
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("persists audio capture through finalization and closes the rejected upstream socket", async () => {
|
||||
proxyReset.captureProxyEnv();
|
||||
const state = await createOpenClawTestState({ layout: "state-only", prefix: "binary-tcp-" });
|
||||
vi.stubEnv("OPENCLAW_DEBUG_PROXY_ENABLED", "1");
|
||||
vi.stubEnv("OPENCLAW_DEBUG_PROXY_SESSION_ID", "binary-audio");
|
||||
let closed = false;
|
||||
let mediaResponses = 0;
|
||||
try {
|
||||
await initializeDebugProxyCaptureAsync("test");
|
||||
await withServer(
|
||||
(request, response) => {
|
||||
request.resume();
|
||||
if (mediaResponses++ === 0) {
|
||||
response.writeHead(200, { "Content-Type": "audio/ogg" });
|
||||
response.end("captured valid media");
|
||||
} else {
|
||||
request.socket.once("close", () => {
|
||||
closed = true;
|
||||
});
|
||||
response.writeHead(200, { "Content-Type": "image/png" });
|
||||
response.write("not the requested media");
|
||||
// Leave the body open: capture must not own the caller's completion.
|
||||
}
|
||||
},
|
||||
async (baseUrl) => {
|
||||
await expect(requestAudio(baseUrl)).resolves.toEqual(Buffer.from("captured valid media"));
|
||||
await expect(requestAudio(baseUrl)).rejects.toThrow("malformed audio response");
|
||||
await vi.waitFor(() => expect(closed).toBe(true));
|
||||
await finalizeDebugProxyCaptureAsync();
|
||||
await closeOpenClawStateDatabaseAsync();
|
||||
const reopened = createDebugProxyCaptureReaderAsync({ env: process.env });
|
||||
const persisted = await reopened.getSessionEvents("binary-audio", 20);
|
||||
expect(persisted.some((event) => event.kind === "request")).toBe(true);
|
||||
expect(persisted.some((event) => event.kind === "response")).toBe(true);
|
||||
expect(persisted).toHaveLength(4);
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await finalizeDebugProxyCaptureAsync();
|
||||
vi.unstubAllEnvs();
|
||||
await state.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ import { OPENAI_QUICKSILVER_OFFER_PATH } from "./realtime-quicksilver-session.js
|
|||
import { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider-factory.js";
|
||||
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider-factory.js";
|
||||
import { buildOpenAISpeechProvider } from "./speech-provider.js";
|
||||
import { buildOpenAIVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "openai",
|
||||
|
|
@ -92,8 +91,5 @@ export default definePluginEntry({
|
|||
});
|
||||
api.registerSpeechProvider(buildOpenAISpeechProvider());
|
||||
api.registerMediaUnderstandingProvider(openaiMediaUnderstandingProvider);
|
||||
api.registerVideoGenerationProvider(
|
||||
buildOpenAIVideoGenerationProvider({ isProviderApiKeyConfigured }),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -377,7 +377,6 @@
|
|||
"embeddingProviders": ["openai"],
|
||||
"mediaUnderstandingProviders": ["openai"],
|
||||
"imageGenerationProviders": ["openai"],
|
||||
"videoGenerationProviders": ["openai"],
|
||||
"usageProviders": ["openai"]
|
||||
},
|
||||
"imageGenerationProviderMetadata": {
|
||||
|
|
@ -389,15 +388,6 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
"videoGenerationProviderMetadata": {
|
||||
"openai": {
|
||||
"authSignals": [
|
||||
{
|
||||
"provider": "openai"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"mediaUnderstandingProviderMetadata": {
|
||||
"openai": {
|
||||
"capabilities": ["image", "audio"],
|
||||
|
|
|
|||
|
|
@ -1,772 +0,0 @@
|
|||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
clearRuntimeAuthProfileStoreSnapshots,
|
||||
saveAuthProfileStore,
|
||||
} from "openclaw/plugin-sdk/agent-runtime";
|
||||
import { createCapturedPluginRegistration } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import {
|
||||
getProviderHttpMocks,
|
||||
installProviderHttpMockCleanup,
|
||||
} from "openclaw/plugin-sdk/provider-http-test-mocks";
|
||||
import { expectExplicitVideoGenerationCapabilities } from "openclaw/plugin-sdk/provider-test-contracts";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing";
|
||||
import { withServer } from "openclaw/plugin-sdk/test-env";
|
||||
import type { VideoGenerationRequest } from "openclaw/plugin-sdk/video-generation";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const {
|
||||
resolveApiKeyForProviderMock,
|
||||
postJsonRequestMock,
|
||||
postMultipartRequestMock,
|
||||
fetchWithTimeoutMock,
|
||||
fetchWithTimeoutGuardedMock,
|
||||
pollProviderOperationJsonMock,
|
||||
assertOkOrThrowHttpErrorMock,
|
||||
executeProviderOperationWithRetryMock,
|
||||
resolveProviderHttpRequestConfigMock,
|
||||
sanitizeConfiguredModelProviderRequestMock,
|
||||
} = getProviderHttpMocks();
|
||||
|
||||
let buildOpenAIVideoGenerationProvider: typeof import("./video-generation-provider.js").buildOpenAIVideoGenerationProvider;
|
||||
|
||||
let modelAuth: Parameters<typeof buildOpenAIVideoGenerationProvider>[0];
|
||||
|
||||
beforeAll(async () => {
|
||||
modelAuth = createCapturedPluginRegistration().api.runtime.modelAuth;
|
||||
({ buildOpenAIVideoGenerationProvider } = await import("./video-generation-provider.js"));
|
||||
});
|
||||
|
||||
installProviderHttpMockCleanup();
|
||||
|
||||
function generateVideo(
|
||||
overrides: Partial<VideoGenerationRequest> & Pick<VideoGenerationRequest, "prompt">,
|
||||
) {
|
||||
return buildOpenAIVideoGenerationProvider(modelAuth).generateVideo({
|
||||
provider: "openai",
|
||||
model: "sora-2",
|
||||
cfg: {},
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function localVideoConfig(allowPrivateNetwork?: boolean, baseUrl = "http://127.0.0.1:44080/v1") {
|
||||
return {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl,
|
||||
...(allowPrivateNetwork === undefined ? {} : { request: { allowPrivateNetwork } }),
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function videoJob(id: string, status: string, fields: { seconds?: string; size?: string } = {}) {
|
||||
return { id, model: "sora-2", status, ...fields };
|
||||
}
|
||||
|
||||
function releasedJson(value: unknown, release = vi.fn(async () => {})) {
|
||||
return { response: Response.json(value), release };
|
||||
}
|
||||
|
||||
function postMultipartRequest(index = 0): Record<string, unknown> {
|
||||
const request = postMultipartRequestMock.mock.calls[index]?.[0] as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!request) {
|
||||
throw new Error(`expected postMultipartRequest call ${index}`);
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
function fetchWithTimeoutCall(index: number): [string, RequestInit | undefined, number, unknown] {
|
||||
const call = fetchWithTimeoutMock.mock.calls[index] as
|
||||
| [string, RequestInit | undefined, number, unknown]
|
||||
| undefined;
|
||||
if (!call) {
|
||||
throw new Error(`expected fetchWithTimeout call ${index}`);
|
||||
}
|
||||
return call;
|
||||
}
|
||||
|
||||
function fetchWithTimeoutGuardedCall(
|
||||
index = 0,
|
||||
): [string, RequestInit | undefined, number, unknown, Record<string, unknown> | undefined] {
|
||||
const call = fetchWithTimeoutGuardedMock.mock.calls[index] as
|
||||
| [string, RequestInit | undefined, number, unknown, Record<string, unknown> | undefined]
|
||||
| undefined;
|
||||
if (!call) {
|
||||
throw new Error(`expected fetchWithTimeoutGuarded call ${index}`);
|
||||
}
|
||||
return call;
|
||||
}
|
||||
|
||||
function pollProviderOperationRequest(index = 0): Record<string, unknown> {
|
||||
const request = pollProviderOperationJsonMock.mock.calls[index]?.[0] as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!request) {
|
||||
throw new Error(`expected pollProviderOperationJson call ${index}`);
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
function providerHttpConfigRequest(): Record<string, unknown> {
|
||||
const [call] = resolveProviderHttpRequestConfigMock.mock.calls;
|
||||
if (!call) {
|
||||
throw new Error("expected provider HTTP config request");
|
||||
}
|
||||
const [request] = call;
|
||||
if (!request || typeof request !== "object" || Array.isArray(request)) {
|
||||
throw new Error("expected provider HTTP config request");
|
||||
}
|
||||
return request as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function streamedVideoResponse(bytes: string): Response {
|
||||
return new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(bytes));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
{ headers: { "content-type": "video/mp4" } },
|
||||
);
|
||||
}
|
||||
|
||||
describe("openai video generation provider", () => {
|
||||
it("declares explicit mode capabilities", () => {
|
||||
expectExplicitVideoGenerationCapabilities(buildOpenAIVideoGenerationProvider(modelAuth));
|
||||
});
|
||||
|
||||
it("does not claim size or duration controls for OpenAI video edits", () => {
|
||||
const provider = buildOpenAIVideoGenerationProvider(modelAuth);
|
||||
|
||||
expect(provider.capabilities.videoToVideo).toEqual({
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputVideos: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("advertises OpenAI video for an actual config-only API key", () => {
|
||||
expect(
|
||||
buildOpenAIVideoGenerationProvider(modelAuth).isConfigured?.({
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: "openai-video-config-key",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not advertise video generation for OAuth-only OpenAI profiles", () => {
|
||||
const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-openai-video-auth-"));
|
||||
const previousOpenAIKey = process.env.OPENAI_API_KEY;
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
try {
|
||||
saveAuthProfileStore(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:chatgpt": {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "chatgpt-oauth-token",
|
||||
refresh: "refresh-token",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
agentDir,
|
||||
{ filterExternalAuthProfiles: false, syncExternalCli: false },
|
||||
);
|
||||
|
||||
expect(buildOpenAIVideoGenerationProvider(modelAuth).isConfigured?.({ agentDir })).toBe(
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
clearRuntimeAuthProfileStoreSnapshots();
|
||||
if (previousOpenAIKey === undefined) {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
} else {
|
||||
process.env.OPENAI_API_KEY = previousOpenAIKey;
|
||||
}
|
||||
// Saving the profile store opens the per-agent database under the temporary agent
|
||||
// dir, and clearing the snapshots does not release it, so Windows fails the removal
|
||||
// with EBUSY unless the cached handles are closed first.
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
fs.rmSync(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("requires an OpenAI API key credential for direct video generation", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: "chatgpt-oauth-token",
|
||||
mode: "oauth",
|
||||
} as never);
|
||||
|
||||
await expect(
|
||||
generateVideo({
|
||||
prompt: "A paper airplane gliding through golden hour light",
|
||||
}),
|
||||
).rejects.toThrow("OpenAI API key missing");
|
||||
|
||||
expect(resolveApiKeyForProviderMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: "openai",
|
||||
modelApi: "openai-responses",
|
||||
}),
|
||||
);
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses SDK-compatible multipart for text-only Sora requests", async () => {
|
||||
const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now());
|
||||
try {
|
||||
postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_123", "queued")));
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce(
|
||||
Response.json(videoJob("vid_123", "completed", { seconds: "4", size: "720x1280" })),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(Buffer.from("webm-bytes"), {
|
||||
headers: new Headers({ "content-type": "video/webm" }),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await generateVideo({
|
||||
prompt: "A paper airplane gliding through golden hour light",
|
||||
durationSeconds: 4,
|
||||
});
|
||||
|
||||
const createRequest = postMultipartRequest();
|
||||
expect(createRequest.url).toBe("https://api.openai.com/v1/videos");
|
||||
const form = createRequest.body as FormData;
|
||||
expect(form.get("prompt")).toBe("A paper airplane gliding through golden hour light");
|
||||
expect(form.get("model")).toBe("sora-2");
|
||||
expect(form.get("seconds")).toBe("4");
|
||||
expect(form.get("input_reference")).toBeNull();
|
||||
const [pollUrl, pollInit, pollTimeout, pollFetch] = fetchWithTimeoutCall(0);
|
||||
expect(pollUrl).toBe("https://api.openai.com/v1/videos/vid_123");
|
||||
expect(pollInit?.method).toBe("GET");
|
||||
expect(pollTimeout).toBe(120000);
|
||||
expect(pollFetch).toBe(fetch);
|
||||
expect(result.videos).toHaveLength(1);
|
||||
expect(result.videos[0]?.mimeType).toBe("video/webm");
|
||||
expect(result.videos[0]?.fileName).toBe("video-1.webm");
|
||||
expect(result.metadata?.videoId).toBe("vid_123");
|
||||
expect(result.metadata?.status).toBe("completed");
|
||||
} finally {
|
||||
clock.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([undefined])(
|
||||
"surfaces an immediately failed OpenAI submission before polling or validating id (%s)",
|
||||
async (videoId) => {
|
||||
const release = vi.fn(async () => {});
|
||||
postMultipartRequestMock.mockResolvedValueOnce(
|
||||
releasedJson(
|
||||
{
|
||||
...(videoId ? { id: videoId } : {}),
|
||||
status: "failed",
|
||||
error: { message: "OpenAI video generation was rejected" },
|
||||
},
|
||||
release,
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
generateVideo({
|
||||
prompt: "A scene that cannot be generated",
|
||||
}),
|
||||
).rejects.toThrow("OpenAI video generation was rejected");
|
||||
|
||||
expect(pollProviderOperationJsonMock).not.toHaveBeenCalled();
|
||||
expect(fetchWithTimeoutMock).not.toHaveBeenCalled();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
},
|
||||
);
|
||||
|
||||
it("downloads an immediately completed OpenAI submission without polling it again", async () => {
|
||||
const release = vi.fn(async () => {});
|
||||
const cancel = vi.fn();
|
||||
postMultipartRequestMock.mockResolvedValueOnce(
|
||||
releasedJson(
|
||||
videoJob("vid_completed", "completed", { seconds: "4", size: "720x1280" }),
|
||||
release,
|
||||
),
|
||||
);
|
||||
fetchWithTimeoutMock.mockResolvedValueOnce(
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("completed-video"));
|
||||
controller.close();
|
||||
},
|
||||
cancel,
|
||||
}),
|
||||
{ headers: { "content-type": "video/mp4" } },
|
||||
),
|
||||
);
|
||||
|
||||
const result = await generateVideo({
|
||||
prompt: "A scene already generated",
|
||||
});
|
||||
|
||||
expect(pollProviderOperationJsonMock).not.toHaveBeenCalled();
|
||||
expect(fetchWithTimeoutMock).toHaveBeenCalledOnce();
|
||||
expect(result).toMatchObject({
|
||||
model: "sora-2",
|
||||
metadata: { seconds: "4", size: "720x1280", status: "completed", videoId: "vid_completed" },
|
||||
});
|
||||
expect(cancel).not.toHaveBeenCalled();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "JSON error",
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ error: "not a rendered video" }),
|
||||
},
|
||||
{
|
||||
label: "problem JSON error",
|
||||
contentType: "application/problem+json",
|
||||
body: JSON.stringify({ detail: "render failed" }),
|
||||
},
|
||||
{ label: "plain-text error", contentType: "text/plain", body: "render failed" },
|
||||
{ label: "empty video", contentType: "video/mp4", body: "" },
|
||||
])(
|
||||
"rejects a successful $label download and releases both requests",
|
||||
async ({ contentType, body }) => {
|
||||
const submissionRelease = vi.fn(async () => {});
|
||||
const downloadRelease = vi.fn(async () => {});
|
||||
postMultipartRequestMock.mockResolvedValueOnce(
|
||||
releasedJson(videoJob("vid_malformed", "completed"), submissionRelease),
|
||||
);
|
||||
fetchWithTimeoutGuardedMock.mockResolvedValueOnce({
|
||||
response: new Response(body, { headers: { "content-type": contentType } }),
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_malformed/content?variant=video",
|
||||
release: downloadRelease,
|
||||
});
|
||||
|
||||
await expect(
|
||||
generateVideo({
|
||||
prompt: "Reject an invalid generated video",
|
||||
cfg: localVideoConfig(true),
|
||||
}),
|
||||
).rejects.toThrow("OpenAI generated video download: malformed video response");
|
||||
|
||||
expect(pollProviderOperationJsonMock).not.toHaveBeenCalled();
|
||||
expect(submissionRelease).toHaveBeenCalledOnce();
|
||||
expect(downloadRelease).toHaveBeenCalledOnce();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ mode: "public", allowPrivateNetwork: false },
|
||||
{ mode: "guarded private", allowPrivateNetwork: true },
|
||||
])(
|
||||
"cancels unread malformed $mode video responses and closes their upstream socket",
|
||||
async ({ allowPrivateNetwork }) => {
|
||||
let notifySocketClosed: ((closed: boolean) => void) | undefined;
|
||||
const socketClosed = new Promise<boolean>((resolve) => {
|
||||
notifySocketClosed = resolve;
|
||||
});
|
||||
await withServer(
|
||||
(request, response) => {
|
||||
request.socket.once("close", () => notifySocketClosed?.(true));
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.write('{"error":"still streaming');
|
||||
},
|
||||
async (baseUrl) => {
|
||||
postMultipartRequestMock.mockResolvedValueOnce(
|
||||
releasedJson({ id: "vid_unread", status: "completed" }),
|
||||
);
|
||||
const upstreamUrl = `${baseUrl}/videos/vid_unread/content`;
|
||||
const downloadRelease = vi.fn(async () => {});
|
||||
if (allowPrivateNetwork) {
|
||||
fetchWithTimeoutGuardedMock.mockImplementationOnce(async () => ({
|
||||
response: await fetch(upstreamUrl),
|
||||
finalUrl: upstreamUrl,
|
||||
release: downloadRelease,
|
||||
}));
|
||||
} else {
|
||||
fetchWithTimeoutMock.mockImplementationOnce(async () => await fetch(upstreamUrl));
|
||||
}
|
||||
|
||||
await expect(
|
||||
generateVideo({
|
||||
prompt: "Reject an unending public video error response",
|
||||
cfg: allowPrivateNetwork ? localVideoConfig(true, `${baseUrl}/v1`) : {},
|
||||
}),
|
||||
).rejects.toThrow("OpenAI generated video download: malformed video response");
|
||||
|
||||
await expect(
|
||||
Promise.race([
|
||||
socketClosed,
|
||||
new Promise<boolean>((resolve) => {
|
||||
setTimeout(() => resolve(false), 250);
|
||||
}),
|
||||
]),
|
||||
).resolves.toBe(true);
|
||||
if (allowPrivateNetwork) {
|
||||
expect(fetchWithTimeoutGuardedMock).toHaveBeenCalledOnce();
|
||||
expect(downloadRelease).toHaveBeenCalledOnce();
|
||||
} else {
|
||||
expect(fetchWithTimeoutGuardedMock).not.toHaveBeenCalled();
|
||||
}
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ mode: "public", allowPrivateNetwork: false },
|
||||
{ mode: "guarded private", allowPrivateNetwork: true },
|
||||
])(
|
||||
"rejects cloned endless $mode video errors without waiting for capture cancellation",
|
||||
async ({ allowPrivateNetwork }) => {
|
||||
const response = new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode('{"error":"still streaming'));
|
||||
},
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
);
|
||||
const captureClone = response.clone();
|
||||
const submissionRelease = vi.fn(async () => {});
|
||||
const downloadRelease = vi.fn(async () => {});
|
||||
postMultipartRequestMock.mockResolvedValueOnce(
|
||||
releasedJson({ id: "vid_cloned", status: "completed" }, submissionRelease),
|
||||
);
|
||||
if (allowPrivateNetwork) {
|
||||
fetchWithTimeoutGuardedMock.mockResolvedValueOnce({
|
||||
response,
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_cloned/content",
|
||||
release: downloadRelease,
|
||||
});
|
||||
} else {
|
||||
fetchWithTimeoutMock.mockResolvedValueOnce(response);
|
||||
}
|
||||
|
||||
const generation = generateVideo({
|
||||
prompt: "Reject a cloned, unending video error",
|
||||
cfg: allowPrivateNetwork ? localVideoConfig(true) : {},
|
||||
});
|
||||
const captureCancellationPending = Symbol("capture cancellation pending");
|
||||
|
||||
try {
|
||||
const result = await Promise.race([
|
||||
generation.then(
|
||||
() => undefined,
|
||||
(error: unknown) => error,
|
||||
),
|
||||
new Promise<symbol>((resolve) => {
|
||||
setImmediate(() => resolve(captureCancellationPending));
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(result).not.toBe(captureCancellationPending);
|
||||
expect(result).toMatchObject({
|
||||
message: "OpenAI generated video download: malformed video response",
|
||||
});
|
||||
expect(submissionRelease).toHaveBeenCalledOnce();
|
||||
if (allowPrivateNetwork) {
|
||||
expect(downloadRelease).toHaveBeenCalledOnce();
|
||||
}
|
||||
} finally {
|
||||
void captureClone.body?.cancel().catch(() => undefined);
|
||||
await generation.catch(() => undefined);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["application/json"])(
|
||||
"keeps the malformed public video error when %s body cancellation fails",
|
||||
async (contentType) => {
|
||||
const cancel = vi.fn(async () => {
|
||||
throw new Error("upstream cancellation failed");
|
||||
});
|
||||
const submissionRelease = vi.fn(async () => {});
|
||||
postMultipartRequestMock.mockResolvedValueOnce(
|
||||
releasedJson({ id: "vid_cancel_failed", status: "completed" }, submissionRelease),
|
||||
);
|
||||
fetchWithTimeoutMock.mockResolvedValueOnce(
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("still streaming"));
|
||||
},
|
||||
cancel,
|
||||
}),
|
||||
{ headers: { "content-type": contentType } },
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
generateVideo({
|
||||
prompt: "Preserve the malformed public video response error",
|
||||
}),
|
||||
).rejects.toThrow("OpenAI generated video download: malformed video response");
|
||||
|
||||
expect(cancel).toHaveBeenCalledOnce();
|
||||
expect(submissionRelease).toHaveBeenCalledOnce();
|
||||
expect(fetchWithTimeoutGuardedMock).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects generated video downloads that exceed the configured media cap", async () => {
|
||||
postMultipartRequestMock.mockResolvedValueOnce(
|
||||
releasedJson(videoJob("vid_too_large", "queued")),
|
||||
);
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce(Response.json(videoJob("vid_too_large", "completed")))
|
||||
.mockResolvedValueOnce(streamedVideoResponse("too-large"));
|
||||
|
||||
await expect(
|
||||
generateVideo({
|
||||
prompt: "short video",
|
||||
cfg: { agents: { defaults: { mediaMaxMb: 0.000001 } } },
|
||||
}),
|
||||
).rejects.toThrow("OpenAI generated video download exceeds 1 bytes");
|
||||
});
|
||||
|
||||
it("uploads the SDK-compatible image reference in a multipart video request", async () => {
|
||||
const clock = vi.spyOn(Date, "now").mockReturnValue(Date.now());
|
||||
try {
|
||||
postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_456", "queued")));
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce(Response.json(videoJob("vid_456", "completed")))
|
||||
.mockResolvedValueOnce(
|
||||
new Response(Buffer.from("mp4-bytes"), {
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
}),
|
||||
);
|
||||
|
||||
const input = Buffer.from("!png-bytes?").subarray(1, -1);
|
||||
await generateVideo({
|
||||
prompt: "Animate this frame",
|
||||
inputImages: [{ buffer: input, mimeType: "image/png" }],
|
||||
});
|
||||
input.fill(0);
|
||||
|
||||
const createRequest = postMultipartRequest();
|
||||
expect(createRequest.url).toBe("https://api.openai.com/v1/videos");
|
||||
const form = createRequest.body as FormData;
|
||||
const reference = form.get("input_reference");
|
||||
expect(reference).toBeInstanceOf(File);
|
||||
const referenceFile = reference as File;
|
||||
expect(referenceFile.name).toBe("reference-image.png");
|
||||
expect(referenceFile.type).toBe("image/png");
|
||||
expect(Buffer.from(await referenceFile.arrayBuffer())).toEqual(Buffer.from("png-bytes"));
|
||||
const [pollUrl, pollInit, pollTimeout, pollFetch] = fetchWithTimeoutCall(0);
|
||||
expect(pollUrl).toBe("https://api.openai.com/v1/videos/vid_456");
|
||||
expect(pollInit?.method).toBe("GET");
|
||||
expect(pollTimeout).toBe(120000);
|
||||
expect(pollFetch).toBe(fetch);
|
||||
} finally {
|
||||
clock.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps configured local baseUrl private-network blocked unless explicitly enabled", async () => {
|
||||
postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_local", "queued")));
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce(Response.json(videoJob("vid_local", "completed")))
|
||||
.mockResolvedValueOnce(
|
||||
new Response(Buffer.from("mp4-bytes"), {
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
}),
|
||||
);
|
||||
|
||||
await generateVideo({
|
||||
prompt: "Render via local relay",
|
||||
cfg: localVideoConfig(),
|
||||
});
|
||||
|
||||
expect(providerHttpConfigRequest().baseUrl).toBe("http://127.0.0.1:44080/v1");
|
||||
expect(providerHttpConfigRequest().request).toBeUndefined();
|
||||
const createRequest = postMultipartRequest();
|
||||
expect(createRequest.url).toBe("http://127.0.0.1:44080/v1/videos");
|
||||
expect(createRequest.allowPrivateNetwork).toBe(false);
|
||||
});
|
||||
|
||||
it("honors configured request allowPrivateNetwork for local video providers", async () => {
|
||||
postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_local", "queued")));
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce(Response.json(videoJob("vid_local", "completed")))
|
||||
.mockResolvedValueOnce({
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
});
|
||||
|
||||
await generateVideo({
|
||||
prompt: "Render via local relay",
|
||||
cfg: localVideoConfig(true),
|
||||
});
|
||||
|
||||
expect(sanitizeConfiguredModelProviderRequestMock).toHaveBeenCalledWith({
|
||||
allowPrivateNetwork: true,
|
||||
});
|
||||
expect(providerHttpConfigRequest().baseUrl).toBe("http://127.0.0.1:44080/v1");
|
||||
expect(providerHttpConfigRequest().request).toEqual({ allowPrivateNetwork: true });
|
||||
const createRequest = postMultipartRequest();
|
||||
expect(createRequest.url).toBe("http://127.0.0.1:44080/v1/videos");
|
||||
expect(createRequest.allowPrivateNetwork).toBe(true);
|
||||
const statusRequest = pollProviderOperationRequest();
|
||||
expect(statusRequest.url).toBe("http://127.0.0.1:44080/v1/videos/vid_local");
|
||||
expect(statusRequest.allowPrivateNetwork).toBe(true);
|
||||
expect(statusRequest.auditContext).toBe("openai-video-status");
|
||||
const [downloadUrl, downloadInit, downloadTimeout, downloadFetch, downloadOptions] =
|
||||
fetchWithTimeoutGuardedCall();
|
||||
expect(downloadUrl).toBe("http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video");
|
||||
expect(downloadInit?.method).toBe("GET");
|
||||
// Download shares the generation deadline, so earlier phases consume part of this budget.
|
||||
expect(downloadTimeout).toBeGreaterThan(0);
|
||||
expect(downloadTimeout).toBeLessThanOrEqual(120_000);
|
||||
expect(downloadFetch).toBe(fetch);
|
||||
expect(downloadOptions).toEqual({
|
||||
ssrfPolicy: { allowPrivateNetwork: true },
|
||||
auditContext: "openai-video-download",
|
||||
});
|
||||
});
|
||||
|
||||
it("retries guarded local video downloads after transient HTTP errors", async () => {
|
||||
const firstRelease = vi.fn(async () => {});
|
||||
const secondRelease = vi.fn(async () => {});
|
||||
assertOkOrThrowHttpErrorMock
|
||||
.mockImplementationOnce(async () => {})
|
||||
.mockImplementationOnce(async (_response, label) => {
|
||||
throw Object.assign(new Error(label), { status: _response.status });
|
||||
})
|
||||
.mockImplementationOnce(async () => {});
|
||||
postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_local", "queued")));
|
||||
fetchWithTimeoutMock.mockResolvedValueOnce(Response.json(videoJob("vid_local", "completed")));
|
||||
fetchWithTimeoutGuardedMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response("busy", { status: 503, statusText: "Service Unavailable" }),
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video",
|
||||
release: firstRelease,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
},
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video",
|
||||
release: secondRelease,
|
||||
});
|
||||
|
||||
const result = await generateVideo({
|
||||
prompt: "Render via local relay",
|
||||
cfg: localVideoConfig(true),
|
||||
});
|
||||
|
||||
expect(result.videos[0]?.buffer?.toString()).toBe("mp4-bytes");
|
||||
expect(executeProviderOperationWithRetryMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ provider: "openai", stage: "download" }),
|
||||
);
|
||||
expect(postMultipartRequestMock).toHaveBeenCalledOnce();
|
||||
expect(fetchWithTimeoutGuardedMock).toHaveBeenCalledTimes(2);
|
||||
expect(firstRelease).toHaveBeenCalledTimes(1);
|
||||
expect(secondRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("releases guarded local video download requests when HTTP errors throw", async () => {
|
||||
const firstRelease = vi.fn(async () => {});
|
||||
const secondRelease = vi.fn(async () => {});
|
||||
assertOkOrThrowHttpErrorMock
|
||||
.mockImplementationOnce(async () => {})
|
||||
.mockImplementationOnce(async (_response, label) => {
|
||||
throw Object.assign(new Error(label), { status: _response.status });
|
||||
})
|
||||
.mockImplementationOnce(async (_response, label) => {
|
||||
throw Object.assign(new Error(label), { status: _response.status });
|
||||
});
|
||||
postMultipartRequestMock.mockResolvedValueOnce(releasedJson(videoJob("vid_local", "queued")));
|
||||
fetchWithTimeoutMock.mockResolvedValueOnce(Response.json(videoJob("vid_local", "completed")));
|
||||
fetchWithTimeoutGuardedMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response("busy", { status: 503, statusText: "Service Unavailable" }),
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video",
|
||||
release: firstRelease,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response("busy", { status: 503, statusText: "Service Unavailable" }),
|
||||
finalUrl: "http://127.0.0.1:44080/v1/videos/vid_local/content?variant=video",
|
||||
release: secondRelease,
|
||||
});
|
||||
|
||||
await expect(
|
||||
generateVideo({
|
||||
prompt: "Render via local relay",
|
||||
cfg: localVideoConfig(true),
|
||||
}),
|
||||
).rejects.toThrow("OpenAI video download failed");
|
||||
|
||||
expect(postMultipartRequestMock).toHaveBeenCalledOnce();
|
||||
expect(fetchWithTimeoutGuardedMock).toHaveBeenCalledTimes(2);
|
||||
expect(firstRelease).toHaveBeenCalledTimes(1);
|
||||
expect(secondRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uses the video edits endpoint for video-to-video uploads", async () => {
|
||||
fetchWithTimeoutMock
|
||||
.mockResolvedValueOnce(Response.json(videoJob("vid_789", "queued")))
|
||||
.mockResolvedValueOnce(Response.json(videoJob("vid_789", "completed")))
|
||||
.mockResolvedValueOnce(
|
||||
new Response(Buffer.from("mp4-bytes"), {
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
}),
|
||||
);
|
||||
|
||||
const input = Buffer.from("!mp4-bytes?").subarray(1, -1);
|
||||
await generateVideo({
|
||||
prompt: "Remix this clip",
|
||||
inputVideos: [{ buffer: input, mimeType: "video/mp4" }],
|
||||
});
|
||||
input.fill(0);
|
||||
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
const createRequest = postMultipartRequest();
|
||||
expect(createRequest.url).toBe("https://api.openai.com/v1/videos/edits");
|
||||
expect(createRequest.body).toBeInstanceOf(FormData);
|
||||
const form = createRequest.body as FormData;
|
||||
expect(form.get("prompt")).toBe("Remix this clip");
|
||||
expect(form.get("model")).toBeNull();
|
||||
expect(form.get("video")).toBeInstanceOf(File);
|
||||
expect(Buffer.from(await (form.get("video") as File).arrayBuffer())).toEqual(
|
||||
Buffer.from("mp4-bytes"),
|
||||
);
|
||||
expect(form.get("input_reference")).toBeNull();
|
||||
expect(createRequest.timeoutMs).toBe(120000);
|
||||
expect(createRequest.fetchFn).toBe(fetch);
|
||||
expect(createRequest.allowPrivateNetwork).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects multiple reference assets", async () => {
|
||||
await expect(
|
||||
generateVideo({
|
||||
prompt: "Animate these",
|
||||
inputImages: [{ buffer: Buffer.from("a"), mimeType: "image/png" }],
|
||||
inputVideos: [{ buffer: Buffer.from("b"), mimeType: "video/mp4" }],
|
||||
}),
|
||||
).rejects.toThrow("OpenAI video generation supports at most one reference image or video.");
|
||||
});
|
||||
});
|
||||
|
|
@ -1,392 +0,0 @@
|
|||
import { bufferToBlobPart } from "openclaw/plugin-sdk/blob-runtime";
|
||||
import { extensionForMime, type MediaKind } from "openclaw/plugin-sdk/media-mime";
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import type {
|
||||
createProviderOperationDeadline,
|
||||
postMultipartRequest,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
import { resolveConfiguredOpenAIBaseUrl } from "./shared.js";
|
||||
|
||||
const DEFAULT_OPENAI_VIDEO_BASE_URL = "https://api.openai.com/v1";
|
||||
const DEFAULT_OPENAI_VIDEO_MODEL = "sora-2";
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const POLL_INTERVAL_MS = 2_500;
|
||||
const MAX_POLL_ATTEMPTS = 120;
|
||||
const OPENAI_VIDEO_SECONDS = [4, 8, 12] as const;
|
||||
const OPENAI_VIDEO_SIZES = ["720x1280", "1280x720", "1024x1792", "1792x1024"] as const;
|
||||
|
||||
type OpenAIVideoRequestPolicy = {
|
||||
allowPrivateNetwork: boolean;
|
||||
dispatcherPolicy?: Parameters<typeof postMultipartRequest>[0]["dispatcherPolicy"];
|
||||
};
|
||||
|
||||
type OpenAIVideoStatus = "queued" | "in_progress" | "completed" | "failed";
|
||||
|
||||
type OpenAIReferenceAsset = {
|
||||
kind: Extract<MediaKind, "image" | "video">;
|
||||
file: File;
|
||||
};
|
||||
|
||||
type OpenAIVideoResponse = {
|
||||
id?: string;
|
||||
model?: string;
|
||||
status?: OpenAIVideoStatus;
|
||||
prompt?: string | null;
|
||||
seconds?: string;
|
||||
size?: string;
|
||||
error?: {
|
||||
code?: string;
|
||||
message?: string;
|
||||
} | null;
|
||||
};
|
||||
|
||||
function readOpenAIVideoFailureMessage(payload: OpenAIVideoResponse): string | undefined {
|
||||
return payload.status === "failed"
|
||||
? (normalizeOptionalString(payload.error?.message) ?? "OpenAI video generation failed")
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function resolveDurationSeconds(durationSeconds: number | undefined): "4" | "8" | "12" | undefined {
|
||||
if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
|
||||
return undefined;
|
||||
}
|
||||
const rounded = Math.max(OPENAI_VIDEO_SECONDS[0], Math.round(durationSeconds));
|
||||
const nearest = OPENAI_VIDEO_SECONDS.reduce((best, current) =>
|
||||
Math.abs(current - rounded) < Math.abs(best - rounded) ? current : best,
|
||||
);
|
||||
return String(nearest) as "4" | "8" | "12";
|
||||
}
|
||||
|
||||
function resolveSize(params: {
|
||||
size?: string;
|
||||
aspectRatio?: string;
|
||||
resolution?: string;
|
||||
}): (typeof OPENAI_VIDEO_SIZES)[number] | undefined {
|
||||
const explicitSize = normalizeOptionalString(params.size);
|
||||
const supportedSize = OPENAI_VIDEO_SIZES.find((size) => size === explicitSize);
|
||||
if (supportedSize) {
|
||||
return supportedSize;
|
||||
}
|
||||
switch (normalizeOptionalString(params.aspectRatio)) {
|
||||
case "9:16":
|
||||
return "720x1280";
|
||||
case "16:9":
|
||||
return "1280x720";
|
||||
case "4:7":
|
||||
return "1024x1792";
|
||||
case "7:4":
|
||||
return "1792x1024";
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (params.resolution === "1080P") {
|
||||
return "1792x1024";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveReferenceAsset(req: VideoGenerationRequest): OpenAIReferenceAsset | null {
|
||||
const allAssets = [...(req.inputImages ?? []), ...(req.inputVideos ?? [])];
|
||||
if (allAssets.length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (allAssets.length > 1) {
|
||||
throw new Error("OpenAI video generation supports at most one reference image or video.");
|
||||
}
|
||||
const [asset] = allAssets;
|
||||
if (!asset?.buffer) {
|
||||
throw new Error(
|
||||
"OpenAI video generation currently requires local image/video uploads for reference assets.",
|
||||
);
|
||||
}
|
||||
const kind = (req.inputVideos?.length ?? 0) > 0 ? "video" : "image";
|
||||
const mimeType =
|
||||
normalizeOptionalString(asset.mimeType) || (kind === "video" ? "video/mp4" : "image/png");
|
||||
const extension =
|
||||
extensionForMime(mimeType)?.slice(1) ?? (mimeType.startsWith("video/") ? "mp4" : "png");
|
||||
const fileName = normalizeOptionalString(asset.fileName) || `reference-${kind}.${extension}`;
|
||||
return {
|
||||
kind,
|
||||
file: new File([bufferToBlobPart(asset.buffer)], fileName, { type: mimeType }),
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchOpenAIVideoDownload(
|
||||
params: {
|
||||
url: string;
|
||||
init: RequestInit;
|
||||
deadline: ReturnType<typeof createProviderOperationDeadline>;
|
||||
fetchFn: typeof fetch;
|
||||
} & OpenAIVideoRequestPolicy,
|
||||
) {
|
||||
const {
|
||||
assertOkOrThrowHttpError,
|
||||
createProviderOperationTimeoutResolver,
|
||||
executeProviderOperationWithRetry,
|
||||
fetchProviderDownloadResponse,
|
||||
fetchWithTimeoutGuarded,
|
||||
} = await import("openclaw/plugin-sdk/provider-http");
|
||||
const timeoutMs = createProviderOperationTimeoutResolver({
|
||||
deadline: params.deadline,
|
||||
defaultTimeoutMs: params.deadline.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
});
|
||||
if (!params.allowPrivateNetwork && !params.dispatcherPolicy) {
|
||||
const response = await fetchProviderDownloadResponse({
|
||||
url: params.url,
|
||||
init: params.init,
|
||||
deadline: params.deadline,
|
||||
fetchFn: params.fetchFn,
|
||||
provider: "openai",
|
||||
requestFailedMessage: "OpenAI video download failed",
|
||||
});
|
||||
return {
|
||||
response,
|
||||
release: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
return await executeProviderOperationWithRetry({
|
||||
provider: "openai",
|
||||
stage: "download",
|
||||
operation: async () => {
|
||||
const result = await fetchWithTimeoutGuarded(
|
||||
params.url,
|
||||
params.init,
|
||||
timeoutMs(),
|
||||
params.fetchFn,
|
||||
{
|
||||
...(params.allowPrivateNetwork ? { ssrfPolicy: { allowPrivateNetwork: true } } : {}),
|
||||
...(params.dispatcherPolicy ? { dispatcherPolicy: params.dispatcherPolicy } : {}),
|
||||
auditContext: "openai-video-download",
|
||||
},
|
||||
);
|
||||
try {
|
||||
await assertOkOrThrowHttpError(result.response, "OpenAI video download failed");
|
||||
return result;
|
||||
} catch (error) {
|
||||
await result.release();
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function buildOpenAIVideoGenerationProvider({
|
||||
isProviderApiKeyConfigured,
|
||||
}: Pick<
|
||||
OpenClawPluginApi["runtime"]["modelAuth"],
|
||||
"isProviderApiKeyConfigured"
|
||||
>): VideoGenerationProvider {
|
||||
return {
|
||||
id: "openai",
|
||||
label: "OpenAI",
|
||||
defaultModel: DEFAULT_OPENAI_VIDEO_MODEL,
|
||||
models: [DEFAULT_OPENAI_VIDEO_MODEL, "sora-2-pro"],
|
||||
isConfigured: (ctx) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: "openai",
|
||||
...ctx,
|
||||
profileTypes: ["api_key"],
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
maxDurationSeconds: 12,
|
||||
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||
supportsSize: true,
|
||||
sizes: OPENAI_VIDEO_SIZES,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: 12,
|
||||
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||
supportsSize: true,
|
||||
sizes: OPENAI_VIDEO_SIZES,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputVideos: 1,
|
||||
},
|
||||
},
|
||||
async generateVideo(req) {
|
||||
const { resolveApiKeyForProvider } =
|
||||
await import("openclaw/plugin-sdk/provider-auth-runtime");
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "openai",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
modelApi: "openai-responses",
|
||||
});
|
||||
if (!auth.apiKey || (auth.mode !== undefined && auth.mode !== "api-key")) {
|
||||
throw new Error("OpenAI API key missing");
|
||||
}
|
||||
|
||||
const [
|
||||
{
|
||||
assertOkOrThrowHttpError,
|
||||
createProviderOperationDeadline,
|
||||
createProviderOperationTimeoutResolver,
|
||||
pollProviderOperationJson,
|
||||
postMultipartRequest,
|
||||
readProviderJsonResponse,
|
||||
resolveProviderOperationTimeoutMs,
|
||||
resolveProviderHttpRequestConfig,
|
||||
sanitizeConfiguredModelProviderRequest,
|
||||
},
|
||||
{ downloadGeneratedVideoAsset, resolveGeneratedMediaMaxBytes },
|
||||
] = await Promise.all([
|
||||
import("openclaw/plugin-sdk/provider-http"),
|
||||
import("openclaw/plugin-sdk/media-generation-runtime"),
|
||||
]);
|
||||
const fetchFn = fetch;
|
||||
const deadline = createProviderOperationDeadline({
|
||||
timeoutMs: req.timeoutMs,
|
||||
label: "OpenAI video generation",
|
||||
});
|
||||
const providerConfig = req.cfg.models?.providers?.openai;
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveProviderHttpRequestConfig({
|
||||
baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg),
|
||||
defaultBaseUrl: DEFAULT_OPENAI_VIDEO_BASE_URL,
|
||||
request: sanitizeConfiguredModelProviderRequest(providerConfig?.request),
|
||||
defaultHeaders: {
|
||||
Authorization: `Bearer ${auth.apiKey}`,
|
||||
},
|
||||
provider: "openai",
|
||||
capability: "video",
|
||||
transport: "http",
|
||||
});
|
||||
|
||||
const model = normalizeOptionalString(req.model) ?? DEFAULT_OPENAI_VIDEO_MODEL;
|
||||
const seconds = resolveDurationSeconds(req.durationSeconds);
|
||||
const size = resolveSize(req);
|
||||
const referenceAsset = resolveReferenceAsset(req);
|
||||
const isVideoEdit = referenceAsset?.kind === "video";
|
||||
const form = new FormData();
|
||||
form.set("prompt", req.prompt);
|
||||
if (isVideoEdit) {
|
||||
form.set("video", referenceAsset.file);
|
||||
} else {
|
||||
form.set("model", model);
|
||||
if (seconds) {
|
||||
form.set("seconds", seconds);
|
||||
}
|
||||
if (size) {
|
||||
form.set("size", size);
|
||||
}
|
||||
if (referenceAsset) {
|
||||
form.set("input_reference", referenceAsset.file);
|
||||
}
|
||||
}
|
||||
const multipartHeaders = new Headers(headers);
|
||||
multipartHeaders.delete("Content-Type");
|
||||
const { response, release } = await postMultipartRequest({
|
||||
url: `${baseUrl}/videos${isVideoEdit ? "/edits" : ""}`,
|
||||
headers: multipartHeaders,
|
||||
body: form,
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
|
||||
let submitted: OpenAIVideoResponse;
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "OpenAI video generation failed");
|
||||
submitted = await readProviderJsonResponse<OpenAIVideoResponse>(
|
||||
response,
|
||||
"OpenAI video generation failed",
|
||||
);
|
||||
} finally {
|
||||
// A consumed submission no longer owns transport during polling or download.
|
||||
await release();
|
||||
}
|
||||
const failureMessage = readOpenAIVideoFailureMessage(submitted);
|
||||
if (failureMessage) {
|
||||
throw new Error(failureMessage);
|
||||
}
|
||||
const videoId = normalizeOptionalString(submitted.id);
|
||||
if (!videoId) {
|
||||
throw new Error("OpenAI video generation response missing video id");
|
||||
}
|
||||
const completed =
|
||||
submitted.status === "completed"
|
||||
? submitted
|
||||
: await pollProviderOperationJson<OpenAIVideoResponse>({
|
||||
url: `${baseUrl}/videos/${videoId}`,
|
||||
headers,
|
||||
deadline: createProviderOperationDeadline({
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
label: `OpenAI video generation task ${videoId}`,
|
||||
}),
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
fetchFn,
|
||||
maxAttempts: MAX_POLL_ATTEMPTS,
|
||||
pollIntervalMs: POLL_INTERVAL_MS,
|
||||
requestFailedMessage: "OpenAI video status request failed",
|
||||
timeoutMessage: `OpenAI video generation task ${videoId} did not finish in time`,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
auditContext: "openai-video-status",
|
||||
isComplete: (payload) => payload.status === "completed",
|
||||
getFailureMessage: readOpenAIVideoFailureMessage,
|
||||
});
|
||||
const url = new URL(`${baseUrl}/videos/${videoId}/content`);
|
||||
url.searchParams.set("variant", "video");
|
||||
const video = await downloadGeneratedVideoAsset({
|
||||
url: url.toString(),
|
||||
timeoutMs: createProviderOperationTimeoutResolver({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
fetchFn,
|
||||
provider: "openai",
|
||||
label: "OpenAI generated video download",
|
||||
requestFailedMessage: "OpenAI video download failed",
|
||||
maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "video"),
|
||||
validateBinaryResponse: true,
|
||||
fetchResponse: async ({ deadline: downloadDeadline }) =>
|
||||
await fetchOpenAIVideoDownload({
|
||||
url: url.toString(),
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: new Headers({
|
||||
...Object.fromEntries(headers.entries()),
|
||||
Accept: "application/binary",
|
||||
}),
|
||||
},
|
||||
deadline: downloadDeadline,
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
}),
|
||||
});
|
||||
return {
|
||||
videos: [video],
|
||||
model: completed.model ?? submitted.model ?? model,
|
||||
metadata: {
|
||||
videoId,
|
||||
status: completed.status,
|
||||
seconds: completed.seconds ?? submitted.seconds,
|
||||
size: completed.size ?? submitted.size,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -50,7 +50,6 @@ import deepinfraPlugin from "./deepinfra/index.js";
|
|||
import falPlugin from "./fal/index.js";
|
||||
import googlePlugin from "./google/index.js";
|
||||
import minimaxPlugin from "./minimax/index.js";
|
||||
import openaiPlugin from "./openai/index.js";
|
||||
import openrouterPlugin from "./openrouter/index.js";
|
||||
import pixversePlugin from "./pixverse/index.js";
|
||||
import qwenPlugin from "./qwen/index.js";
|
||||
|
|
@ -123,7 +122,6 @@ const CASES: LiveProviderCase[] = [
|
|||
pluginName: "MiniMax Provider",
|
||||
providerId: "minimax",
|
||||
},
|
||||
{ plugin: openaiPlugin, pluginId: "openai", pluginName: "OpenAI Provider", providerId: "openai" },
|
||||
{
|
||||
plugin: openrouterPlugin,
|
||||
pluginId: "openrouter",
|
||||
|
|
@ -168,9 +166,9 @@ function withPluginsEnabled(cfg: OpenClawConfig): OpenClawConfig {
|
|||
};
|
||||
}
|
||||
|
||||
function createEditReferencePng(params?: { width?: number; height?: number }): Buffer {
|
||||
const width = params?.width ?? 384;
|
||||
const height = params?.height ?? 384;
|
||||
function createEditReferencePng(): Buffer {
|
||||
const width = 384;
|
||||
const height = 384;
|
||||
const buf = Buffer.alloc(width * height * 4, 255);
|
||||
|
||||
for (let y = 0; y < height; y += 1) {
|
||||
|
|
@ -229,11 +227,9 @@ function expectGeneratedVideo(video: GeneratedVideoAsset | undefined): LiveGener
|
|||
function buildLiveCapabilityOverrides(params: {
|
||||
caps: VideoGenerationModeCapabilities | undefined;
|
||||
liveResolution: VideoGenerationRequest["resolution"];
|
||||
liveSize: string | undefined;
|
||||
}): Pick<VideoGenerationRequest, "size" | "aspectRatio" | "resolution" | "audio" | "watermark"> {
|
||||
const { caps, liveResolution, liveSize } = params;
|
||||
}): Pick<VideoGenerationRequest, "aspectRatio" | "resolution" | "audio" | "watermark"> {
|
||||
const { caps, liveResolution } = params;
|
||||
return {
|
||||
...(caps?.supportsSize && liveSize ? { size: liveSize } : undefined),
|
||||
...(caps?.supportsAspectRatio ? { aspectRatio: "16:9" } : undefined),
|
||||
...(caps?.supportsResolution ? { resolution: liveResolution } : undefined),
|
||||
...(caps?.supportsAudio ? { audio: false } : undefined),
|
||||
|
|
@ -465,7 +461,6 @@ async function runLiveVideoProviderCase(
|
|||
providerId: testCase.providerId,
|
||||
modelRef,
|
||||
});
|
||||
const liveSize = testCase.providerId === "openai" ? "1280x720" : undefined;
|
||||
const logPrefix = `[live:video-generation] provider=${testCase.providerId} model=${providerModel}`;
|
||||
|
||||
const generateAttempt = await runLiveVideoAttempt({
|
||||
|
|
@ -486,7 +481,7 @@ async function runLiveVideoProviderCase(
|
|||
authStore,
|
||||
timeoutMs: LIVE_VIDEO_OPERATION_TIMEOUT_MS,
|
||||
durationSeconds,
|
||||
...buildLiveCapabilityOverrides({ caps: generateCaps, liveResolution, liveSize }),
|
||||
...buildLiveCapabilityOverrides({ caps: generateCaps, liveResolution }),
|
||||
},
|
||||
skipped,
|
||||
});
|
||||
|
|
@ -516,10 +511,7 @@ async function runLiveVideoProviderCase(
|
|||
return;
|
||||
}
|
||||
|
||||
const referenceImage =
|
||||
testCase.providerId === "openai"
|
||||
? createEditReferencePng({ width: 1280, height: 720 })
|
||||
: createEditReferencePng();
|
||||
const referenceImage = createEditReferencePng();
|
||||
const imageAttempt = await runLiveVideoAttempt({
|
||||
authLabel,
|
||||
attempted,
|
||||
|
|
@ -552,7 +544,6 @@ async function runLiveVideoProviderCase(
|
|||
...buildLiveCapabilityOverrides({
|
||||
caps: imageToVideoCaps,
|
||||
liveResolution,
|
||||
liveSize,
|
||||
}),
|
||||
},
|
||||
skipped,
|
||||
|
|
@ -608,7 +599,6 @@ async function runLiveVideoProviderCase(
|
|||
...buildLiveCapabilityOverrides({
|
||||
caps: videoToVideoCaps,
|
||||
liveResolution,
|
||||
liveSize: undefined,
|
||||
}),
|
||||
},
|
||||
skipped,
|
||||
|
|
|
|||
|
|
@ -309,13 +309,13 @@ describe("createVideoGenerateTool", () => {
|
|||
expect(emptyConfigTool).toBeNull();
|
||||
});
|
||||
|
||||
it("treats legacy OpenAI-Codex auth profiles as canonical OpenAI video auth", () => {
|
||||
it("exposes video generation for an auth-backed video provider", () => {
|
||||
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([]);
|
||||
|
||||
expectVideoGenerateTool(
|
||||
createVideoGenerateTool({
|
||||
config: asConfig({}),
|
||||
authProfileStore: createAuthStore(["openai"]),
|
||||
authProfileStore: createAuthStore(["runway"]),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
|
@ -341,7 +341,7 @@ describe("createVideoGenerateTool", () => {
|
|||
const properties = toolParameterProperties(
|
||||
createVideoGenerateTool({
|
||||
config: configWithDefaults({
|
||||
videoGenerationModel: { primary: "openai/sora-2" },
|
||||
videoGenerationModel: { primary: "runway/gen4.5" },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
|
@ -428,7 +428,7 @@ describe("createVideoGenerateTool", () => {
|
|||
const properties = toolParameterProperties(
|
||||
createVideoGenerateTool({
|
||||
config: configWithDefaults({
|
||||
videoGenerationModel: { primary: "openai/sora-2" },
|
||||
videoGenerationModel: { primary: "runway/gen4.5" },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
|
@ -449,7 +449,7 @@ describe("createVideoGenerateTool", () => {
|
|||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
videoGenerationModel: { primary: "openai/sora-2" },
|
||||
videoGenerationModel: { primary: "runway/gen4.5" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -138,7 +138,6 @@ export const pluginRegistrationContractCases = {
|
|||
realtimeVoiceProviderIds: ["openai"],
|
||||
mediaUnderstandingProviderIds: ["openai"],
|
||||
imageGenerationProviderIds: ["openai"],
|
||||
videoGenerationProviderIds: ["openai"],
|
||||
},
|
||||
"opencode-go": {
|
||||
pluginId: "opencode-go",
|
||||
|
|
|
|||
|
|
@ -23,16 +23,18 @@ describe("video-generation live-test helpers", () => {
|
|||
it("parses provider filters and treats empty/all as unfiltered", () => {
|
||||
expect(parseVideoProviderFilter()).toBeNull();
|
||||
expect(parseVideoProviderFilter("all")).toBeNull();
|
||||
expect(parseVideoProviderFilter(" google , openai ")).toEqual(new Set(["google", "openai"]));
|
||||
expect(parseVideoProviderFilter(" google , xai ")).toEqual(new Set(["google", "xai"]));
|
||||
});
|
||||
|
||||
it("parses provider model overrides by provider id", () => {
|
||||
expect(
|
||||
parseProviderModelMap("google/veo-3.1-fast-generate-preview, openai/sora-2, invalid"),
|
||||
parseProviderModelMap(
|
||||
"google/veo-3.1-fast-generate-preview, xai/grok-imagine-video, invalid",
|
||||
),
|
||||
).toEqual(
|
||||
new Map([
|
||||
["google", "google/veo-3.1-fast-generate-preview"],
|
||||
["openai", "openai/sora-2"],
|
||||
["xai", "xai/grok-imagine-video"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
|
@ -44,7 +46,7 @@ describe("video-generation live-test helpers", () => {
|
|||
mediaModels: {
|
||||
video: {
|
||||
primary: "google/veo-3.1-fast-generate-preview",
|
||||
fallbacks: ["openai/sora-2", "invalid"],
|
||||
fallbacks: ["xai/grok-imagine-video", "invalid"],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -54,7 +56,7 @@ describe("video-generation live-test helpers", () => {
|
|||
expect(resolveConfiguredLiveVideoModels(cfg)).toEqual(
|
||||
new Map([
|
||||
["google", "google/veo-3.1-fast-generate-preview"],
|
||||
["openai", "openai/sora-2"],
|
||||
["xai", "xai/grok-imagine-video"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
|
@ -62,7 +64,6 @@ describe("video-generation live-test helpers", () => {
|
|||
it("runs buffer-backed video-to-video only for supported providers/models", () => {
|
||||
for (const [providerId, modelRef, expected] of [
|
||||
["google", "google/veo-3.1-fast-generate-preview", false],
|
||||
["openai", "openai/sora-2", false],
|
||||
["runway", "runway/gen4_aleph", true],
|
||||
["runway", "runway/gen4.5", false],
|
||||
["alibaba", "alibaba/wan2.6-r2v", false],
|
||||
|
|
@ -77,7 +78,7 @@ describe("video-generation live-test helpers", () => {
|
|||
|
||||
it("runs buffer-backed image-to-video only for providers that accept bundled image inputs", () => {
|
||||
for (const [providerId, modelRef, expected] of [
|
||||
["openai", "openai/sora-2", true],
|
||||
["xai", "xai/grok-imagine-video", true],
|
||||
["vydra", "vydra/veo3", false],
|
||||
["together", "together/Wan-AI/Wan2.2-T2V-A14B", false],
|
||||
["together", "together/Wan-AI/Wan2.2-I2V-A14B", true],
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ export const DEFAULT_LIVE_VIDEO_MODELS: Record<string, string> = {
|
|||
fal: "fal/fal-ai/minimax/video-01-live",
|
||||
google: "google/veo-3.1-fast-generate-preview",
|
||||
minimax: "minimax/MiniMax-Hailuo-2.3",
|
||||
openai: "openai/sora-2",
|
||||
openrouter: "openrouter/google/veo-3.1-fast",
|
||||
pixverse: "pixverse/v6",
|
||||
qwen: "qwen/wan2.6-t2v",
|
||||
|
|
@ -29,7 +28,7 @@ export const DEFAULT_LIVE_VIDEO_MODELS: Record<string, string> = {
|
|||
xai: "xai/grok-imagine-video",
|
||||
};
|
||||
|
||||
const REMOTE_URL_VIDEO_TO_VIDEO_PROVIDERS = new Set(["alibaba", "google", "openai", "qwen", "xai"]);
|
||||
const REMOTE_URL_VIDEO_TO_VIDEO_PROVIDERS = new Set(["alibaba", "google", "qwen", "xai"]);
|
||||
const BUFFER_BACKED_IMAGE_TO_VIDEO_UNSUPPORTED_PROVIDERS = new Set(["vydra"]);
|
||||
const TOGETHER_BUFFER_BACKED_IMAGE_TO_VIDEO_MODEL = "Wan-AI/Wan2.2-I2V-A14B";
|
||||
|
||||
|
|
|
|||
|
|
@ -11600,19 +11600,19 @@ surfaces:
|
|||
description: "Covers typed providerOptions across video generation request normalization before provider execution: `generate`, `imageToVideo`, and `videoToVideo` modes, reference media typing and roles, and related video generation modes behavior."
|
||||
- name: queue-backed jobs
|
||||
coverageIds: [media-generation.queue-backed-jobs]
|
||||
description: "Covers queue-backed jobs across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
description: "Covers queue-backed jobs across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
- name: polling/timeout handling
|
||||
coverageIds: [media-generation.polling-timeout-handling]
|
||||
description: "Covers polling/timeout handling across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
description: "Covers polling/timeout handling across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
- name: Hosted URL download
|
||||
coverageIds: [media-generation.hosted-url-download]
|
||||
description: "Covers hosted URL download across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
description: "Covers hosted URL download across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
- name: provider skip explanations
|
||||
coverageIds: [media-generation.provider-skip-explanations]
|
||||
description: "Covers provider skip explanations across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
description: "Covers provider skip explanations across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
- name: returned asset metadata
|
||||
coverageIds: [media-generation.returned-asset-metadata]
|
||||
description: "Covers returned asset metadata across provider integration and async polling for video generation after request normalization: OpenAI Sora, OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
description: "Covers returned asset metadata across provider integration and async polling for video generation after request normalization: OpenRouter, fal, Runway, and related video providers and polling behavior."
|
||||
docs:
|
||||
- docs/tools/video-generation.md
|
||||
- docs/providers/runway.md
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ describe("hosted media provider live CLI", () => {
|
|||
providers: [],
|
||||
skippedReason: "no providers selected",
|
||||
},
|
||||
{ suite: MEDIA_SUITES.video, providers: ["openai"] },
|
||||
{ suite: MEDIA_SUITES.video, providers: [], skippedReason: "no providers selected" },
|
||||
]);
|
||||
|
||||
expect(skipped).toEqual([]);
|
||||
|
|
@ -228,17 +228,12 @@ describe("hosted media provider live CLI", () => {
|
|||
"vydra",
|
||||
]);
|
||||
expect(requirePlanEntry(plan, "music").providers).toEqual(["fal", "google", "minimax"]);
|
||||
expect(requirePlanEntry(plan, "video").providers).toEqual([
|
||||
"google",
|
||||
"minimax",
|
||||
"openai",
|
||||
"vydra",
|
||||
]);
|
||||
expect(requirePlanEntry(plan, "video").providers).toEqual(["google", "minimax", "vydra"]);
|
||||
});
|
||||
|
||||
it("supports suite-specific provider filters without auth narrowing", async () => {
|
||||
const plan = await buildRunPlan(
|
||||
parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]),
|
||||
parseArgs(["video", "--video-providers", "fal,google,runway", "--all-providers"]),
|
||||
{
|
||||
collectProviderApiKeysImpl: collectProviderApiKeysMock,
|
||||
getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`],
|
||||
|
|
@ -249,7 +244,7 @@ describe("hosted media provider live CLI", () => {
|
|||
expect(plan).toHaveLength(1);
|
||||
const [entry] = plan;
|
||||
expect(entry?.suite.id).toBe("video");
|
||||
expect(entry?.providers).toEqual(["fal", "openai", "runway"]);
|
||||
expect(entry?.providers).toEqual(["fal", "google", "runway"]);
|
||||
});
|
||||
|
||||
it("forwards quiet flags separately from passthrough args", () => {
|
||||
|
|
|
|||
|
|
@ -52,7 +52,6 @@ export const MEDIA_SUITES: Record<MediaSuiteId, MediaSuiteConfig> = {
|
|||
"fal",
|
||||
"google",
|
||||
"minimax",
|
||||
"openai",
|
||||
"openrouter",
|
||||
"qwen",
|
||||
"runway",
|
||||
|
|
@ -66,7 +65,6 @@ export const MEDIA_SUITES: Record<MediaSuiteId, MediaSuiteConfig> = {
|
|||
"deepinfra",
|
||||
"google",
|
||||
"minimax",
|
||||
"openai",
|
||||
"openrouter",
|
||||
"qwen",
|
||||
"runway",
|
||||
|
|
@ -476,8 +474,8 @@ function formatHelp(): string {
|
|||
Usage:
|
||||
pnpm test:live:media
|
||||
pnpm test:live:media image
|
||||
pnpm test:live:media image video --providers openai,google,minimax
|
||||
pnpm test:live:media video --video-providers openai,runway --all-providers
|
||||
pnpm test:live:media image video --providers google,minimax,xai
|
||||
pnpm test:live:media video --video-providers runway,xai --all-providers
|
||||
|
||||
QA evidence mode:
|
||||
node --import tsx ${SOURCE_PATH} --qa-evidence --suite image --artifact-base <dir>
|
||||
|
|
|
|||
|
|
@ -10318,7 +10318,7 @@ describe("package artifact reuse", () => {
|
|||
expect(workflow).toContain("suite_id: native-live-extensions-media-video");
|
||||
expect(workflow).toContain("suite_group: native-live-extensions-media-video");
|
||||
expect(workflow).toContain("OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=google,minimax");
|
||||
expect(workflow).toContain("OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=openai,openrouter,xai");
|
||||
expect(workflow).toContain("OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS=openrouter,xai");
|
||||
expect(workflow).toContain(
|
||||
"inputs.live_suite_filter == 'native-live-src-gateway-profiles-anthropic'",
|
||||
);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue