From 05d83125c91284bbfff39bafdca83c83f1dffe8c Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 29 Jun 2026 13:01:47 -0700 Subject: [PATCH] test: add hosted media provider QA live evidence (#97900) --- package.json | 8 +- ...osted-image-generation-providers-live.yaml | 36 ++ ...osted-video-generation-providers-live.yaml | 41 ++ scripts/test-projects.test-support.mjs | 5 +- src/scripts/test-live-media.test.ts | 83 ---- .../media/hosted-media-provider-live.test.ts | 283 ++++++++++++ .../media/hosted-media-provider-live.ts | 414 ++++++++++++++++-- test/scripts/test-live-media.test.ts | 141 ------ 8 files changed, 741 insertions(+), 270 deletions(-) create mode 100644 qa/scenarios/media/hosted-image-generation-providers-live.yaml create mode 100644 qa/scenarios/media/hosted-video-generation-providers-live.yaml delete mode 100644 src/scripts/test-live-media.test.ts create mode 100644 test/e2e/qa-lab/media/hosted-media-provider-live.test.ts rename scripts/test-live-media.ts => test/e2e/qa-lab/media/hosted-media-provider-live.ts (54%) delete mode 100644 test/scripts/test-live-media.test.ts diff --git a/package.json b/package.json index dc8d3e5f24a..fd92064eb8c 100644 --- a/package.json +++ b/package.json @@ -1880,10 +1880,10 @@ "test:live:codex-harness": "node scripts/test-live.mjs --codex-harness -- src/gateway/gateway-codex-harness.live.test.ts", "test:live:crestodian-rescue-channel": "node scripts/run-with-env.mjs OPENCLAW_LIVE_CRESTODIAN_RESCUE_CHANNEL=1 -- node scripts/test-live.mjs -- src/crestodian/rescue-channel.live.test.ts", "test:live:gateway-profiles": "node scripts/test-live.mjs -- src/gateway/gateway-models.profiles.live.test.ts", - "test:live:media": "node --import tsx scripts/test-live-media.ts", - "test:live:media:image": "node --import tsx scripts/test-live-media.ts image", - "test:live:media:music": "node --import tsx scripts/test-live-media.ts music", - "test:live:media:video": "node --import tsx scripts/test-live-media.ts video", + "test:live:media": "node --import tsx test/e2e/qa-lab/media/hosted-media-provider-live.ts", + "test:live:media:image": "node --import tsx test/e2e/qa-lab/media/hosted-media-provider-live.ts image", + "test:live:media:music": "node --import tsx test/e2e/qa-lab/media/hosted-media-provider-live.ts music", + "test:live:media:video": "node --import tsx test/e2e/qa-lab/media/hosted-media-provider-live.ts video", "test:live:models-profiles": "node scripts/test-live.mjs -- src/agents/models.profiles.live.test.ts", "test:macos:ci": "node scripts/test-projects.mjs src/daemon/launchd.test.ts src/daemon/runtime-paths.test.ts src/daemon/runtime-binary.test.ts src/infra/brew.test.ts src/infra/stable-node-path.test.ts test/scripts/vitest-process-group.test.ts test/scripts/package-mac-app.test.ts test/scripts/package-mac-dist.test.ts test/scripts/create-dmg.test.ts test/scripts/codesign-mac-app.test.ts test/scripts/notarize-mac-artifact.test.ts", "test:max": "node scripts/test-projects-max.mjs", diff --git a/qa/scenarios/media/hosted-image-generation-providers-live.yaml b/qa/scenarios/media/hosted-image-generation-providers-live.yaml new file mode 100644 index 00000000000..420215eef72 --- /dev/null +++ b/qa/scenarios/media/hosted-image-generation-providers-live.yaml @@ -0,0 +1,36 @@ +title: Hosted image generation providers live + +scenario: + id: hosted-image-generation-providers-live + surface: long-tail-hosted-providers + category: long-tail-hosted-providers.hosted-media-providers + runtimeParityTier: live-only + coverage: + primary: + - hosted-providers.image-generation-providers + objective: Verify hosted image generation providers can generate real images through the live provider runtime sweep. + successCriteria: + - The QA producer runs the canonical live media image harness instead of duplicating provider assertions. + - At least one auth-available hosted image provider runs through `provider.generateImage`. + - Generated image output includes image MIME type and non-trivial bytes. + - Missing live provider credentials produce blocked evidence rather than false passing primary coverage. + docsRefs: + - docs/help/testing.md + - docs/tools/image-generation.md + - docs/tools/media-overview.md + codeRefs: + - test/e2e/qa-lab/media/hosted-media-provider-live.ts + - test/image-generation.runtime.live.test.ts + - src/image-generation/live-test-helpers.ts + execution: + kind: script + path: test/e2e/qa-lab/media/hosted-media-provider-live.ts + summary: Runs the existing hosted image-generation live provider sweep and emits QA Lab evidence. + allowBlockedEvidence: true + timeoutMs: 7200000 + args: + - --qa-evidence + - --suite + - image + - --artifact-base + - ${outputDir} diff --git a/qa/scenarios/media/hosted-video-generation-providers-live.yaml b/qa/scenarios/media/hosted-video-generation-providers-live.yaml new file mode 100644 index 00000000000..c76ecdc8049 --- /dev/null +++ b/qa/scenarios/media/hosted-video-generation-providers-live.yaml @@ -0,0 +1,41 @@ +title: Hosted video generation providers live + +scenario: + id: hosted-video-generation-providers-live + surface: long-tail-hosted-providers + category: long-tail-hosted-providers.hosted-media-providers + runtimeParityTier: live-only + coverage: + primary: + - hosted-providers.video-generation-providers + - media.reference-image-video-and-audio-inputs + secondary: + - media.video-generation-tool-invocation + objective: Verify hosted video generation providers can generate real videos and exercise full reference-input modes where supported. + successCriteria: + - The QA producer runs the canonical live media video harness instead of duplicating provider assertions. + - At least one auth-available hosted video provider runs through `provider.generateVideo`. + - Full live video modes are enabled so image-to-video and video-to-video reference inputs are attempted where provider capabilities allow them. + - Generated video output includes video MIME type with bytes or an HTTP(S) URL. + - Missing live provider credentials produce blocked evidence rather than false passing primary coverage. + docsRefs: + - docs/help/testing.md + - docs/tools/video-generation.md + - docs/tools/media-overview.md + codeRefs: + - test/e2e/qa-lab/media/hosted-media-provider-live.ts + - extensions/video-generation-providers.live.test.ts + - src/video-generation/runtime.ts + - src/agents/tools/video-generate-tool.ts + execution: + kind: script + path: test/e2e/qa-lab/media/hosted-media-provider-live.ts + summary: Runs the existing hosted video-generation live provider sweep with full reference-input modes and emits QA Lab evidence. + allowBlockedEvidence: true + timeoutMs: 14400000 + args: + - --qa-evidence + - --suite + - video + - --artifact-base + - ${outputDir} diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 086684e3cb8..2f00ca0382a 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -1924,7 +1924,10 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ["scripts/test-projects.test-support.mjs", ["test/scripts/test-projects.test.ts"]], ["scripts/dev/gateway-smoke.ts", ["test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts"]], ["scripts/dev/test-device-pair-telegram.ts", ["test/scripts/test-device-pair-telegram.test.ts"]], - ["scripts/test-live-media.ts", ["test/scripts/test-live-media.test.ts"]], + [ + "test/e2e/qa-lab/media/hosted-media-provider-live.ts", + ["test/e2e/qa-lab/media/hosted-media-provider-live.test.ts"], + ], ["scripts/profile-extension-memory.mjs", ["test/scripts/profile-extension-memory.test.ts"]], [ "scripts/openclaw-performance-source-summary.mjs", diff --git a/src/scripts/test-live-media.test.ts b/src/scripts/test-live-media.test.ts deleted file mode 100644 index dd0fdbda442..00000000000 --- a/src/scripts/test-live-media.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -// Live media script tests cover live media test command construction. -import { afterEach, describe, expect, it, vi } from "vitest"; - -const loadShellEnvFallbackMock = vi.fn(); -const collectProviderApiKeysMock = vi.fn((provider: string) => - process.env[`TEST_AUTH_${provider.toUpperCase()}`] ? ["test-key"] : [], -); - -function requirePlanEntry( - plan: Awaited>, - suiteId: string, -) { - const entry = plan.find((candidate) => candidate.suite.id === suiteId); - if (!entry) { - throw new Error(`expected ${suiteId} run plan entry`); - } - return entry; -} - -describe("test-live-media", () => { - afterEach(() => { - collectProviderApiKeysMock.mockClear(); - loadShellEnvFallbackMock.mockReset(); - vi.unstubAllEnvs(); - }); - - it("defaults to all suites with auth filtering", async () => { - vi.stubEnv("TEST_AUTH_OPENAI", "1"); - vi.stubEnv("TEST_AUTH_GOOGLE", "1"); - vi.stubEnv("TEST_AUTH_MINIMAX", "1"); - vi.stubEnv("TEST_AUTH_FAL", "1"); - vi.stubEnv("TEST_AUTH_VYDRA", "1"); - - const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts"); - const plan = await buildRunPlan(parseArgs([]), { - collectProviderApiKeysImpl: collectProviderApiKeysMock, - getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], - loadShellEnvFallbackImpl: loadShellEnvFallbackMock, - }); - - expect(plan.map((entry) => entry.suite.id)).toEqual(["image", "music", "video"]); - expect(requirePlanEntry(plan, "image").providers).toEqual([ - "fal", - "google", - "minimax", - "openai", - "vydra", - ]); - expect(requirePlanEntry(plan, "music").providers).toEqual(["fal", "google", "minimax"]); - expect(requirePlanEntry(plan, "video").providers).toEqual([ - "google", - "minimax", - "openai", - "vydra", - ]); - }); - - it("supports suite-specific provider filters without auth narrowing", async () => { - const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts"); - const plan = await buildRunPlan( - parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]), - { - collectProviderApiKeysImpl: collectProviderApiKeysMock, - getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], - loadShellEnvFallbackImpl: loadShellEnvFallbackMock, - }, - ); - - expect(plan).toHaveLength(1); - const [entry] = plan; - expect(entry?.suite.id).toBe("video"); - expect(entry?.providers).toEqual(["fal", "openai", "runway"]); - }); - - it("forwards quiet flags separately from passthrough args", async () => { - const { parseArgs } = await import("../../scripts/test-live-media.ts"); - const options = parseArgs(["image", "--quiet", "--reporter", "dot"]); - - expect(options.suites).toEqual(["image"]); - expect(options.quietArgs).toEqual(["--quiet"]); - expect(options.passthroughArgs).toEqual(["--reporter", "dot"]); - }); -}); diff --git a/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts b/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts new file mode 100644 index 00000000000..6f525a1da6b --- /dev/null +++ b/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts @@ -0,0 +1,283 @@ +// Hosted media provider live producer tests cover QA evidence wiring. +import { spawnSync } from "node:child_process"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + MEDIA_SUITES, + buildRunPlan, + buildHostedMediaCommand, + buildHostedMediaEvidence, + classifyHostedMediaFailureStatus, + findSkippedExplicitProviderSelections, + parseArgs, + parseHostedMediaOptions, + runCli, + type SuiteRunPlan, +} from "./hosted-media-provider-live.js"; + +const SOURCE_PATH = "test/e2e/qa-lab/media/hosted-media-provider-live.ts"; +const loadShellEnvFallbackMock = vi.fn(); +const collectProviderApiKeysMock = vi.fn((provider: string) => + process.env[`TEST_AUTH_${provider.toUpperCase()}`] ? ["test-key"] : [], +); + +function requirePlanEntry(plan: SuiteRunPlan[], suiteId: string) { + const entry = plan.find((candidate) => candidate.suite.id === suiteId); + if (!entry) { + throw new Error(`expected ${suiteId} run plan entry`); + } + return entry; +} + +afterEach(() => { + collectProviderApiKeysMock.mockClear(); + loadShellEnvFallbackMock.mockReset(); + vi.unstubAllEnvs(); +}); + +describe("hosted media provider live QA producer", () => { + it("builds the image live media command with provider filters from env", () => { + const options = parseHostedMediaOptions([ + "--suite", + "image", + "--artifact-base", + ".artifacts/qa-e2e/image", + "--providers-env", + "OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS", + ]); + const command = buildHostedMediaCommand({ + env: { OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS: "openai,google" }, + options, + }); + + expect(command.args).toContain(SOURCE_PATH); + expect(command.args).toContain("image"); + expect(command.args).toContain("--image-providers"); + expect(command.args).toContain("openai,google"); + expect(command.env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES).toBeUndefined(); + }); + + it("forces full video modes so reference image and video inputs are covered", () => { + const options = parseHostedMediaOptions([ + "--suite", + "video", + "--artifact-base", + ".artifacts/qa-e2e/video", + ]); + const command = buildHostedMediaCommand({ env: {}, options }); + + expect(command.args).toContain("video"); + expect(command.env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES).toBe("1"); + }); + + it("classifies missing live media auth as blocked evidence", () => { + expect( + classifyHostedMediaFailureStatus( + "[live:media] no runnable providers matched available auth; pass --allow-empty", + ), + ).toBe("blocked"); + expect(classifyHostedMediaFailureStatus("provider response was malformed")).toBe("fail"); + }); + + it("maps video provider live coverage roles without making tool invocation primary", () => { + const artifactBase = path.join(os.tmpdir(), "openclaw-hosted-media-live-test"); + const options = parseHostedMediaOptions(["--suite", "video", "--artifact-base", artifactBase]); + const evidence = buildHostedMediaEvidence({ + options, + result: { + artifacts: [{ kind: "log", path: "hosted-media-live.log" }], + durationMs: 10, + status: "pass", + }, + }); + + expect(evidence.entries[0]?.coverage).toEqual([ + { id: "hosted-providers.video-generation-providers", role: "primary" }, + { id: "media.reference-image-video-and-audio-inputs", role: "primary" }, + { id: "media.video-generation-tool-invocation", role: "secondary" }, + ]); + }); +}); + +describe("hosted media provider live CLI", () => { + it("prints help through the real node --import tsx entrypoint", () => { + const result = spawnSync(process.execPath, ["--import", "tsx", SOURCE_PATH, "--help"], { + cwd: process.cwd(), + encoding: "utf8", + }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("Media live harness"); + expect(result.stdout).toContain("pnpm test:live:media"); + expect(result.stderr).toBe(""); + }); + + it("rejects unknown global providers for the selected suites", () => { + expect(() => + parseArgs(["image", "--providers", "definitely-not-a-provider", "--all-providers"]), + ).toThrow("Unknown provider(s) for selected media suite(s): definitely-not-a-provider"); + }); + + it("rejects unknown suite-specific providers", () => { + expect(() => parseArgs(["image", "--image-providers", "runway", "--all-providers"])).toThrow( + "Unknown image provider(s): runway", + ); + }); + + it("accepts providers supported by the wrapped live suites", () => { + expect( + parseArgs(["image", "--image-providers", "openrouter", "--all-providers"]).suiteProviders + .image, + ).toEqual(new Set(["openrouter"])); + expect( + parseArgs(["music", "--music-providers", "fal,openrouter", "--all-providers"]).suiteProviders + .music, + ).toEqual(new Set(["fal", "openrouter"])); + expect( + parseArgs(["video", "--video-providers", "openrouter", "--all-providers"]).suiteProviders + .video, + ).toEqual(new Set(["openrouter"])); + }); + + it("rejects suite-specific provider filters for unselected suites", () => { + expect(() => parseArgs(["image", "--music-providers", "fal", "--all-providers"])).toThrow( + "Provider filter(s) target unselected media suite(s): music", + ); + }); + + it("passes single-dash Vitest args after the option separator", () => { + expect( + parseArgs(["image", "--all-providers", "--project", "tooling", "--", "-t", "media-smoke"]), + ).toMatchObject({ + suites: ["image"], + requireAuth: false, + passthroughArgs: ["--project", "tooling", "-t", "media-smoke"], + }); + }); + + it("parses the explicit empty-run escape hatch", () => { + expect(parseArgs(["--allow-empty"])).toMatchObject({ + allowEmpty: true, + requireAuth: true, + }); + }); + + it("fails explicit suite selections that auth filtering would skip", () => { + const options = parseArgs([ + "image", + "music", + "--image-providers", + "openai", + "--music-providers", + "minimax", + ]); + const skipped = findSkippedExplicitProviderSelections(options, [ + { suite: MEDIA_SUITES.image, providers: ["openai"] }, + { + suite: MEDIA_SUITES.music, + providers: [], + skippedReason: "no providers with usable auth", + }, + ]); + + expect(skipped.map((entry) => entry.suite.id)).toEqual(["music"]); + }); + + it("does not fail global provider filters for suites without provider overlap", () => { + const options = parseArgs(["image", "music", "video", "--providers", "openai"]); + const skipped = findSkippedExplicitProviderSelections(options, [ + { suite: MEDIA_SUITES.image, providers: ["openai"] }, + { + suite: MEDIA_SUITES.music, + providers: [], + skippedReason: "no providers selected", + }, + { suite: MEDIA_SUITES.video, providers: ["openai"] }, + ]); + + expect(skipped).toEqual([]); + }); + + it("fails default live media runs when auth filtering leaves no providers", async () => { + await expect( + runCli(["image"], { + buildRunPlanImpl: () => [ + { + providers: [], + skippedReason: "no providers with usable auth", + suite: MEDIA_SUITES.image, + }, + ], + }), + ).resolves.toBe(1); + }); + + it("allows empty live media runs only with an explicit escape hatch", async () => { + await expect( + runCli(["image", "--allow-empty"], { + buildRunPlanImpl: () => [ + { + providers: [], + skippedReason: "no providers with usable auth", + suite: MEDIA_SUITES.image, + }, + ], + }), + ).resolves.toBe(0); + }); + + it("defaults to all suites with auth filtering", async () => { + vi.stubEnv("TEST_AUTH_OPENAI", "1"); + vi.stubEnv("TEST_AUTH_GOOGLE", "1"); + vi.stubEnv("TEST_AUTH_MINIMAX", "1"); + vi.stubEnv("TEST_AUTH_FAL", "1"); + vi.stubEnv("TEST_AUTH_VYDRA", "1"); + + const plan = await buildRunPlan(parseArgs([]), { + collectProviderApiKeysImpl: collectProviderApiKeysMock, + getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], + loadShellEnvFallbackImpl: loadShellEnvFallbackMock, + }); + + expect(plan.map((entry) => entry.suite.id)).toEqual(["image", "music", "video"]); + expect(requirePlanEntry(plan, "image").providers).toEqual([ + "fal", + "google", + "minimax", + "openai", + "vydra", + ]); + expect(requirePlanEntry(plan, "music").providers).toEqual(["fal", "google", "minimax"]); + expect(requirePlanEntry(plan, "video").providers).toEqual([ + "google", + "minimax", + "openai", + "vydra", + ]); + }); + + it("supports suite-specific provider filters without auth narrowing", async () => { + const plan = await buildRunPlan( + parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]), + { + collectProviderApiKeysImpl: collectProviderApiKeysMock, + getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], + loadShellEnvFallbackImpl: loadShellEnvFallbackMock, + }, + ); + + expect(plan).toHaveLength(1); + const [entry] = plan; + expect(entry?.suite.id).toBe("video"); + expect(entry?.providers).toEqual(["fal", "openai", "runway"]); + }); + + it("forwards quiet flags separately from passthrough args", () => { + const options = parseArgs(["image", "--quiet", "--reporter", "dot"]); + + expect(options.suites).toEqual(["image"]); + expect(options.quietArgs).toEqual(["--quiet"]); + expect(options.passthroughArgs).toEqual(["--reporter", "dot"]); + }); +}); diff --git a/scripts/test-live-media.ts b/test/e2e/qa-lab/media/hosted-media-provider-live.ts similarity index 54% rename from scripts/test-live-media.ts rename to test/e2e/qa-lab/media/hosted-media-provider-live.ts index 9d511e041de..c8db37df333 100644 --- a/scripts/test-live-media.ts +++ b/test/e2e/qa-lab/media/hosted-media-provider-live.ts @@ -1,10 +1,21 @@ -// Test Live Media script supports OpenClaw repository automation. - +// Hosted media provider live runner and QA Lab evidence producer. +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; import { pathToFileURL } from "node:url"; -import { formatErrorMessage } from "../src/infra/errors.ts"; -import { spawnPnpmRunner as _spawnPnpmRunner } from "./pnpm-runner.mjs"; +import { + buildScriptEvidenceSummary, + QA_EVIDENCE_FILENAME, + type QaEvidenceStatus, + type QaEvidenceSummaryJson, +} from "../../../../extensions/qa-lab/src/evidence-summary.js"; +import { spawnPnpmRunner as _spawnPnpmRunner } from "../../../../scripts/pnpm-runner.mjs"; + +const SOURCE_PATH = "test/e2e/qa-lab/media/hosted-media-provider-live.ts"; +const DEFAULT_PROVIDERS_ENV = "OPENCLAW_QA_HOSTED_MEDIA_PROVIDERS"; export type MediaSuiteId = "image" | "music" | "video"; +type EvidenceSuiteId = "image" | "video"; export type MediaSuiteConfig = { id: MediaSuiteId; @@ -67,13 +78,13 @@ const DEFAULT_SUITES: MediaSuiteId[] = ["image", "music", "video"]; export type CliOptions = { allowEmpty: boolean; - suites: MediaSuiteId[]; globalProviders: Set | null; - suiteProviders: Partial>>; - requireAuth: boolean; - quietArgs: string[]; - passthroughArgs: string[]; help: boolean; + passthroughArgs: string[]; + quietArgs: string[]; + requireAuth: boolean; + suiteProviders: Partial>>; + suites: MediaSuiteId[]; }; export type SuiteRunPlan = { @@ -98,10 +109,77 @@ export type RunCliDeps = { runSuiteImpl?: typeof runSuite; }; +type HostedMediaOptions = { + artifactBase: string; + providersEnv: string; + repoRoot: string; + suiteId: EvidenceSuiteId; +}; + +type HostedMediaSuiteDefinition = { + codeRefs: string[]; + docsRefs: string[]; + primaryCoverageIds: string[]; + secondaryCoverageIds?: string[]; + scenarioId: string; + title: string; + videoFullModes?: boolean; +}; + +type HostedMediaProofResult = { + artifacts: Array<{ kind: string; path: string }>; + details?: string; + durationMs: number; + status: QaEvidenceStatus; +}; + +const EVIDENCE_SUITES: Record = { + image: { + scenarioId: "hosted-image-generation-providers-live", + title: "Hosted image generation providers live", + primaryCoverageIds: ["hosted-providers.image-generation-providers"], + docsRefs: [ + "docs/help/testing.md", + "docs/tools/image-generation.md", + "docs/tools/media-overview.md", + ], + codeRefs: [ + SOURCE_PATH, + "test/image-generation.runtime.live.test.ts", + "src/image-generation/live-test-helpers.ts", + ], + }, + video: { + scenarioId: "hosted-video-generation-providers-live", + title: "Hosted video generation providers live", + primaryCoverageIds: [ + "hosted-providers.video-generation-providers", + "media.reference-image-video-and-audio-inputs", + ], + secondaryCoverageIds: ["media.video-generation-tool-invocation"], + docsRefs: [ + "docs/help/testing.md", + "docs/tools/video-generation.md", + "docs/tools/media-overview.md", + ], + codeRefs: [ + SOURCE_PATH, + "extensions/video-generation-providers.live.test.ts", + "src/video-generation/runtime.ts", + "src/agents/tools/video-generate-tool.ts", + ], + videoFullModes: true, + }, +}; + function formatProviderList(providers: Iterable): string { return [...providers].toSorted().join(", "); } +function formatErrorMessage(error: unknown) { + return error instanceof Error ? error.message : String(error); +} + function spawnLivePnpm(params: { pnpmArgs: string[]; env: NodeJS.ProcessEnv }) { return _spawnPnpmRunner({ pnpmArgs: params.pnpmArgs, @@ -111,12 +189,12 @@ function spawnLivePnpm(params: { pnpmArgs: string[]; env: NodeJS.ProcessEnv }) { } async function collectProviderApiKeysForLiveMedia(provider: string): Promise { - const { collectProviderApiKeys } = await import("../src/agents/live-auth-keys.js"); + const { collectProviderApiKeys } = await import("../../../../src/agents/live-auth-keys.js"); return collectProviderApiKeys(provider); } async function getProviderEnvVarsForLiveMedia(provider: string): Promise { - const { getProviderEnvVars } = await import("../src/secrets/provider-env-vars.js"); + const { getProviderEnvVars } = await import("../../../../src/secrets/provider-env-vars.js"); return getProviderEnvVars(provider); } @@ -126,7 +204,7 @@ async function loadShellEnvFallbackForLiveMedia(params: { expectedKeys: string[]; logger: { warn: (message: string) => void }; }): Promise { - const { loadShellEnvFallback } = await import("../src/infra/shell-env.js"); + const { loadShellEnvFallback } = await import("../../../../src/infra/shell-env.js"); loadShellEnvFallback(params); } @@ -150,6 +228,22 @@ function parseSuiteToken(raw: string): MediaSuiteId | null { return null; } +function parseEvidenceSuiteToken(raw: string): EvidenceSuiteId { + const normalized = raw.trim().toLowerCase(); + if (normalized === "image" || normalized === "video") { + return normalized; + } + throw new Error(`unsupported hosted media evidence suite: ${raw}`); +} + +function readOptionValue(argv: readonly string[], index: number, arg: string) { + const value = argv[index + 1] ?? ""; + if (!value || value.startsWith("-")) { + throw new Error(`${arg} requires a value`); + } + return value; +} + export function parseArgs(argv: string[]): CliOptions { const separatorIndex = argv.indexOf("--"); const optionArgs = separatorIndex >= 0 ? argv.slice(0, separatorIndex) : argv; @@ -158,18 +252,10 @@ export function parseArgs(argv: string[]): CliOptions { const suiteProviders: Partial>> = {}; const passthroughArgs: string[] = []; const quietArgs: string[] = []; - let globalProviders: Set | null = null; - let requireAuth = true; - let help = false; let allowEmpty = false; - - const readValue = (index: number): string => { - const value = optionArgs[index + 1]?.trim(); - if (!value) { - throw new Error(`Missing value for ${optionArgs[index]}`); - } - return value; - }; + let globalProviders: Set | null = null; + let help = false; + let requireAuth = true; for (let index = 0; index < optionArgs.length; index += 1) { const arg = optionArgs[index] ?? ""; @@ -190,7 +276,7 @@ export function parseArgs(argv: string[]): CliOptions { continue; } if (arg === "--providers") { - globalProviders = parseCsv(readValue(index)); + globalProviders = parseCsv(readOptionValue(optionArgs, index, arg)); index += 1; continue; } @@ -199,7 +285,8 @@ export function parseArgs(argv: string[]): CliOptions { if (!suite) { throw new Error(`Unknown suite flag: ${arg}`); } - suiteProviders[suite] = parseCsv(readValue(index)) ?? new Set(); + suiteProviders[suite] = + parseCsv(readOptionValue(optionArgs, index, arg)) ?? new Set(); index += 1; continue; } @@ -217,7 +304,7 @@ export function parseArgs(argv: string[]): CliOptions { } if (arg.startsWith("--")) { passthroughArgs.push(arg); - const next = argv[index + 1]; + const next = optionArgs[index + 1]; if (next && !next.startsWith("--")) { passthroughArgs.push(next); index += 1; @@ -240,13 +327,13 @@ export function parseArgs(argv: string[]): CliOptions { const options = { allowEmpty, - suites: (suites.size ? [...suites] : DEFAULT_SUITES).toSorted(), globalProviders, - suiteProviders, - requireAuth, - quietArgs, - passthroughArgs: [...passthroughArgs, ...separatorPassthroughArgs], help, + passthroughArgs: [...passthroughArgs, ...separatorPassthroughArgs], + quietArgs, + requireAuth, + suiteProviders, + suites: (suites.size ? [...suites] : DEFAULT_SUITES).toSorted(), }; validateProviderFilters(options); return options; @@ -316,10 +403,10 @@ export function findSkippedExplicitProviderSelections( async function selectProviders(params: { collectProviderApiKeysImpl?: BuildRunPlanDeps["collectProviderApiKeysImpl"]; - suite: MediaSuiteConfig; globalProviders: Set | null; - suiteProviders: Set | undefined; requireAuth: boolean; + suite: MediaSuiteConfig; + suiteProviders: Set | undefined; }): Promise { const explicit = params.suiteProviders ?? params.globalProviders; const candidates = explicit @@ -337,8 +424,7 @@ async function selectProviders(params: { .length > 0, })), ); - providers = providerAuth.filter((entry) => entry.hasAuth).map((entry) => entry.provider); - return providers; + return providerAuth.filter((entry) => entry.hasAuth).map((entry) => entry.provider); } export async function buildRunPlan( @@ -373,10 +459,10 @@ export async function buildRunPlan( const suite = MEDIA_SUITES[suiteId]; const providers = await selectProviders({ collectProviderApiKeysImpl: deps.collectProviderApiKeysImpl, - suite, globalProviders: options.globalProviders, - suiteProviders: options.suiteProviders[suiteId], requireAuth: options.requireAuth, + suite, + suiteProviders: options.suiteProviders[suiteId], }); return { suite, @@ -402,6 +488,10 @@ Usage: pnpm test:live:media image video --providers openai,google,minimax pnpm test:live:media video --video-providers openai,runway --all-providers +QA evidence mode: + node --import tsx ${SOURCE_PATH} --qa-evidence --suite image --artifact-base + node --import tsx ${SOURCE_PATH} --qa-evidence --suite video --artifact-base + Defaults: - runs image + music + video - auto-loads missing provider env vars from ~/.profile @@ -420,10 +510,10 @@ Flags: `); } -async function runSuite(params: { +export async function runSuite(params: { + passthroughArgs: string[]; plan: SuiteRunPlan; quietArgs: string[]; - passthroughArgs: string[]; }): Promise { const { plan } = params; if (!plan.providers.length) { @@ -499,9 +589,9 @@ export async function runCli(argv: string[], deps: RunCliDeps = {}): Promise(); + const recordOnce = (flag: string) => { + if (seen.has(flag)) { + throw new Error(`${flag} was provided more than once`); + } + seen.add(flag); + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--help" || arg === "-h") { + printHelp(); + process.exit(0); + } + if (arg === "--qa-evidence") { + continue; + } + if (arg === "--artifact-base") { + recordOnce(arg); + artifactBase = readOptionValue(argv, index, arg); + index += 1; + continue; + } + if (arg === "--repo-root") { + recordOnce(arg); + repoRoot = readOptionValue(argv, index, arg); + index += 1; + continue; + } + if (arg === "--providers-env") { + recordOnce(arg); + providersEnv = readOptionValue(argv, index, arg); + index += 1; + continue; + } + if (arg === "--suite") { + recordOnce(arg); + suiteId = parseEvidenceSuiteToken(readOptionValue(argv, index, arg)); + index += 1; + continue; + } + throw new Error(`unsupported hosted media evidence arg: ${arg}`); + } + + if (!artifactBase.trim()) { + throw new Error("--artifact-base is required"); + } + if (!suiteId) { + throw new Error("--suite is required"); + } + if (!providersEnv.trim()) { + throw new Error("--providers-env requires a non-empty env var name"); + } + + return { + artifactBase: path.resolve(repoRoot, artifactBase), + providersEnv, + repoRoot: path.resolve(repoRoot), + suiteId, + }; +} + +async function writeJson(filePath: string, value: unknown) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +async function appendText(filePath: string, text: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.appendFile(filePath, text, "utf8"); +} + +function relativeArtifactPath(options: HostedMediaOptions, filePath: string) { + return path.relative(options.artifactBase, filePath) || path.basename(filePath); +} + +function suiteProviderFilter(options: HostedMediaOptions, env: NodeJS.ProcessEnv) { + const suiteEnv = `OPENCLAW_QA_HOSTED_${options.suiteId.toUpperCase()}_PROVIDERS`; + return env[suiteEnv]?.trim() || env[options.providersEnv]?.trim() || ""; +} + +export function buildHostedMediaCommand(params: { + env?: NodeJS.ProcessEnv; + options: HostedMediaOptions; +}) { + const definition = EVIDENCE_SUITES[params.options.suiteId]; + const env = { ...(params.env ?? process.env) }; + const args = ["--import", "tsx", SOURCE_PATH, params.options.suiteId]; + const providerFilter = suiteProviderFilter(params.options, env); + if (providerFilter) { + args.push(`--${params.options.suiteId}-providers`, providerFilter); + } + if (definition.videoFullModes) { + env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES = "1"; + } + return { + args, + command: process.execPath, + env, + }; +} + +export function classifyHostedMediaFailureStatus(message: string): QaEvidenceStatus { + const blockedPatterns = [ + /no runnable providers matched available auth/i, + /no runnable providers matched the explicit provider selection/i, + /no runnable providers matched explicit provider selection/i, + /no providers with usable auth/i, + ]; + return blockedPatterns.some((pattern) => pattern.test(message)) ? "blocked" : "fail"; +} + +function formatCommand(command: string, args: readonly string[]) { + return [command, ...args].map((arg) => JSON.stringify(arg)).join(" "); +} + +async function runHostedMediaProof(options: HostedMediaOptions): Promise { + const startedAt = Date.now(); + await fs.mkdir(options.artifactBase, { recursive: true }); + const logPath = path.join(options.artifactBase, "hosted-media-live.log"); + const artifacts = [{ kind: "log", path: relativeArtifactPath(options, logPath) }]; + const command = buildHostedMediaCommand({ options }); + + await appendText(logPath, `$ ${formatCommand(command.command, command.args)}\n`); + await appendText( + logPath, + `suite: ${options.suiteId}\nprovidersEnv: ${options.providersEnv}\nvideoFullModes: ${String(EVIDENCE_SUITES[options.suiteId].videoFullModes === true)}\n`, + ); + + return await new Promise((resolve, reject) => { + const child = spawn(command.command, command.args, { + cwd: options.repoRoot, + env: command.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (status, signal) => { + const output = [ + stdout ? `\n--- stdout ---\n${stdout}` : "", + stderr ? `\n--- stderr ---\n${stderr}` : "", + ].join(""); + appendText(logPath, output) + .then(() => { + const durationMs = Math.max(1, Date.now() - startedAt); + if (status === 0 && !signal) { + resolve({ + artifacts, + details: `${options.suiteId} hosted media live suite passed`, + durationMs, + status: "pass", + }); + return; + } + const details = signal + ? `${options.suiteId} hosted media live suite terminated by ${signal}` + : `${options.suiteId} hosted media live suite exited with ${status ?? 1}`; + const combined = `${details}\n${stderr || stdout}`; + resolve({ + artifacts, + details: combined, + durationMs, + status: classifyHostedMediaFailureStatus(combined), + }); + }) + .catch(reject); + }); + }); +} + +export function buildHostedMediaEvidence(params: { + options: HostedMediaOptions; + result: HostedMediaProofResult; +}): QaEvidenceSummaryJson { + const definition = EVIDENCE_SUITES[params.options.suiteId]; + return buildScriptEvidenceSummary({ + artifactPaths: params.result.artifacts, + evidenceMode: "full", + env: process.env, + generatedAt: new Date().toISOString(), + primaryModel: "live-media/hosted-media-provider", + providerMode: "live-frontier", + repoRoot: params.options.repoRoot, + runner: "script", + targets: [ + { + id: definition.scenarioId, + title: definition.title, + sourcePath: SOURCE_PATH, + primaryCoverageIds: definition.primaryCoverageIds, + secondaryCoverageIds: definition.secondaryCoverageIds, + docsRefs: definition.docsRefs, + codeRefs: definition.codeRefs, + }, + ], + results: [ + { + id: definition.scenarioId, + status: params.result.status, + durationMs: params.result.durationMs, + failureMessage: params.result.details, + }, + ], + }); +} + +export async function runHostedMediaProviderLiveProducer( + options: HostedMediaOptions, +): Promise { + const result = await runHostedMediaProof(options); + const evidence = buildHostedMediaEvidence({ options, result }); + await writeJson(path.join(options.artifactBase, QA_EVIDENCE_FILENAME), evidence); + await writeJson(path.join(options.artifactBase, "latest-run.json"), { + qaEvidence: QA_EVIDENCE_FILENAME, + }); + return evidence; +} + +async function main(argv: string[]) { + if (argv.includes("--qa-evidence")) { + const evidence = await runHostedMediaProviderLiveProducer(parseHostedMediaOptions(argv)); + console.log(`Hosted media provider live evidence: ${QA_EVIDENCE_FILENAME}`); + console.log(`Hosted media provider live status: ${evidence.entries[0]?.result.status}`); + return evidence.entries[0]?.result.status === "fail" ? 1 : 0; + } + return await runCli(argv); +} + if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { - runCli(process.argv.slice(2)) + main(process.argv.slice(2)) .then((code) => process.exit(code)) .catch((error: unknown) => { console.error(formatErrorMessage(error)); diff --git a/test/scripts/test-live-media.test.ts b/test/scripts/test-live-media.test.ts deleted file mode 100644 index b69d190a15a..00000000000 --- a/test/scripts/test-live-media.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -// Test Live Media tests cover test live media script behavior. -import { spawnSync } from "node:child_process"; -import { describe, expect, it } from "vitest"; -import { - MEDIA_SUITES, - findSkippedExplicitProviderSelections, - parseArgs, - runCli, -} from "../../scripts/test-live-media.ts"; - -describe("scripts/test-live-media", () => { - it("prints help through the real node --import tsx entrypoint", () => { - const result = spawnSync( - process.execPath, - ["--import", "tsx", "scripts/test-live-media.ts", "--help"], - { - cwd: process.cwd(), - encoding: "utf8", - }, - ); - - expect(result.status).toBe(0); - expect(result.stdout).toContain("Media live harness"); - expect(result.stdout).toContain("pnpm test:live:media"); - expect(result.stderr).toBe(""); - }); - - it("rejects unknown global providers for the selected suites", () => { - expect(() => - parseArgs(["image", "--providers", "definitely-not-a-provider", "--all-providers"]), - ).toThrow("Unknown provider(s) for selected media suite(s): definitely-not-a-provider"); - }); - - it("rejects unknown suite-specific providers", () => { - expect(() => parseArgs(["image", "--image-providers", "runway", "--all-providers"])).toThrow( - "Unknown image provider(s): runway", - ); - }); - - it("accepts providers supported by the wrapped live suites", () => { - expect( - parseArgs(["image", "--image-providers", "openrouter", "--all-providers"]).suiteProviders - .image, - ).toEqual(new Set(["openrouter"])); - expect( - parseArgs(["music", "--music-providers", "fal,openrouter", "--all-providers"]).suiteProviders - .music, - ).toEqual(new Set(["fal", "openrouter"])); - expect( - parseArgs(["video", "--video-providers", "openrouter", "--all-providers"]).suiteProviders - .video, - ).toEqual(new Set(["openrouter"])); - }); - - it("rejects suite-specific provider filters for unselected suites", () => { - expect(() => parseArgs(["image", "--music-providers", "fal", "--all-providers"])).toThrow( - "Provider filter(s) target unselected media suite(s): music", - ); - }); - - it("passes single-dash Vitest args after the option separator", () => { - expect( - parseArgs(["image", "--all-providers", "--project", "tooling", "--", "-t", "media-smoke"]), - ).toMatchObject({ - suites: ["image"], - requireAuth: false, - passthroughArgs: ["--project", "tooling", "-t", "media-smoke"], - }); - }); - - it("parses the explicit empty-run escape hatch", () => { - expect(parseArgs(["--allow-empty"])).toMatchObject({ - allowEmpty: true, - requireAuth: true, - }); - }); - - it("fails explicit suite selections that auth filtering would skip", () => { - const options = parseArgs([ - "image", - "music", - "--image-providers", - "openai", - "--music-providers", - "minimax", - ]); - const skipped = findSkippedExplicitProviderSelections(options, [ - { suite: MEDIA_SUITES.image, providers: ["openai"] }, - { - suite: MEDIA_SUITES.music, - providers: [], - skippedReason: "no providers with usable auth", - }, - ]); - - expect(skipped.map((entry) => entry.suite.id)).toEqual(["music"]); - }); - - it("does not fail global provider filters for suites without provider overlap", () => { - const options = parseArgs(["image", "music", "video", "--providers", "openai"]); - const skipped = findSkippedExplicitProviderSelections(options, [ - { suite: MEDIA_SUITES.image, providers: ["openai"] }, - { - suite: MEDIA_SUITES.music, - providers: [], - skippedReason: "no providers selected", - }, - { suite: MEDIA_SUITES.video, providers: ["openai"] }, - ]); - - expect(skipped).toEqual([]); - }); - - it("fails default live media runs when auth filtering leaves no providers", async () => { - await expect( - runCli(["image"], { - buildRunPlanImpl: () => [ - { - providers: [], - skippedReason: "no providers with usable auth", - suite: MEDIA_SUITES.image, - }, - ], - }), - ).resolves.toBe(1); - }); - - it("allows empty live media runs only with an explicit escape hatch", async () => { - await expect( - runCli(["image", "--allow-empty"], { - buildRunPlanImpl: () => [ - { - providers: [], - skippedReason: "no providers with usable auth", - suite: MEDIA_SUITES.image, - }, - ], - }), - ).resolves.toBe(0); - }); -});