diff --git a/docs/help/testing-live.md b/docs/help/testing-live.md index fb688e67132c..ad3bb13e4614 100644 --- a/docs/help/testing-live.md +++ b/docs/help/testing-live.md @@ -666,6 +666,7 @@ Live tests discover credentials the same way the CLI does. Practical implication - If the CLI works, live tests should find the same keys. - If a live test says "no creds", debug the same way you'd debug `openclaw models list` / model selection. +- An OpenClaw live suite that cannot resolve its credentials must skip visibly in the reporter with Vitest test-context `skip(reason)` or fail; it must never pass green without reaching the provider, because a green run that did not reach the provider is not live evidence. - Per-agent auth profiles: SQLite credential rows in `~/.openclaw/agents//agent/openclaw-agent.sqlite` (this is what "profile keys" means in the live tests) - Config: `~/.openclaw/openclaw.json` (or `OPENCLAW_CONFIG_PATH`) diff --git a/extensions/github-copilot/connection-bound-ids.live.test.ts b/extensions/github-copilot/connection-bound-ids.live.test.ts index 00b9c470a454..c66925b35510 100644 --- a/extensions/github-copilot/connection-bound-ids.live.test.ts +++ b/extensions/github-copilot/connection-bound-ids.live.test.ts @@ -136,11 +136,13 @@ async function resolveGithubTokenCandidates(): Promise { - it("rewrites replayed item IDs and preserves streamed tool arguments", async () => { + it("rewrites replayed item IDs and preserves streamed tool arguments", async ({ skip }) => { logProgress("start"); const candidates = await resolveGithubTokenCandidates(); if (candidates.length === 0) { - logProgress("skip (no GitHub Copilot token found in env or auth profile)"); + skip( + "No GitHub Copilot token found in env vars OPENCLAW_LIVE_GITHUB_COPILOT_TOKEN / COPILOT_GITHUB_TOKEN / GH_TOKEN / GITHUB_TOKEN or the github-copilot auth profile", + ); return; } diff --git a/extensions/music-generation-providers.live.test.ts b/extensions/music-generation-providers.live.test.ts index 48f08dbc18a9..a9ddb407adef 100644 --- a/extensions/music-generation-providers.live.test.ts +++ b/extensions/music-generation-providers.live.test.ts @@ -31,7 +31,7 @@ import { resolveConfiguredLiveMusicModels, resolveLiveMusicAuthStore, } from "openclaw/plugin-sdk/test-media-generation"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import falPlugin from "./fal/index.js"; import googlePlugin from "./google/index.js"; import minimaxPlugin from "./minimax/index.js"; @@ -137,6 +137,7 @@ function expectMusicLiveSweepPassed(params: { attempted: string[]; failures: string[]; providerFilter: Set | null; + skip: (note: string) => void; skipped: string[]; }): void { if (params.attempted.length === 0) { @@ -146,7 +147,9 @@ function expectMusicLiveSweepPassed(params: { `[live:music-generation] requested provider filter produced no live attempts: ${formatProviderFilter(params.providerFilter)}; skipped=${params.skipped.join(", ") || "none"}`, ); } - console.warn("[live:music-generation] no provider had usable auth; skipping assertions"); + params.skip( + `[live:music-generation] no live music attempt completed; skipped=${params.skipped.join(", ") || "none"}`, + ); return; } expect(params.failures).toStrictEqual([]); @@ -195,7 +198,7 @@ function resolveLiveMusicSkipReason(providerId: string, error: unknown): string describeLive("music generation provider live", () => { it( "covers generate plus declared edit paths with shell/profile auth", - async () => { + async ({ skip }) => { const cfg = withPluginsEnabled(await readLiveTestConfig()); const configuredModels = resolveConfiguredLiveMusicModels(cfg); const agentDir = resolveDefaultAgentDir(cfg as never); @@ -325,21 +328,40 @@ describeLive("music generation provider live", () => { `[live:music-generation] attempted=${attempted.join(", ") || "none"} skipped=${skipped.join(", ") || "none"} failures=${failures.join(" | ") || "none"} shellEnv=${getShellEnvAppliedKeys().join(", ") || "none"}`, ); - expectMusicLiveSweepPassed({ attempted, failures, providerFilter, skipped }); + expectMusicLiveSweepPassed({ attempted, failures, providerFilter, skip, skipped }); }, 10 * 60_000, ); }); describe("music generation live provider filter coverage", () => { + it("skips unfiltered sweeps when no provider is attempted", () => { + const skip = vi.fn(); + expect(() => + expectMusicLiveSweepPassed({ + attempted: [], + failures: [], + providerFilter: null, + skip, + skipped: ["minimax: no usable auth", "google: no model configured"], + }), + ).not.toThrow(); + expect(skip).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("minimax: no usable auth, google: no model configured"), + ); + }); + it("fails filtered sweeps when no requested provider is attempted", () => { + const skip = vi.fn(); expect(() => expectMusicLiveSweepPassed({ attempted: [], failures: [], providerFilter: new Set(["minimax"]), + skip, skipped: ["minimax: no usable auth"], }), ).toThrow(/requested provider filter produced no live attempts: minimax/u); + expect(skip).not.toHaveBeenCalled(); }); }); diff --git a/extensions/video-generation-providers.live.test.ts b/extensions/video-generation-providers.live.test.ts index 37f833906936..84cb729d26b6 100644 --- a/extensions/video-generation-providers.live.test.ts +++ b/extensions/video-generation-providers.live.test.ts @@ -43,7 +43,7 @@ import type { VideoGenerationProvider, VideoGenerationRequest, } from "openclaw/plugin-sdk/test-media-generation"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import alibabaPlugin from "./alibaba/index.js"; import byteplusPlugin from "./byteplus/index.js"; import deepinfraPlugin from "./deepinfra/index.js"; @@ -365,6 +365,7 @@ function expectLiveVideoCasePassed( attempted: string[]; failures: string[]; providerId: string; + skip: (note: string) => void; skipped: string[]; }, activeProviderFilter = providerFilter, @@ -377,7 +378,9 @@ function expectLiveVideoCasePassed( `[live:video-generation] requested provider produced no live attempts: ${params.providerId}; skipped=${params.skipped.join(", ") || "none"}`, ); } - console.warn("[live:video-generation] no live video attempt completed; skipping assertions"); + params.skip( + `[live:video-generation] no live video attempt completed for ${params.providerId}; skipped=${params.skipped.join(", ") || "none"}`, + ); return; } expect(params.failures).toStrictEqual([]); @@ -400,14 +403,17 @@ function resolveLiveSmokeDurationSeconds(params: { ); } -async function runLiveVideoProviderCase(testCase: LiveProviderCase): Promise { +async function runLiveVideoProviderCase( + testCase: LiveProviderCase, + skip: (note: string) => void, +): Promise { const cfg = withPluginsEnabled(await readLiveTestConfig()); const configuredModels = resolveConfiguredLiveVideoModels(cfg); const agentDir = resolveDefaultAgentDir(cfg as never); const attempted: string[] = []; const skipped: string[] = []; const failures: string[] = []; - const summaryParams = { attempted, failures, providerId: testCase.providerId, skipped }; + const summaryParams = { attempted, failures, providerId: testCase.providerId, skip, skipped }; maybeLoadShellEnvForVideoProviders([testCase.providerId]); @@ -627,8 +633,8 @@ describeLive("video generation provider live", () => { // One provider per test keeps cumulative suite runtime from tripping a single timeout cap. it( `covers declared video-generation modes with shell/profile auth (${testCase.providerId})`, - async () => { - await runLiveVideoProviderCase(testCase); + async ({ skip }) => { + await runLiveVideoProviderCase(testCase, skip); }, LIVE_VIDEO_TEST_TIMEOUT_MS, ); @@ -642,26 +648,39 @@ describe("video generation live provider filter coverage", () => { ); }); - it("keeps unfiltered zero-attempt provider cases advisory", () => { - expectLiveVideoCasePassed({ - attempted: [], - failures: [], - providerId: "local-only", - skipped: ["local-only: no usable auth"], - }); + it("skips unfiltered zero-attempt provider cases", () => { + const skip = vi.fn(); + expect(() => + expectLiveVideoCasePassed( + { + attempted: [], + failures: [], + providerId: "local-only", + skip, + skipped: ["local-only: no usable auth"], + }, + null, + ), + ).not.toThrow(); + expect(skip).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("local-only: no usable auth"), + ); }); it("fails filtered provider cases when the requested provider is not attempted", () => { + const skip = vi.fn(); expect(() => expectLiveVideoCasePassed( { attempted: [], failures: [], providerId: "minimax", + skip, skipped: ["minimax: no usable auth"], }, new Set(["minimax"]), ), ).toThrow(/requested provider produced no live attempts: minimax/u); + expect(skip).not.toHaveBeenCalled(); }); });