fix: live test suites pass green when provider credentials are missing (#141024)

The github-copilot connection-bound-ids live suite returned green when
OPENCLAW_LIVE_TEST=1 was set but no Copilot token could be resolved from env or
the auth profile, so a Testbox run could report a passing live test that never
contacted the provider. The music and video generation live sweeps had the same
shape: an unfiltered run with zero attempts warned and passed.

Use the Vitest test-context skip(reason) in the Copilot test, and thread the
context skip through the music/video sweep summary helpers so unfiltered
zero-attempt runs show as skipped in the reporter with the skipped entries.
Filtered zero-attempt runs still throw; a present-but-invalid token still fails.
Unit coverage asserts the skip callback contract for both sweep helpers, and
docs/help/testing-live.md records the rule that a live suite without
credentials must skip visibly or fail, never pass green.

docs/help/testing-live.md

@@ -666,6 +666,7 @@ Live tests discover credentials the same way the CLI does. Practical implication
 
 - If the CLI works, live tests should find the same keys.
 - If a live test says "no creds", debug the same way you'd debug `openclaw models list` / model selection.
+- An OpenClaw live suite that cannot resolve its credentials must skip visibly in the reporter with Vitest test-context `skip(reason)` or fail; it must never pass green without reaching the provider, because a green run that did not reach the provider is not live evidence.
 
 - Per-agent auth profiles: SQLite credential rows in `~/.openclaw/agents/<agentId>/agent/openclaw-agent.sqlite` (this is what "profile keys" means in the live tests)
 - Config: `~/.openclaw/openclaw.json` (or `OPENCLAW_CONFIG_PATH`)

extensions/github-copilot/connection-bound-ids.live.test.ts

@@ -136,11 +136,13 @@ async function resolveGithubTokenCandidates(): Promise<Array<{ source: string; t
 }
 
 describeLive("github-copilot connection-bound Responses IDs live", () => {
-  it("rewrites replayed item IDs and preserves streamed tool arguments", async () => {
+  it("rewrites replayed item IDs and preserves streamed tool arguments", async ({ skip }) => {
     logProgress("start");
     const candidates = await resolveGithubTokenCandidates();
     if (candidates.length === 0) {
-      logProgress("skip (no GitHub Copilot token found in env or auth profile)");
+      skip(
+        "No GitHub Copilot token found in env vars OPENCLAW_LIVE_GITHUB_COPILOT_TOKEN / COPILOT_GITHUB_TOKEN / GH_TOKEN / GITHUB_TOKEN or the github-copilot auth profile",
+      );
       return;
     }
 

extensions/music-generation-providers.live.test.ts

@@ -31,7 +31,7 @@ import {
   resolveConfiguredLiveMusicModels,
   resolveLiveMusicAuthStore,
 } from "openclaw/plugin-sdk/test-media-generation";
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import falPlugin from "./fal/index.js";
 import googlePlugin from "./google/index.js";
 import minimaxPlugin from "./minimax/index.js";
@@ -137,6 +137,7 @@ function expectMusicLiveSweepPassed(params: {
   attempted: string[];
   failures: string[];
   providerFilter: Set<string> | null;
+  skip: (note: string) => void;
   skipped: string[];
 }): void {
   if (params.attempted.length === 0) {
@@ -146,7 +147,9 @@ function expectMusicLiveSweepPassed(params: {
         `[live:music-generation] requested provider filter produced no live attempts: ${formatProviderFilter(params.providerFilter)}; skipped=${params.skipped.join(", ") || "none"}`,
       );
     }
-    console.warn("[live:music-generation] no provider had usable auth; skipping assertions");
+    params.skip(
+      `[live:music-generation] no live music attempt completed; skipped=${params.skipped.join(", ") || "none"}`,
+    );
     return;
   }
   expect(params.failures).toStrictEqual([]);
@@ -195,7 +198,7 @@ function resolveLiveMusicSkipReason(providerId: string, error: unknown): string
 describeLive("music generation provider live", () => {
   it(
     "covers generate plus declared edit paths with shell/profile auth",
-    async () => {
+    async ({ skip }) => {
       const cfg = withPluginsEnabled(await readLiveTestConfig());
       const configuredModels = resolveConfiguredLiveMusicModels(cfg);
       const agentDir = resolveDefaultAgentDir(cfg as never);
@@ -325,21 +328,40 @@ describeLive("music generation provider live", () => {
         `[live:music-generation] attempted=${attempted.join(", ") || "none"} skipped=${skipped.join(", ") || "none"} failures=${failures.join(" | ") || "none"} shellEnv=${getShellEnvAppliedKeys().join(", ") || "none"}`,
       );
 
-      expectMusicLiveSweepPassed({ attempted, failures, providerFilter, skipped });
+      expectMusicLiveSweepPassed({ attempted, failures, providerFilter, skip, skipped });
     },
     10 * 60_000,
   );
 });
 
 describe("music generation live provider filter coverage", () => {
+  it("skips unfiltered sweeps when no provider is attempted", () => {
+    const skip = vi.fn();
+    expect(() =>
+      expectMusicLiveSweepPassed({
+        attempted: [],
+        failures: [],
+        providerFilter: null,
+        skip,
+        skipped: ["minimax: no usable auth", "google: no model configured"],
+      }),
+    ).not.toThrow();
+    expect(skip).toHaveBeenCalledExactlyOnceWith(
+      expect.stringContaining("minimax: no usable auth, google: no model configured"),
+    );
+  });
+
   it("fails filtered sweeps when no requested provider is attempted", () => {
+    const skip = vi.fn();
     expect(() =>
       expectMusicLiveSweepPassed({
         attempted: [],
         failures: [],
         providerFilter: new Set(["minimax"]),
+        skip,
         skipped: ["minimax: no usable auth"],
       }),
     ).toThrow(/requested provider filter produced no live attempts: minimax/u);
+    expect(skip).not.toHaveBeenCalled();
   });
 });

extensions/video-generation-providers.live.test.ts

@@ -43,7 +43,7 @@ import type {
   VideoGenerationProvider,
   VideoGenerationRequest,
 } from "openclaw/plugin-sdk/test-media-generation";
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import alibabaPlugin from "./alibaba/index.js";
 import byteplusPlugin from "./byteplus/index.js";
 import deepinfraPlugin from "./deepinfra/index.js";
@@ -365,6 +365,7 @@ function expectLiveVideoCasePassed(
     attempted: string[];
     failures: string[];
     providerId: string;
+    skip: (note: string) => void;
     skipped: string[];
   },
   activeProviderFilter = providerFilter,
@@ -377,7 +378,9 @@ function expectLiveVideoCasePassed(
         `[live:video-generation] requested provider produced no live attempts: ${params.providerId}; skipped=${params.skipped.join(", ") || "none"}`,
       );
     }
-    console.warn("[live:video-generation] no live video attempt completed; skipping assertions");
+    params.skip(
+      `[live:video-generation] no live video attempt completed for ${params.providerId}; skipped=${params.skipped.join(", ") || "none"}`,
+    );
     return;
   }
   expect(params.failures).toStrictEqual([]);
@@ -400,14 +403,17 @@ function resolveLiveSmokeDurationSeconds(params: {
   );
 }
 
-async function runLiveVideoProviderCase(testCase: LiveProviderCase): Promise<void> {
+async function runLiveVideoProviderCase(
+  testCase: LiveProviderCase,
+  skip: (note: string) => void,
+): Promise<void> {
   const cfg = withPluginsEnabled(await readLiveTestConfig());
   const configuredModels = resolveConfiguredLiveVideoModels(cfg);
   const agentDir = resolveDefaultAgentDir(cfg as never);
   const attempted: string[] = [];
   const skipped: string[] = [];
   const failures: string[] = [];
-  const summaryParams = { attempted, failures, providerId: testCase.providerId, skipped };
+  const summaryParams = { attempted, failures, providerId: testCase.providerId, skip, skipped };
 
   maybeLoadShellEnvForVideoProviders([testCase.providerId]);
 
@@ -627,8 +633,8 @@ describeLive("video generation provider live", () => {
     // One provider per test keeps cumulative suite runtime from tripping a single timeout cap.
     it(
       `covers declared video-generation modes with shell/profile auth (${testCase.providerId})`,
-      async () => {
-        await runLiveVideoProviderCase(testCase);
+      async ({ skip }) => {
+        await runLiveVideoProviderCase(testCase, skip);
       },
       LIVE_VIDEO_TEST_TIMEOUT_MS,
     );
@@ -642,26 +648,39 @@ describe("video generation live provider filter coverage", () => {
     );
   });
 
-  it("keeps unfiltered zero-attempt provider cases advisory", () => {
-    expectLiveVideoCasePassed({
-      attempted: [],
-      failures: [],
-      providerId: "local-only",
-      skipped: ["local-only: no usable auth"],
-    });
+  it("skips unfiltered zero-attempt provider cases", () => {
+    const skip = vi.fn();
+    expect(() =>
+      expectLiveVideoCasePassed(
+        {
+          attempted: [],
+          failures: [],
+          providerId: "local-only",
+          skip,
+          skipped: ["local-only: no usable auth"],
+        },
+        null,
+      ),
+    ).not.toThrow();
+    expect(skip).toHaveBeenCalledExactlyOnceWith(
+      expect.stringContaining("local-only: no usable auth"),
+    );
   });
 
   it("fails filtered provider cases when the requested provider is not attempted", () => {
+    const skip = vi.fn();
     expect(() =>
       expectLiveVideoCasePassed(
         {
           attempted: [],
           failures: [],
           providerId: "minimax",
+          skip,
           skipped: ["minimax: no usable auth"],
         },
         new Set(["minimax"]),
       ),
     ).toThrow(/requested provider produced no live attempts: minimax/u);
+    expect(skip).not.toHaveBeenCalled();
   });
 });