diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index 19828fe67d27..62b61d289db9 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -120,6 +120,7 @@ const qaTestFileScenarioExecutionSchema = z.discriminatedUnion("kind", [ kind: z.literal("script"), allowBlockedEvidence: z.boolean().optional(), args: z.array(z.string()).optional(), + dockerLane: z.string().trim().min(1).optional(), parallelSafe: z.boolean().optional(), timeoutMs: z.number().int().positive().optional(), }), diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index dcc5814906c9..eaafa2c72225 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -18,12 +18,14 @@ const { crablineRuntimeLoads, prepareDockerE2eEnvironment, replaceFileAtomicMock, + runPluginCommandWithTimeout, runQaFlowSuite, runQaTestFileScenarios, } = vi.hoisted(() => ({ crablineRuntimeLoads: vi.fn(), prepareDockerE2eEnvironment: vi.fn(), replaceFileAtomicMock: vi.fn(), + runPluginCommandWithTimeout: vi.fn(), runQaFlowSuite: vi.fn(), runQaTestFileScenarios: vi.fn(), })); @@ -48,6 +50,8 @@ vi.mock("./test-file-scenario-docker-batch.js", async (importOriginal) => ({ prepareDockerE2eEnvironment, })); +vi.mock("openclaw/plugin-sdk/run-command", () => ({ runPluginCommandWithTimeout })); + vi.mock("openclaw/plugin-sdk/security-runtime", async (importOriginal) => { const actual = await importOriginal(); replaceFileAtomicMock.mockImplementation(actual.replaceFileAtomic); @@ -246,6 +250,8 @@ describe("qa suite runtime launcher", () => { runQaTestFileScenarios.mockReset(); prepareDockerE2eEnvironment.mockReset(); prepareDockerE2eEnvironment.mockResolvedValue(undefined); + runPluginCommandWithTimeout.mockReset(); + runPluginCommandWithTimeout.mockResolvedValue({ code: 0, stdout: "", stderr: "" }); runQaFlowSuite.mockImplementation( async ( params: @@ -1200,6 +1206,7 @@ describe("qa suite runtime launcher", () => { }, }); expect(runQaFlowSuite).not.toHaveBeenCalled(); + expect(runPluginCommandWithTimeout).not.toHaveBeenCalled(); expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1); const [call] = runQaTestFileScenarios.mock.calls[0] ?? []; expect(call).toMatchObject({ @@ -1224,6 +1231,47 @@ describe("qa suite runtime launcher", () => { ); }); + it("prepares a missing native runtime before marking the child prebuilt", async () => { + const repoRoot = await makeTempRepo("qa-suite-prepared-vitest-"); + const aiRuntimePath = path.join(repoRoot, "packages/ai/dist/internal/runtime.mjs"); + runPluginCommandWithTimeout.mockImplementation(async ({ argv }) => { + if (argv.includes("tsdown.ai.config.ts")) { + await fs.mkdir(path.dirname(aiRuntimePath), { recursive: true }); + await fs.writeFile(aiRuntimePath, "export {};\n", "utf8"); + } + return { code: 0, stdout: "", stderr: "" }; + }); + const defaultTestFileImplementation = requireDefaultQaTestFileImplementation(); + runQaTestFileScenarios.mockImplementationOnce(async (params) => { + await expect(fs.stat(aiRuntimePath)).resolves.toBeDefined(); + return await defaultTestFileImplementation(params); + }); + + await runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/prepared-vitest", + scenarioIds: ["auth-profile-doctor-migration-safety"], + }); + + expect(runQaTestFileScenarios).toHaveBeenCalledWith( + expect.objectContaining({ + env: { OPENCLAW_E2E_USE_PREBUILT_DIST: "1" }, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "prepared-vitest", "vitest"), + }), + ); + expect(runQaTestFileScenarios.mock.calls[0]?.[0]).not.toHaveProperty("envMode"); + expect(runPluginCommandWithTimeout.mock.calls.map(([options]) => options.argv)).toEqual([ + [ + process.execPath, + "--import", + "tsx", + "scripts/tsdown-build.mts", + "--config", + "tsdown.ai.config.ts", + ], + ]); + }); + it("projects a skipped native producer as a skipped unified scenario", async () => { const repoRoot = await makeTempRepo("qa-suite-native-skip-"); const defaultImplementation = requireDefaultQaTestFileImplementation(); @@ -1994,6 +2042,27 @@ describe("qa suite runtime launcher", () => { ); }); + it("leaves nested E2E script runtime preparation to the script owner", async () => { + const repoRoot = await makeTempRepo("qa-suite-script-runtime-owner-"); + + await runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/script-runtime-owner", + scenarioIds: ["managed-gateway-service-lifecycle"], + }); + + expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1); + const [call] = runQaTestFileScenarios.mock.calls[0] ?? []; + expect(call.scenarios).toEqual([ + expect.objectContaining({ + id: "managed-gateway-service-lifecycle", + execution: expect.objectContaining({ kind: "script" }), + }), + ]); + expect(call).not.toHaveProperty("env"); + expect(call).not.toHaveProperty("envMode"); + }); + it("streams native owner progress without exposing child output to CI", async () => { const repoRoot = await makeTempRepo("qa-suite-safe-native-progress-"); vi.stubEnv("OPENCLAW_QA_SUITE_PROGRESS", "1"); @@ -2192,6 +2261,23 @@ describe("qa suite runtime launcher", () => { expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1); }); + it("prepares the Docker candidate before a script-owned Docker lane", async () => { + const repoRoot = await makeTempRepo("qa-suite-script-docker-prep-"); + const preparedEnv = Object.freeze({ OPENCLAW_CURRENT_PACKAGE_TGZ: "/tmp/candidate.tgz" }); + prepareDockerE2eEnvironment.mockResolvedValueOnce(preparedEnv); + + await runQaSuite({ repoRoot, scenarioIds: ["cli-onboarding"] }); + + expect(prepareDockerE2eEnvironment).toHaveBeenCalledWith( + expect.objectContaining({ + scenarios: [expect.objectContaining({ id: "cli-onboarding" })], + }), + ); + expect(runQaTestFileScenarios).toHaveBeenCalledWith( + expect.objectContaining({ env: preparedEnv, envMode: "replace" }), + ); + }); + it("keeps selected evidence order and successful siblings when a parallel script rejects", async () => { const repoRoot = await makeTempRepo("qa-suite-parallel-script-rejection-"); const defaultTestFileImplementation = requireDefaultQaTestFileImplementation(); diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index b9032db3d862..171e9184b187 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { formatErrorMessage, toErrorObject } from "openclaw/plugin-sdk/error-runtime"; +import { runPluginCommandWithTimeout } from "openclaw/plugin-sdk/run-command"; import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js"; import { QaSuiteArtifactError, QaSuiteInfraError } from "./errors.js"; import { @@ -417,6 +418,7 @@ async function resolveSuiteExecutionPlan( async function runQaTestFileSuiteFromRuntime(params: { env?: NodeJS.ProcessEnv; + kind: QaTestFileExecutionKind; runParams: QaSuiteRunParams | undefined; scenarios: readonly QaTestFileScenario[]; }): Promise { @@ -428,7 +430,15 @@ async function runQaTestFileSuiteFromRuntime(params: { const primaryModel = runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode); return await runQaTestFileScenarios({ evidenceMode: runParams?.evidenceMode, - ...(params.env ? { env: params.env, envMode: "replace" as const } : {}), + ...(params.env + ? { env: params.env, envMode: "replace" as const } + : params.kind !== "script" + ? { + // The owning QA process already loaded the prepared runtime. Native + // child setup must not clean or rebuild those files under live gateways. + env: { OPENCLAW_E2E_USE_PREBUILT_DIST: "1" }, + } + : {}), ...(runParams?.failFast ? { failFast: true } : {}), ...(shouldLogQaSuiteProgress() ? { progress: (message: string) => writeQaSuiteProgress(true, message) } @@ -442,6 +452,21 @@ async function runQaTestFileSuiteFromRuntime(params: { }); } +async function prepareQaSuiteNativeRuntime(repoRoot: string) { + const argv = [ + process.execPath, + "--import", + "tsx", + "scripts/tsdown-build.mts", + "--config", + "tsdown.ai.config.ts", + ]; + const result = await runPluginCommandWithTimeout({ argv, cwd: repoRoot, timeoutMs: 20 * 60_000 }); + if (result.code !== 0) { + throw new Error(`QA suite runtime preparation failed (${argv.join(" ")}): ${result.stderr}`); + } +} + function rejectFlowOnlySuiteOptionsForUnifiedRun(runParams: QaSuiteRunParams | undefined) { if (runParams?.runtimePair) { throw new Error("--runtime-pair requires execution.kind: flow scenarios."); @@ -1062,6 +1087,7 @@ async function runUnifiedQaSuite(params: { ); const result = await runQaTestFileSuiteFromRuntime({ env: kind === "script" ? preparedScriptEnv : undefined, + kind, runParams: { ...params.runParams, outputDir: suitePartitionOutputDir(outputDir, kind), @@ -1306,10 +1332,15 @@ async function runUnifiedQaSuite(params: { }) : await runWeightedUnifiedPartitionTasks(retryingTasks, maxWeight); }; + // Native children opt out of their destructive global build only after this + // scheduler has established the shared runtime they consume concurrently. + if (concurrentTestFileScenariosByKind.has("vitest")) { + await prepareQaSuiteNativeRuntime(repoRoot); + } const concurrentPartitionResults = await runPartitionTasks(concurrentPartitionTasks, concurrency); const concurrentFailed = failFast && concurrentPartitionResults.some(partitionFailed); let scriptPreparationFailure: QaUnifiedPartitionResult | undefined; - if (!concurrentFailed && scriptScenarios?.some(dockerBatch.isDockerE2eScenario)) { + if (!concurrentFailed && scriptScenarios?.some(dockerBatch.dockerLaneName)) { try { preparedScriptEnv = await dockerBatch.prepareDockerE2eEnvironment({ env: process.env, diff --git a/extensions/qa-lab/src/test-file-scenario-docker-batch.ts b/extensions/qa-lab/src/test-file-scenario-docker-batch.ts index dc1f69b674ea..ecbacee2c6ba 100644 --- a/extensions/qa-lab/src/test-file-scenario-docker-batch.ts +++ b/extensions/qa-lab/src/test-file-scenario-docker-batch.ts @@ -68,6 +68,13 @@ export function isDockerE2eScenario( return dockerE2eLaneName(scenario) !== undefined; } +export function dockerLaneName(scenario: QaSeedScenarioWithSource) { + if (scenario.execution.kind !== "script") { + return undefined; + } + return scenario.execution.dockerLane ?? dockerE2eLaneName(scenario); +} + export async function prepareDockerE2eEnvironment(params: { env: NodeJS.ProcessEnv; outputDir: string; @@ -76,7 +83,7 @@ export async function prepareDockerE2eEnvironment(params: { scenarios: readonly QaSeedScenarioWithSource[]; }): Promise | undefined> { const laneNames = [ - ...new Set(params.scenarios.flatMap((scenario) => dockerE2eLaneName(scenario) ?? [])), + ...new Set(params.scenarios.flatMap((scenario) => dockerLaneName(scenario) ?? [])), ]; if (laneNames.length === 0) { return undefined; diff --git a/extensions/qa-lab/src/test-file-scenario-runner.docker.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.docker.test.ts index a4064cf7ba96..a57c5c11cb9d 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.docker.test.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.docker.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { + dockerLaneName, dockerE2eLaneName, prepareDockerE2eEnvironment, } from "./test-file-scenario-docker-batch.js"; @@ -26,6 +27,19 @@ afterEach(async () => { await harness.cleanup(); }); +it("prepares declared candidates for scripts that own their Docker invocation", () => { + const scenario = makeTestFileScenario("script", "scripts/e2e/qa-cli-onboarding.mjs"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + expect( + dockerLaneName({ + ...scenario, + execution: { ...scenario.execution, dockerLane: "onboard" }, + }), + ).toBe("onboard"); +}); + it("only batches the canonical Docker lane argument shape", () => { const scenario = makeDockerE2eScenario("docker-lane", "gateway-network"); if (scenario.execution.kind !== "script") { @@ -45,10 +59,14 @@ it("prepares the exact Docker lane union in a sanitized bound environment", asyn const outputDir = path.join(repoRoot, "out"); const packagePath = path.join(repoRoot, "openclaw.tgz"); const registryDir = path.join(repoRoot, "registry"); + const onboardingScenario = makeTestFileScenario("script", "scripts/e2e/qa-cli-onboarding.mjs"); + if (onboardingScenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } const runCommand = vi.fn(async (command: QaScenarioCommandExecution) => { expect(command.env).toMatchObject({ KEEP_ME: "yes", - OPENCLAW_DOCKER_ALL_LANES: "gateway-network,openai-chat-tools", + OPENCLAW_DOCKER_ALL_LANES: "gateway-network,openai-chat-tools,onboard", OPENCLAW_DOCKER_E2E_REPO_ROOT: repoRoot, }); expect(command.env).not.toHaveProperty("OPENCLAW_DOCKER_ALL_BUILD"); @@ -87,6 +105,10 @@ it("prepares the exact Docker lane union in a sanitized bound environment", asyn makeDockerE2eScenario("one", "gateway-network"), makeDockerE2eScenario("duplicate", "gateway-network"), makeDockerE2eScenario("two", "openai-chat-tools"), + { + ...onboardingScenario, + execution: { ...onboardingScenario.execution, dockerLane: "onboard" }, + }, ], }); diff --git a/qa/scenarios/cli/cli-onboarding.yaml b/qa/scenarios/cli/cli-onboarding.yaml index 0224eb0bbb20..76113440f912 100644 --- a/qa/scenarios/cli/cli-onboarding.yaml +++ b/qa/scenarios/cli/cli-onboarding.yaml @@ -32,6 +32,7 @@ scenario: execution: kind: script path: scripts/e2e/qa-cli-onboarding.mjs + dockerLane: onboard summary: Runs the executable onboarding Docker lane with only the guided, Gateway-storage, remote, reset, and skills cases selected. timeoutMs: 1800000 args: diff --git a/qa/scenarios/runtime/compose-setup.yaml b/qa/scenarios/runtime/compose-setup.yaml index c0a6dc62a4dd..9320fe6230eb 100644 --- a/qa/scenarios/runtime/compose-setup.yaml +++ b/qa/scenarios/runtime/compose-setup.yaml @@ -38,6 +38,7 @@ scenario: execution: kind: script path: test/e2e/qa-lab/runtime/docker-artifact-proof.ts + dockerLane: compose-setup summary: Builds the package-backed functional image, launches the documented Compose services, validates endpoint and authenticated CLI health, forces and recovers Docker health state, captures diagnostics, and records artifact identities. timeoutMs: 1800000 args: diff --git a/qa/scenarios/runtime/docker-package-install.yaml b/qa/scenarios/runtime/docker-package-install.yaml index 584be2636f53..537d5204ec0d 100644 --- a/qa/scenarios/runtime/docker-package-install.yaml +++ b/qa/scenarios/runtime/docker-package-install.yaml @@ -31,6 +31,7 @@ scenario: execution: kind: script path: test/e2e/qa-lab/runtime/docker-artifact-proof.ts + dockerLane: docker-package-install summary: Builds one real package artifact, installs and executes it through npm, pnpm, and Bun PATH entrypoints in clean containers, and records artifact identities. timeoutMs: 1800000 args: