vrr/openclaw
2
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-10 00:11:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

qa-lab: support script-backed evidence scenarios (#94276)

Browse source 
* qa: add script scenario execution kind

* fix(qa-lab): carry suite profile into script producer evidence and simplify artifact path resolution

* fix(qa-lab): keep out-of-repo producer artifacts absolute to avoid ../ traversal refs

---------

Co-authored-by: Dallin Romney <dallinromney@gmail.com>
This commit is contained in:
Colin Johnson 2026-06-17 18:09:25 -04:00 • committed by GitHub
parent 8288b4d4c9
commit 591313e80a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 811 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

7
docs/concepts/qa-e2e-automation.md
View file

@ -942,7 +942,12 @@ Every `qa suite` run writes top-level `qa-evidence.json`,
`qa-suite-summary.json`, and `qa-suite-report.md` artifacts for the selected
scenario set. Scenarios that declare `execution.kind: vitest` or
`execution.kind: playwright` run the matching test path and also write
per-scenario logs. When `qa suite` is reached through
per-scenario logs. Scenarios that declare `execution.kind: script` run the
evidence producer at `execution.path` through `node --import tsx` (with
`${outputDir}` and `${scenarioId}` expanded in `execution.args`); the producer
writes its own `qa-evidence.json`, whose entries are imported into the suite
output and whose artifact paths are resolved relative to that producer
`qa-evidence.json`. When `qa suite` is reached through
`qa run --qa-profile`, the same `qa-evidence.json` also includes the profile
scorecard summary for the selected taxonomy categories.
Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test.

37
extensions/qa-lab/src/coverage-report.test.ts
View file

@ -67,11 +67,13 @@ function scenarioWithCoverage(params: {
primary?: readonly string[];
secondary?: readonly string[];
sourcePath?: string;
executionKind?: "flow" | "vitest" | "playwright";
executionKind?: "flow" | "script" | "vitest" | "playwright";
executionPath?: string;
}): QaSeedScenarioWithSource {
const execution =
params.executionKind === "vitest" || params.executionKind === "playwright"
params.executionKind === "script" ||
params.executionKind === "vitest" ||
params.executionKind === "playwright"
? {
kind: params.executionKind,
path: params.executionPath ?? "src/test.test.ts",
@ -328,6 +330,37 @@ describe("qa coverage report", () => {
]);
});
it("uses script producer evidence as coverage fulfillment", () => {
const report = buildQaScorecardTaxonomyReport({
taxonomy: testMaturityTaxonomy({
categoryId: TEST_BROWSER_CATEGORY_ID,
coverageIds: [TEST_BROWSER_COVERAGE_ID],
}),
repoRoot: process.cwd(),
scenarios: [
scenarioWithCoverage({
primary: [TEST_BROWSER_COVERAGE_ID],
sourcePath: "qa/scenarios/ui/script-evidence-producer.yaml",
executionKind: "script",
executionPath: "scripts/check-no-conflict-markers.mjs",
}),
],
});
expect(report.validationIssues).toStrictEqual([]);
expect(report.fulfilledCategoryCount).toBe(1);
expect(report.fulfilledFeatureCount).toBe(1);
expect(report.categories[0]?.evidence).toStrictEqual([
{
coverageId: TEST_BROWSER_COVERAGE_ID,
kind: "script",
path: "scripts/check-no-conflict-markers.mjs",
role: "primary",
scenarioRefs: ["qa/scenarios/ui/script-evidence-producer.yaml"],
},
]);
});
it("reports profile membership refs missing from taxonomy categories", () => {
const report = buildQaScorecardTaxonomyReport({
taxonomy: testMaturityTaxonomy({

7
extensions/qa-lab/src/coverage-report.ts
View file

@ -465,7 +465,12 @@ function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) {
group.push(match);
groups.set(match.executionKind, group);
}
const executionOrder: QaScenarioSearchMatch["executionKind"][] = ["flow", "vitest", "playwright"];
const executionOrder: QaScenarioSearchMatch["executionKind"][] = [
"flow",
"script",
"vitest",
"playwright",
];
return executionOrder.flatMap((executionKind) => {
const group = groups.get(executionKind);
return group && group.length > 0 ? [{ executionKind, matches: group }] : [];

16
extensions/qa-lab/src/evidence-summary.ts
View file

@ -349,7 +349,7 @@ function uniqueSortedStrings(values: readonly (string | undefined)[]) {
);
}
function resolveQaEvidenceProfile(params: {
export function resolveQaEvidenceProfile(params: {
env?: NodeJS.ProcessEnv;
explicit?: QaEvidenceProfile;
}) {
@ -705,6 +705,20 @@ export function buildPlaywrightEvidenceSummary(
});
}
export function buildScriptEvidenceSummary(
params: QaEvidenceBuildBase & {
targets: readonly QaEvidenceTestTargetInput[];
results: readonly QaEvidenceTestResultInput[];
},
): QaEvidenceSummaryJson {
return buildTestRunnerEvidenceSummary({
...params,
defaultRunner: "script",
testKind: "script-test",
runner: params.runner ?? "script",
});
}
export function buildLiveTransportEvidenceSummary(
params: QaEvidenceBuildBase & {
checks: readonly QaEvidenceLiveTransportCheckInput[];

4
extensions/qa-lab/src/scenario-catalog.ts
View file

@ -73,6 +73,10 @@ const qaTestFileScenarioExecutionBaseSchema = z.object({
const qaTestFileScenarioExecutionSchema = z.discriminatedUnion("kind", [
qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("vitest") }),
qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("playwright") }),
qaTestFileScenarioExecutionBaseSchema.extend({
kind: z.literal("script"),
args: z.array(z.string()).optional(),
}),
]);
const qaScenarioExecutionSchema = z.union([

2
extensions/qa-lab/src/scorecard-taxonomy.ts
View file

@ -90,7 +90,7 @@ const qaMaturityTaxonomySchema = z
}
});
export type QaNativeCoverageEvidenceKind = "vitest" | "playwright";
export type QaNativeCoverageEvidenceKind = "script" | "vitest" | "playwright";
export type QaScorecardEvidenceKind = QaNativeCoverageEvidenceKind | "qa-scenario";
export type QaScorecardEvidenceMode = z.infer<typeof qaScorecardEvidenceModeSchema>;
type QaCoverageEvidenceRole = z.infer<typeof qaCoverageEvidenceRoleSchema>;

2
extensions/qa-lab/src/suite-launch.runtime.test.ts
View file

@ -74,7 +74,7 @@ describe("qa suite runtime launcher", () => {
runQaTestFileScenarios.mockImplementation(
async (params: {
outputDir: string;
scenarios: Array<{ id: string; execution: { kind: "vitest" | "playwright" } }>;
scenarios: Array<{ id: string; execution: { kind: "script" | "vitest" | "playwright" } }>;
}) => {
const [scenario] = params.scenarios;
if (!scenario) {

487
extensions/qa-lab/src/test-file-scenario-runner.test.ts
View file

@ -12,7 +12,7 @@ import {
const tempRoots: string[] = [];
function makeTestFileScenario(
executionKind: "vitest" | "playwright",
executionKind: "script" | "vitest" | "playwright",
pathLocal: string,
): QaSeedScenarioWithSource {
return {
@ -35,6 +35,9 @@ function makeTestFileScenario(
execution: {
kind: executionKind,
path: pathLocal,
...(executionKind === "script"
? { args: ["--once", "--artifact-base", "${outputDir}"] }
: {}),
},
};
}
@ -204,4 +207,486 @@ describe("qa test file scenario runner", () => {
},
});
});
it("runs script scenarios and imports producer QA evidence artifacts", async () => {
const repoRoot = await makeTempRepo("qa-script-scenario-");
const commands: QaScenarioCommandExecution[] = [];
const result = await runQaTestFileScenarios({
repoRoot,
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script"),
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.5",
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
runCommand: async (command) => {
commands.push(command);
const scenarioArtifactBase = path.join(
repoRoot,
".artifacts",
"qa-e2e",
"scenario-script",
"scenario-script",
);
const runRoot = path.join(scenarioArtifactBase, "run-1");
await fs.mkdir(path.join(runRoot, "surfaces", "web-ui"), { recursive: true });
await fs.writeFile(path.join(runRoot, "surfaces", "web-ui", "screenshot.png"), "png");
await fs.writeFile(
path.join(runRoot, "qa-evidence.json"),
`${JSON.stringify(
{
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-06-14T00:00:00.000Z",
evidenceMode: "full",
entries: [
{
test: {
kind: "script-producer-check",
id: "script-producer.web-ui.smoke",
title: "Script producer: web-ui smoke",
source: { path: "scripts/evidence-producer.ts" },
},
coverage: [{ id: "ui.control", role: "primary" }],
execution: {
runner: "evidence-producer-script",
environment: {
ref: "scenario-ref",
os: "darwin",
nodeVersion: "v24.0.0",
},
provider: {
id: "script-producer",
live: false,
model: { name: null, ref: null },
fixture: "mocked-script-evidence",
},
packageSource: { kind: "source-checkout", sha: "abc123" },
artifacts: [
{
kind: "screenshot",
path: "surfaces/web-ui/screenshot.png",
source: "script-producer:web-ui:smoke",
},
],
},
result: { status: "pass", timing: { wallMs: 1 } },
},
],
},
null,
2,
)}\n`,
"utf8",
);
await fs.writeFile(
path.join(scenarioArtifactBase, "latest-run.json"),
`${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`,
"utf8",
);
return {
exitCode: 0,
stdout: "script pass\n",
stderr: "",
};
},
env: {
OPENCLAW_QA_REF: "scenario-ref",
} as NodeJS.ProcessEnv,
});
expect(result.executionKind).toBe("script");
expect(commands.map((command) => command.args)).toEqual([
[
"--import",
"tsx",
"scripts/evidence-producer.ts",
"--once",
"--artifact-base",
path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script", "scenario-script"),
],
]);
const evidence = validateQaEvidenceSummaryJson(
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
);
expect(evidence.entries).toHaveLength(1);
expect(evidence.entries[0]).toMatchObject({
test: {
kind: "script-producer-check",
id: "script-producer.web-ui.smoke",
},
coverage: [
{
id: "ui.control",
role: "primary",
},
],
execution: {
runner: "evidence-producer-script",
artifacts: [
{
kind: "screenshot",
path: ".artifacts/qa-e2e/scenario-script/scenario-script/run-1/surfaces/web-ui/screenshot.png",
source: "script-producer:web-ui:smoke",
},
],
},
result: {
status: "pass",
},
});
});
it("imports producer QA evidence artifacts from failed script scenarios", async () => {
const repoRoot = await makeTempRepo("qa-script-failed-scenario-");
const result = await runQaTestFileScenarios({
repoRoot,
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-failed"),
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.5",
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
runCommand: async () => {
const scenarioArtifactBase = path.join(
repoRoot,
".artifacts",
"qa-e2e",
"scenario-script-failed",
"scenario-script",
);
const runRoot = path.join(scenarioArtifactBase, "run-1");
await fs.mkdir(runRoot, { recursive: true });
await fs.writeFile(
path.join(runRoot, "qa-evidence.json"),
`${JSON.stringify(
{
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-06-14T00:00:00.000Z",
evidenceMode: "full",
entries: [
{
test: {
kind: "script-producer-check",
id: "script-producer.web-ui.smoke",
title: "Script producer: web-ui smoke",
source: { path: "scripts/evidence-producer.ts" },
},
coverage: [{ id: "ui.control", role: "primary" }],
execution: {
runner: "evidence-producer-script",
environment: {
ref: "scenario-ref",
os: "darwin",
nodeVersion: "v24.0.0",
},
provider: {
id: "script-producer",
live: false,
model: { name: null, ref: null },
fixture: "failed-producer-evidence",
},
packageSource: { kind: "source-checkout", sha: "abc123" },
artifacts: [],
},
result: {
status: "fail",
failure: {
reason: "Script producer check failed.",
},
timing: { wallMs: 1 },
},
},
],
},
null,
2,
)}\n`,
"utf8",
);
await fs.writeFile(
path.join(scenarioArtifactBase, "latest-run.json"),
`${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`,
"utf8",
);
return {
exitCode: 1,
stdout: "",
stderr: "script failed\n",
};
},
env: {
OPENCLAW_QA_REF: "scenario-ref",
} as NodeJS.ProcessEnv,
});
expect(result.results[0]).toMatchObject({
status: "fail",
failureMessage: "node exited with 1",
producerEvidence: {
entries: [
{
test: {
id: "script-producer.web-ui.smoke",
},
result: {
status: "fail",
},
},
],
},
});
const evidence = validateQaEvidenceSummaryJson(
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
);
expect(evidence.entries).toHaveLength(2);
expect(evidence.entries[0]).toMatchObject({
test: {
kind: "script-producer-check",
id: "script-producer.web-ui.smoke",
},
result: {
status: "fail",
failure: {
reason: "Script producer check failed.",
},
},
});
expect(evidence.entries[1]).toMatchObject({
test: {
kind: "script-test",
id: "scenario-script",
source: {
path: "scripts/evidence-producer.ts",
},
},
result: {
status: "fail",
failure: {
reason: "node exited with 1",
},
},
});
});
it("fails script scenario results when imported producer evidence fails", async () => {
const repoRoot = await makeTempRepo("qa-script-producer-fail-");
const result = await runQaTestFileScenarios({
repoRoot,
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-producer-fail"),
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.5",
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
runCommand: async () => {
const scenarioArtifactBase = path.join(
repoRoot,
".artifacts",
"qa-e2e",
"scenario-script-producer-fail",
"scenario-script",
);
const runRoot = path.join(scenarioArtifactBase, "run-1");
await fs.mkdir(runRoot, { recursive: true });
await fs.writeFile(
path.join(runRoot, "qa-evidence.json"),
`${JSON.stringify(
{
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-06-14T00:00:00.000Z",
evidenceMode: "full",
entries: [
{
test: {
kind: "script-producer-check",
id: "script-producer.web-ui.smoke",
title: "Script producer: web-ui smoke",
source: { path: "scripts/evidence-producer.ts" },
},
coverage: [{ id: "ui.control", role: "primary" }],
execution: {
runner: "evidence-producer-script",
environment: {
ref: "scenario-ref",
os: "darwin",
nodeVersion: "v24.0.0",
},
provider: {
id: "script-producer",
live: false,
model: { name: null, ref: null },
fixture: "mocked-script-evidence",
},
packageSource: { kind: "source-checkout", sha: "abc123" },
artifacts: [],
},
result: {
status: "fail",
failure: {
reason: "Script producer check failed.",
},
timing: { wallMs: 1 },
},
},
],
},
null,
2,
)}\n`,
"utf8",
);
await fs.writeFile(
path.join(scenarioArtifactBase, "latest-run.json"),
`${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`,
"utf8",
);
return {
exitCode: 0,
stdout: "script pass\n",
stderr: "",
};
},
env: {
OPENCLAW_QA_REF: "scenario-ref",
} as NodeJS.ProcessEnv,
});
expect(result.results[0]).toMatchObject({
status: "fail",
failureMessage: "Script producer check failed.",
});
const evidence = validateQaEvidenceSummaryJson(
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
);
expect(evidence.entries).toHaveLength(1);
expect(evidence.entries[0]).toMatchObject({
test: {
id: "script-producer.web-ui.smoke",
},
result: {
status: "fail",
},
});
});
it("carries the suite profile into merged producer evidence", async () => {
const repoRoot = await makeTempRepo("qa-script-profile-");
const result = await runQaTestFileScenarios({
repoRoot,
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-profile"),
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.5",
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
runCommand: async () => {
const scenarioOutputDir = path.join(
repoRoot,
".artifacts",
"qa-e2e",
"scenario-script-profile",
"scenario-script",
);
await fs.mkdir(scenarioOutputDir, { recursive: true });
await fs.writeFile(
path.join(scenarioOutputDir, "qa-evidence.json"),
`${JSON.stringify({
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-06-14T00:00:00.000Z",
evidenceMode: "full",
entries: [
{
test: {
kind: "script-producer-check",
id: "script-producer.web-ui.smoke",
title: "Script producer: web-ui smoke",
source: { path: "scripts/evidence-producer.ts" },
},
coverage: [{ id: "ui.control", role: "primary" }],
result: { status: "pass", timing: { wallMs: 1 } },
},
],
})}\n`,
"utf8",
);
return { exitCode: 0, stdout: "script pass\n", stderr: "" };
},
env: {
OPENCLAW_QA_REF: "scenario-ref",
OPENCLAW_QA_PROFILE: "smoke-ci",
} as NodeJS.ProcessEnv,
});
const evidence = validateQaEvidenceSummaryJson(
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
);
expect(evidence.profile).toBe("smoke-ci");
});
it("keeps producer artifacts outside the repo root absolute instead of emitting ../ paths", async () => {
const repoRoot = await makeTempRepo("qa-script-external-artifact-");
const externalArtifact = path.join(os.tmpdir(), "qa-external-artifact.png");
const result = await runQaTestFileScenarios({
repoRoot,
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-external"),
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.5",
scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")],
runCommand: async () => {
const scenarioOutputDir = path.join(
repoRoot,
".artifacts",
"qa-e2e",
"scenario-script-external",
"scenario-script",
);
await fs.mkdir(scenarioOutputDir, { recursive: true });
await fs.writeFile(
path.join(scenarioOutputDir, "qa-evidence.json"),
`${JSON.stringify({
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-06-14T00:00:00.000Z",
evidenceMode: "full",
entries: [
{
test: {
kind: "script-producer-check",
id: "script-producer.web-ui.smoke",
title: "Script producer: web-ui smoke",
source: { path: "scripts/evidence-producer.ts" },
},
coverage: [{ id: "ui.control", role: "primary" }],
execution: {
runner: "evidence-producer-script",
environment: { ref: "scenario-ref", os: "darwin", nodeVersion: "v24.0.0" },
provider: {
id: "script-producer",
live: false,
model: { name: null, ref: null },
fixture: "mocked-script-evidence",
},
packageSource: { kind: "source-checkout", sha: "abc123" },
artifacts: [
{
kind: "screenshot",
path: externalArtifact,
source: "script-producer:web-ui:smoke",
},
],
},
result: { status: "pass", timing: { wallMs: 1 } },
},
],
})}\n`,
"utf8",
);
return { exitCode: 0, stdout: "script pass\n", stderr: "" };
},
env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv,
});
const evidence = validateQaEvidenceSummaryJson(
JSON.parse(await fs.readFile(result.evidencePath, "utf8")),
);
const artifactPath = evidence.entries[0]?.execution?.artifacts[0]?.path;
expect(artifactPath).toBe(path.normalize(externalArtifact));
expect(artifactPath?.includes("..")).toBe(false);
});
});

263
extensions/qa-lab/src/test-file-scenario-runner.ts
View file

@ -2,15 +2,18 @@ import { spawn } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { toRepoRelativePath } from "./cli-paths.js";
import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js";
import { QaSuiteArtifactError } from "./errors.js";
import {
buildPlaywrightEvidenceSummary,
buildScriptEvidenceSummary,
buildVitestEvidenceSummary,
QA_EVIDENCE_FILENAME,
QA_EVIDENCE_SUMMARY_KIND,
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
type QaEvidenceStatus,
type QaEvidenceSummaryJson,
resolveQaEvidenceProfile,
validateQaEvidenceSummaryJson,
} from "./evidence-summary.js";
import type { QaProviderMode } from "./providers/index.js";
@ -19,10 +22,13 @@ import type { QaScorecardEvidenceMode } from "./scorecard-taxonomy.js";
import { shellQuote } from "./shell-quote.js";
export type QaTestFileScenario = QaSeedScenarioWithSource & {
execution: Extract<QaSeedScenarioWithSource["execution"], { kind: "vitest" | "playwright" }>;
execution: Extract<
QaSeedScenarioWithSource["execution"],
{ kind: "script" | "vitest" | "playwright" }
>;
};
export type QaTestFileExecutionKind = "vitest" | "playwright";
export type QaTestFileExecutionKind = "script" | "vitest" | "playwright";
export type QaTestFileScenarioRunParams = {
evidenceMode?: QaScorecardEvidenceMode;
@ -61,7 +67,9 @@ type QaScenarioCommandStep = {
type QaTestFileScenarioResult = {
durationMs: number;
failureMessage?: string;
includeFallbackEvidence?: boolean;
logPath: string;
producerEvidence?: QaEvidenceSummaryJson;
scenario: QaTestFileScenario;
status: QaEvidenceStatus;
};
@ -75,13 +83,17 @@ export type QaTestFileScenarioRunResult = {
type QaTestFileRunnerDefinition = {
buildEvidenceSummary: typeof buildVitestEvidenceSummary;
buildSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[];
buildSteps(scenario: QaTestFileScenario, context: { outputDir: string }): QaScenarioCommandStep[];
};
export function isQaTestFileScenario(
scenario: QaSeedScenarioWithSource,
): scenario is QaTestFileScenario {
return scenario.execution.kind === "vitest" || scenario.execution.kind === "playwright";
return (
scenario.execution.kind === "vitest" ||
scenario.execution.kind === "playwright" ||
scenario.execution.kind === "script"
);
}
function vitestSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[] {
@ -115,7 +127,42 @@ function playwrightSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[]
];
}
function replaceScriptArgTokens(
args: readonly string[] | undefined,
context: { outputDir: string; scenarioId: string },
) {
return (args ?? []).map((arg) =>
arg
.replaceAll("${outputDir}", context.outputDir)
.replaceAll("${scenarioId}", context.scenarioId),
);
}
function scriptSteps(
scenario: QaTestFileScenario,
context: { outputDir: string },
): QaScenarioCommandStep[] {
const scenarioOutputDir = path.join(context.outputDir, scenario.id);
const scriptArgs =
scenario.execution.kind === "script"
? replaceScriptArgTokens(scenario.execution.args, {
outputDir: scenarioOutputDir,
scenarioId: scenario.id,
})
: [];
return [
{
command: process.execPath,
args: ["--import", "tsx", scenario.execution.path, ...scriptArgs],
},
];
}
const testFileRunnerDefinitions: Record<QaTestFileExecutionKind, QaTestFileRunnerDefinition> = {
script: {
buildEvidenceSummary: buildScriptEvidenceSummary,
buildSteps: scriptSteps,
},
vitest: {
buildEvidenceSummary: buildVitestEvidenceSummary,
buildSteps: vitestSteps,
@ -230,16 +277,64 @@ async function runQaTestFileScenario(params: {
scenario: QaTestFileScenario;
}) {
const definition = testFileRunnerDefinitions[params.scenario.execution.kind];
return await runScenarioCommandSteps({
const result = await runScenarioCommandSteps({
...params,
steps: definition.buildSteps(params.scenario),
steps: definition.buildSteps(params.scenario, { outputDir: params.outputDir }),
});
if (params.scenario.execution.kind !== "script") {
return result;
}
const producerEvidenceResult = await readScriptProducerEvidence({
outputDir: params.outputDir,
repoRoot: params.repoRoot,
scenario: params.scenario,
});
if (!producerEvidenceResult.producerEvidence) {
return result;
}
if (result.status !== "pass") {
return {
...result,
...producerEvidenceResult,
includeFallbackEvidence: true,
};
}
return {
...result,
...producerEvidenceResult,
...statusFromProducerEvidence(producerEvidenceResult.producerEvidence),
};
}
function statusFromProducerEvidence(
producerEvidence: QaEvidenceSummaryJson | undefined,
): Pick<QaTestFileScenarioResult, "failureMessage" | "status"> {
if (!producerEvidence || producerEvidence.entries.length === 0) {
return { status: "pass" };
}
const blockingEntry = producerEvidence.entries.find(
(entry) => entry.result.status === "fail" || entry.result.status === "blocked",
);
if (blockingEntry) {
return {
failureMessage:
blockingEntry.result.failure?.reason ??
`${blockingEntry.test.id} reported ${blockingEntry.result.status}`,
status: blockingEntry.result.status,
};
}
if (producerEvidence.entries.every((entry) => entry.result.status === "skipped")) {
return { status: "skipped" };
}
return { status: "pass" };
}
function resolveTestFileExecutionKind(scenarios: readonly QaTestFileScenario[]) {
const kinds = new Set(scenarios.map((scenario) => scenario.execution.kind));
if (kinds.size > 1) {
throw new Error("qa suite cannot mix Vitest and Playwright scenarios in one invocation.");
throw new Error(
"qa suite cannot mix script, Vitest, and Playwright scenarios in one invocation.",
);
}
const [kind] = kinds;
return kind;
@ -255,6 +350,55 @@ function buildTestFileEvidence(params: {
evidenceMode?: QaScorecardEvidenceMode;
env?: NodeJS.ProcessEnv;
}) {
const producerEntries = params.results.flatMap(
(result) => result.producerEvidence?.entries ?? [],
);
if (producerEntries.length > 0) {
const definition = testFileRunnerDefinitions[params.kind];
const fallbackResults = params.results.filter(
(result) => !result.producerEvidence || result.includeFallbackEvidence,
);
const evidenceMode =
params.evidenceMode ??
(params.results.every((result) => result.producerEvidence?.evidenceMode === "slim")
? "slim"
: "full");
const fallbackEvidence =
fallbackResults.length > 0
? definition.buildEvidenceSummary({
artifactPaths: params.artifactPaths,
evidenceMode,
env: params.env,
generatedAt: params.generatedAt,
primaryModel: params.primaryModel,
providerMode: params.providerMode,
targets: fallbackResults.map((result) => buildScenarioEvidenceTarget(result.scenario)),
results: fallbackResults.map((result) => ({
id: result.scenario.id,
status: result.status,
durationMs: result.durationMs,
failureMessage: result.failureMessage,
})),
})
: undefined;
return validateQaEvidenceSummaryJson({
kind: QA_EVIDENCE_SUMMARY_KIND,
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
generatedAt: params.generatedAt,
evidenceMode,
profile: resolveQaEvidenceProfile({ env: params.env }),
entries: [
...producerEntries.map((entry) => {
if (evidenceMode !== "slim") {
return entry;
}
const { execution: _execution, ...withoutExecution } = entry;
return withoutExecution;
}),
...(fallbackEvidence?.entries ?? []),
],
});
}
const definition = testFileRunnerDefinitions[params.kind];
const evidence = definition.buildEvidenceSummary({
artifactPaths: params.artifactPaths,
@ -281,6 +425,107 @@ function buildTestFileEvidence(params: {
});
}
async function readJsonFileIfExists(filePath: string): Promise<unknown> {
let text: string;
try {
text = await fs.readFile(filePath, "utf8");
} catch (error) {
if (
error &&
typeof error === "object" &&
"code" in error &&
(error as { code?: unknown }).code === "ENOENT"
) {
return undefined;
}
throw error;
}
try {
return JSON.parse(text) as unknown;
} catch (error) {
throw new Error(`invalid JSON in ${filePath}: ${formatErrorMessage(error)}`, { cause: error });
}
}
// Producer artifact paths follow one convention: relative paths resolve against the
// qa-evidence.json directory, absolute paths are taken as-is. Paths under the repo root
// become repo-relative; paths outside it stay absolute so downstream consumers never see
// `../` segments that would read as path traversal.
function resolveScriptProducerArtifactPath(params: {
evidenceDir: string;
repoRoot: string;
artifactPath: string;
}) {
const absolutePath = path.isAbsolute(params.artifactPath)
? params.artifactPath
: path.join(params.evidenceDir, params.artifactPath);
const repoRelativePath = toRepoRelativePath(params.repoRoot, absolutePath);
return isRepoRootRelativeRef(repoRelativePath) ? repoRelativePath : path.normalize(absolutePath);
}
function normalizeScriptProducerEvidence(params: {
evidence: QaEvidenceSummaryJson;
evidencePath: string;
repoRoot: string;
}): QaEvidenceSummaryJson {
// Input is already validated by the caller; this only rewrites artifact path strings,
// so the transformed shape stays schema-valid without re-parsing.
const evidenceDir = path.dirname(params.evidencePath);
return {
...params.evidence,
entries: params.evidence.entries.map((entry) => ({
...entry,
execution: entry.execution
? {
...entry.execution,
artifacts: entry.execution.artifacts.map((artifact) => ({
...artifact,
path: resolveScriptProducerArtifactPath({
artifactPath: artifact.path,
evidenceDir,
repoRoot: params.repoRoot,
}),
})),
}
: undefined,
})),
};
}
async function readScriptProducerEvidence(params: {
outputDir: string;
repoRoot: string;
scenario: QaTestFileScenario;
}): Promise<Pick<QaTestFileScenarioResult, "producerEvidence">> {
const scenarioOutputDir = path.join(params.outputDir, params.scenario.id);
const latestRun = (await readJsonFileIfExists(
path.join(scenarioOutputDir, "latest-run.json"),
)) as { qaEvidence?: unknown } | undefined;
const candidates = [
typeof latestRun?.qaEvidence === "string" ? latestRun.qaEvidence : undefined,
path.join(scenarioOutputDir, QA_EVIDENCE_FILENAME),
].filter((candidate): candidate is string => Boolean(candidate));
for (const candidate of candidates) {
const evidencePath = path.isAbsolute(candidate)
? candidate
: path.join(scenarioOutputDir, candidate);
const rawEvidence = await readJsonFileIfExists(evidencePath);
if (!rawEvidence) {
continue;
}
const evidence = validateQaEvidenceSummaryJson(rawEvidence);
return {
producerEvidence: normalizeScriptProducerEvidence({
evidence,
evidencePath,
repoRoot: params.repoRoot,
}),
};
}
return {};
}
function buildScenarioArtifactPaths(params: {
repoRoot: string;
results: readonly QaTestFileScenarioResult[];
@ -319,7 +564,7 @@ export async function runQaTestFileScenarios(
const scenarios = params.scenarios.filter(isQaTestFileScenario);
const kind = resolveTestFileExecutionKind(scenarios);
if (!kind) {
throw new Error("qa suite found no Vitest or Playwright scenarios to run.");
throw new Error("qa suite found no script, Vitest, or Playwright scenarios to run.");
}
await fs.mkdir(params.outputDir, { recursive: true });
const runCommand = params.runCommand ?? runQaScenarioCommand;

6
qa/scenarios/index.yaml
View file

@ -22,9 +22,9 @@ title: OpenClaw QA Scenario Pack
# - prefer reusing an existing coverage ID over minting a scenario-shaped ID
# - avoid copying the scenario title into coverage IDs
# - use `pnpm openclaw qa coverage` to render the current inventory
# - use `scenario.execution.kind: vitest` or `scenario.execution.kind: playwright`
# plus `scenario.execution.path` for native test files that provide evidence without
# a top-level `flow`
# - use `scenario.execution.kind: vitest`, `playwright`, or `script`
# plus `scenario.execution.path` for native tests or evidence producers that
# provide evidence without a top-level `flow`
# - use `runtimeParityTier` for runtime-pair gate membership: `standard`,
# `optional`, `live-only`, or `soak`
# - treat the old `coverage: ["id"]` / `coverage: - id` list shape as invalid