vrr/openclaw
2
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-09 15:59:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

Add slim evidence mode for QA profile evidence (#93179)

Browse source 
* test(qa): compact profile evidence execution metadata

* docs(qa): document compact profile evidence

* test(qa): support compact evidence mode

* test(qa): rename compact evidence mode to slim

* docs(qa): trim slim evidence wording

* fix(qa): avoid commander runtime import
This commit is contained in:
Dallin Romney 2026-06-15 14:50:40 -07:00 • committed by GitHub
parent f018945eff
commit e32929e12c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 288 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

5
docs/concepts/qa-e2e-automation.md
View file

@ -56,8 +56,9 @@ the resolved scenarios through `qa suite`. `--surface` and
`--category` filter the selected profile instead of defining separate lanes.
The resulting `qa-evidence.json` includes a profile scorecard summary with
selected-category counts and missing coverage IDs; the individual evidence
entries remain the source of truth for the tests, coverage roles, artifacts,
and results:
entries remain the source of truth for the tests, coverage roles, and results.
Slim evidence omits per-entry `execution` and sets `evidenceMode: "slim"`;
`smoke-ci` defaults to slim, and `--evidence-mode full` restores full entries:
```bash
pnpm openclaw qa run \

2
docs/help/testing.md
View file

@ -189,6 +189,8 @@ inside every shard.
mixed flow, Vitest, and Playwright scenario selections.
- When dispatched by `pnpm openclaw qa run --qa-profile <profile>`, embeds the
selected taxonomy profile scorecard in the same `qa-evidence.json`.
`smoke-ci` writes slim evidence, which sets `evidenceMode: "slim"` and omits
per-entry `execution`.
- Runs multiple selected scenarios in parallel by default with isolated
gateway workers. `qa-channel` defaults to concurrency 4 (bounded by the
selected scenario count). Use `--concurrency <count>` to tune the worker

5
extensions/qa-lab/src/cli.runtime.test.ts
View file

@ -124,6 +124,7 @@ function makeQaEvidence(entries: unknown[] = []) {
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-06-14T00:00:00.000Z",
evidenceMode: "full",
entries,
};
}
@ -441,6 +442,8 @@ describe("qa cli runtime", () => {
expect(suiteArgs.scenarioIds).not.toContain("thinking-slash-model-remap");
expect(process.env.OPENCLAW_QA_PROFILE).toBe("release");
const evidence = JSON.parse(await fs.readFile(suiteEvidencePath, "utf8")) as {
evidenceMode?: unknown;
entries?: unknown[];
profile?: unknown;
scorecard?: {
run?: { evidenceEntryCount?: unknown };
@ -453,6 +456,7 @@ describe("qa cli runtime", () => {
};
};
expect(evidence.profile).toBe("smoke-ci");
expect(evidence.evidenceMode).toBe("slim");
expect(evidence.scorecard).toMatchObject({
run: {
evidenceEntryCount: 1,
@ -468,6 +472,7 @@ describe("qa cli runtime", () => {
fulfilled: 1,
},
});
expect(evidence.entries?.[0]).not.toHaveProperty("execution");
expect(JSON.stringify(evidence.scorecard)).not.toContain("dm-chat-baseline");
expectWriteContains(stdoutWrite, "QA run profile: smoke-ci; categories: 1; scenarios:");
expectWriteContains(stdoutWrite, `QA profile scorecard: ${suiteEvidencePath}`);

5
extensions/qa-lab/src/cli.runtime.ts
View file

@ -72,6 +72,7 @@ import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js"
import {
readQaScorecardTaxonomyReport,
type QaScorecardCategoryCoverageReport,
type QaScorecardEvidenceMode,
} from "./scorecard-taxonomy.js";
import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js";
import { scenarioMatchesQaProviderLane } from "./suite-planning.js";
@ -115,6 +116,7 @@ type QaScenarioProviderCommandOptions = {
};
type QaScenarioRunCommandOptions = QaScenarioProviderCommandOptions & {
evidenceMode?: QaScorecardEvidenceMode;
repoRoot?: string;
outputDir?: string;
concurrency?: number;
@ -669,6 +671,7 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
const suiteResult = await runQaSuiteCommand({
repoRoot,
outputDir: opts.outputDir,
evidenceMode: opts.evidenceMode,
transportId: opts.transportId,
providerMode,
primaryModel: opts.primaryModel,
@ -686,6 +689,7 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
}
await attachQaProfileScorecardEvidenceToFile({
evidencePath,
evidenceMode: opts.evidenceMode,
profile,
filters: {
surface: opts.surface,
@ -849,6 +853,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
runQaSuite({
repoRoot,
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
evidenceMode: opts.evidenceMode,
transportId,
...(opts.providerMode !== undefined ? { providerMode } : {}),
primaryModel,

71
extensions/qa-lab/src/cli.test.ts
View file

@ -217,6 +217,8 @@ describe("qa cli registration", () => {
"agent-runtime-and-provider-execution",
"--category",
"agent-runtime-and-provider-execution.agent-turn-execution",
"--evidence-mode",
"slim",
"--transport",
"qa-channel",
"--provider-mode",
@ -237,6 +239,7 @@ describe("qa cli registration", () => {
profile: "smoke-ci",
surface: "agent-runtime-and-provider-execution",
category: "agent-runtime-and-provider-execution.agent-turn-execution",
evidenceMode: "slim",
transportId: "qa-channel",
providerMode: "mock-openai",
primaryModel: "openai/gpt-5.5",
@ -252,6 +255,8 @@ describe("qa cli registration", () => {
["--output-dir", [".artifacts/qa-e2e/smoke-ci"]],
["--surface", ["agent-runtime-and-provider-execution"]],
["--category", ["agent-runtime-and-provider-execution.agent-turn-execution"]],
["--evidence-mode", ["slim"]],
["--exclude-test-execution-evidence", []],
["--transport", ["qa-channel"]],
["--provider-mode", ["mock-openai"]],
["--model", ["openai/gpt-5.5"]],
@ -268,6 +273,72 @@ describe("qa cli registration", () => {
expect(runQaProfileCommand).not.toHaveBeenCalled();
});
it.each([["--evidence-mode", "compact"], ["--exclude-test-execution-evidence"]])(
"maps deprecated compact evidence flag %s to slim",
async (...flags) => {
await program.parseAsync([
"node",
"openclaw",
"qa",
"run",
"--qa-profile",
"release",
...flags.filter(Boolean),
]);
expect(runQaProfileCommand).toHaveBeenCalledWith(
expect.objectContaining({
evidenceMode: "slim",
profile: "release",
}),
);
},
);
it("rejects conflicting deprecated evidence flags", async () => {
await expect(
program.parseAsync([
"node",
"openclaw",
"qa",
"run",
"--qa-profile",
"release",
"--evidence-mode",
"full",
"--exclude-test-execution-evidence",
]),
).rejects.toThrow("--exclude-test-execution-evidence conflicts with --evidence-mode full");
expect(runQaProfileCommand).not.toHaveBeenCalled();
});
it("rejects unknown qa evidence modes", async () => {
const invalidProgram = new Command();
invalidProgram.exitOverride();
invalidProgram.configureOutput({
writeErr: () => {},
writeOut: () => {},
});
registerQaLabCli(invalidProgram);
await expect(
invalidProgram.parseAsync([
"node",
"openclaw",
"qa",
"run",
"--qa-profile",
"smoke-ci",
"--evidence-mode",
"tiny",
]),
).rejects.toThrow("--evidence-mode must be one of full, slim.");
expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled();
expect(runQaProfileCommand).not.toHaveBeenCalled();
});
it("rejects an empty qa run --qa-profile instead of falling back to self-check", async () => {
await expect(
program.parseAsync(["node", "openclaw", "qa", "run", "--qa-profile", ""]),

87
extensions/qa-lab/src/cli.ts
View file

@ -40,12 +40,16 @@ type QaRunCliOptions = QaLabSelfCheckCommandOptions &
qaProfile?: QaProfileCommandOptions["profile"];
surface?: QaProfileCommandOptions["surface"];
category?: QaProfileCommandOptions["category"];
evidenceMode?: QaProfileCommandOptions["evidenceMode"];
excludeTestExecutionEvidence?: boolean;
};
const QA_RUN_PROFILE_ONLY_OPTIONS = [
{ optionName: "outputDir", flag: "--output-dir" },
{ optionName: "surface", flag: "--surface" },
{ optionName: "category", flag: "--category" },
{ optionName: "evidenceMode", flag: "--evidence-mode" },
{ optionName: "excludeTestExecutionEvidence", flag: "--exclude-test-execution-evidence" },
{ optionName: "transport", flag: "--transport" },
{ optionName: "providerMode", flag: "--provider-mode" },
{ optionName: "model", flag: "--model" },
@ -97,6 +101,29 @@ function parseQaCliPositiveIntegerOption(value: string, flag: string): number {
return parsed;
}
function parseQaEvidenceModeOption(value: string): QaProfileCommandOptions["evidenceMode"] {
const evidenceMode = value.trim();
if (evidenceMode === "full" || evidenceMode === "slim") {
return evidenceMode;
}
if (evidenceMode === "compact") {
return "slim";
}
throw invalidQaCliArgument("--evidence-mode must be one of full, slim.");
}
function resolveQaEvidenceModeOptions(opts: QaRunCliOptions) {
if (opts.excludeTestExecutionEvidence !== true) {
return opts.evidenceMode;
}
if (opts.evidenceMode === "full") {
throw invalidQaCliArgument(
"--exclude-test-execution-evidence conflicts with --evidence-mode full.",
);
}
return "slim";
}
function collectCliSuppliedQaRunFlags(
command: Command,
options: readonly { optionName: string; flag: string }[],
@ -360,7 +387,8 @@ export function registerQaLabCli(program: Command) {
.description("Run private QA automation flows and launch the QA debugger");
registerMantisCli(qa);
qa.command("run")
const qaRun = qa
.command("run")
.description("Run the bundled QA self-check and write a Markdown report")
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
.option("--output <path>", "Report output path")
@ -368,6 +396,18 @@ export function registerQaLabCli(program: Command) {
.option("--qa-profile <id>", "Run the QA profile from taxonomy.yaml")
.option("--surface <id>", "Limit --qa-profile to a taxonomy surface id")
.option("--category <id>", "Limit --qa-profile to a taxonomy category id")
.option(
"--evidence-mode <mode>",
"Set profile qa-evidence.json mode: full or slim",
parseQaEvidenceModeOption,
)
.option(
"--exclude-test-execution-evidence",
"Deprecated alias for --evidence-mode slim",
false,
);
qaRun.options.at(-1)?.hideHelp();
qaRun
.option("--transport <id>", "QA transport id", "qa-channel")
.option("--provider-mode <mode>", formatQaProviderModeHelp())
.option("--model <ref>", "Primary provider/model ref")
@ -380,31 +420,32 @@ export function registerQaLabCli(program: Command) {
"Write artifacts without setting a failing exit code when scenarios fail",
false,
)
.option("--fast", "Enable provider fast mode where supported", false)
.action(async (opts: QaRunCliOptions, command: Command) => {
validateQaRunMode(opts, command);
if (opts.qaProfile?.trim()) {
await runQaProfile({
repoRoot: opts.repoRoot,
outputDir: opts.outputDir,
profile: opts.qaProfile,
surface: opts.surface,
category: opts.category,
transportId: opts.transport,
providerMode: opts.providerMode,
primaryModel: opts.model,
alternateModel: opts.altModel,
concurrency: opts.concurrency,
allowFailures: opts.allowFailures,
fastMode: opts.fast,
});
return;
}
await runQaSelfCheck({
.option("--fast", "Enable provider fast mode where supported", false);
qaRun.action(async (opts: QaRunCliOptions, command: Command) => {
validateQaRunMode(opts, command);
if (opts.qaProfile?.trim()) {
await runQaProfile({
repoRoot: opts.repoRoot,
output: opts.output,
outputDir: opts.outputDir,
profile: opts.qaProfile,
surface: opts.surface,
category: opts.category,
evidenceMode: resolveQaEvidenceModeOptions(opts),
transportId: opts.transport,
providerMode: opts.providerMode,
primaryModel: opts.model,
alternateModel: opts.altModel,
concurrency: opts.concurrency,
allowFailures: opts.allowFailures,
fastMode: opts.fast,
});
return;
}
await runQaSelfCheck({
repoRoot: opts.repoRoot,
output: opts.output,
});
});
qa.command("suite")
.description("Run repo-backed QA scenarios against the QA gateway lane")

53
extensions/qa-lab/src/evidence-summary.test.ts
View file

@ -47,6 +47,7 @@ describe("evidence summary", () => {
expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
expect(evidence.kind).toBe(QA_EVIDENCE_SUMMARY_KIND);
expect(evidence.schemaVersion).toBe(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION);
expect(evidence.evidenceMode).toBe("full");
expect(evidence.profile).toBeUndefined();
expect(evidence.entries).toHaveLength(1);
expect(evidence.entries[0]).toMatchObject({
@ -440,6 +441,38 @@ describe("evidence summary", () => {
expect(evidence.profile).toBe("experimental-profile");
});
it.each([
{ evidenceMode: undefined, expectedMode: "slim", hasExecution: false },
{ evidenceMode: "full" as const, expectedMode: "full", hasExecution: true },
])(
"resolves profile evidence mode $expectedMode",
({ evidenceMode, expectedMode, hasExecution }) => {
const evidence = buildQaSuiteEvidenceSummary({
artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }],
...(evidenceMode ? { evidenceMode } : {}),
profile: "smoke-ci",
scenarioDefinitions: [
{
id: "dm-chat-baseline",
title: "DM baseline conversation",
coverage: {
primary: ["channels.dm"],
},
},
],
channelId: "qa-channel",
generatedAt: "2026-06-07T12:09:00.000Z",
primaryModel: "mock-openai/gpt-5.5",
providerMode: "mock-openai",
scenarioResults: [{ name: "DM baseline conversation", status: "pass" }],
});
expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence);
expect(evidence.evidenceMode).toBe(expectedMode);
expect("execution" in evidence.entries[0]).toBe(hasExecution);
},
);
it("keeps mock non-OpenAI model refs attributed to their model provider", () => {
const evidence = buildQaSuiteEvidenceSummary({
artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }],
@ -460,11 +493,13 @@ describe("evidence summary", () => {
scenarioResults: [{ name: "Anthropic parity", status: "pass" }],
});
expect(evidence.entries[0]?.execution.provider).toMatchObject({
id: "anthropic",
model: {
name: "claude-opus-4-8",
ref: "anthropic/claude-opus-4-8",
expect(evidence.entries[0]?.execution).toMatchObject({
provider: {
id: "anthropic",
model: {
name: "claude-opus-4-8",
ref: "anthropic/claude-opus-4-8",
},
},
});
expect(evidence.entries[0]).toMatchObject({
@ -500,7 +535,7 @@ describe("evidence summary", () => {
transportId: "telegram",
});
expect(evidence.entries[0]?.execution.packageSource).toEqual({
expect(evidence.entries[0]?.execution?.packageSource).toEqual({
kind: "packed-tarball",
spec: "/tmp/openclaw.tgz",
sha: "abc123",
@ -530,7 +565,7 @@ describe("evidence summary", () => {
transportId: "telegram",
});
expect(evidence.entries[0]?.execution.packageSource).toEqual({
expect(evidence.entries[0]?.execution?.packageSource).toEqual({
kind: "npm-package",
spec: "openclaw@beta",
sha: "def456",
@ -558,7 +593,7 @@ describe("evidence summary", () => {
transportId: "telegram",
});
expect(evidence.entries[0]?.execution.packageSource).toEqual({
expect(evidence.entries[0]?.execution?.packageSource).toEqual({
kind: "source-checkout",
spec: undefined,
sha: undefined,
@ -589,7 +624,7 @@ describe("evidence summary", () => {
transportId: "discord",
});
expect(evidence.entries[0]?.execution.artifacts).toEqual(
expect(evidence.entries[0]?.execution?.artifacts).toEqual(
expect.arrayContaining([
{
kind: "screenshot",

52
extensions/qa-lab/src/evidence-summary.ts
View file

@ -2,6 +2,11 @@
import { z } from "zod";
import { splitQaModelRef } from "./model-selection.js";
import { getQaProvider, type QaProviderMode } from "./providers/index.js";
import {
qaScorecardEvidenceModeSchema,
readQaScorecardProfileOptions,
type QaScorecardEvidenceMode,
} from "./scorecard-taxonomy.js";
export const QA_EVIDENCE_SUMMARY_KIND = "openclaw.qa.evidence-summary";
export const QA_EVIDENCE_FILENAME = "qa-evidence.json";
@ -176,7 +181,7 @@ export const qaEvidenceSummaryEntrySchema = z
coverage: z.array(qaEvidenceCoverageSchema),
refs: z.array(qaEvidenceRefSchema).optional(),
runtimeParityTier: nonEmptyStringSchema.optional(),
execution: qaEvidenceExecutionSchema,
execution: qaEvidenceExecutionSchema.optional(),
result: qaEvidenceResultSchema,
})
.strict();
@ -186,6 +191,7 @@ export const qaEvidenceSummarySchema = z
kind: z.literal(QA_EVIDENCE_SUMMARY_KIND),
schemaVersion: z.literal(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION),
generatedAt: nonEmptyStringSchema,
evidenceMode: qaScorecardEvidenceModeSchema,
entries: z.array(qaEvidenceSummaryEntrySchema),
profile: qaEvidenceProfileIdSchema.optional(),
scorecard: qaEvidenceScorecardSchema.optional(),
@ -274,6 +280,7 @@ type QaEvidenceArtifactInput = {
type QaEvidenceBuildBase = {
artifactPaths: readonly QaEvidenceArtifactInput[];
evidenceMode?: QaScorecardEvidenceMode;
env?: NodeJS.ProcessEnv;
generatedAt: string;
primaryModel: string;
@ -491,15 +498,28 @@ function resultForEvidence(
function buildQaEvidenceSummary(params: {
entries: QaEvidenceSummaryEntry[];
evidenceMode?: QaScorecardEvidenceMode;
generatedAt: string;
profile?: QaEvidenceProfile;
scorecard?: QaEvidenceScorecardJson;
}): QaEvidenceSummaryJson {
const profileOptions = readQaScorecardProfileOptions(params.profile);
const evidenceMode = params.evidenceMode ?? profileOptions.evidenceMode;
const entries =
evidenceMode === "slim"
? params.entries.map((entry) => {
const { execution: _execution, ...withoutExecution } = entry;
return withoutExecution;
})
: params.entries;
return qaEvidenceSummarySchema.parse({
kind: QA_EVIDENCE_SUMMARY_KIND,
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
generatedAt: params.generatedAt,
entries: params.entries,
evidenceMode,
entries,
profile: params.profile,
scorecard: params.scorecard,
});
}
@ -508,12 +528,15 @@ export function validateQaEvidenceSummaryJson(summary: unknown): QaEvidenceSumma
}
export function attachQaEvidenceScorecard(params: {
evidenceMode?: QaScorecardEvidenceMode;
summary: QaEvidenceSummaryJson;
profile: QaEvidenceProfile;
scorecard: QaEvidenceScorecardJson;
}): QaEvidenceSummaryJson {
return validateQaEvidenceSummaryJson({
...params.summary,
return buildQaEvidenceSummary({
entries: params.summary.entries,
evidenceMode: params.evidenceMode,
generatedAt: params.summary.generatedAt,
profile: params.profile,
scorecard: params.scorecard,
});
@ -582,7 +605,12 @@ export function buildQaSuiteEvidenceSummary(
result: resultForEvidence(result, timing),
};
});
return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries, profile });
return buildQaEvidenceSummary({
entries,
evidenceMode: params.evidenceMode,
generatedAt: params.generatedAt,
profile,
});
}
function buildTestRunnerEvidenceSummary(
@ -641,7 +669,12 @@ function buildTestRunnerEvidenceSummary(
result: resultForEvidence(result, timing),
};
});
return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries, profile });
return buildQaEvidenceSummary({
entries,
evidenceMode: params.evidenceMode,
generatedAt: params.generatedAt,
profile,
});
}
export function buildVitestEvidenceSummary(
@ -734,5 +767,10 @@ export function buildLiveTransportEvidenceSummary(
result: resultForEvidence(check, timing),
};
});
return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries, profile });
return buildQaEvidenceSummary({
entries,
evidenceMode: params.evidenceMode,
generatedAt: params.generatedAt,
profile,
});
}

1
extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts
View file

@ -49,6 +49,7 @@ function makeEvidenceSummary(status: "pass" | "fail" | "blocked" | "skipped") {
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-05-01T00:00:00.000Z",
evidenceMode: "full",
entries: [
{
test: {

2
extensions/qa-lab/src/mantis/run.runtime.ts
View file

@ -251,7 +251,7 @@ async function readNormalizedLaneResult(params: {
const entry =
summary.entries.find((candidate) => candidate.test.id === params.scenario) ??
summary.entries[0];
const artifacts = entry?.execution.artifacts ?? [];
const artifacts = entry?.execution?.artifacts ?? [];
return {
details: entry?.result.failure?.reason,
screenshotPath: artifacts.find((artifact) => artifact.kind === "screenshot")?.path,

7
extensions/qa-lab/src/scorecard-evidence.ts
View file

@ -7,7 +7,10 @@ import {
type QaEvidenceSummaryEntry,
type QaEvidenceSummaryJson,
} from "./evidence-summary.js";
import type { QaScorecardCategoryCoverageReport } from "./scorecard-taxonomy.js";
import type {
QaScorecardCategoryCoverageReport,
QaScorecardEvidenceMode,
} from "./scorecard-taxonomy.js";
import { readQaScorecardFeatureCoverageByCategory } from "./scorecard-taxonomy.js";
type QaProfileScorecardFilters = {
@ -155,6 +158,7 @@ export function buildQaProfileScorecardEvidence(params: {
export async function attachQaProfileScorecardEvidenceToFile(params: {
evidencePath: string;
evidenceMode?: QaScorecardEvidenceMode;
profile: string;
filters: QaProfileScorecardFilters;
categories: readonly QaScorecardCategoryCoverageReport[];
@ -170,6 +174,7 @@ export async function attachQaProfileScorecardEvidenceToFile(params: {
});
const nextEvidence = attachQaEvidenceScorecard({
summary: evidence,
evidenceMode: params.evidenceMode,
profile: params.profile,
scorecard,
});

17
extensions/qa-lab/src/scorecard-taxonomy.ts
View file

@ -19,10 +19,12 @@ function isRepoRootRelativeRef(value: string) {
}
const qaCoverageEvidenceRoleSchema = z.enum(["primary", "secondary"]);
export const qaScorecardEvidenceModeSchema = z.enum(["full", "slim"]);
const qaScorecardProfileSchema = z.object({
id: qaScorecardIdSchema,
description: z.string().trim().min(1),
evidenceMode: qaScorecardEvidenceModeSchema.optional(),
categoryIds: z.array(qaScorecardIdSchema).default([]),
});
@ -81,6 +83,7 @@ const qaMaturityTaxonomySchema = z
export type QaNativeCoverageEvidenceKind = "vitest" | "playwright";
export type QaScorecardEvidenceKind = QaNativeCoverageEvidenceKind | "qa-scenario";
export type QaScorecardEvidenceMode = z.infer<typeof qaScorecardEvidenceModeSchema>;
type QaCoverageEvidenceRole = z.infer<typeof qaCoverageEvidenceRoleSchema>;
type QaMaturityTaxonomy = z.infer<typeof qaMaturityTaxonomySchema>;
@ -125,6 +128,7 @@ export type QaScorecardCategoryCoverageReport = {
export type QaScorecardProfileReport = {
id: string;
evidenceMode: QaScorecardEvidenceMode;
categoryIds: string[];
};
@ -332,6 +336,18 @@ export function readQaScorecardFeatureCoverageByCategory(repoRoot?: string) {
);
}
export function readQaScorecardProfileOptions(profileId: string | undefined, repoRoot?: string) {
const profile = profileId?.trim();
if (!profile) {
return { evidenceMode: "full" as const };
}
return {
evidenceMode:
readQaMaturityTaxonomy(repoRoot)?.profiles.find((entry) => entry.id === profile)
?.evidenceMode ?? "full",
};
}
function pushMissingPrimaryIssues(params: {
issues: QaScorecardValidationIssue[];
category: MaturityCategoryRef;
@ -467,6 +483,7 @@ export function buildQaScorecardTaxonomyReport(params: {
}
return {
id: profile.id,
evidenceMode: profile.evidenceMode ?? "full",
categoryIds: validCategoryIds,
};
}) ?? [];

1
extensions/qa-lab/src/suite-launch.runtime.test.ts
View file

@ -37,6 +37,7 @@ async function writeEvidence(pathLocal: string) {
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-06-14T00:00:00.000Z",
evidenceMode: "full",
entries: [],
},
null,

6
extensions/qa-lab/src/suite-launch.runtime.ts
View file

@ -131,6 +131,7 @@ async function runQaTestFileSuiteFromRuntime(params: {
const providerMode = normalizeQaProviderMode(runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
const primaryModel = runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode);
return await runQaTestFileScenarios({
evidenceMode: runParams?.evidenceMode,
repoRoot,
outputDir,
providerMode,
@ -174,6 +175,11 @@ function mergeQaEvidenceSummaries(params: {
kind: QA_EVIDENCE_SUMMARY_KIND,
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
generatedAt: params.generatedAt,
evidenceMode:
params.evidenceSummaries.length > 0 &&
params.evidenceSummaries.every((summary) => summary.evidenceMode === "slim")
? "slim"
: "full",
entries: params.evidenceSummaries.flatMap((summary) => summary.entries),
profile: profiles.length === 1 ? profiles[0] : undefined,
});

10
extensions/qa-lab/src/suite.ts
View file

@ -51,6 +51,7 @@ import {
type QaSeedScenarioWithSource,
} from "./scenario-catalog.js";
import { runScenarioFlow } from "./scenario-flow-runner.js";
import type { QaScorecardEvidenceMode } from "./scorecard-taxonomy.js";
import {
applyQaMergePatch,
collectQaSuiteGatewayConfigPatch,
@ -101,6 +102,7 @@ type QaSuiteEnvironment = {
export type QaSuiteStartLabFn = (params?: QaLabServerStartParams) => Promise<QaLabServerHandle>;
export type QaSuiteRunParams = {
evidenceMode?: QaScorecardEvidenceMode;
repoRoot?: string;
outputDir?: string;
providerMode?: QaProviderMode;
@ -615,6 +617,7 @@ export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSui
}
async function runQaRuntimeParitySuite(params: {
evidenceMode?: QaScorecardEvidenceMode;
repoRoot: string;
outputDir: string;
startedAt: Date;
@ -792,6 +795,7 @@ async function runQaRuntimeParitySuite(params: {
finishedAt,
scenarios,
scenarioDefinitions: params.selectedScenarios,
evidenceMode: params.evidenceMode,
transport,
providerMode: params.providerMode,
primaryModel: params.primaryModel,
@ -838,6 +842,7 @@ async function writeQaSuiteArtifacts(params: {
finishedAt: Date;
scenarios: QaSuiteScenarioResult[];
scenarioDefinitions?: readonly QaSeedScenarioWithSource[];
evidenceMode?: QaScorecardEvidenceMode;
metrics?: QaSuiteSummaryJson["metrics"];
transport: QaTransportAdapter;
// Reuse the canonical QaProviderMode union instead of re-declaring it
@ -876,6 +881,7 @@ async function writeQaSuiteArtifacts(params: {
{ kind: "summary", path: path.basename(summaryPath) },
{ kind: "report", path: path.basename(reportPath) },
],
evidenceMode: params.evidenceMode,
channelId: params.transport.id,
env: process.env,
generatedAt: params.finishedAt.toISOString(),
@ -1105,6 +1111,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
if (params?.runtimePair) {
return await runQaRuntimeParitySuite({
evidenceMode: params.evidenceMode,
repoRoot,
outputDir,
startedAt,
@ -1176,6 +1183,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
finishedAt: partialFinishedAt,
scenarios: partialScenarios,
scenarioDefinitions: completedScenarioDefinitions,
evidenceMode: params?.evidenceMode,
transport,
providerMode,
primaryModel,
@ -1320,6 +1328,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
finishedAt,
scenarios,
scenarioDefinitions: selectedScenarios,
evidenceMode: params?.evidenceMode,
transport,
providerMode,
primaryModel,
@ -1581,6 +1590,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
scenarios,
metrics,
scenarioDefinitions: selectedScenarios,
evidenceMode: params?.evidenceMode,
transport,
providerMode,
primaryModel,

6
extensions/qa-lab/src/test-file-scenario-runner.ts
View file

@ -15,6 +15,7 @@ import {
} from "./evidence-summary.js";
import type { QaProviderMode } from "./providers/index.js";
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
import type { QaScorecardEvidenceMode } from "./scorecard-taxonomy.js";
import { shellQuote } from "./shell-quote.js";
export type QaTestFileScenario = QaSeedScenarioWithSource & {
@ -24,6 +25,7 @@ export type QaTestFileScenario = QaSeedScenarioWithSource & {
export type QaTestFileExecutionKind = "vitest" | "playwright";
export type QaTestFileScenarioRunParams = {
evidenceMode?: QaScorecardEvidenceMode;
env?: NodeJS.ProcessEnv;
outputDir: string;
primaryModel: string;
@ -250,11 +252,13 @@ function buildTestFileEvidence(params: {
primaryModel: string;
providerMode: QaProviderMode;
results: readonly QaTestFileScenarioResult[];
evidenceMode?: QaScorecardEvidenceMode;
env?: NodeJS.ProcessEnv;
}) {
const definition = testFileRunnerDefinitions[params.kind];
const evidence = definition.buildEvidenceSummary({
artifactPaths: params.artifactPaths,
evidenceMode: params.evidenceMode,
env: params.env,
generatedAt: params.generatedAt,
primaryModel: params.primaryModel,
@ -271,6 +275,7 @@ function buildTestFileEvidence(params: {
kind: QA_EVIDENCE_SUMMARY_KIND,
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
generatedAt: params.generatedAt,
evidenceMode: evidence.evidenceMode,
profile: evidence.profile,
entries: evidence.entries,
});
@ -341,6 +346,7 @@ export async function runQaTestFileScenarios(
});
const evidence = buildTestFileEvidence({
artifactPaths,
evidenceMode: params.evidenceMode,
env,
generatedAt,
kind,

1
taxonomy.yaml
View file

@ -9,6 +9,7 @@ profiles:
- id: smoke-ci
description: Deterministic PR and merge proof with mock model providers, synthetic qa-channel,
and local SDK-backed channel upstreams; no live external service required.
evidenceMode: slim
categoryIds:
- agent-runtime-and-provider-execution.agent-turn-execution
- session-memory-and-context-engine.token-management