From e32929e12cd470d238a97afbb7b6d97c532c8415 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 15 Jun 2026 14:50:40 -0700 Subject: [PATCH] Add slim evidence mode for QA profile evidence (#93179) * test(qa): compact profile evidence execution metadata * docs(qa): document compact profile evidence * test(qa): support compact evidence mode * test(qa): rename compact evidence mode to slim * docs(qa): trim slim evidence wording * fix(qa): avoid commander runtime import --- docs/concepts/qa-e2e-automation.md | 5 +- docs/help/testing.md | 2 + extensions/qa-lab/src/cli.runtime.test.ts | 5 ++ extensions/qa-lab/src/cli.runtime.ts | 5 ++ extensions/qa-lab/src/cli.test.ts | 71 +++++++++++++++ extensions/qa-lab/src/cli.ts | 87 ++++++++++++++----- .../qa-lab/src/evidence-summary.test.ts | 53 +++++++++-- extensions/qa-lab/src/evidence-summary.ts | 52 +++++++++-- .../whatsapp/cli.runtime.test.ts | 1 + extensions/qa-lab/src/mantis/run.runtime.ts | 2 +- extensions/qa-lab/src/scorecard-evidence.ts | 7 +- extensions/qa-lab/src/scorecard-taxonomy.ts | 17 ++++ .../qa-lab/src/suite-launch.runtime.test.ts | 1 + extensions/qa-lab/src/suite-launch.runtime.ts | 6 ++ extensions/qa-lab/src/suite.ts | 10 +++ .../qa-lab/src/test-file-scenario-runner.ts | 6 ++ taxonomy.yaml | 1 + 17 files changed, 288 insertions(+), 43 deletions(-) diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 00c2dbda918..666f1f2f9b1 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -56,8 +56,9 @@ the resolved scenarios through `qa suite`. `--surface` and `--category` filter the selected profile instead of defining separate lanes. The resulting `qa-evidence.json` includes a profile scorecard summary with selected-category counts and missing coverage IDs; the individual evidence -entries remain the source of truth for the tests, coverage roles, artifacts, -and results: +entries remain the source of truth for the tests, coverage roles, and results. +Slim evidence omits per-entry `execution` and sets `evidenceMode: "slim"`; +`smoke-ci` defaults to slim, and `--evidence-mode full` restores full entries: ```bash pnpm openclaw qa run \ diff --git a/docs/help/testing.md b/docs/help/testing.md index 231486b52be..605de266407 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -189,6 +189,8 @@ inside every shard. mixed flow, Vitest, and Playwright scenario selections. - When dispatched by `pnpm openclaw qa run --qa-profile `, embeds the selected taxonomy profile scorecard in the same `qa-evidence.json`. + `smoke-ci` writes slim evidence, which sets `evidenceMode: "slim"` and omits + per-entry `execution`. - Runs multiple selected scenarios in parallel by default with isolated gateway workers. `qa-channel` defaults to concurrency 4 (bounded by the selected scenario count). Use `--concurrency ` to tune the worker diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 34e4bf232dd..0b5d7d481c8 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -124,6 +124,7 @@ function makeQaEvidence(entries: unknown[] = []) { kind: "openclaw.qa.evidence-summary", schemaVersion: 2, generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", entries, }; } @@ -441,6 +442,8 @@ describe("qa cli runtime", () => { expect(suiteArgs.scenarioIds).not.toContain("thinking-slash-model-remap"); expect(process.env.OPENCLAW_QA_PROFILE).toBe("release"); const evidence = JSON.parse(await fs.readFile(suiteEvidencePath, "utf8")) as { + evidenceMode?: unknown; + entries?: unknown[]; profile?: unknown; scorecard?: { run?: { evidenceEntryCount?: unknown }; @@ -453,6 +456,7 @@ describe("qa cli runtime", () => { }; }; expect(evidence.profile).toBe("smoke-ci"); + expect(evidence.evidenceMode).toBe("slim"); expect(evidence.scorecard).toMatchObject({ run: { evidenceEntryCount: 1, @@ -468,6 +472,7 @@ describe("qa cli runtime", () => { fulfilled: 1, }, }); + expect(evidence.entries?.[0]).not.toHaveProperty("execution"); expect(JSON.stringify(evidence.scorecard)).not.toContain("dm-chat-baseline"); expectWriteContains(stdoutWrite, "QA run profile: smoke-ci; categories: 1; scenarios:"); expectWriteContains(stdoutWrite, `QA profile scorecard: ${suiteEvidencePath}`); diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 355f70196fb..9fd656aa001 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -72,6 +72,7 @@ import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js" import { readQaScorecardTaxonomyReport, type QaScorecardCategoryCoverageReport, + type QaScorecardEvidenceMode, } from "./scorecard-taxonomy.js"; import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js"; import { scenarioMatchesQaProviderLane } from "./suite-planning.js"; @@ -115,6 +116,7 @@ type QaScenarioProviderCommandOptions = { }; type QaScenarioRunCommandOptions = QaScenarioProviderCommandOptions & { + evidenceMode?: QaScorecardEvidenceMode; repoRoot?: string; outputDir?: string; concurrency?: number; @@ -669,6 +671,7 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) { const suiteResult = await runQaSuiteCommand({ repoRoot, outputDir: opts.outputDir, + evidenceMode: opts.evidenceMode, transportId: opts.transportId, providerMode, primaryModel: opts.primaryModel, @@ -686,6 +689,7 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) { } await attachQaProfileScorecardEvidenceToFile({ evidencePath, + evidenceMode: opts.evidenceMode, profile, filters: { surface: opts.surface, @@ -849,6 +853,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { runQaSuite({ repoRoot, outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir), + evidenceMode: opts.evidenceMode, transportId, ...(opts.providerMode !== undefined ? { providerMode } : {}), primaryModel, diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index 5dd1d87d15c..55ae164284f 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -217,6 +217,8 @@ describe("qa cli registration", () => { "agent-runtime-and-provider-execution", "--category", "agent-runtime-and-provider-execution.agent-turn-execution", + "--evidence-mode", + "slim", "--transport", "qa-channel", "--provider-mode", @@ -237,6 +239,7 @@ describe("qa cli registration", () => { profile: "smoke-ci", surface: "agent-runtime-and-provider-execution", category: "agent-runtime-and-provider-execution.agent-turn-execution", + evidenceMode: "slim", transportId: "qa-channel", providerMode: "mock-openai", primaryModel: "openai/gpt-5.5", @@ -252,6 +255,8 @@ describe("qa cli registration", () => { ["--output-dir", [".artifacts/qa-e2e/smoke-ci"]], ["--surface", ["agent-runtime-and-provider-execution"]], ["--category", ["agent-runtime-and-provider-execution.agent-turn-execution"]], + ["--evidence-mode", ["slim"]], + ["--exclude-test-execution-evidence", []], ["--transport", ["qa-channel"]], ["--provider-mode", ["mock-openai"]], ["--model", ["openai/gpt-5.5"]], @@ -268,6 +273,72 @@ describe("qa cli registration", () => { expect(runQaProfileCommand).not.toHaveBeenCalled(); }); + it.each([["--evidence-mode", "compact"], ["--exclude-test-execution-evidence"]])( + "maps deprecated compact evidence flag %s to slim", + async (...flags) => { + await program.parseAsync([ + "node", + "openclaw", + "qa", + "run", + "--qa-profile", + "release", + ...flags.filter(Boolean), + ]); + + expect(runQaProfileCommand).toHaveBeenCalledWith( + expect.objectContaining({ + evidenceMode: "slim", + profile: "release", + }), + ); + }, + ); + + it("rejects conflicting deprecated evidence flags", async () => { + await expect( + program.parseAsync([ + "node", + "openclaw", + "qa", + "run", + "--qa-profile", + "release", + "--evidence-mode", + "full", + "--exclude-test-execution-evidence", + ]), + ).rejects.toThrow("--exclude-test-execution-evidence conflicts with --evidence-mode full"); + + expect(runQaProfileCommand).not.toHaveBeenCalled(); + }); + + it("rejects unknown qa evidence modes", async () => { + const invalidProgram = new Command(); + invalidProgram.exitOverride(); + invalidProgram.configureOutput({ + writeErr: () => {}, + writeOut: () => {}, + }); + registerQaLabCli(invalidProgram); + + await expect( + invalidProgram.parseAsync([ + "node", + "openclaw", + "qa", + "run", + "--qa-profile", + "smoke-ci", + "--evidence-mode", + "tiny", + ]), + ).rejects.toThrow("--evidence-mode must be one of full, slim."); + + expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled(); + expect(runQaProfileCommand).not.toHaveBeenCalled(); + }); + it("rejects an empty qa run --qa-profile instead of falling back to self-check", async () => { await expect( program.parseAsync(["node", "openclaw", "qa", "run", "--qa-profile", ""]), diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index 1e1ed1289eb..056b3f6011b 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -40,12 +40,16 @@ type QaRunCliOptions = QaLabSelfCheckCommandOptions & qaProfile?: QaProfileCommandOptions["profile"]; surface?: QaProfileCommandOptions["surface"]; category?: QaProfileCommandOptions["category"]; + evidenceMode?: QaProfileCommandOptions["evidenceMode"]; + excludeTestExecutionEvidence?: boolean; }; const QA_RUN_PROFILE_ONLY_OPTIONS = [ { optionName: "outputDir", flag: "--output-dir" }, { optionName: "surface", flag: "--surface" }, { optionName: "category", flag: "--category" }, + { optionName: "evidenceMode", flag: "--evidence-mode" }, + { optionName: "excludeTestExecutionEvidence", flag: "--exclude-test-execution-evidence" }, { optionName: "transport", flag: "--transport" }, { optionName: "providerMode", flag: "--provider-mode" }, { optionName: "model", flag: "--model" }, @@ -97,6 +101,29 @@ function parseQaCliPositiveIntegerOption(value: string, flag: string): number { return parsed; } +function parseQaEvidenceModeOption(value: string): QaProfileCommandOptions["evidenceMode"] { + const evidenceMode = value.trim(); + if (evidenceMode === "full" || evidenceMode === "slim") { + return evidenceMode; + } + if (evidenceMode === "compact") { + return "slim"; + } + throw invalidQaCliArgument("--evidence-mode must be one of full, slim."); +} + +function resolveQaEvidenceModeOptions(opts: QaRunCliOptions) { + if (opts.excludeTestExecutionEvidence !== true) { + return opts.evidenceMode; + } + if (opts.evidenceMode === "full") { + throw invalidQaCliArgument( + "--exclude-test-execution-evidence conflicts with --evidence-mode full.", + ); + } + return "slim"; +} + function collectCliSuppliedQaRunFlags( command: Command, options: readonly { optionName: string; flag: string }[], @@ -360,7 +387,8 @@ export function registerQaLabCli(program: Command) { .description("Run private QA automation flows and launch the QA debugger"); registerMantisCli(qa); - qa.command("run") + const qaRun = qa + .command("run") .description("Run the bundled QA self-check and write a Markdown report") .option("--repo-root ", "Repository root to target when running from a neutral cwd") .option("--output ", "Report output path") @@ -368,6 +396,18 @@ export function registerQaLabCli(program: Command) { .option("--qa-profile ", "Run the QA profile from taxonomy.yaml") .option("--surface ", "Limit --qa-profile to a taxonomy surface id") .option("--category ", "Limit --qa-profile to a taxonomy category id") + .option( + "--evidence-mode ", + "Set profile qa-evidence.json mode: full or slim", + parseQaEvidenceModeOption, + ) + .option( + "--exclude-test-execution-evidence", + "Deprecated alias for --evidence-mode slim", + false, + ); + qaRun.options.at(-1)?.hideHelp(); + qaRun .option("--transport ", "QA transport id", "qa-channel") .option("--provider-mode ", formatQaProviderModeHelp()) .option("--model ", "Primary provider/model ref") @@ -380,31 +420,32 @@ export function registerQaLabCli(program: Command) { "Write artifacts without setting a failing exit code when scenarios fail", false, ) - .option("--fast", "Enable provider fast mode where supported", false) - .action(async (opts: QaRunCliOptions, command: Command) => { - validateQaRunMode(opts, command); - if (opts.qaProfile?.trim()) { - await runQaProfile({ - repoRoot: opts.repoRoot, - outputDir: opts.outputDir, - profile: opts.qaProfile, - surface: opts.surface, - category: opts.category, - transportId: opts.transport, - providerMode: opts.providerMode, - primaryModel: opts.model, - alternateModel: opts.altModel, - concurrency: opts.concurrency, - allowFailures: opts.allowFailures, - fastMode: opts.fast, - }); - return; - } - await runQaSelfCheck({ + .option("--fast", "Enable provider fast mode where supported", false); + qaRun.action(async (opts: QaRunCliOptions, command: Command) => { + validateQaRunMode(opts, command); + if (opts.qaProfile?.trim()) { + await runQaProfile({ repoRoot: opts.repoRoot, - output: opts.output, + outputDir: opts.outputDir, + profile: opts.qaProfile, + surface: opts.surface, + category: opts.category, + evidenceMode: resolveQaEvidenceModeOptions(opts), + transportId: opts.transport, + providerMode: opts.providerMode, + primaryModel: opts.model, + alternateModel: opts.altModel, + concurrency: opts.concurrency, + allowFailures: opts.allowFailures, + fastMode: opts.fast, }); + return; + } + await runQaSelfCheck({ + repoRoot: opts.repoRoot, + output: opts.output, }); + }); qa.command("suite") .description("Run repo-backed QA scenarios against the QA gateway lane") diff --git a/extensions/qa-lab/src/evidence-summary.test.ts b/extensions/qa-lab/src/evidence-summary.test.ts index b4d05cbb7a1..696597a4aad 100644 --- a/extensions/qa-lab/src/evidence-summary.test.ts +++ b/extensions/qa-lab/src/evidence-summary.test.ts @@ -47,6 +47,7 @@ describe("evidence summary", () => { expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence); expect(evidence.kind).toBe(QA_EVIDENCE_SUMMARY_KIND); expect(evidence.schemaVersion).toBe(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION); + expect(evidence.evidenceMode).toBe("full"); expect(evidence.profile).toBeUndefined(); expect(evidence.entries).toHaveLength(1); expect(evidence.entries[0]).toMatchObject({ @@ -440,6 +441,38 @@ describe("evidence summary", () => { expect(evidence.profile).toBe("experimental-profile"); }); + it.each([ + { evidenceMode: undefined, expectedMode: "slim", hasExecution: false }, + { evidenceMode: "full" as const, expectedMode: "full", hasExecution: true }, + ])( + "resolves profile evidence mode $expectedMode", + ({ evidenceMode, expectedMode, hasExecution }) => { + const evidence = buildQaSuiteEvidenceSummary({ + artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }], + ...(evidenceMode ? { evidenceMode } : {}), + profile: "smoke-ci", + scenarioDefinitions: [ + { + id: "dm-chat-baseline", + title: "DM baseline conversation", + coverage: { + primary: ["channels.dm"], + }, + }, + ], + channelId: "qa-channel", + generatedAt: "2026-06-07T12:09:00.000Z", + primaryModel: "mock-openai/gpt-5.5", + providerMode: "mock-openai", + scenarioResults: [{ name: "DM baseline conversation", status: "pass" }], + }); + + expect(validateQaEvidenceSummaryJson(evidence)).toEqual(evidence); + expect(evidence.evidenceMode).toBe(expectedMode); + expect("execution" in evidence.entries[0]).toBe(hasExecution); + }, + ); + it("keeps mock non-OpenAI model refs attributed to their model provider", () => { const evidence = buildQaSuiteEvidenceSummary({ artifactPaths: [{ kind: "summary", path: "qa-suite-summary.json" }], @@ -460,11 +493,13 @@ describe("evidence summary", () => { scenarioResults: [{ name: "Anthropic parity", status: "pass" }], }); - expect(evidence.entries[0]?.execution.provider).toMatchObject({ - id: "anthropic", - model: { - name: "claude-opus-4-8", - ref: "anthropic/claude-opus-4-8", + expect(evidence.entries[0]?.execution).toMatchObject({ + provider: { + id: "anthropic", + model: { + name: "claude-opus-4-8", + ref: "anthropic/claude-opus-4-8", + }, }, }); expect(evidence.entries[0]).toMatchObject({ @@ -500,7 +535,7 @@ describe("evidence summary", () => { transportId: "telegram", }); - expect(evidence.entries[0]?.execution.packageSource).toEqual({ + expect(evidence.entries[0]?.execution?.packageSource).toEqual({ kind: "packed-tarball", spec: "/tmp/openclaw.tgz", sha: "abc123", @@ -530,7 +565,7 @@ describe("evidence summary", () => { transportId: "telegram", }); - expect(evidence.entries[0]?.execution.packageSource).toEqual({ + expect(evidence.entries[0]?.execution?.packageSource).toEqual({ kind: "npm-package", spec: "openclaw@beta", sha: "def456", @@ -558,7 +593,7 @@ describe("evidence summary", () => { transportId: "telegram", }); - expect(evidence.entries[0]?.execution.packageSource).toEqual({ + expect(evidence.entries[0]?.execution?.packageSource).toEqual({ kind: "source-checkout", spec: undefined, sha: undefined, @@ -589,7 +624,7 @@ describe("evidence summary", () => { transportId: "discord", }); - expect(evidence.entries[0]?.execution.artifacts).toEqual( + expect(evidence.entries[0]?.execution?.artifacts).toEqual( expect.arrayContaining([ { kind: "screenshot", diff --git a/extensions/qa-lab/src/evidence-summary.ts b/extensions/qa-lab/src/evidence-summary.ts index 74011ac0a74..7c8d3858285 100644 --- a/extensions/qa-lab/src/evidence-summary.ts +++ b/extensions/qa-lab/src/evidence-summary.ts @@ -2,6 +2,11 @@ import { z } from "zod"; import { splitQaModelRef } from "./model-selection.js"; import { getQaProvider, type QaProviderMode } from "./providers/index.js"; +import { + qaScorecardEvidenceModeSchema, + readQaScorecardProfileOptions, + type QaScorecardEvidenceMode, +} from "./scorecard-taxonomy.js"; export const QA_EVIDENCE_SUMMARY_KIND = "openclaw.qa.evidence-summary"; export const QA_EVIDENCE_FILENAME = "qa-evidence.json"; @@ -176,7 +181,7 @@ export const qaEvidenceSummaryEntrySchema = z coverage: z.array(qaEvidenceCoverageSchema), refs: z.array(qaEvidenceRefSchema).optional(), runtimeParityTier: nonEmptyStringSchema.optional(), - execution: qaEvidenceExecutionSchema, + execution: qaEvidenceExecutionSchema.optional(), result: qaEvidenceResultSchema, }) .strict(); @@ -186,6 +191,7 @@ export const qaEvidenceSummarySchema = z kind: z.literal(QA_EVIDENCE_SUMMARY_KIND), schemaVersion: z.literal(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION), generatedAt: nonEmptyStringSchema, + evidenceMode: qaScorecardEvidenceModeSchema, entries: z.array(qaEvidenceSummaryEntrySchema), profile: qaEvidenceProfileIdSchema.optional(), scorecard: qaEvidenceScorecardSchema.optional(), @@ -274,6 +280,7 @@ type QaEvidenceArtifactInput = { type QaEvidenceBuildBase = { artifactPaths: readonly QaEvidenceArtifactInput[]; + evidenceMode?: QaScorecardEvidenceMode; env?: NodeJS.ProcessEnv; generatedAt: string; primaryModel: string; @@ -491,15 +498,28 @@ function resultForEvidence( function buildQaEvidenceSummary(params: { entries: QaEvidenceSummaryEntry[]; + evidenceMode?: QaScorecardEvidenceMode; generatedAt: string; profile?: QaEvidenceProfile; + scorecard?: QaEvidenceScorecardJson; }): QaEvidenceSummaryJson { + const profileOptions = readQaScorecardProfileOptions(params.profile); + const evidenceMode = params.evidenceMode ?? profileOptions.evidenceMode; + const entries = + evidenceMode === "slim" + ? params.entries.map((entry) => { + const { execution: _execution, ...withoutExecution } = entry; + return withoutExecution; + }) + : params.entries; return qaEvidenceSummarySchema.parse({ kind: QA_EVIDENCE_SUMMARY_KIND, schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, generatedAt: params.generatedAt, - entries: params.entries, + evidenceMode, + entries, profile: params.profile, + scorecard: params.scorecard, }); } @@ -508,12 +528,15 @@ export function validateQaEvidenceSummaryJson(summary: unknown): QaEvidenceSumma } export function attachQaEvidenceScorecard(params: { + evidenceMode?: QaScorecardEvidenceMode; summary: QaEvidenceSummaryJson; profile: QaEvidenceProfile; scorecard: QaEvidenceScorecardJson; }): QaEvidenceSummaryJson { - return validateQaEvidenceSummaryJson({ - ...params.summary, + return buildQaEvidenceSummary({ + entries: params.summary.entries, + evidenceMode: params.evidenceMode, + generatedAt: params.summary.generatedAt, profile: params.profile, scorecard: params.scorecard, }); @@ -582,7 +605,12 @@ export function buildQaSuiteEvidenceSummary( result: resultForEvidence(result, timing), }; }); - return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries, profile }); + return buildQaEvidenceSummary({ + entries, + evidenceMode: params.evidenceMode, + generatedAt: params.generatedAt, + profile, + }); } function buildTestRunnerEvidenceSummary( @@ -641,7 +669,12 @@ function buildTestRunnerEvidenceSummary( result: resultForEvidence(result, timing), }; }); - return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries, profile }); + return buildQaEvidenceSummary({ + entries, + evidenceMode: params.evidenceMode, + generatedAt: params.generatedAt, + profile, + }); } export function buildVitestEvidenceSummary( @@ -734,5 +767,10 @@ export function buildLiveTransportEvidenceSummary( result: resultForEvidence(check, timing), }; }); - return buildQaEvidenceSummary({ generatedAt: params.generatedAt, entries, profile }); + return buildQaEvidenceSummary({ + entries, + evidenceMode: params.evidenceMode, + generatedAt: params.generatedAt, + profile, + }); } diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts index 37ebdeb7a97..be43be164c2 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts @@ -49,6 +49,7 @@ function makeEvidenceSummary(status: "pass" | "fail" | "blocked" | "skipped") { kind: "openclaw.qa.evidence-summary", schemaVersion: 2, generatedAt: "2026-05-01T00:00:00.000Z", + evidenceMode: "full", entries: [ { test: { diff --git a/extensions/qa-lab/src/mantis/run.runtime.ts b/extensions/qa-lab/src/mantis/run.runtime.ts index cd5800ee119..aa4779bb0d1 100644 --- a/extensions/qa-lab/src/mantis/run.runtime.ts +++ b/extensions/qa-lab/src/mantis/run.runtime.ts @@ -251,7 +251,7 @@ async function readNormalizedLaneResult(params: { const entry = summary.entries.find((candidate) => candidate.test.id === params.scenario) ?? summary.entries[0]; - const artifacts = entry?.execution.artifacts ?? []; + const artifacts = entry?.execution?.artifacts ?? []; return { details: entry?.result.failure?.reason, screenshotPath: artifacts.find((artifact) => artifact.kind === "screenshot")?.path, diff --git a/extensions/qa-lab/src/scorecard-evidence.ts b/extensions/qa-lab/src/scorecard-evidence.ts index b9252387f44..5bed4688065 100644 --- a/extensions/qa-lab/src/scorecard-evidence.ts +++ b/extensions/qa-lab/src/scorecard-evidence.ts @@ -7,7 +7,10 @@ import { type QaEvidenceSummaryEntry, type QaEvidenceSummaryJson, } from "./evidence-summary.js"; -import type { QaScorecardCategoryCoverageReport } from "./scorecard-taxonomy.js"; +import type { + QaScorecardCategoryCoverageReport, + QaScorecardEvidenceMode, +} from "./scorecard-taxonomy.js"; import { readQaScorecardFeatureCoverageByCategory } from "./scorecard-taxonomy.js"; type QaProfileScorecardFilters = { @@ -155,6 +158,7 @@ export function buildQaProfileScorecardEvidence(params: { export async function attachQaProfileScorecardEvidenceToFile(params: { evidencePath: string; + evidenceMode?: QaScorecardEvidenceMode; profile: string; filters: QaProfileScorecardFilters; categories: readonly QaScorecardCategoryCoverageReport[]; @@ -170,6 +174,7 @@ export async function attachQaProfileScorecardEvidenceToFile(params: { }); const nextEvidence = attachQaEvidenceScorecard({ summary: evidence, + evidenceMode: params.evidenceMode, profile: params.profile, scorecard, }); diff --git a/extensions/qa-lab/src/scorecard-taxonomy.ts b/extensions/qa-lab/src/scorecard-taxonomy.ts index 764e8bf37e4..cf47f9a603a 100644 --- a/extensions/qa-lab/src/scorecard-taxonomy.ts +++ b/extensions/qa-lab/src/scorecard-taxonomy.ts @@ -19,10 +19,12 @@ function isRepoRootRelativeRef(value: string) { } const qaCoverageEvidenceRoleSchema = z.enum(["primary", "secondary"]); +export const qaScorecardEvidenceModeSchema = z.enum(["full", "slim"]); const qaScorecardProfileSchema = z.object({ id: qaScorecardIdSchema, description: z.string().trim().min(1), + evidenceMode: qaScorecardEvidenceModeSchema.optional(), categoryIds: z.array(qaScorecardIdSchema).default([]), }); @@ -81,6 +83,7 @@ const qaMaturityTaxonomySchema = z export type QaNativeCoverageEvidenceKind = "vitest" | "playwright"; export type QaScorecardEvidenceKind = QaNativeCoverageEvidenceKind | "qa-scenario"; +export type QaScorecardEvidenceMode = z.infer; type QaCoverageEvidenceRole = z.infer; type QaMaturityTaxonomy = z.infer; @@ -125,6 +128,7 @@ export type QaScorecardCategoryCoverageReport = { export type QaScorecardProfileReport = { id: string; + evidenceMode: QaScorecardEvidenceMode; categoryIds: string[]; }; @@ -332,6 +336,18 @@ export function readQaScorecardFeatureCoverageByCategory(repoRoot?: string) { ); } +export function readQaScorecardProfileOptions(profileId: string | undefined, repoRoot?: string) { + const profile = profileId?.trim(); + if (!profile) { + return { evidenceMode: "full" as const }; + } + return { + evidenceMode: + readQaMaturityTaxonomy(repoRoot)?.profiles.find((entry) => entry.id === profile) + ?.evidenceMode ?? "full", + }; +} + function pushMissingPrimaryIssues(params: { issues: QaScorecardValidationIssue[]; category: MaturityCategoryRef; @@ -467,6 +483,7 @@ export function buildQaScorecardTaxonomyReport(params: { } return { id: profile.id, + evidenceMode: profile.evidenceMode ?? "full", categoryIds: validCategoryIds, }; }) ?? []; diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 4b7beaf867a..8d56824e162 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -37,6 +37,7 @@ async function writeEvidence(pathLocal: string) { kind: "openclaw.qa.evidence-summary", schemaVersion: 2, generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", entries: [], }, null, diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index 5eebc5ce352..605cba5ca5c 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -131,6 +131,7 @@ async function runQaTestFileSuiteFromRuntime(params: { const providerMode = normalizeQaProviderMode(runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE); const primaryModel = runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode); return await runQaTestFileScenarios({ + evidenceMode: runParams?.evidenceMode, repoRoot, outputDir, providerMode, @@ -174,6 +175,11 @@ function mergeQaEvidenceSummaries(params: { kind: QA_EVIDENCE_SUMMARY_KIND, schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, generatedAt: params.generatedAt, + evidenceMode: + params.evidenceSummaries.length > 0 && + params.evidenceSummaries.every((summary) => summary.evidenceMode === "slim") + ? "slim" + : "full", entries: params.evidenceSummaries.flatMap((summary) => summary.entries), profile: profiles.length === 1 ? profiles[0] : undefined, }); diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index f62ba209a89..22c9e4583d9 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -51,6 +51,7 @@ import { type QaSeedScenarioWithSource, } from "./scenario-catalog.js"; import { runScenarioFlow } from "./scenario-flow-runner.js"; +import type { QaScorecardEvidenceMode } from "./scorecard-taxonomy.js"; import { applyQaMergePatch, collectQaSuiteGatewayConfigPatch, @@ -101,6 +102,7 @@ type QaSuiteEnvironment = { export type QaSuiteStartLabFn = (params?: QaLabServerStartParams) => Promise; export type QaSuiteRunParams = { + evidenceMode?: QaScorecardEvidenceMode; repoRoot?: string; outputDir?: string; providerMode?: QaProviderMode; @@ -615,6 +617,7 @@ export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSui } async function runQaRuntimeParitySuite(params: { + evidenceMode?: QaScorecardEvidenceMode; repoRoot: string; outputDir: string; startedAt: Date; @@ -792,6 +795,7 @@ async function runQaRuntimeParitySuite(params: { finishedAt, scenarios, scenarioDefinitions: params.selectedScenarios, + evidenceMode: params.evidenceMode, transport, providerMode: params.providerMode, primaryModel: params.primaryModel, @@ -838,6 +842,7 @@ async function writeQaSuiteArtifacts(params: { finishedAt: Date; scenarios: QaSuiteScenarioResult[]; scenarioDefinitions?: readonly QaSeedScenarioWithSource[]; + evidenceMode?: QaScorecardEvidenceMode; metrics?: QaSuiteSummaryJson["metrics"]; transport: QaTransportAdapter; // Reuse the canonical QaProviderMode union instead of re-declaring it @@ -876,6 +881,7 @@ async function writeQaSuiteArtifacts(params: { { kind: "summary", path: path.basename(summaryPath) }, { kind: "report", path: path.basename(reportPath) }, ], + evidenceMode: params.evidenceMode, channelId: params.transport.id, env: process.env, generatedAt: params.finishedAt.toISOString(), @@ -1105,6 +1111,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise